lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller,apiserver,xcecli): expose job CPU peak and efficiency to the apiserver and xcecli (bug-1736)

Browse source 
The controller computed run usage (CPU time, memory, CPU peak and
efficiency, LSF bjobs -l definitions) but the capnp Job struct had no
field for it, so the job pushed to the apiserver never carried it, and
snapshots (the same struct) dropped it: a job restored from a checkpoint
lost its accumulated usage.

- controller.capnp: Job.runUsage @13 (JobRunusage); old entries decode
  with no run usage.
- apiserver: map run_usage into JobInfo; max_mem, avg_mem, used_mem,
  cpu_time, peak_cpu, peak_cpu_time, peak_cpu_duration, mem_efficiency,
  avg_cpu_efficiency, peak_cpu_efficiency become queryable columns.
- xcecli: -o fields MAX_MEM, AVG_MEM, MEM_EFFICIENCY, CPU_PEAK,
  CPU_PEAK_DURATION, CPU_EFFICIENCY, CPU_PEAK_EFFICIENCY (CPU_TIME now
  renders too), and a MEMORY USAGE / CPU USAGE block in jobs list -l.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-06 13:54:43 +00:00
parent e2472aa73c
commit 1cedd7f791
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
9 changed files with 466 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

123
src/controller/testsuite/test_persist_snapshot_entities.c
View file

@ -739,6 +739,128 @@ static void test_snapent_checkpoint_exit_info_roundtrip(void)
snapent_teardown(src); snapent_teardown(src);
} }
/* ------------------------------------------------------------------ */
/* bug-1736: job.run_usage must round-trip through a checkpoint, every */
/* field. The capnp Job struct had no run-usage field, so a job */
/* restored from a snapshot lost its accumulated CPU/memory usage and */
/* the CPU peak/efficiency values (the WAL_JOB_RUSAGE facts behind the */
/* checkpoint are trimmed), and the same struct is what the apiserver */
/* receives, so they never reached it either. */
/* ------------------------------------------------------------------ */
static xce_job_runusage_t* snapent_make_runusage(void)
{
xce_job_runusage_t* ru = calloc(1, sizeof(*ru));
TEST_ASSERT_NOT_NULL(ru);
ru->max_mem = 512.5;
ru->avg_mem = 300.25;
ru->used_mem = 410.0;
ru->cpu_time = 1234;
ru->peak_cpu_time = 4;
ru->peak_cpu_duration = 30;
ru->mem_efficiency = 80.5f;
ru->avg_cpu_efficiency = 62.25f;
ru->peak_cpu_efficiency = 95.0f;
ru->peak_cpu = 3.75f;
ru->num_hosts = 2;
ru->host_runusage = calloc(2, sizeof(xce_host_runusage_t*));
TEST_ASSERT_NOT_NULL(ru->host_runusage);
for (int h = 0; h < 2; h++) {
xce_host_runusage_t* hr = calloc(1, sizeof(*hr));
TEST_ASSERT_NOT_NULL(hr);
hr->host_name = strdup(h == 0 ? "hostA" : "hostB");
hr->cpu_time = 600 + h;
hr->mem_usage = 200.5 + h;
hr->mem_swap = 10.0 + h;
hr->tasks_current = 3 + h;
hr->sample_time = 1790000000 + h;
hr->cpu_rate = 1.5f + (float)h;
ru->host_runusage[h] = hr;
}
return ru;
}
static void snapent_assert_runusage(const xce_job_runusage_t* ru)
{
TEST_ASSERT_NOT_NULL_MESSAGE(ru,
"bug-1736: a checkpointed job must keep its run_usage");
TEST_ASSERT_EQUAL_FLOAT((float)(512.5), (float)(ru->max_mem));
TEST_ASSERT_EQUAL_FLOAT((float)(300.25), (float)(ru->avg_mem));
TEST_ASSERT_EQUAL_FLOAT((float)(410.0), (float)(ru->used_mem));
TEST_ASSERT_EQUAL_INT(1234, ru->cpu_time);
TEST_ASSERT_EQUAL_INT(4, ru->peak_cpu_time);
TEST_ASSERT_EQUAL_INT(30, ru->peak_cpu_duration);
TEST_ASSERT_EQUAL_FLOAT(80.5f, ru->mem_efficiency);
TEST_ASSERT_EQUAL_FLOAT(62.25f, ru->avg_cpu_efficiency);
TEST_ASSERT_EQUAL_FLOAT(95.0f, ru->peak_cpu_efficiency);
TEST_ASSERT_EQUAL_FLOAT(3.75f, ru->peak_cpu);
TEST_ASSERT_EQUAL_INT(2, ru->num_hosts);
for (int h = 0; h < 2; h++) {
const xce_host_runusage_t* hr = ru->host_runusage[h];
TEST_ASSERT_NOT_NULL(hr);
TEST_ASSERT_EQUAL_STRING(h == 0 ? "hostA" : "hostB", hr->host_name);
TEST_ASSERT_EQUAL_INT(600 + h, hr->cpu_time);
TEST_ASSERT_EQUAL_FLOAT((float)(200.5 + h), (float)(hr->mem_usage));
TEST_ASSERT_EQUAL_FLOAT((float)(10.0 + h), (float)(hr->mem_swap));
TEST_ASSERT_EQUAL_INT(3 + h, hr->tasks_current);
TEST_ASSERT_EQUAL_INT64(1790000000 + h, hr->sample_time);
TEST_ASSERT_EQUAL_FLOAT(1.5f + (float)h, hr->cpu_rate);
}
}
static void test_snapent_checkpoint_run_usage_roundtrip(void)
{
const int n_jobs = 2;
snapent_fixture_t* src = snapent_make(n_jobs, 0);
TEST_ASSERT_NOT_NULL(src);
/* job 0: full run usage; job 1: none reported yet (NULL) */
job_t* j = snapent_find_job(src->c, snapent_jobid(0));
TEST_ASSERT_NOT_NULL(j);
lock(j, RTTI_LOCK_WRITE);
j->job.run_usage = snapent_make_runusage();
unlock(j);
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&src->c->persist));
uint64_t lsn = src->c->persist.checkpoint_lsn;
char path[4096];
snprintf(path, sizeof(path), "%s/snap-%016" PRIx64 ".capnp",
src->c->persist.snap_path, lsn);
/* 1. the checkpoint's own job frames carry it */
xce_job_t decoded[8];
int count = snapent_read_checkpoint_jobs(path, decoded, 8);
TEST_ASSERT_EQUAL_INT(n_jobs, count);
xce_job_t* d0 = snapent_find_decoded_job(decoded, count, snapent_jobid(0));
xce_job_t* d1 = snapent_find_decoded_job(decoded, count, snapent_jobid(1));
TEST_ASSERT_NOT_NULL(d0);
TEST_ASSERT_NOT_NULL(d1);
snapent_assert_runusage(d0->run_usage);
TEST_ASSERT_NULL_MESSAGE(d1->run_usage,
"a job with no run usage must come back with none");
snapent_free_decoded_jobs(decoded, count);
/* 2. a real persist_load_snapshot restore into an empty member */
snapent_fixture_t* dst = snapent_make(0, 0);
TEST_ASSERT_NOT_NULL(dst);
bool entities_restored = false;
TEST_ASSERT_EQUAL_INT(0, persist_load_snapshot(&dst->c->persist, path, lsn,
&entities_restored));
TEST_ASSERT_TRUE(entities_restored);
job_t* r0 = snapent_find_job(dst->c, snapent_jobid(0));
job_t* r1 = snapent_find_job(dst->c, snapent_jobid(1));
TEST_ASSERT_NOT_NULL(r0);
TEST_ASSERT_NOT_NULL(r1);
snapent_assert_runusage(r0->job.run_usage);
TEST_ASSERT_NULL(r1->job.run_usage);
snapent_teardown(dst);
snapent_teardown(src);
}
/* ------------------------------------------------------------------ */ /* ------------------------------------------------------------------ */
/* Test 3 (task 2.1, RED): concurrent per-entity mutation during the */ /* Test 3 (task 2.1, RED): concurrent per-entity mutation during the */
/* checkpoint walk must never be captured as a torn mix of fields */ /* checkpoint walk must never be captured as a torn mix of fields */
@ -2299,6 +2421,7 @@ void test_persist_snapshot_entities(void)
RUN_TEST(test_snapent_fixture_seeds_jobs_and_queues); RUN_TEST(test_snapent_fixture_seeds_jobs_and_queues);
RUN_TEST(test_snapent_checkpoint_job_content_roundtrip); RUN_TEST(test_snapent_checkpoint_job_content_roundtrip);
RUN_TEST(test_snapent_checkpoint_exit_info_roundtrip); RUN_TEST(test_snapent_checkpoint_exit_info_roundtrip);
RUN_TEST(test_snapent_checkpoint_run_usage_roundtrip);
RUN_TEST(test_snapent_checkpoint_concurrent_mutation_not_torn); RUN_TEST(test_snapent_checkpoint_concurrent_mutation_not_torn);
RUN_TEST(test_snapent_checkpoint_write_failure_leaves_no_final_file); RUN_TEST(test_snapent_checkpoint_write_failure_leaves_no_final_file);
RUN_TEST(test_snapent_checkpoint_mid_encode_failure_discards_whole_file); RUN_TEST(test_snapent_checkpoint_mid_encode_failure_discards_whole_file);

14
src/frontend/apiserver/backend/convert.go
View file

@ -114,6 +114,20 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
} }
r.submit = rd r.submit = rd
} }
if u := job.run_usage; u != nil {
r.runUsage = &rawRunUsage{
maxMem: float64(u.max_mem),
avgMem: float64(u.avg_mem),
usedMem: float64(u.used_mem),
cpuTime: int(u.cpu_time),
peakCPUTime: int(u.peak_cpu_time),
peakCPUDuration: int(u.peak_cpu_duration),
memEfficiency: float64(u.mem_efficiency),
avgCPUEfficiency: float64(u.avg_cpu_efficiency),
peakCPUEfficiency: float64(u.peak_cpu_efficiency),
peakCPU: float64(u.peak_cpu),
}
}
if job.allocs != nil { if job.allocs != nil {
n := int(job.allocs.nhosts) n := int(job.allocs.nhosts)
for i := 0; i < n; i++ { for i := 0; i < n; i++ {

30
src/frontend/apiserver/backend/convert_pure.go
View file

@ -263,6 +263,23 @@ type rawHostSlot struct {
slots int slots int
} }
// rawRunUsage mirrors the C xce_job_runusage_t job-level fields used by the
// Go model (bug-1736): accumulated CPU time and memory, and the CPU peak and
// efficiency values the controller computes with LSF bjobs -l definitions
// (efficiencies in percent, memory in MB as the worker reports it).
type rawRunUsage struct {
maxMem float64
avgMem float64
usedMem float64
cpuTime int
peakCPUTime int
peakCPUDuration int
memEfficiency float64
avgCPUEfficiency float64
peakCPUEfficiency float64
peakCPU float64
}
// rawJobData mirrors the C xce_job_t fields used by the Go model. // rawJobData mirrors the C xce_job_t fields used by the Go model.
type rawJobData struct { type rawJobData struct {
jobID int64 jobID int64
@ -274,6 +291,7 @@ type rawJobData struct {
finishTime int64 finishTime int64
submit *rawSubmitData submit *rawSubmitData
execHosts []rawHostSlot execHosts []rawHostSlot
runUsage *rawRunUsage // nil until the job's first usage report
} }
// rawWorkerData mirrors the C xce_worker_t fields used by the Go model. // rawWorkerData mirrors the C xce_worker_t fields used by the Go model.
@ -363,6 +381,18 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)}) data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)})
info.ExecHosts = models.Jsonb{String: string(data)} info.ExecHosts = models.Jsonb{String: string(data)}
} }
if u := r.runUsage; u != nil {
info.MaxMem = u.maxMem
info.AvgMem = u.avgMem
info.UsedMem = u.usedMem
info.CPUTime = u.cpuTime
info.PeakCPU = u.peakCPU
info.PeakCPUTime = u.peakCPUTime
info.PeakCPUDuration = float64(u.peakCPUDuration)
info.MemEfficiency = u.memEfficiency
info.AvgCPUEfficiency = u.avgCPUEfficiency
info.PeakCPUEfficiency = u.peakCPUEfficiency
}
return info return info
} }

66
src/frontend/apiserver/backend/convert_test.go
View file

@ -5,6 +5,8 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"reflect"
"strings"
"testing" "testing"
) )
@ -1185,3 +1187,67 @@ func TestRawClusterToModel(t *testing.T) {
}) })
} }
} }
// bug-1736: the controller's accumulated run usage (CPU time, memory, CPU
// peak and efficiency, LSF bjobs -l definitions) reaches the job model, so
// REST field selection and xcecli can show it.
func TestRawJobToModel_RunUsage(t *testing.T) {
r := rawJobData{
jobID: 42,
state: 4,
runUsage: &rawRunUsage{
maxMem: 512.5,
avgMem: 300.25,
usedMem: 410,
cpuTime: 1234,
peakCPUTime: 4,
peakCPUDuration: 30,
memEfficiency: 80.5,
avgCPUEfficiency: 62.25,
peakCPUEfficiency: 95,
peakCPU: 3.75,
},
}
info := rawJobToModel(r)
if info.MaxMem != 512.5 || info.AvgMem != 300.25 || info.UsedMem != 410 {
t.Errorf("memory: got max=%v avg=%v used=%v", info.MaxMem, info.AvgMem, info.UsedMem)
}
if info.CPUTime != 1234 {
t.Errorf("CPUTime = %d, want 1234", info.CPUTime)
}
if info.PeakCPU != 3.75 || info.PeakCPUTime != 4 || info.PeakCPUDuration != 30 {
t.Errorf("CPU peak: got peak=%v peak_time=%d duration=%v",
info.PeakCPU, info.PeakCPUTime, info.PeakCPUDuration)
}
if info.MemEfficiency != 80.5 || info.AvgCPUEfficiency != 62.25 || info.PeakCPUEfficiency != 95 {
t.Errorf("efficiency: got mem=%v avg_cpu=%v peak_cpu=%v",
info.MemEfficiency, info.AvgCPUEfficiency, info.PeakCPUEfficiency)
}
}
// bug-1736: a job with no run usage reported yet maps to zero values.
func TestRawJobToModel_NoRunUsage(t *testing.T) {
info := rawJobToModel(rawJobData{jobID: 42})
if info.CPUTime != 0 || info.PeakCPU != 0 || info.MemEfficiency != 0 || info.MaxMem != 0 {
t.Errorf("expected zero run usage, got %+v", info)
}
}
// bug-1736: the run-usage fields are JobInfo columns, so a REST query can
// select them (store rows are built from JobInfo's json tags).
func TestJobInfoExposesRunUsageColumns(t *testing.T) {
cols := map[string]bool{}
ty := reflect.TypeOf(models.JobInfo{})
for i := 0; i < ty.NumField(); i++ {
cols[strings.Split(ty.Field(i).Tag.Get("json"), ",")[0]] = true
}
for _, c := range []string{"max_mem", "avg_mem", "used_mem", "cpu_time",
"peak_cpu", "peak_cpu_time", "peak_cpu_duration", "mem_efficiency",
"avg_cpu_efficiency", "peak_cpu_efficiency"} {
if !cols[c] {
t.Errorf("JobInfo has no %q column", c)
}
}
}

66
src/frontend/apiserver/models/jobs.go
View file

@ -546,18 +546,29 @@ type JobInfo struct {
ReserveTime int64 `json:"reserve_time"` ReserveTime int64 `json:"reserve_time"`
Duration int `json:"duration"` Duration int `json:"duration"`
CPUTime int `json:"cpu_time"` CPUTime int `json:"cpu_time"`
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte // Run usage (bug-1736), from the controller's accumulated job usage with
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte // LSF bjobs -l definitions; zero until the first usage report.
ExecUID int `json:"exec_uid"` MaxMem float64 `json:"max_mem"` // MB
ExecHome string `json:"exec_home"` AvgMem float64 `json:"avg_mem"` // MB
ExecCWD string `json:"exec_cwd"` UsedMem float64 `json:"used_mem"` // MB, latest
ExecUser string `json:"exec_user"` //name change PeakCPU float64 `json:"peak_cpu"` // highest cores used at once
ChargedSaap string `json:"charged_saap"` PeakCPUTime int `json:"peak_cpu_time"` // peak_cpu rounded
RunTime int `json:"run_time"` PeakCPUDuration float64 `json:"peak_cpu_duration"` // seconds, the interval that set the peak
RunTimeEstimation int `json:"run_time_estimation"` MemEfficiency float64 `json:"mem_efficiency"` // percent of the rusage[mem] request
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // percent, cpu_time / elapsed / slots
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // percent, peak_cpu / slots
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
ExecUID int `json:"exec_uid"`
ExecHome string `json:"exec_home"`
ExecCWD string `json:"exec_cwd"`
ExecUser string `json:"exec_user"` //name change
ChargedSaap string `json:"charged_saap"`
RunTime int `json:"run_time"`
RunTimeEstimation int `json:"run_time_estimation"`
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
} }
type JobExecHosts struct { type JobExecHosts struct {
@ -598,25 +609,18 @@ type JobAdditionalFields struct {
FromHost string `json:"from_host"` FromHost string `json:"from_host"`
ExecHosts string `json:"exec_hosts"` ExecHosts string `json:"exec_hosts"`
StartTime int64 `json:"start_time"` StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"` EndTime int64 `json:"end_time"`
JobRusage Jsonb `json:"job_rusage"` JobRusage Jsonb `json:"job_rusage"`
MaxMem float64 `json:"max_mem"` // max_mem/avg_mem/used_mem/cpu_time/peak_cpu*/..._efficiency are
AvgMem float64 `json:"avg_mem"` // JobInfo columns now (bug-1736).
UsedMem float64 `json:"used_mem"` NumHosts int `json:"num_hosts"` // Number of hosts used by the job
CPUTime int `json:"cpu_time"` FirstHost string `json:"first_host"` // First host where the job ran
PeakCPUTime int `json:"peak_cpu_time"` AllocSlot string `json:"alloc_slots"` // Allocated slot information
PeakCPUDuration float64 `json:"peak_cpu_duration"` PIDInfo Jsonb `json:"pid_info"`
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage Pids []int `json:"pids"` // List of process IDs associated with the job
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage Pgids []int `json:"pgids"` // List of process group IDs associated with the job
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage InteractiveJobURL string `json:"interactive_job_url"`
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
FirstHost string `json:"first_host"` // First host where the job ran
AllocSlot string `json:"alloc_slots"` // Allocated slot information
PIDInfo Jsonb `json:"pid_info"`
Pids []int `json:"pids"` // List of process IDs associated with the job
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
InteractiveJobURL string `json:"interactive_job_url"`
// CoreLimit/MemLimit: JobInfo has no equivalent field (these are // CoreLimit/MemLimit: JobInfo has no equivalent field (these are
// submission-time resource limits, not job-state fields), but xcecli's // submission-time resource limits, not job-state fields), but xcecli's
// JobFieldsMap accepts them as valid -o/-l display fields (bug-1674) -- // JobFieldsMap accepts them as valid -o/-l display fields (bug-1674) --

68
src/frontend/xcecli/common/output/page_print_jobs.go
View file

@ -103,6 +103,10 @@ func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
width = len(fmt.Sprintf("%f", job.CoreLimit)) width = len(fmt.Sprintf("%f", job.CoreLimit))
case models.HEADER_JOB_MEM_LIMIT: case models.HEADER_JOB_MEM_LIMIT:
width = len(fmt.Sprintf("%f", job.MemLimit)) width = len(fmt.Sprintf("%f", job.MemLimit))
default:
if v, ok := jobRunUsageColumn(format.name, job); ok {
width = len(v)
}
} }
if !p.wideFormat && width > 15 { if !p.wideFormat && width > 15 {
@ -237,7 +241,11 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
case models.HEADER_JOB_MEM_LIMIT: case models.HEADER_JOB_MEM_LIMIT:
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.MemLimit)), "-") mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.MemLimit)), "-")
default: default:
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*v", width, "-"), "-") v, ok := jobRunUsageColumn(format.name, job)
if !ok {
v = "-"
}
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, v), "-")
} }
} }
rows = append(rows, mainRow) rows = append(rows, mainRow)
@ -409,6 +417,9 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
} }
PrintWrappedLine(w, endLine, 80, 18) PrintWrappedLine(w, endLine, 80, 18)
for _, l := range jobResourceUsageLines(job) {
fmt.Fprintln(w, l)
}
fmt.Println("") fmt.Println("")
// print the scheduling parameters table // print the scheduling parameters table
fmt.Printf("SCHEDULING PARAMETERS\n") fmt.Printf("SCHEDULING PARAMETERS\n")
@ -443,3 +454,58 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
} }
} }
} }
// jobResourceUsageLines renders a started job's memory and CPU usage in the
// LSF bjobs -l shape (bug-1736): MAX/AVG MEM in Mbytes and MEM EFFICIENCY
// against the rusage[mem] request; CPU PEAK in cores with the interval that
// set it; average and peak CPU efficiency against the allocated slots. A job
// that has not started (PEND/PSUSP) has nothing to report.
func jobResourceUsageLines(job models.QueryJob) []string {
if job.JobStatus == "" || job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
return nil
}
return []string{
"",
" MEMORY USAGE:",
fmt.Sprintf(" MAX MEM: %.0f Mbytes; AVG MEM: %.0f Mbytes; MEM EFFICIENCY: %.2f%%",
job.MaxMem, job.AvgMem, job.MemEfficiency),
"",
" CPU USAGE:",
fmt.Sprintf(" CPU PEAK: %.2f ; CPU PEAK DURATION: %.0f second(s)",
job.PeakCPU, job.PeakCPUDuration),
fmt.Sprintf(" CPU AVERAGE EFFICIENCY: %.2f%% ; CPU PEAK EFFICIENCY: %.2f%%",
job.AvgCPUEfficiency, job.PeakCPUEfficiency),
}
}
// jobRunUsageColumn formats one run-usage column of `jobs list -o` (bug-1736),
// in the units of the -l view: memory in Mbytes, CPU peak in cores, durations
// in seconds, efficiencies in percent. ok is false for any other column. A
// job that has not started shows "-".
func jobRunUsageColumn(header string, job models.QueryJob) (string, bool) {
var v string
switch header {
case models.HEADER_JOB_CPU_TIME:
v = fmt.Sprintf("%d", job.CPUTime)
case models.HEADER_JOB_MAX_MEM:
v = fmt.Sprintf("%.0f", job.MaxMem)
case models.HEADER_JOB_AVG_MEM:
v = fmt.Sprintf("%.0f", job.AvgMem)
case models.HEADER_JOB_MEM_EFFICIENCY:
v = fmt.Sprintf("%.2f%%", job.MemEfficiency)
case models.HEADER_JOB_CPU_PEAK:
v = fmt.Sprintf("%.2f", job.PeakCPU)
case models.HEADER_JOB_CPU_PEAK_DURATION:
v = fmt.Sprintf("%.0f", job.PeakCPUDuration)
case models.HEADER_JOB_CPU_EFFICIENCY:
v = fmt.Sprintf("%.2f%%", job.AvgCPUEfficiency)
case models.HEADER_JOB_CPU_PEAK_EFFICIENCY:
v = fmt.Sprintf("%.2f%%", job.PeakCPUEfficiency)
default:
return "", false
}
if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
v = "-"
}
return v, true
}

90
src/frontend/xcecli/common/output/page_print_jobs_test.go Normal file
View file

@ -0,0 +1,90 @@
package output
import (
"strings"
"testing"
"xcecli/models"
)
// bug-1736: `xcecli jobs list -l` shows the job's memory and CPU usage in the
// LSF bjobs -l shape once the job has started.
func TestJobResourceUsageLines_StartedJob(t *testing.T) {
job := models.QueryJob{
JobStatus: "RUN",
MaxMem: 512,
AvgMem: 300.4,
MemEfficiency: 80.5,
PeakCPU: 3.75,
PeakCPUDuration: 30,
AvgCPUEfficiency: 62.25,
PeakCPUEfficiency: 95,
}
got := strings.Join(jobResourceUsageLines(job), "\n")
for _, want := range []string{
"MEMORY USAGE:",
"MAX MEM: 512 Mbytes; AVG MEM: 300 Mbytes; MEM EFFICIENCY: 80.50%",
"CPU USAGE:",
"CPU PEAK: 3.75 ; CPU PEAK DURATION: 30 second(s)",
"CPU AVERAGE EFFICIENCY: 62.25% ; CPU PEAK EFFICIENCY: 95.00%",
} {
if !strings.Contains(got, want) {
t.Errorf("missing %q in:\n%s", want, got)
}
}
}
// A job that has not started has no usage to report.
func TestJobResourceUsageLines_PendingJob(t *testing.T) {
for _, st := range []string{"PEND", "PSUSP"} {
if lines := jobResourceUsageLines(models.QueryJob{JobStatus: st}); len(lines) != 0 {
t.Errorf("%s job: expected no usage lines, got %q", st, lines)
}
}
}
// bug-1736: the usage fields can be selected with -o and are requested by -l.
func TestJobFieldsMapIncludesRunUsage(t *testing.T) {
have := map[string]bool{}
for _, f := range models.JobFieldsMap {
have[f] = true
}
for _, f := range []string{"max_mem", "avg_mem", "mem_efficiency",
"peak_cpu", "peak_cpu_duration", "avg_cpu_efficiency",
"peak_cpu_efficiency"} {
if !have[f] {
t.Errorf("JobFieldsMap has no field %q", f)
}
}
}
// bug-1736: the run-usage columns render in `jobs list -o` (they printed "-").
func TestJobRunUsageColumn(t *testing.T) {
job := models.QueryJob{
JobStatus: "RUN", CPUTime: 1234, MaxMem: 512, AvgMem: 300.4,
MemEfficiency: 80.5, PeakCPU: 3.75, PeakCPUDuration: 30,
AvgCPUEfficiency: 62.25, PeakCPUEfficiency: 95,
}
cases := map[string]string{
models.HEADER_JOB_CPU_TIME: "1234",
models.HEADER_JOB_MAX_MEM: "512",
models.HEADER_JOB_AVG_MEM: "300",
models.HEADER_JOB_MEM_EFFICIENCY: "80.50%",
models.HEADER_JOB_CPU_PEAK: "3.75",
models.HEADER_JOB_CPU_PEAK_DURATION: "30",
models.HEADER_JOB_CPU_EFFICIENCY: "62.25%",
models.HEADER_JOB_CPU_PEAK_EFFICIENCY: "95.00%",
}
for h, want := range cases {
got, ok := jobRunUsageColumn(h, job)
if !ok || got != want {
t.Errorf("%s: got %q (ok=%v), want %q", h, got, ok, want)
}
}
if _, ok := jobRunUsageColumn(models.HEADER_JOB_USER, job); ok {
t.Errorf("USER is not a run-usage column")
}
// a job that has not started shows "-", not zeros
if got, _ := jobRunUsageColumn(models.HEADER_JOB_CPU_PEAK, models.QueryJob{JobStatus: "PEND"}); got != "-" {
t.Errorf("PEND job CPU_PEAK: got %q, want -", got)
}
}

64
src/frontend/xcecli/models/jobs.go
View file

@ -31,6 +31,14 @@ const (
HEADER_JOB_NUM_PROCESSORS = "NUM_PROCESSORS" HEADER_JOB_NUM_PROCESSORS = "NUM_PROCESSORS"
HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL" HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL"
HEADER_JOB_PENDING_REASON = "PENDING_REASON" HEADER_JOB_PENDING_REASON = "PENDING_REASON"
// run usage (bug-1736)
HEADER_JOB_MAX_MEM = "MAX_MEM"
HEADER_JOB_AVG_MEM = "AVG_MEM"
HEADER_JOB_MEM_EFFICIENCY = "MEM_EFFICIENCY"
HEADER_JOB_CPU_PEAK = "CPU_PEAK"
HEADER_JOB_CPU_PEAK_DURATION = "CPU_PEAK_DURATION"
HEADER_JOB_CPU_EFFICIENCY = "CPU_EFFICIENCY"
HEADER_JOB_CPU_PEAK_EFFICIENCY = "CPU_PEAK_EFFICIENCY"
// second line // second line
HEADER_JOB_R15S = "r15s" HEADER_JOB_R15S = "r15s"
@ -120,6 +128,7 @@ type QueryJob struct {
AvgMem float64 `json:"avg_mem"` AvgMem float64 `json:"avg_mem"`
UsedMem float64 `json:"used_mem"` UsedMem float64 `json:"used_mem"`
CPUTime int `json:"cpu_time"` CPUTime int `json:"cpu_time"`
PeakCPU float64 `json:"peak_cpu"` // highest cores used at once (bug-1736)
PeakCPUTime int `json:"peak_cpu_time"` PeakCPUTime int `json:"peak_cpu_time"`
PeakCPUDuration float64 `json:"peak_cpu_duration"` PeakCPUDuration float64 `json:"peak_cpu_duration"`
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage
@ -184,30 +193,37 @@ type QueryJob struct {
} }
var JobFieldsMap = map[string]string{ var JobFieldsMap = map[string]string{
HEADER_JOB_JOBID: "job_id", HEADER_JOB_JOBID: "job_id",
HEADER_JOB_USER: "submit_user", HEADER_JOB_USER: "submit_user",
HEADER_JOB_STATUS: "job_status", HEADER_JOB_STATUS: "job_status",
HEADER_JOB_QUEUE: "queue_name", HEADER_JOB_QUEUE: "queue_name",
HEADER_JOB_COMMAND: "command", HEADER_JOB_COMMAND: "command",
HEADER_JOB_NAME: "job_name", HEADER_JOB_NAME: "job_name",
HEADER_JOB_SUBMIT_TIME: "submit_time", HEADER_JOB_SUBMIT_TIME: "submit_time",
HEADER_JOB_FROM_HOST: "from_host", HEADER_JOB_FROM_HOST: "from_host",
HEADER_JOB_EXEC_HOST: "exec_hosts", HEADER_JOB_EXEC_HOST: "exec_hosts",
HEADER_JOB_CORE_LIMIT: "core_limit", HEADER_JOB_CORE_LIMIT: "core_limit",
HEADER_JOB_MEM_LIMIT: "mem_limit", HEADER_JOB_MEM_LIMIT: "mem_limit",
HEADER_JOB_PID: "pids", HEADER_JOB_PID: "pids",
HEADER_JOB_PGID: "pgids", HEADER_JOB_PGID: "pgids",
HEADER_JOB_PROJECT_NAME: "project_name", HEADER_JOB_PROJECT_NAME: "project_name",
HEADER_JOB_START_TIME: "start_time", HEADER_JOB_START_TIME: "start_time",
HEADER_JOB_END_TIME: "end_time", HEADER_JOB_END_TIME: "end_time",
HEADER_JOB_CWD: "cwd", HEADER_JOB_CWD: "cwd",
HEADER_JOB_CPU_TIME: "cpu_time", HEADER_JOB_CPU_TIME: "cpu_time",
HEADER_JOB_RESREQ: "resreq", HEADER_JOB_RESREQ: "resreq",
HEADER_JOB_DESCRIPTION: "job_description", HEADER_JOB_DESCRIPTION: "job_description",
HEADER_JOB_LAUNCH_TYPE: "launch_type", HEADER_JOB_LAUNCH_TYPE: "launch_type",
HEADER_JOB_NUM_PROCESSORS: "num_processors", HEADER_JOB_NUM_PROCESSORS: "num_processors",
HEADER_JOB_INTERACTIVE_URL: "interactive_job_url", HEADER_JOB_INTERACTIVE_URL: "interactive_job_url",
HEADER_JOB_PENDING_REASON: "pending_reason", HEADER_JOB_PENDING_REASON: "pending_reason",
HEADER_JOB_MAX_MEM: "max_mem",
HEADER_JOB_AVG_MEM: "avg_mem",
HEADER_JOB_MEM_EFFICIENCY: "mem_efficiency",
HEADER_JOB_CPU_PEAK: "peak_cpu",
HEADER_JOB_CPU_PEAK_DURATION: "peak_cpu_duration",
HEADER_JOB_CPU_EFFICIENCY: "avg_cpu_efficiency",
HEADER_JOB_CPU_PEAK_EFFICIENCY: "peak_cpu_efficiency",
//job_filter.HEADER_JOB_PRIORITY: "job_priority", //job_filter.HEADER_JOB_PRIORITY: "job_priority",
} }

1
src/schema/controller.capnp
View file

@ -338,6 +338,7 @@ struct Job $C.mapname("xce_job_t"){
pendreason @10: JobPendReason $C.mapname("pend_reasons"); pendreason @10: JobPendReason $C.mapname("pend_reasons");
transientSince @11: UInt64 = 0 $C.mapname("transient_since"); # epoch seconds; set when job enters JOB_STAT_TRANSIENT transientSince @11: UInt64 = 0 $C.mapname("transient_since"); # epoch seconds; set when job enters JOB_STAT_TRANSIENT
exitInfo @12: JobExitInfo $C.mapname("exit_info"); # bug-1867: null until the job's exit code is known exitInfo @12: JobExitInfo $C.mapname("exit_info"); # bug-1867: null until the job's exit code is known
runUsage @13: JobRunusage $C.mapname("run_usage"); # bug-1736: null until the first usage report; snapshot + apiserver
} }
# bug-1867: job_exitinfo_t, so a checkpointed Job keeps its exit code. The # bug-1867: job_exitinfo_t, so a checkpointed Job keeps its exit code. The