From 1cedd7f79103eed8ed3765aa834d5c1daf6e0dd1 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 6 Oct 2026 13:54:43 +0000 Subject: [PATCH] fix(controller,apiserver,xcecli): expose job CPU peak and efficiency to the apiserver and xcecli (bug-1736) The controller computed run usage (CPU time, memory, CPU peak and efficiency, LSF bjobs -l definitions) but the capnp Job struct had no field for it, so the job pushed to the apiserver never carried it, and snapshots (the same struct) dropped it: a job restored from a checkpoint lost its accumulated usage. - controller.capnp: Job.runUsage @13 (JobRunusage); old entries decode with no run usage. - apiserver: map run_usage into JobInfo; max_mem, avg_mem, used_mem, cpu_time, peak_cpu, peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency become queryable columns. - xcecli: -o fields MAX_MEM, AVG_MEM, MEM_EFFICIENCY, CPU_PEAK, CPU_PEAK_DURATION, CPU_EFFICIENCY, CPU_PEAK_EFFICIENCY (CPU_TIME now renders too), and a MEMORY USAGE / CPU USAGE block in jobs list -l. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../test_persist_snapshot_entities.c | 123 ++++++++++++++++++ src/frontend/apiserver/backend/convert.go | 14 ++ .../apiserver/backend/convert_pure.go | 30 +++++ .../apiserver/backend/convert_test.go | 66 ++++++++++ src/frontend/apiserver/models/jobs.go | 66 +++++----- .../xcecli/common/output/page_print_jobs.go | 68 +++++++++- .../common/output/page_print_jobs_test.go | 90 +++++++++++++ src/frontend/xcecli/models/jobs.go | 64 +++++---- src/schema/controller.capnp | 1 + 9 files changed, 466 insertions(+), 56 deletions(-) create mode 100644 src/frontend/xcecli/common/output/page_print_jobs_test.go diff --git a/src/controller/testsuite/test_persist_snapshot_entities.c b/src/controller/testsuite/test_persist_snapshot_entities.c index 19d543ee..ea8e8d42 100644 --- a/src/controller/testsuite/test_persist_snapshot_entities.c +++ b/src/controller/testsuite/test_persist_snapshot_entities.c @@ -739,6 +739,128 @@ static void test_snapent_checkpoint_exit_info_roundtrip(void) snapent_teardown(src); } + +/* ------------------------------------------------------------------ */ +/* bug-1736: job.run_usage must round-trip through a checkpoint, every */ +/* field. The capnp Job struct had no run-usage field, so a job */ +/* restored from a snapshot lost its accumulated CPU/memory usage and */ +/* the CPU peak/efficiency values (the WAL_JOB_RUSAGE facts behind the */ +/* checkpoint are trimmed), and the same struct is what the apiserver */ +/* receives, so they never reached it either. */ +/* ------------------------------------------------------------------ */ + +static xce_job_runusage_t* snapent_make_runusage(void) +{ + xce_job_runusage_t* ru = calloc(1, sizeof(*ru)); + TEST_ASSERT_NOT_NULL(ru); + ru->max_mem = 512.5; + ru->avg_mem = 300.25; + ru->used_mem = 410.0; + ru->cpu_time = 1234; + ru->peak_cpu_time = 4; + ru->peak_cpu_duration = 30; + ru->mem_efficiency = 80.5f; + ru->avg_cpu_efficiency = 62.25f; + ru->peak_cpu_efficiency = 95.0f; + ru->peak_cpu = 3.75f; + ru->num_hosts = 2; + ru->host_runusage = calloc(2, sizeof(xce_host_runusage_t*)); + TEST_ASSERT_NOT_NULL(ru->host_runusage); + for (int h = 0; h < 2; h++) { + xce_host_runusage_t* hr = calloc(1, sizeof(*hr)); + TEST_ASSERT_NOT_NULL(hr); + hr->host_name = strdup(h == 0 ? "hostA" : "hostB"); + hr->cpu_time = 600 + h; + hr->mem_usage = 200.5 + h; + hr->mem_swap = 10.0 + h; + hr->tasks_current = 3 + h; + hr->sample_time = 1790000000 + h; + hr->cpu_rate = 1.5f + (float)h; + ru->host_runusage[h] = hr; + } + return ru; +} + +static void snapent_assert_runusage(const xce_job_runusage_t* ru) +{ + TEST_ASSERT_NOT_NULL_MESSAGE(ru, + "bug-1736: a checkpointed job must keep its run_usage"); + TEST_ASSERT_EQUAL_FLOAT((float)(512.5), (float)(ru->max_mem)); + TEST_ASSERT_EQUAL_FLOAT((float)(300.25), (float)(ru->avg_mem)); + TEST_ASSERT_EQUAL_FLOAT((float)(410.0), (float)(ru->used_mem)); + TEST_ASSERT_EQUAL_INT(1234, ru->cpu_time); + TEST_ASSERT_EQUAL_INT(4, ru->peak_cpu_time); + TEST_ASSERT_EQUAL_INT(30, ru->peak_cpu_duration); + TEST_ASSERT_EQUAL_FLOAT(80.5f, ru->mem_efficiency); + TEST_ASSERT_EQUAL_FLOAT(62.25f, ru->avg_cpu_efficiency); + TEST_ASSERT_EQUAL_FLOAT(95.0f, ru->peak_cpu_efficiency); + TEST_ASSERT_EQUAL_FLOAT(3.75f, ru->peak_cpu); + TEST_ASSERT_EQUAL_INT(2, ru->num_hosts); + for (int h = 0; h < 2; h++) { + const xce_host_runusage_t* hr = ru->host_runusage[h]; + TEST_ASSERT_NOT_NULL(hr); + TEST_ASSERT_EQUAL_STRING(h == 0 ? "hostA" : "hostB", hr->host_name); + TEST_ASSERT_EQUAL_INT(600 + h, hr->cpu_time); + TEST_ASSERT_EQUAL_FLOAT((float)(200.5 + h), (float)(hr->mem_usage)); + TEST_ASSERT_EQUAL_FLOAT((float)(10.0 + h), (float)(hr->mem_swap)); + TEST_ASSERT_EQUAL_INT(3 + h, hr->tasks_current); + TEST_ASSERT_EQUAL_INT64(1790000000 + h, hr->sample_time); + TEST_ASSERT_EQUAL_FLOAT(1.5f + (float)h, hr->cpu_rate); + } +} + +static void test_snapent_checkpoint_run_usage_roundtrip(void) +{ + const int n_jobs = 2; + + snapent_fixture_t* src = snapent_make(n_jobs, 0); + TEST_ASSERT_NOT_NULL(src); + + /* job 0: full run usage; job 1: none reported yet (NULL) */ + job_t* j = snapent_find_job(src->c, snapent_jobid(0)); + TEST_ASSERT_NOT_NULL(j); + lock(j, RTTI_LOCK_WRITE); + j->job.run_usage = snapent_make_runusage(); + unlock(j); + + TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&src->c->persist)); + + uint64_t lsn = src->c->persist.checkpoint_lsn; + char path[4096]; + snprintf(path, sizeof(path), "%s/snap-%016" PRIx64 ".capnp", + src->c->persist.snap_path, lsn); + + /* 1. the checkpoint's own job frames carry it */ + xce_job_t decoded[8]; + int count = snapent_read_checkpoint_jobs(path, decoded, 8); + TEST_ASSERT_EQUAL_INT(n_jobs, count); + xce_job_t* d0 = snapent_find_decoded_job(decoded, count, snapent_jobid(0)); + xce_job_t* d1 = snapent_find_decoded_job(decoded, count, snapent_jobid(1)); + TEST_ASSERT_NOT_NULL(d0); + TEST_ASSERT_NOT_NULL(d1); + snapent_assert_runusage(d0->run_usage); + TEST_ASSERT_NULL_MESSAGE(d1->run_usage, + "a job with no run usage must come back with none"); + snapent_free_decoded_jobs(decoded, count); + + /* 2. a real persist_load_snapshot restore into an empty member */ + snapent_fixture_t* dst = snapent_make(0, 0); + TEST_ASSERT_NOT_NULL(dst); + bool entities_restored = false; + TEST_ASSERT_EQUAL_INT(0, persist_load_snapshot(&dst->c->persist, path, lsn, + &entities_restored)); + TEST_ASSERT_TRUE(entities_restored); + + job_t* r0 = snapent_find_job(dst->c, snapent_jobid(0)); + job_t* r1 = snapent_find_job(dst->c, snapent_jobid(1)); + TEST_ASSERT_NOT_NULL(r0); + TEST_ASSERT_NOT_NULL(r1); + snapent_assert_runusage(r0->job.run_usage); + TEST_ASSERT_NULL(r1->job.run_usage); + + snapent_teardown(dst); + snapent_teardown(src); +} /* ------------------------------------------------------------------ */ /* Test 3 (task 2.1, RED): concurrent per-entity mutation during the */ /* checkpoint walk must never be captured as a torn mix of fields */ @@ -2299,6 +2421,7 @@ void test_persist_snapshot_entities(void) RUN_TEST(test_snapent_fixture_seeds_jobs_and_queues); RUN_TEST(test_snapent_checkpoint_job_content_roundtrip); RUN_TEST(test_snapent_checkpoint_exit_info_roundtrip); + RUN_TEST(test_snapent_checkpoint_run_usage_roundtrip); RUN_TEST(test_snapent_checkpoint_concurrent_mutation_not_torn); RUN_TEST(test_snapent_checkpoint_write_failure_leaves_no_final_file); RUN_TEST(test_snapent_checkpoint_mid_encode_failure_discards_whole_file); diff --git a/src/frontend/apiserver/backend/convert.go b/src/frontend/apiserver/backend/convert.go index b0cb999a..9862c85a 100644 --- a/src/frontend/apiserver/backend/convert.go +++ b/src/frontend/apiserver/backend/convert.go @@ -114,6 +114,20 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo { } r.submit = rd } + if u := job.run_usage; u != nil { + r.runUsage = &rawRunUsage{ + maxMem: float64(u.max_mem), + avgMem: float64(u.avg_mem), + usedMem: float64(u.used_mem), + cpuTime: int(u.cpu_time), + peakCPUTime: int(u.peak_cpu_time), + peakCPUDuration: int(u.peak_cpu_duration), + memEfficiency: float64(u.mem_efficiency), + avgCPUEfficiency: float64(u.avg_cpu_efficiency), + peakCPUEfficiency: float64(u.peak_cpu_efficiency), + peakCPU: float64(u.peak_cpu), + } + } if job.allocs != nil { n := int(job.allocs.nhosts) for i := 0; i < n; i++ { diff --git a/src/frontend/apiserver/backend/convert_pure.go b/src/frontend/apiserver/backend/convert_pure.go index 163402d4..88595198 100644 --- a/src/frontend/apiserver/backend/convert_pure.go +++ b/src/frontend/apiserver/backend/convert_pure.go @@ -263,6 +263,23 @@ type rawHostSlot struct { slots int } +// rawRunUsage mirrors the C xce_job_runusage_t job-level fields used by the +// Go model (bug-1736): accumulated CPU time and memory, and the CPU peak and +// efficiency values the controller computes with LSF bjobs -l definitions +// (efficiencies in percent, memory in MB as the worker reports it). +type rawRunUsage struct { + maxMem float64 + avgMem float64 + usedMem float64 + cpuTime int + peakCPUTime int + peakCPUDuration int + memEfficiency float64 + avgCPUEfficiency float64 + peakCPUEfficiency float64 + peakCPU float64 +} + // rawJobData mirrors the C xce_job_t fields used by the Go model. type rawJobData struct { jobID int64 @@ -274,6 +291,7 @@ type rawJobData struct { finishTime int64 submit *rawSubmitData execHosts []rawHostSlot + runUsage *rawRunUsage // nil until the job's first usage report } // rawWorkerData mirrors the C xce_worker_t fields used by the Go model. @@ -363,6 +381,18 @@ func rawJobToModel(r rawJobData) *models.JobInfo { data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)}) info.ExecHosts = models.Jsonb{String: string(data)} } + if u := r.runUsage; u != nil { + info.MaxMem = u.maxMem + info.AvgMem = u.avgMem + info.UsedMem = u.usedMem + info.CPUTime = u.cpuTime + info.PeakCPU = u.peakCPU + info.PeakCPUTime = u.peakCPUTime + info.PeakCPUDuration = float64(u.peakCPUDuration) + info.MemEfficiency = u.memEfficiency + info.AvgCPUEfficiency = u.avgCPUEfficiency + info.PeakCPUEfficiency = u.peakCPUEfficiency + } return info } diff --git a/src/frontend/apiserver/backend/convert_test.go b/src/frontend/apiserver/backend/convert_test.go index d212cf21..d8446bd0 100644 --- a/src/frontend/apiserver/backend/convert_test.go +++ b/src/frontend/apiserver/backend/convert_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "reflect" + "strings" "testing" ) @@ -1185,3 +1187,67 @@ func TestRawClusterToModel(t *testing.T) { }) } } + +// bug-1736: the controller's accumulated run usage (CPU time, memory, CPU +// peak and efficiency, LSF bjobs -l definitions) reaches the job model, so +// REST field selection and xcecli can show it. +func TestRawJobToModel_RunUsage(t *testing.T) { + r := rawJobData{ + jobID: 42, + state: 4, + runUsage: &rawRunUsage{ + maxMem: 512.5, + avgMem: 300.25, + usedMem: 410, + cpuTime: 1234, + peakCPUTime: 4, + peakCPUDuration: 30, + memEfficiency: 80.5, + avgCPUEfficiency: 62.25, + peakCPUEfficiency: 95, + peakCPU: 3.75, + }, + } + + info := rawJobToModel(r) + + if info.MaxMem != 512.5 || info.AvgMem != 300.25 || info.UsedMem != 410 { + t.Errorf("memory: got max=%v avg=%v used=%v", info.MaxMem, info.AvgMem, info.UsedMem) + } + if info.CPUTime != 1234 { + t.Errorf("CPUTime = %d, want 1234", info.CPUTime) + } + if info.PeakCPU != 3.75 || info.PeakCPUTime != 4 || info.PeakCPUDuration != 30 { + t.Errorf("CPU peak: got peak=%v peak_time=%d duration=%v", + info.PeakCPU, info.PeakCPUTime, info.PeakCPUDuration) + } + if info.MemEfficiency != 80.5 || info.AvgCPUEfficiency != 62.25 || info.PeakCPUEfficiency != 95 { + t.Errorf("efficiency: got mem=%v avg_cpu=%v peak_cpu=%v", + info.MemEfficiency, info.AvgCPUEfficiency, info.PeakCPUEfficiency) + } +} + +// bug-1736: a job with no run usage reported yet maps to zero values. +func TestRawJobToModel_NoRunUsage(t *testing.T) { + info := rawJobToModel(rawJobData{jobID: 42}) + if info.CPUTime != 0 || info.PeakCPU != 0 || info.MemEfficiency != 0 || info.MaxMem != 0 { + t.Errorf("expected zero run usage, got %+v", info) + } +} + +// bug-1736: the run-usage fields are JobInfo columns, so a REST query can +// select them (store rows are built from JobInfo's json tags). +func TestJobInfoExposesRunUsageColumns(t *testing.T) { + cols := map[string]bool{} + ty := reflect.TypeOf(models.JobInfo{}) + for i := 0; i < ty.NumField(); i++ { + cols[strings.Split(ty.Field(i).Tag.Get("json"), ",")[0]] = true + } + for _, c := range []string{"max_mem", "avg_mem", "used_mem", "cpu_time", + "peak_cpu", "peak_cpu_time", "peak_cpu_duration", "mem_efficiency", + "avg_cpu_efficiency", "peak_cpu_efficiency"} { + if !cols[c] { + t.Errorf("JobInfo has no %q column", c) + } + } +} diff --git a/src/frontend/apiserver/models/jobs.go b/src/frontend/apiserver/models/jobs.go index 93e60427..b3bb0617 100644 --- a/src/frontend/apiserver/models/jobs.go +++ b/src/frontend/apiserver/models/jobs.go @@ -546,18 +546,29 @@ type JobInfo struct { ReserveTime int64 `json:"reserve_time"` Duration int `json:"duration"` CPUTime int `json:"cpu_time"` - ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte - ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte - ExecUID int `json:"exec_uid"` - ExecHome string `json:"exec_home"` - ExecCWD string `json:"exec_cwd"` - ExecUser string `json:"exec_user"` //name change - ChargedSaap string `json:"charged_saap"` - RunTime int `json:"run_time"` - RunTimeEstimation int `json:"run_time_estimation"` - PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte - PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte - JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte + // Run usage (bug-1736), from the controller's accumulated job usage with + // LSF bjobs -l definitions; zero until the first usage report. + MaxMem float64 `json:"max_mem"` // MB + AvgMem float64 `json:"avg_mem"` // MB + UsedMem float64 `json:"used_mem"` // MB, latest + PeakCPU float64 `json:"peak_cpu"` // highest cores used at once + PeakCPUTime int `json:"peak_cpu_time"` // peak_cpu rounded + PeakCPUDuration float64 `json:"peak_cpu_duration"` // seconds, the interval that set the peak + MemEfficiency float64 `json:"mem_efficiency"` // percent of the rusage[mem] request + AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // percent, cpu_time / elapsed / slots + PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // percent, peak_cpu / slots + ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte + ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte + ExecUID int `json:"exec_uid"` + ExecHome string `json:"exec_home"` + ExecCWD string `json:"exec_cwd"` + ExecUser string `json:"exec_user"` //name change + ChargedSaap string `json:"charged_saap"` + RunTime int `json:"run_time"` + RunTimeEstimation int `json:"run_time_estimation"` + PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte + PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte + JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte } type JobExecHosts struct { @@ -598,25 +609,18 @@ type JobAdditionalFields struct { FromHost string `json:"from_host"` ExecHosts string `json:"exec_hosts"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - JobRusage Jsonb `json:"job_rusage"` - MaxMem float64 `json:"max_mem"` - AvgMem float64 `json:"avg_mem"` - UsedMem float64 `json:"used_mem"` - CPUTime int `json:"cpu_time"` - PeakCPUTime int `json:"peak_cpu_time"` - PeakCPUDuration float64 `json:"peak_cpu_duration"` - MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage - AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage - PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage - NumHosts int `json:"num_hosts"` // Number of hosts used by the job - FirstHost string `json:"first_host"` // First host where the job ran - AllocSlot string `json:"alloc_slots"` // Allocated slot information - PIDInfo Jsonb `json:"pid_info"` - Pids []int `json:"pids"` // List of process IDs associated with the job - Pgids []int `json:"pgids"` // List of process group IDs associated with the job - InteractiveJobURL string `json:"interactive_job_url"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + JobRusage Jsonb `json:"job_rusage"` + // max_mem/avg_mem/used_mem/cpu_time/peak_cpu*/..._efficiency are + // JobInfo columns now (bug-1736). + NumHosts int `json:"num_hosts"` // Number of hosts used by the job + FirstHost string `json:"first_host"` // First host where the job ran + AllocSlot string `json:"alloc_slots"` // Allocated slot information + PIDInfo Jsonb `json:"pid_info"` + Pids []int `json:"pids"` // List of process IDs associated with the job + Pgids []int `json:"pgids"` // List of process group IDs associated with the job + InteractiveJobURL string `json:"interactive_job_url"` // CoreLimit/MemLimit: JobInfo has no equivalent field (these are // submission-time resource limits, not job-state fields), but xcecli's // JobFieldsMap accepts them as valid -o/-l display fields (bug-1674) -- diff --git a/src/frontend/xcecli/common/output/page_print_jobs.go b/src/frontend/xcecli/common/output/page_print_jobs.go index 487936f2..ec02f26d 100644 --- a/src/frontend/xcecli/common/output/page_print_jobs.go +++ b/src/frontend/xcecli/common/output/page_print_jobs.go @@ -103,6 +103,10 @@ func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) { width = len(fmt.Sprintf("%f", job.CoreLimit)) case models.HEADER_JOB_MEM_LIMIT: width = len(fmt.Sprintf("%f", job.MemLimit)) + default: + if v, ok := jobRunUsageColumn(format.name, job); ok { + width = len(v) + } } if !p.wideFormat && width > 15 { @@ -237,7 +241,11 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool case models.HEADER_JOB_MEM_LIMIT: mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.MemLimit)), "-") default: - mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*v", width, "-"), "-") + v, ok := jobRunUsageColumn(format.name, job) + if !ok { + v = "-" + } + mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, v), "-") } } rows = append(rows, mainRow) @@ -409,6 +417,9 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) { } PrintWrappedLine(w, endLine, 80, 18) + for _, l := range jobResourceUsageLines(job) { + fmt.Fprintln(w, l) + } fmt.Println("") // print the scheduling parameters table fmt.Printf("SCHEDULING PARAMETERS\n") @@ -443,3 +454,58 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) { } } } + +// jobResourceUsageLines renders a started job's memory and CPU usage in the +// LSF bjobs -l shape (bug-1736): MAX/AVG MEM in Mbytes and MEM EFFICIENCY +// against the rusage[mem] request; CPU PEAK in cores with the interval that +// set it; average and peak CPU efficiency against the allocated slots. A job +// that has not started (PEND/PSUSP) has nothing to report. +func jobResourceUsageLines(job models.QueryJob) []string { + if job.JobStatus == "" || job.JobStatus == "PEND" || job.JobStatus == "PSUSP" { + return nil + } + return []string{ + "", + " MEMORY USAGE:", + fmt.Sprintf(" MAX MEM: %.0f Mbytes; AVG MEM: %.0f Mbytes; MEM EFFICIENCY: %.2f%%", + job.MaxMem, job.AvgMem, job.MemEfficiency), + "", + " CPU USAGE:", + fmt.Sprintf(" CPU PEAK: %.2f ; CPU PEAK DURATION: %.0f second(s)", + job.PeakCPU, job.PeakCPUDuration), + fmt.Sprintf(" CPU AVERAGE EFFICIENCY: %.2f%% ; CPU PEAK EFFICIENCY: %.2f%%", + job.AvgCPUEfficiency, job.PeakCPUEfficiency), + } +} + +// jobRunUsageColumn formats one run-usage column of `jobs list -o` (bug-1736), +// in the units of the -l view: memory in Mbytes, CPU peak in cores, durations +// in seconds, efficiencies in percent. ok is false for any other column. A +// job that has not started shows "-". +func jobRunUsageColumn(header string, job models.QueryJob) (string, bool) { + var v string + switch header { + case models.HEADER_JOB_CPU_TIME: + v = fmt.Sprintf("%d", job.CPUTime) + case models.HEADER_JOB_MAX_MEM: + v = fmt.Sprintf("%.0f", job.MaxMem) + case models.HEADER_JOB_AVG_MEM: + v = fmt.Sprintf("%.0f", job.AvgMem) + case models.HEADER_JOB_MEM_EFFICIENCY: + v = fmt.Sprintf("%.2f%%", job.MemEfficiency) + case models.HEADER_JOB_CPU_PEAK: + v = fmt.Sprintf("%.2f", job.PeakCPU) + case models.HEADER_JOB_CPU_PEAK_DURATION: + v = fmt.Sprintf("%.0f", job.PeakCPUDuration) + case models.HEADER_JOB_CPU_EFFICIENCY: + v = fmt.Sprintf("%.2f%%", job.AvgCPUEfficiency) + case models.HEADER_JOB_CPU_PEAK_EFFICIENCY: + v = fmt.Sprintf("%.2f%%", job.PeakCPUEfficiency) + default: + return "", false + } + if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" { + v = "-" + } + return v, true +} diff --git a/src/frontend/xcecli/common/output/page_print_jobs_test.go b/src/frontend/xcecli/common/output/page_print_jobs_test.go new file mode 100644 index 00000000..f7599353 --- /dev/null +++ b/src/frontend/xcecli/common/output/page_print_jobs_test.go @@ -0,0 +1,90 @@ +package output + +import ( + "strings" + "testing" + "xcecli/models" +) + +// bug-1736: `xcecli jobs list -l` shows the job's memory and CPU usage in the +// LSF bjobs -l shape once the job has started. +func TestJobResourceUsageLines_StartedJob(t *testing.T) { + job := models.QueryJob{ + JobStatus: "RUN", + MaxMem: 512, + AvgMem: 300.4, + MemEfficiency: 80.5, + PeakCPU: 3.75, + PeakCPUDuration: 30, + AvgCPUEfficiency: 62.25, + PeakCPUEfficiency: 95, + } + got := strings.Join(jobResourceUsageLines(job), "\n") + for _, want := range []string{ + "MEMORY USAGE:", + "MAX MEM: 512 Mbytes; AVG MEM: 300 Mbytes; MEM EFFICIENCY: 80.50%", + "CPU USAGE:", + "CPU PEAK: 3.75 ; CPU PEAK DURATION: 30 second(s)", + "CPU AVERAGE EFFICIENCY: 62.25% ; CPU PEAK EFFICIENCY: 95.00%", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } +} + +// A job that has not started has no usage to report. +func TestJobResourceUsageLines_PendingJob(t *testing.T) { + for _, st := range []string{"PEND", "PSUSP"} { + if lines := jobResourceUsageLines(models.QueryJob{JobStatus: st}); len(lines) != 0 { + t.Errorf("%s job: expected no usage lines, got %q", st, lines) + } + } +} + +// bug-1736: the usage fields can be selected with -o and are requested by -l. +func TestJobFieldsMapIncludesRunUsage(t *testing.T) { + have := map[string]bool{} + for _, f := range models.JobFieldsMap { + have[f] = true + } + for _, f := range []string{"max_mem", "avg_mem", "mem_efficiency", + "peak_cpu", "peak_cpu_duration", "avg_cpu_efficiency", + "peak_cpu_efficiency"} { + if !have[f] { + t.Errorf("JobFieldsMap has no field %q", f) + } + } +} + +// bug-1736: the run-usage columns render in `jobs list -o` (they printed "-"). +func TestJobRunUsageColumn(t *testing.T) { + job := models.QueryJob{ + JobStatus: "RUN", CPUTime: 1234, MaxMem: 512, AvgMem: 300.4, + MemEfficiency: 80.5, PeakCPU: 3.75, PeakCPUDuration: 30, + AvgCPUEfficiency: 62.25, PeakCPUEfficiency: 95, + } + cases := map[string]string{ + models.HEADER_JOB_CPU_TIME: "1234", + models.HEADER_JOB_MAX_MEM: "512", + models.HEADER_JOB_AVG_MEM: "300", + models.HEADER_JOB_MEM_EFFICIENCY: "80.50%", + models.HEADER_JOB_CPU_PEAK: "3.75", + models.HEADER_JOB_CPU_PEAK_DURATION: "30", + models.HEADER_JOB_CPU_EFFICIENCY: "62.25%", + models.HEADER_JOB_CPU_PEAK_EFFICIENCY: "95.00%", + } + for h, want := range cases { + got, ok := jobRunUsageColumn(h, job) + if !ok || got != want { + t.Errorf("%s: got %q (ok=%v), want %q", h, got, ok, want) + } + } + if _, ok := jobRunUsageColumn(models.HEADER_JOB_USER, job); ok { + t.Errorf("USER is not a run-usage column") + } + // a job that has not started shows "-", not zeros + if got, _ := jobRunUsageColumn(models.HEADER_JOB_CPU_PEAK, models.QueryJob{JobStatus: "PEND"}); got != "-" { + t.Errorf("PEND job CPU_PEAK: got %q, want -", got) + } +} diff --git a/src/frontend/xcecli/models/jobs.go b/src/frontend/xcecli/models/jobs.go index e95d895c..e074cc7e 100755 --- a/src/frontend/xcecli/models/jobs.go +++ b/src/frontend/xcecli/models/jobs.go @@ -31,6 +31,14 @@ const ( HEADER_JOB_NUM_PROCESSORS = "NUM_PROCESSORS" HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL" HEADER_JOB_PENDING_REASON = "PENDING_REASON" + // run usage (bug-1736) + HEADER_JOB_MAX_MEM = "MAX_MEM" + HEADER_JOB_AVG_MEM = "AVG_MEM" + HEADER_JOB_MEM_EFFICIENCY = "MEM_EFFICIENCY" + HEADER_JOB_CPU_PEAK = "CPU_PEAK" + HEADER_JOB_CPU_PEAK_DURATION = "CPU_PEAK_DURATION" + HEADER_JOB_CPU_EFFICIENCY = "CPU_EFFICIENCY" + HEADER_JOB_CPU_PEAK_EFFICIENCY = "CPU_PEAK_EFFICIENCY" // second line HEADER_JOB_R15S = "r15s" @@ -120,6 +128,7 @@ type QueryJob struct { AvgMem float64 `json:"avg_mem"` UsedMem float64 `json:"used_mem"` CPUTime int `json:"cpu_time"` + PeakCPU float64 `json:"peak_cpu"` // highest cores used at once (bug-1736) PeakCPUTime int `json:"peak_cpu_time"` PeakCPUDuration float64 `json:"peak_cpu_duration"` MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage @@ -184,30 +193,37 @@ type QueryJob struct { } var JobFieldsMap = map[string]string{ - HEADER_JOB_JOBID: "job_id", - HEADER_JOB_USER: "submit_user", - HEADER_JOB_STATUS: "job_status", - HEADER_JOB_QUEUE: "queue_name", - HEADER_JOB_COMMAND: "command", - HEADER_JOB_NAME: "job_name", - HEADER_JOB_SUBMIT_TIME: "submit_time", - HEADER_JOB_FROM_HOST: "from_host", - HEADER_JOB_EXEC_HOST: "exec_hosts", - HEADER_JOB_CORE_LIMIT: "core_limit", - HEADER_JOB_MEM_LIMIT: "mem_limit", - HEADER_JOB_PID: "pids", - HEADER_JOB_PGID: "pgids", - HEADER_JOB_PROJECT_NAME: "project_name", - HEADER_JOB_START_TIME: "start_time", - HEADER_JOB_END_TIME: "end_time", - HEADER_JOB_CWD: "cwd", - HEADER_JOB_CPU_TIME: "cpu_time", - HEADER_JOB_RESREQ: "resreq", - HEADER_JOB_DESCRIPTION: "job_description", - HEADER_JOB_LAUNCH_TYPE: "launch_type", - HEADER_JOB_NUM_PROCESSORS: "num_processors", - HEADER_JOB_INTERACTIVE_URL: "interactive_job_url", - HEADER_JOB_PENDING_REASON: "pending_reason", + HEADER_JOB_JOBID: "job_id", + HEADER_JOB_USER: "submit_user", + HEADER_JOB_STATUS: "job_status", + HEADER_JOB_QUEUE: "queue_name", + HEADER_JOB_COMMAND: "command", + HEADER_JOB_NAME: "job_name", + HEADER_JOB_SUBMIT_TIME: "submit_time", + HEADER_JOB_FROM_HOST: "from_host", + HEADER_JOB_EXEC_HOST: "exec_hosts", + HEADER_JOB_CORE_LIMIT: "core_limit", + HEADER_JOB_MEM_LIMIT: "mem_limit", + HEADER_JOB_PID: "pids", + HEADER_JOB_PGID: "pgids", + HEADER_JOB_PROJECT_NAME: "project_name", + HEADER_JOB_START_TIME: "start_time", + HEADER_JOB_END_TIME: "end_time", + HEADER_JOB_CWD: "cwd", + HEADER_JOB_CPU_TIME: "cpu_time", + HEADER_JOB_RESREQ: "resreq", + HEADER_JOB_DESCRIPTION: "job_description", + HEADER_JOB_LAUNCH_TYPE: "launch_type", + HEADER_JOB_NUM_PROCESSORS: "num_processors", + HEADER_JOB_INTERACTIVE_URL: "interactive_job_url", + HEADER_JOB_PENDING_REASON: "pending_reason", + HEADER_JOB_MAX_MEM: "max_mem", + HEADER_JOB_AVG_MEM: "avg_mem", + HEADER_JOB_MEM_EFFICIENCY: "mem_efficiency", + HEADER_JOB_CPU_PEAK: "peak_cpu", + HEADER_JOB_CPU_PEAK_DURATION: "peak_cpu_duration", + HEADER_JOB_CPU_EFFICIENCY: "avg_cpu_efficiency", + HEADER_JOB_CPU_PEAK_EFFICIENCY: "peak_cpu_efficiency", //job_filter.HEADER_JOB_PRIORITY: "job_priority", } diff --git a/src/schema/controller.capnp b/src/schema/controller.capnp index 8a12e3a8..539d523d 100644 --- a/src/schema/controller.capnp +++ b/src/schema/controller.capnp @@ -338,6 +338,7 @@ struct Job $C.mapname("xce_job_t"){ pendreason @10: JobPendReason $C.mapname("pend_reasons"); transientSince @11: UInt64 = 0 $C.mapname("transient_since"); # epoch seconds; set when job enters JOB_STAT_TRANSIENT exitInfo @12: JobExitInfo $C.mapname("exit_info"); # bug-1867: null until the job's exit code is known + runUsage @13: JobRunusage $C.mapname("run_usage"); # bug-1736: null until the first usage report; snapshot + apiserver } # bug-1867: job_exitinfo_t, so a checkpointed Job keeps its exit code. The