fix(controller,apiserver,xcecli): expose job CPU peak and efficiency to the apiserver and xcecli (bug-1736)
The controller computed run usage (CPU time, memory, CPU peak and efficiency, LSF bjobs -l definitions) but the capnp Job struct had no field for it, so the job pushed to the apiserver never carried it, and snapshots (the same struct) dropped it: a job restored from a checkpoint lost its accumulated usage. - controller.capnp: Job.runUsage @13 (JobRunusage); old entries decode with no run usage. - apiserver: map run_usage into JobInfo; max_mem, avg_mem, used_mem, cpu_time, peak_cpu, peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency become queryable columns. - xcecli: -o fields MAX_MEM, AVG_MEM, MEM_EFFICIENCY, CPU_PEAK, CPU_PEAK_DURATION, CPU_EFFICIENCY, CPU_PEAK_EFFICIENCY (CPU_TIME now renders too), and a MEMORY USAGE / CPU USAGE block in jobs list -l. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e2472aa73c
commit
1cedd7f791
GPG key ID:
80533536B0EB651C
9 changed files with 466 additions and 56 deletions
|
|
@ -739,6 +739,128 @@ static void test_snapent_checkpoint_exit_info_roundtrip(void)
|
|||
snapent_teardown(src);
|
||||
}
|
||||
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* bug-1736: job.run_usage must round-trip through a checkpoint, every */
|
||||
/* field. The capnp Job struct had no run-usage field, so a job */
|
||||
/* restored from a snapshot lost its accumulated CPU/memory usage and */
|
||||
/* the CPU peak/efficiency values (the WAL_JOB_RUSAGE facts behind the */
|
||||
/* checkpoint are trimmed), and the same struct is what the apiserver */
|
||||
/* receives, so they never reached it either. */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
static xce_job_runusage_t* snapent_make_runusage(void)
|
||||
{
|
||||
xce_job_runusage_t* ru = calloc(1, sizeof(*ru));
|
||||
TEST_ASSERT_NOT_NULL(ru);
|
||||
ru->max_mem = 512.5;
|
||||
ru->avg_mem = 300.25;
|
||||
ru->used_mem = 410.0;
|
||||
ru->cpu_time = 1234;
|
||||
ru->peak_cpu_time = 4;
|
||||
ru->peak_cpu_duration = 30;
|
||||
ru->mem_efficiency = 80.5f;
|
||||
ru->avg_cpu_efficiency = 62.25f;
|
||||
ru->peak_cpu_efficiency = 95.0f;
|
||||
ru->peak_cpu = 3.75f;
|
||||
ru->num_hosts = 2;
|
||||
ru->host_runusage = calloc(2, sizeof(xce_host_runusage_t*));
|
||||
TEST_ASSERT_NOT_NULL(ru->host_runusage);
|
||||
for (int h = 0; h < 2; h++) {
|
||||
xce_host_runusage_t* hr = calloc(1, sizeof(*hr));
|
||||
TEST_ASSERT_NOT_NULL(hr);
|
||||
hr->host_name = strdup(h == 0 ? "hostA" : "hostB");
|
||||
hr->cpu_time = 600 + h;
|
||||
hr->mem_usage = 200.5 + h;
|
||||
hr->mem_swap = 10.0 + h;
|
||||
hr->tasks_current = 3 + h;
|
||||
hr->sample_time = 1790000000 + h;
|
||||
hr->cpu_rate = 1.5f + (float)h;
|
||||
ru->host_runusage[h] = hr;
|
||||
}
|
||||
return ru;
|
||||
}
|
||||
|
||||
static void snapent_assert_runusage(const xce_job_runusage_t* ru)
|
||||
{
|
||||
TEST_ASSERT_NOT_NULL_MESSAGE(ru,
|
||||
"bug-1736: a checkpointed job must keep its run_usage");
|
||||
TEST_ASSERT_EQUAL_FLOAT((float)(512.5), (float)(ru->max_mem));
|
||||
TEST_ASSERT_EQUAL_FLOAT((float)(300.25), (float)(ru->avg_mem));
|
||||
TEST_ASSERT_EQUAL_FLOAT((float)(410.0), (float)(ru->used_mem));
|
||||
TEST_ASSERT_EQUAL_INT(1234, ru->cpu_time);
|
||||
TEST_ASSERT_EQUAL_INT(4, ru->peak_cpu_time);
|
||||
TEST_ASSERT_EQUAL_INT(30, ru->peak_cpu_duration);
|
||||
TEST_ASSERT_EQUAL_FLOAT(80.5f, ru->mem_efficiency);
|
||||
TEST_ASSERT_EQUAL_FLOAT(62.25f, ru->avg_cpu_efficiency);
|
||||
TEST_ASSERT_EQUAL_FLOAT(95.0f, ru->peak_cpu_efficiency);
|
||||
TEST_ASSERT_EQUAL_FLOAT(3.75f, ru->peak_cpu);
|
||||
TEST_ASSERT_EQUAL_INT(2, ru->num_hosts);
|
||||
for (int h = 0; h < 2; h++) {
|
||||
const xce_host_runusage_t* hr = ru->host_runusage[h];
|
||||
TEST_ASSERT_NOT_NULL(hr);
|
||||
TEST_ASSERT_EQUAL_STRING(h == 0 ? "hostA" : "hostB", hr->host_name);
|
||||
TEST_ASSERT_EQUAL_INT(600 + h, hr->cpu_time);
|
||||
TEST_ASSERT_EQUAL_FLOAT((float)(200.5 + h), (float)(hr->mem_usage));
|
||||
TEST_ASSERT_EQUAL_FLOAT((float)(10.0 + h), (float)(hr->mem_swap));
|
||||
TEST_ASSERT_EQUAL_INT(3 + h, hr->tasks_current);
|
||||
TEST_ASSERT_EQUAL_INT64(1790000000 + h, hr->sample_time);
|
||||
TEST_ASSERT_EQUAL_FLOAT(1.5f + (float)h, hr->cpu_rate);
|
||||
}
|
||||
}
|
||||
|
||||
static void test_snapent_checkpoint_run_usage_roundtrip(void)
|
||||
{
|
||||
const int n_jobs = 2;
|
||||
|
||||
snapent_fixture_t* src = snapent_make(n_jobs, 0);
|
||||
TEST_ASSERT_NOT_NULL(src);
|
||||
|
||||
/* job 0: full run usage; job 1: none reported yet (NULL) */
|
||||
job_t* j = snapent_find_job(src->c, snapent_jobid(0));
|
||||
TEST_ASSERT_NOT_NULL(j);
|
||||
lock(j, RTTI_LOCK_WRITE);
|
||||
j->job.run_usage = snapent_make_runusage();
|
||||
unlock(j);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(&src->c->persist));
|
||||
|
||||
uint64_t lsn = src->c->persist.checkpoint_lsn;
|
||||
char path[4096];
|
||||
snprintf(path, sizeof(path), "%s/snap-%016" PRIx64 ".capnp",
|
||||
src->c->persist.snap_path, lsn);
|
||||
|
||||
/* 1. the checkpoint's own job frames carry it */
|
||||
xce_job_t decoded[8];
|
||||
int count = snapent_read_checkpoint_jobs(path, decoded, 8);
|
||||
TEST_ASSERT_EQUAL_INT(n_jobs, count);
|
||||
xce_job_t* d0 = snapent_find_decoded_job(decoded, count, snapent_jobid(0));
|
||||
xce_job_t* d1 = snapent_find_decoded_job(decoded, count, snapent_jobid(1));
|
||||
TEST_ASSERT_NOT_NULL(d0);
|
||||
TEST_ASSERT_NOT_NULL(d1);
|
||||
snapent_assert_runusage(d0->run_usage);
|
||||
TEST_ASSERT_NULL_MESSAGE(d1->run_usage,
|
||||
"a job with no run usage must come back with none");
|
||||
snapent_free_decoded_jobs(decoded, count);
|
||||
|
||||
/* 2. a real persist_load_snapshot restore into an empty member */
|
||||
snapent_fixture_t* dst = snapent_make(0, 0);
|
||||
TEST_ASSERT_NOT_NULL(dst);
|
||||
bool entities_restored = false;
|
||||
TEST_ASSERT_EQUAL_INT(0, persist_load_snapshot(&dst->c->persist, path, lsn,
|
||||
&entities_restored));
|
||||
TEST_ASSERT_TRUE(entities_restored);
|
||||
|
||||
job_t* r0 = snapent_find_job(dst->c, snapent_jobid(0));
|
||||
job_t* r1 = snapent_find_job(dst->c, snapent_jobid(1));
|
||||
TEST_ASSERT_NOT_NULL(r0);
|
||||
TEST_ASSERT_NOT_NULL(r1);
|
||||
snapent_assert_runusage(r0->job.run_usage);
|
||||
TEST_ASSERT_NULL(r1->job.run_usage);
|
||||
|
||||
snapent_teardown(dst);
|
||||
snapent_teardown(src);
|
||||
}
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Test 3 (task 2.1, RED): concurrent per-entity mutation during the */
|
||||
/* checkpoint walk must never be captured as a torn mix of fields */
|
||||
|
|
@ -2299,6 +2421,7 @@ void test_persist_snapshot_entities(void)
|
|||
RUN_TEST(test_snapent_fixture_seeds_jobs_and_queues);
|
||||
RUN_TEST(test_snapent_checkpoint_job_content_roundtrip);
|
||||
RUN_TEST(test_snapent_checkpoint_exit_info_roundtrip);
|
||||
RUN_TEST(test_snapent_checkpoint_run_usage_roundtrip);
|
||||
RUN_TEST(test_snapent_checkpoint_concurrent_mutation_not_torn);
|
||||
RUN_TEST(test_snapent_checkpoint_write_failure_leaves_no_final_file);
|
||||
RUN_TEST(test_snapent_checkpoint_mid_encode_failure_discards_whole_file);
|
||||
|
|
|
|||
|
|
@ -114,6 +114,20 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
|
|||
}
|
||||
r.submit = rd
|
||||
}
|
||||
if u := job.run_usage; u != nil {
|
||||
r.runUsage = &rawRunUsage{
|
||||
maxMem: float64(u.max_mem),
|
||||
avgMem: float64(u.avg_mem),
|
||||
usedMem: float64(u.used_mem),
|
||||
cpuTime: int(u.cpu_time),
|
||||
peakCPUTime: int(u.peak_cpu_time),
|
||||
peakCPUDuration: int(u.peak_cpu_duration),
|
||||
memEfficiency: float64(u.mem_efficiency),
|
||||
avgCPUEfficiency: float64(u.avg_cpu_efficiency),
|
||||
peakCPUEfficiency: float64(u.peak_cpu_efficiency),
|
||||
peakCPU: float64(u.peak_cpu),
|
||||
}
|
||||
}
|
||||
if job.allocs != nil {
|
||||
n := int(job.allocs.nhosts)
|
||||
for i := 0; i < n; i++ {
|
||||
|
|
|
|||
|
|
@ -263,6 +263,23 @@ type rawHostSlot struct {
|
|||
slots int
|
||||
}
|
||||
|
||||
// rawRunUsage mirrors the C xce_job_runusage_t job-level fields used by the
|
||||
// Go model (bug-1736): accumulated CPU time and memory, and the CPU peak and
|
||||
// efficiency values the controller computes with LSF bjobs -l definitions
|
||||
// (efficiencies in percent, memory in MB as the worker reports it).
|
||||
type rawRunUsage struct {
|
||||
maxMem float64
|
||||
avgMem float64
|
||||
usedMem float64
|
||||
cpuTime int
|
||||
peakCPUTime int
|
||||
peakCPUDuration int
|
||||
memEfficiency float64
|
||||
avgCPUEfficiency float64
|
||||
peakCPUEfficiency float64
|
||||
peakCPU float64
|
||||
}
|
||||
|
||||
// rawJobData mirrors the C xce_job_t fields used by the Go model.
|
||||
type rawJobData struct {
|
||||
jobID int64
|
||||
|
|
@ -274,6 +291,7 @@ type rawJobData struct {
|
|||
finishTime int64
|
||||
submit *rawSubmitData
|
||||
execHosts []rawHostSlot
|
||||
runUsage *rawRunUsage // nil until the job's first usage report
|
||||
}
|
||||
|
||||
// rawWorkerData mirrors the C xce_worker_t fields used by the Go model.
|
||||
|
|
@ -363,6 +381,18 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
|
|||
data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)})
|
||||
info.ExecHosts = models.Jsonb{String: string(data)}
|
||||
}
|
||||
if u := r.runUsage; u != nil {
|
||||
info.MaxMem = u.maxMem
|
||||
info.AvgMem = u.avgMem
|
||||
info.UsedMem = u.usedMem
|
||||
info.CPUTime = u.cpuTime
|
||||
info.PeakCPU = u.peakCPU
|
||||
info.PeakCPUTime = u.peakCPUTime
|
||||
info.PeakCPUDuration = float64(u.peakCPUDuration)
|
||||
info.MemEfficiency = u.memEfficiency
|
||||
info.AvgCPUEfficiency = u.avgCPUEfficiency
|
||||
info.PeakCPUEfficiency = u.peakCPUEfficiency
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -1185,3 +1187,67 @@ func TestRawClusterToModel(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: the controller's accumulated run usage (CPU time, memory, CPU
|
||||
// peak and efficiency, LSF bjobs -l definitions) reaches the job model, so
|
||||
// REST field selection and xcecli can show it.
|
||||
func TestRawJobToModel_RunUsage(t *testing.T) {
|
||||
r := rawJobData{
|
||||
jobID: 42,
|
||||
state: 4,
|
||||
runUsage: &rawRunUsage{
|
||||
maxMem: 512.5,
|
||||
avgMem: 300.25,
|
||||
usedMem: 410,
|
||||
cpuTime: 1234,
|
||||
peakCPUTime: 4,
|
||||
peakCPUDuration: 30,
|
||||
memEfficiency: 80.5,
|
||||
avgCPUEfficiency: 62.25,
|
||||
peakCPUEfficiency: 95,
|
||||
peakCPU: 3.75,
|
||||
},
|
||||
}
|
||||
|
||||
info := rawJobToModel(r)
|
||||
|
||||
if info.MaxMem != 512.5 || info.AvgMem != 300.25 || info.UsedMem != 410 {
|
||||
t.Errorf("memory: got max=%v avg=%v used=%v", info.MaxMem, info.AvgMem, info.UsedMem)
|
||||
}
|
||||
if info.CPUTime != 1234 {
|
||||
t.Errorf("CPUTime = %d, want 1234", info.CPUTime)
|
||||
}
|
||||
if info.PeakCPU != 3.75 || info.PeakCPUTime != 4 || info.PeakCPUDuration != 30 {
|
||||
t.Errorf("CPU peak: got peak=%v peak_time=%d duration=%v",
|
||||
info.PeakCPU, info.PeakCPUTime, info.PeakCPUDuration)
|
||||
}
|
||||
if info.MemEfficiency != 80.5 || info.AvgCPUEfficiency != 62.25 || info.PeakCPUEfficiency != 95 {
|
||||
t.Errorf("efficiency: got mem=%v avg_cpu=%v peak_cpu=%v",
|
||||
info.MemEfficiency, info.AvgCPUEfficiency, info.PeakCPUEfficiency)
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: a job with no run usage reported yet maps to zero values.
|
||||
func TestRawJobToModel_NoRunUsage(t *testing.T) {
|
||||
info := rawJobToModel(rawJobData{jobID: 42})
|
||||
if info.CPUTime != 0 || info.PeakCPU != 0 || info.MemEfficiency != 0 || info.MaxMem != 0 {
|
||||
t.Errorf("expected zero run usage, got %+v", info)
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: the run-usage fields are JobInfo columns, so a REST query can
|
||||
// select them (store rows are built from JobInfo's json tags).
|
||||
func TestJobInfoExposesRunUsageColumns(t *testing.T) {
|
||||
cols := map[string]bool{}
|
||||
ty := reflect.TypeOf(models.JobInfo{})
|
||||
for i := 0; i < ty.NumField(); i++ {
|
||||
cols[strings.Split(ty.Field(i).Tag.Get("json"), ",")[0]] = true
|
||||
}
|
||||
for _, c := range []string{"max_mem", "avg_mem", "used_mem", "cpu_time",
|
||||
"peak_cpu", "peak_cpu_time", "peak_cpu_duration", "mem_efficiency",
|
||||
"avg_cpu_efficiency", "peak_cpu_efficiency"} {
|
||||
if !cols[c] {
|
||||
t.Errorf("JobInfo has no %q column", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -546,18 +546,29 @@ type JobInfo struct {
|
|||
ReserveTime int64 `json:"reserve_time"`
|
||||
Duration int `json:"duration"`
|
||||
CPUTime int `json:"cpu_time"`
|
||||
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
|
||||
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
|
||||
ExecUID int `json:"exec_uid"`
|
||||
ExecHome string `json:"exec_home"`
|
||||
ExecCWD string `json:"exec_cwd"`
|
||||
ExecUser string `json:"exec_user"` //name change
|
||||
ChargedSaap string `json:"charged_saap"`
|
||||
RunTime int `json:"run_time"`
|
||||
RunTimeEstimation int `json:"run_time_estimation"`
|
||||
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
|
||||
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
|
||||
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
|
||||
// Run usage (bug-1736), from the controller's accumulated job usage with
|
||||
// LSF bjobs -l definitions; zero until the first usage report.
|
||||
MaxMem float64 `json:"max_mem"` // MB
|
||||
AvgMem float64 `json:"avg_mem"` // MB
|
||||
UsedMem float64 `json:"used_mem"` // MB, latest
|
||||
PeakCPU float64 `json:"peak_cpu"` // highest cores used at once
|
||||
PeakCPUTime int `json:"peak_cpu_time"` // peak_cpu rounded
|
||||
PeakCPUDuration float64 `json:"peak_cpu_duration"` // seconds, the interval that set the peak
|
||||
MemEfficiency float64 `json:"mem_efficiency"` // percent of the rusage[mem] request
|
||||
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // percent, cpu_time / elapsed / slots
|
||||
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // percent, peak_cpu / slots
|
||||
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
|
||||
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
|
||||
ExecUID int `json:"exec_uid"`
|
||||
ExecHome string `json:"exec_home"`
|
||||
ExecCWD string `json:"exec_cwd"`
|
||||
ExecUser string `json:"exec_user"` //name change
|
||||
ChargedSaap string `json:"charged_saap"`
|
||||
RunTime int `json:"run_time"`
|
||||
RunTimeEstimation int `json:"run_time_estimation"`
|
||||
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
|
||||
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
|
||||
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
|
||||
}
|
||||
|
||||
type JobExecHosts struct {
|
||||
|
|
@ -598,25 +609,18 @@ type JobAdditionalFields struct {
|
|||
FromHost string `json:"from_host"`
|
||||
ExecHosts string `json:"exec_hosts"`
|
||||
|
||||
StartTime int64 `json:"start_time"`
|
||||
EndTime int64 `json:"end_time"`
|
||||
JobRusage Jsonb `json:"job_rusage"`
|
||||
MaxMem float64 `json:"max_mem"`
|
||||
AvgMem float64 `json:"avg_mem"`
|
||||
UsedMem float64 `json:"used_mem"`
|
||||
CPUTime int `json:"cpu_time"`
|
||||
PeakCPUTime int `json:"peak_cpu_time"`
|
||||
PeakCPUDuration float64 `json:"peak_cpu_duration"`
|
||||
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage
|
||||
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage
|
||||
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage
|
||||
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
|
||||
FirstHost string `json:"first_host"` // First host where the job ran
|
||||
AllocSlot string `json:"alloc_slots"` // Allocated slot information
|
||||
PIDInfo Jsonb `json:"pid_info"`
|
||||
Pids []int `json:"pids"` // List of process IDs associated with the job
|
||||
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
|
||||
InteractiveJobURL string `json:"interactive_job_url"`
|
||||
StartTime int64 `json:"start_time"`
|
||||
EndTime int64 `json:"end_time"`
|
||||
JobRusage Jsonb `json:"job_rusage"`
|
||||
// max_mem/avg_mem/used_mem/cpu_time/peak_cpu*/..._efficiency are
|
||||
// JobInfo columns now (bug-1736).
|
||||
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
|
||||
FirstHost string `json:"first_host"` // First host where the job ran
|
||||
AllocSlot string `json:"alloc_slots"` // Allocated slot information
|
||||
PIDInfo Jsonb `json:"pid_info"`
|
||||
Pids []int `json:"pids"` // List of process IDs associated with the job
|
||||
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
|
||||
InteractiveJobURL string `json:"interactive_job_url"`
|
||||
// CoreLimit/MemLimit: JobInfo has no equivalent field (these are
|
||||
// submission-time resource limits, not job-state fields), but xcecli's
|
||||
// JobFieldsMap accepts them as valid -o/-l display fields (bug-1674) --
|
||||
|
|
|
|||
|
|
@ -103,6 +103,10 @@ func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
|
|||
width = len(fmt.Sprintf("%f", job.CoreLimit))
|
||||
case models.HEADER_JOB_MEM_LIMIT:
|
||||
width = len(fmt.Sprintf("%f", job.MemLimit))
|
||||
default:
|
||||
if v, ok := jobRunUsageColumn(format.name, job); ok {
|
||||
width = len(v)
|
||||
}
|
||||
}
|
||||
|
||||
if !p.wideFormat && width > 15 {
|
||||
|
|
@ -237,7 +241,11 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
|
|||
case models.HEADER_JOB_MEM_LIMIT:
|
||||
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.MemLimit)), "-")
|
||||
default:
|
||||
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*v", width, "-"), "-")
|
||||
v, ok := jobRunUsageColumn(format.name, job)
|
||||
if !ok {
|
||||
v = "-"
|
||||
}
|
||||
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, v), "-")
|
||||
}
|
||||
}
|
||||
rows = append(rows, mainRow)
|
||||
|
|
@ -409,6 +417,9 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
|
|||
|
||||
}
|
||||
PrintWrappedLine(w, endLine, 80, 18)
|
||||
for _, l := range jobResourceUsageLines(job) {
|
||||
fmt.Fprintln(w, l)
|
||||
}
|
||||
fmt.Println("")
|
||||
// print the scheduling parameters table
|
||||
fmt.Printf("SCHEDULING PARAMETERS\n")
|
||||
|
|
@ -443,3 +454,58 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// jobResourceUsageLines renders a started job's memory and CPU usage in the
|
||||
// LSF bjobs -l shape (bug-1736): MAX/AVG MEM in Mbytes and MEM EFFICIENCY
|
||||
// against the rusage[mem] request; CPU PEAK in cores with the interval that
|
||||
// set it; average and peak CPU efficiency against the allocated slots. A job
|
||||
// that has not started (PEND/PSUSP) has nothing to report.
|
||||
func jobResourceUsageLines(job models.QueryJob) []string {
|
||||
if job.JobStatus == "" || job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
|
||||
return nil
|
||||
}
|
||||
return []string{
|
||||
"",
|
||||
" MEMORY USAGE:",
|
||||
fmt.Sprintf(" MAX MEM: %.0f Mbytes; AVG MEM: %.0f Mbytes; MEM EFFICIENCY: %.2f%%",
|
||||
job.MaxMem, job.AvgMem, job.MemEfficiency),
|
||||
"",
|
||||
" CPU USAGE:",
|
||||
fmt.Sprintf(" CPU PEAK: %.2f ; CPU PEAK DURATION: %.0f second(s)",
|
||||
job.PeakCPU, job.PeakCPUDuration),
|
||||
fmt.Sprintf(" CPU AVERAGE EFFICIENCY: %.2f%% ; CPU PEAK EFFICIENCY: %.2f%%",
|
||||
job.AvgCPUEfficiency, job.PeakCPUEfficiency),
|
||||
}
|
||||
}
|
||||
|
||||
// jobRunUsageColumn formats one run-usage column of `jobs list -o` (bug-1736),
|
||||
// in the units of the -l view: memory in Mbytes, CPU peak in cores, durations
|
||||
// in seconds, efficiencies in percent. ok is false for any other column. A
|
||||
// job that has not started shows "-".
|
||||
func jobRunUsageColumn(header string, job models.QueryJob) (string, bool) {
|
||||
var v string
|
||||
switch header {
|
||||
case models.HEADER_JOB_CPU_TIME:
|
||||
v = fmt.Sprintf("%d", job.CPUTime)
|
||||
case models.HEADER_JOB_MAX_MEM:
|
||||
v = fmt.Sprintf("%.0f", job.MaxMem)
|
||||
case models.HEADER_JOB_AVG_MEM:
|
||||
v = fmt.Sprintf("%.0f", job.AvgMem)
|
||||
case models.HEADER_JOB_MEM_EFFICIENCY:
|
||||
v = fmt.Sprintf("%.2f%%", job.MemEfficiency)
|
||||
case models.HEADER_JOB_CPU_PEAK:
|
||||
v = fmt.Sprintf("%.2f", job.PeakCPU)
|
||||
case models.HEADER_JOB_CPU_PEAK_DURATION:
|
||||
v = fmt.Sprintf("%.0f", job.PeakCPUDuration)
|
||||
case models.HEADER_JOB_CPU_EFFICIENCY:
|
||||
v = fmt.Sprintf("%.2f%%", job.AvgCPUEfficiency)
|
||||
case models.HEADER_JOB_CPU_PEAK_EFFICIENCY:
|
||||
v = fmt.Sprintf("%.2f%%", job.PeakCPUEfficiency)
|
||||
default:
|
||||
return "", false
|
||||
}
|
||||
if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
|
||||
v = "-"
|
||||
}
|
||||
return v, true
|
||||
}
|
||||
|
|
|
|||
90
src/frontend/xcecli/common/output/page_print_jobs_test.go
Normal file
90
src/frontend/xcecli/common/output/page_print_jobs_test.go
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
package output
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"xcecli/models"
|
||||
)
|
||||
|
||||
// bug-1736: `xcecli jobs list -l` shows the job's memory and CPU usage in the
|
||||
// LSF bjobs -l shape once the job has started.
|
||||
func TestJobResourceUsageLines_StartedJob(t *testing.T) {
|
||||
job := models.QueryJob{
|
||||
JobStatus: "RUN",
|
||||
MaxMem: 512,
|
||||
AvgMem: 300.4,
|
||||
MemEfficiency: 80.5,
|
||||
PeakCPU: 3.75,
|
||||
PeakCPUDuration: 30,
|
||||
AvgCPUEfficiency: 62.25,
|
||||
PeakCPUEfficiency: 95,
|
||||
}
|
||||
got := strings.Join(jobResourceUsageLines(job), "\n")
|
||||
for _, want := range []string{
|
||||
"MEMORY USAGE:",
|
||||
"MAX MEM: 512 Mbytes; AVG MEM: 300 Mbytes; MEM EFFICIENCY: 80.50%",
|
||||
"CPU USAGE:",
|
||||
"CPU PEAK: 3.75 ; CPU PEAK DURATION: 30 second(s)",
|
||||
"CPU AVERAGE EFFICIENCY: 62.25% ; CPU PEAK EFFICIENCY: 95.00%",
|
||||
} {
|
||||
if !strings.Contains(got, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A job that has not started has no usage to report.
|
||||
func TestJobResourceUsageLines_PendingJob(t *testing.T) {
|
||||
for _, st := range []string{"PEND", "PSUSP"} {
|
||||
if lines := jobResourceUsageLines(models.QueryJob{JobStatus: st}); len(lines) != 0 {
|
||||
t.Errorf("%s job: expected no usage lines, got %q", st, lines)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: the usage fields can be selected with -o and are requested by -l.
|
||||
func TestJobFieldsMapIncludesRunUsage(t *testing.T) {
|
||||
have := map[string]bool{}
|
||||
for _, f := range models.JobFieldsMap {
|
||||
have[f] = true
|
||||
}
|
||||
for _, f := range []string{"max_mem", "avg_mem", "mem_efficiency",
|
||||
"peak_cpu", "peak_cpu_duration", "avg_cpu_efficiency",
|
||||
"peak_cpu_efficiency"} {
|
||||
if !have[f] {
|
||||
t.Errorf("JobFieldsMap has no field %q", f)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: the run-usage columns render in `jobs list -o` (they printed "-").
|
||||
func TestJobRunUsageColumn(t *testing.T) {
|
||||
job := models.QueryJob{
|
||||
JobStatus: "RUN", CPUTime: 1234, MaxMem: 512, AvgMem: 300.4,
|
||||
MemEfficiency: 80.5, PeakCPU: 3.75, PeakCPUDuration: 30,
|
||||
AvgCPUEfficiency: 62.25, PeakCPUEfficiency: 95,
|
||||
}
|
||||
cases := map[string]string{
|
||||
models.HEADER_JOB_CPU_TIME: "1234",
|
||||
models.HEADER_JOB_MAX_MEM: "512",
|
||||
models.HEADER_JOB_AVG_MEM: "300",
|
||||
models.HEADER_JOB_MEM_EFFICIENCY: "80.50%",
|
||||
models.HEADER_JOB_CPU_PEAK: "3.75",
|
||||
models.HEADER_JOB_CPU_PEAK_DURATION: "30",
|
||||
models.HEADER_JOB_CPU_EFFICIENCY: "62.25%",
|
||||
models.HEADER_JOB_CPU_PEAK_EFFICIENCY: "95.00%",
|
||||
}
|
||||
for h, want := range cases {
|
||||
got, ok := jobRunUsageColumn(h, job)
|
||||
if !ok || got != want {
|
||||
t.Errorf("%s: got %q (ok=%v), want %q", h, got, ok, want)
|
||||
}
|
||||
}
|
||||
if _, ok := jobRunUsageColumn(models.HEADER_JOB_USER, job); ok {
|
||||
t.Errorf("USER is not a run-usage column")
|
||||
}
|
||||
// a job that has not started shows "-", not zeros
|
||||
if got, _ := jobRunUsageColumn(models.HEADER_JOB_CPU_PEAK, models.QueryJob{JobStatus: "PEND"}); got != "-" {
|
||||
t.Errorf("PEND job CPU_PEAK: got %q, want -", got)
|
||||
}
|
||||
}
|
||||
|
|
@ -31,6 +31,14 @@ const (
|
|||
HEADER_JOB_NUM_PROCESSORS = "NUM_PROCESSORS"
|
||||
HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL"
|
||||
HEADER_JOB_PENDING_REASON = "PENDING_REASON"
|
||||
// run usage (bug-1736)
|
||||
HEADER_JOB_MAX_MEM = "MAX_MEM"
|
||||
HEADER_JOB_AVG_MEM = "AVG_MEM"
|
||||
HEADER_JOB_MEM_EFFICIENCY = "MEM_EFFICIENCY"
|
||||
HEADER_JOB_CPU_PEAK = "CPU_PEAK"
|
||||
HEADER_JOB_CPU_PEAK_DURATION = "CPU_PEAK_DURATION"
|
||||
HEADER_JOB_CPU_EFFICIENCY = "CPU_EFFICIENCY"
|
||||
HEADER_JOB_CPU_PEAK_EFFICIENCY = "CPU_PEAK_EFFICIENCY"
|
||||
|
||||
// second line
|
||||
HEADER_JOB_R15S = "r15s"
|
||||
|
|
@ -120,6 +128,7 @@ type QueryJob struct {
|
|||
AvgMem float64 `json:"avg_mem"`
|
||||
UsedMem float64 `json:"used_mem"`
|
||||
CPUTime int `json:"cpu_time"`
|
||||
PeakCPU float64 `json:"peak_cpu"` // highest cores used at once (bug-1736)
|
||||
PeakCPUTime int `json:"peak_cpu_time"`
|
||||
PeakCPUDuration float64 `json:"peak_cpu_duration"`
|
||||
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage
|
||||
|
|
@ -184,30 +193,37 @@ type QueryJob struct {
|
|||
}
|
||||
|
||||
var JobFieldsMap = map[string]string{
|
||||
HEADER_JOB_JOBID: "job_id",
|
||||
HEADER_JOB_USER: "submit_user",
|
||||
HEADER_JOB_STATUS: "job_status",
|
||||
HEADER_JOB_QUEUE: "queue_name",
|
||||
HEADER_JOB_COMMAND: "command",
|
||||
HEADER_JOB_NAME: "job_name",
|
||||
HEADER_JOB_SUBMIT_TIME: "submit_time",
|
||||
HEADER_JOB_FROM_HOST: "from_host",
|
||||
HEADER_JOB_EXEC_HOST: "exec_hosts",
|
||||
HEADER_JOB_CORE_LIMIT: "core_limit",
|
||||
HEADER_JOB_MEM_LIMIT: "mem_limit",
|
||||
HEADER_JOB_PID: "pids",
|
||||
HEADER_JOB_PGID: "pgids",
|
||||
HEADER_JOB_PROJECT_NAME: "project_name",
|
||||
HEADER_JOB_START_TIME: "start_time",
|
||||
HEADER_JOB_END_TIME: "end_time",
|
||||
HEADER_JOB_CWD: "cwd",
|
||||
HEADER_JOB_CPU_TIME: "cpu_time",
|
||||
HEADER_JOB_RESREQ: "resreq",
|
||||
HEADER_JOB_DESCRIPTION: "job_description",
|
||||
HEADER_JOB_LAUNCH_TYPE: "launch_type",
|
||||
HEADER_JOB_NUM_PROCESSORS: "num_processors",
|
||||
HEADER_JOB_INTERACTIVE_URL: "interactive_job_url",
|
||||
HEADER_JOB_PENDING_REASON: "pending_reason",
|
||||
HEADER_JOB_JOBID: "job_id",
|
||||
HEADER_JOB_USER: "submit_user",
|
||||
HEADER_JOB_STATUS: "job_status",
|
||||
HEADER_JOB_QUEUE: "queue_name",
|
||||
HEADER_JOB_COMMAND: "command",
|
||||
HEADER_JOB_NAME: "job_name",
|
||||
HEADER_JOB_SUBMIT_TIME: "submit_time",
|
||||
HEADER_JOB_FROM_HOST: "from_host",
|
||||
HEADER_JOB_EXEC_HOST: "exec_hosts",
|
||||
HEADER_JOB_CORE_LIMIT: "core_limit",
|
||||
HEADER_JOB_MEM_LIMIT: "mem_limit",
|
||||
HEADER_JOB_PID: "pids",
|
||||
HEADER_JOB_PGID: "pgids",
|
||||
HEADER_JOB_PROJECT_NAME: "project_name",
|
||||
HEADER_JOB_START_TIME: "start_time",
|
||||
HEADER_JOB_END_TIME: "end_time",
|
||||
HEADER_JOB_CWD: "cwd",
|
||||
HEADER_JOB_CPU_TIME: "cpu_time",
|
||||
HEADER_JOB_RESREQ: "resreq",
|
||||
HEADER_JOB_DESCRIPTION: "job_description",
|
||||
HEADER_JOB_LAUNCH_TYPE: "launch_type",
|
||||
HEADER_JOB_NUM_PROCESSORS: "num_processors",
|
||||
HEADER_JOB_INTERACTIVE_URL: "interactive_job_url",
|
||||
HEADER_JOB_PENDING_REASON: "pending_reason",
|
||||
HEADER_JOB_MAX_MEM: "max_mem",
|
||||
HEADER_JOB_AVG_MEM: "avg_mem",
|
||||
HEADER_JOB_MEM_EFFICIENCY: "mem_efficiency",
|
||||
HEADER_JOB_CPU_PEAK: "peak_cpu",
|
||||
HEADER_JOB_CPU_PEAK_DURATION: "peak_cpu_duration",
|
||||
HEADER_JOB_CPU_EFFICIENCY: "avg_cpu_efficiency",
|
||||
HEADER_JOB_CPU_PEAK_EFFICIENCY: "peak_cpu_efficiency",
|
||||
//job_filter.HEADER_JOB_PRIORITY: "job_priority",
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -338,6 +338,7 @@ struct Job $C.mapname("xce_job_t"){
|
|||
pendreason @10: JobPendReason $C.mapname("pend_reasons");
|
||||
transientSince @11: UInt64 = 0 $C.mapname("transient_since"); # epoch seconds; set when job enters JOB_STAT_TRANSIENT
|
||||
exitInfo @12: JobExitInfo $C.mapname("exit_info"); # bug-1867: null until the job's exit code is known
|
||||
runUsage @13: JobRunusage $C.mapname("run_usage"); # bug-1736: null until the first usage report; snapshot + apiserver
|
||||
}
|
||||
|
||||
# bug-1867: job_exitinfo_t, so a checkpointed Job keeps its exit code. The
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue