fix(controller,apiserver,xcecli): expose job CPU peak and efficiency to the apiserver and xcecli (bug-1736)
The controller computed run usage (CPU time, memory, CPU peak and efficiency, LSF bjobs -l definitions) but the capnp Job struct had no field for it, so the job pushed to the apiserver never carried it, and snapshots (the same struct) dropped it: a job restored from a checkpoint lost its accumulated usage. - controller.capnp: Job.runUsage @13 (JobRunusage); old entries decode with no run usage. - apiserver: map run_usage into JobInfo; max_mem, avg_mem, used_mem, cpu_time, peak_cpu, peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency become queryable columns. - xcecli: -o fields MAX_MEM, AVG_MEM, MEM_EFFICIENCY, CPU_PEAK, CPU_PEAK_DURATION, CPU_EFFICIENCY, CPU_PEAK_EFFICIENCY (CPU_TIME now renders too), and a MEMORY USAGE / CPU USAGE block in jobs list -l. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e2472aa73c
commit
1cedd7f791
GPG key ID:
80533536B0EB651C
9 changed files with 466 additions and 56 deletions
|
|
@ -114,6 +114,20 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
|
|||
}
|
||||
r.submit = rd
|
||||
}
|
||||
if u := job.run_usage; u != nil {
|
||||
r.runUsage = &rawRunUsage{
|
||||
maxMem: float64(u.max_mem),
|
||||
avgMem: float64(u.avg_mem),
|
||||
usedMem: float64(u.used_mem),
|
||||
cpuTime: int(u.cpu_time),
|
||||
peakCPUTime: int(u.peak_cpu_time),
|
||||
peakCPUDuration: int(u.peak_cpu_duration),
|
||||
memEfficiency: float64(u.mem_efficiency),
|
||||
avgCPUEfficiency: float64(u.avg_cpu_efficiency),
|
||||
peakCPUEfficiency: float64(u.peak_cpu_efficiency),
|
||||
peakCPU: float64(u.peak_cpu),
|
||||
}
|
||||
}
|
||||
if job.allocs != nil {
|
||||
n := int(job.allocs.nhosts)
|
||||
for i := 0; i < n; i++ {
|
||||
|
|
|
|||
|
|
@ -263,6 +263,23 @@ type rawHostSlot struct {
|
|||
slots int
|
||||
}
|
||||
|
||||
// rawRunUsage mirrors the C xce_job_runusage_t job-level fields used by the
|
||||
// Go model (bug-1736): accumulated CPU time and memory, and the CPU peak and
|
||||
// efficiency values the controller computes with LSF bjobs -l definitions
|
||||
// (efficiencies in percent, memory in MB as the worker reports it).
|
||||
type rawRunUsage struct {
|
||||
maxMem float64
|
||||
avgMem float64
|
||||
usedMem float64
|
||||
cpuTime int
|
||||
peakCPUTime int
|
||||
peakCPUDuration int
|
||||
memEfficiency float64
|
||||
avgCPUEfficiency float64
|
||||
peakCPUEfficiency float64
|
||||
peakCPU float64
|
||||
}
|
||||
|
||||
// rawJobData mirrors the C xce_job_t fields used by the Go model.
|
||||
type rawJobData struct {
|
||||
jobID int64
|
||||
|
|
@ -274,6 +291,7 @@ type rawJobData struct {
|
|||
finishTime int64
|
||||
submit *rawSubmitData
|
||||
execHosts []rawHostSlot
|
||||
runUsage *rawRunUsage // nil until the job's first usage report
|
||||
}
|
||||
|
||||
// rawWorkerData mirrors the C xce_worker_t fields used by the Go model.
|
||||
|
|
@ -363,6 +381,18 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
|
|||
data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)})
|
||||
info.ExecHosts = models.Jsonb{String: string(data)}
|
||||
}
|
||||
if u := r.runUsage; u != nil {
|
||||
info.MaxMem = u.maxMem
|
||||
info.AvgMem = u.avgMem
|
||||
info.UsedMem = u.usedMem
|
||||
info.CPUTime = u.cpuTime
|
||||
info.PeakCPU = u.peakCPU
|
||||
info.PeakCPUTime = u.peakCPUTime
|
||||
info.PeakCPUDuration = float64(u.peakCPUDuration)
|
||||
info.MemEfficiency = u.memEfficiency
|
||||
info.AvgCPUEfficiency = u.avgCPUEfficiency
|
||||
info.PeakCPUEfficiency = u.peakCPUEfficiency
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,8 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
|
|
@ -1185,3 +1187,67 @@ func TestRawClusterToModel(t *testing.T) {
|
|||
})
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: the controller's accumulated run usage (CPU time, memory, CPU
|
||||
// peak and efficiency, LSF bjobs -l definitions) reaches the job model, so
|
||||
// REST field selection and xcecli can show it.
|
||||
func TestRawJobToModel_RunUsage(t *testing.T) {
|
||||
r := rawJobData{
|
||||
jobID: 42,
|
||||
state: 4,
|
||||
runUsage: &rawRunUsage{
|
||||
maxMem: 512.5,
|
||||
avgMem: 300.25,
|
||||
usedMem: 410,
|
||||
cpuTime: 1234,
|
||||
peakCPUTime: 4,
|
||||
peakCPUDuration: 30,
|
||||
memEfficiency: 80.5,
|
||||
avgCPUEfficiency: 62.25,
|
||||
peakCPUEfficiency: 95,
|
||||
peakCPU: 3.75,
|
||||
},
|
||||
}
|
||||
|
||||
info := rawJobToModel(r)
|
||||
|
||||
if info.MaxMem != 512.5 || info.AvgMem != 300.25 || info.UsedMem != 410 {
|
||||
t.Errorf("memory: got max=%v avg=%v used=%v", info.MaxMem, info.AvgMem, info.UsedMem)
|
||||
}
|
||||
if info.CPUTime != 1234 {
|
||||
t.Errorf("CPUTime = %d, want 1234", info.CPUTime)
|
||||
}
|
||||
if info.PeakCPU != 3.75 || info.PeakCPUTime != 4 || info.PeakCPUDuration != 30 {
|
||||
t.Errorf("CPU peak: got peak=%v peak_time=%d duration=%v",
|
||||
info.PeakCPU, info.PeakCPUTime, info.PeakCPUDuration)
|
||||
}
|
||||
if info.MemEfficiency != 80.5 || info.AvgCPUEfficiency != 62.25 || info.PeakCPUEfficiency != 95 {
|
||||
t.Errorf("efficiency: got mem=%v avg_cpu=%v peak_cpu=%v",
|
||||
info.MemEfficiency, info.AvgCPUEfficiency, info.PeakCPUEfficiency)
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: a job with no run usage reported yet maps to zero values.
|
||||
func TestRawJobToModel_NoRunUsage(t *testing.T) {
|
||||
info := rawJobToModel(rawJobData{jobID: 42})
|
||||
if info.CPUTime != 0 || info.PeakCPU != 0 || info.MemEfficiency != 0 || info.MaxMem != 0 {
|
||||
t.Errorf("expected zero run usage, got %+v", info)
|
||||
}
|
||||
}
|
||||
|
||||
// bug-1736: the run-usage fields are JobInfo columns, so a REST query can
|
||||
// select them (store rows are built from JobInfo's json tags).
|
||||
func TestJobInfoExposesRunUsageColumns(t *testing.T) {
|
||||
cols := map[string]bool{}
|
||||
ty := reflect.TypeOf(models.JobInfo{})
|
||||
for i := 0; i < ty.NumField(); i++ {
|
||||
cols[strings.Split(ty.Field(i).Tag.Get("json"), ",")[0]] = true
|
||||
}
|
||||
for _, c := range []string{"max_mem", "avg_mem", "used_mem", "cpu_time",
|
||||
"peak_cpu", "peak_cpu_time", "peak_cpu_duration", "mem_efficiency",
|
||||
"avg_cpu_efficiency", "peak_cpu_efficiency"} {
|
||||
if !cols[c] {
|
||||
t.Errorf("JobInfo has no %q column", c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -546,18 +546,29 @@ type JobInfo struct {
|
|||
ReserveTime int64 `json:"reserve_time"`
|
||||
Duration int `json:"duration"`
|
||||
CPUTime int `json:"cpu_time"`
|
||||
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
|
||||
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
|
||||
ExecUID int `json:"exec_uid"`
|
||||
ExecHome string `json:"exec_home"`
|
||||
ExecCWD string `json:"exec_cwd"`
|
||||
ExecUser string `json:"exec_user"` //name change
|
||||
ChargedSaap string `json:"charged_saap"`
|
||||
RunTime int `json:"run_time"`
|
||||
RunTimeEstimation int `json:"run_time_estimation"`
|
||||
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
|
||||
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
|
||||
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
|
||||
// Run usage (bug-1736), from the controller's accumulated job usage with
|
||||
// LSF bjobs -l definitions; zero until the first usage report.
|
||||
MaxMem float64 `json:"max_mem"` // MB
|
||||
AvgMem float64 `json:"avg_mem"` // MB
|
||||
UsedMem float64 `json:"used_mem"` // MB, latest
|
||||
PeakCPU float64 `json:"peak_cpu"` // highest cores used at once
|
||||
PeakCPUTime int `json:"peak_cpu_time"` // peak_cpu rounded
|
||||
PeakCPUDuration float64 `json:"peak_cpu_duration"` // seconds, the interval that set the peak
|
||||
MemEfficiency float64 `json:"mem_efficiency"` // percent of the rusage[mem] request
|
||||
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // percent, cpu_time / elapsed / slots
|
||||
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // percent, peak_cpu / slots
|
||||
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
|
||||
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
|
||||
ExecUID int `json:"exec_uid"`
|
||||
ExecHome string `json:"exec_home"`
|
||||
ExecCWD string `json:"exec_cwd"`
|
||||
ExecUser string `json:"exec_user"` //name change
|
||||
ChargedSaap string `json:"charged_saap"`
|
||||
RunTime int `json:"run_time"`
|
||||
RunTimeEstimation int `json:"run_time_estimation"`
|
||||
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
|
||||
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
|
||||
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
|
||||
}
|
||||
|
||||
type JobExecHosts struct {
|
||||
|
|
@ -598,25 +609,18 @@ type JobAdditionalFields struct {
|
|||
FromHost string `json:"from_host"`
|
||||
ExecHosts string `json:"exec_hosts"`
|
||||
|
||||
StartTime int64 `json:"start_time"`
|
||||
EndTime int64 `json:"end_time"`
|
||||
JobRusage Jsonb `json:"job_rusage"`
|
||||
MaxMem float64 `json:"max_mem"`
|
||||
AvgMem float64 `json:"avg_mem"`
|
||||
UsedMem float64 `json:"used_mem"`
|
||||
CPUTime int `json:"cpu_time"`
|
||||
PeakCPUTime int `json:"peak_cpu_time"`
|
||||
PeakCPUDuration float64 `json:"peak_cpu_duration"`
|
||||
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage
|
||||
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage
|
||||
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage
|
||||
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
|
||||
FirstHost string `json:"first_host"` // First host where the job ran
|
||||
AllocSlot string `json:"alloc_slots"` // Allocated slot information
|
||||
PIDInfo Jsonb `json:"pid_info"`
|
||||
Pids []int `json:"pids"` // List of process IDs associated with the job
|
||||
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
|
||||
InteractiveJobURL string `json:"interactive_job_url"`
|
||||
StartTime int64 `json:"start_time"`
|
||||
EndTime int64 `json:"end_time"`
|
||||
JobRusage Jsonb `json:"job_rusage"`
|
||||
// max_mem/avg_mem/used_mem/cpu_time/peak_cpu*/..._efficiency are
|
||||
// JobInfo columns now (bug-1736).
|
||||
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
|
||||
FirstHost string `json:"first_host"` // First host where the job ran
|
||||
AllocSlot string `json:"alloc_slots"` // Allocated slot information
|
||||
PIDInfo Jsonb `json:"pid_info"`
|
||||
Pids []int `json:"pids"` // List of process IDs associated with the job
|
||||
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
|
||||
InteractiveJobURL string `json:"interactive_job_url"`
|
||||
// CoreLimit/MemLimit: JobInfo has no equivalent field (these are
|
||||
// submission-time resource limits, not job-state fields), but xcecli's
|
||||
// JobFieldsMap accepts them as valid -o/-l display fields (bug-1674) --
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue