lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller,apiserver,xcecli): expose job CPU peak and efficiency to the apiserver and xcecli (bug-1736)

Browse source 
The controller computed run usage (CPU time, memory, CPU peak and
efficiency, LSF bjobs -l definitions) but the capnp Job struct had no
field for it, so the job pushed to the apiserver never carried it, and
snapshots (the same struct) dropped it: a job restored from a checkpoint
lost its accumulated usage.

- controller.capnp: Job.runUsage @13 (JobRunusage); old entries decode
  with no run usage.
- apiserver: map run_usage into JobInfo; max_mem, avg_mem, used_mem,
  cpu_time, peak_cpu, peak_cpu_time, peak_cpu_duration, mem_efficiency,
  avg_cpu_efficiency, peak_cpu_efficiency become queryable columns.
- xcecli: -o fields MAX_MEM, AVG_MEM, MEM_EFFICIENCY, CPU_PEAK,
  CPU_PEAK_DURATION, CPU_EFFICIENCY, CPU_PEAK_EFFICIENCY (CPU_TIME now
  renders too), and a MEMORY USAGE / CPU USAGE block in jobs list -l.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-06 13:54:43 +00:00
parent e2472aa73c
commit 1cedd7f791
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
9 changed files with 466 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/frontend/apiserver/backend/convert.go
View file

@ -114,6 +114,20 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
}
r.submit = rd
}
if u := job.run_usage; u != nil {
r.runUsage = &rawRunUsage{
maxMem: float64(u.max_mem),
avgMem: float64(u.avg_mem),
usedMem: float64(u.used_mem),
cpuTime: int(u.cpu_time),
peakCPUTime: int(u.peak_cpu_time),
peakCPUDuration: int(u.peak_cpu_duration),
memEfficiency: float64(u.mem_efficiency),
avgCPUEfficiency: float64(u.avg_cpu_efficiency),
peakCPUEfficiency: float64(u.peak_cpu_efficiency),
peakCPU: float64(u.peak_cpu),
}
}
if job.allocs != nil {
n := int(job.allocs.nhosts)
for i := 0; i < n; i++ {

30
src/frontend/apiserver/backend/convert_pure.go
View file

@ -263,6 +263,23 @@ type rawHostSlot struct {
slots int
}
// rawRunUsage mirrors the C xce_job_runusage_t job-level fields used by the
// Go model (bug-1736): accumulated CPU time and memory, and the CPU peak and
// efficiency values the controller computes with LSF bjobs -l definitions
// (efficiencies in percent, memory in MB as the worker reports it).
type rawRunUsage struct {
maxMem float64
avgMem float64
usedMem float64
cpuTime int
peakCPUTime int
peakCPUDuration int
memEfficiency float64
avgCPUEfficiency float64
peakCPUEfficiency float64
peakCPU float64
}
// rawJobData mirrors the C xce_job_t fields used by the Go model.
type rawJobData struct {
jobID int64
@ -274,6 +291,7 @@ type rawJobData struct {
finishTime int64
submit *rawSubmitData
execHosts []rawHostSlot
runUsage *rawRunUsage // nil until the job's first usage report
}
// rawWorkerData mirrors the C xce_worker_t fields used by the Go model.
@ -363,6 +381,18 @@ func rawJobToModel(r rawJobData) *models.JobInfo {
data, _ := json.Marshal(execHostsJSON{HostSlots: slots, NumHosts: len(slots)})
info.ExecHosts = models.Jsonb{String: string(data)}
}
if u := r.runUsage; u != nil {
info.MaxMem = u.maxMem
info.AvgMem = u.avgMem
info.UsedMem = u.usedMem
info.CPUTime = u.cpuTime
info.PeakCPU = u.peakCPU
info.PeakCPUTime = u.peakCPUTime
info.PeakCPUDuration = float64(u.peakCPUDuration)
info.MemEfficiency = u.memEfficiency
info.AvgCPUEfficiency = u.avgCPUEfficiency
info.PeakCPUEfficiency = u.peakCPUEfficiency
}
return info
}

66
src/frontend/apiserver/backend/convert_test.go
View file

@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"reflect"
"strings"
"testing"
)
@ -1185,3 +1187,67 @@ func TestRawClusterToModel(t *testing.T) {
})
}
}
// bug-1736: the controller's accumulated run usage (CPU time, memory, CPU
// peak and efficiency, LSF bjobs -l definitions) reaches the job model, so
// REST field selection and xcecli can show it.
func TestRawJobToModel_RunUsage(t *testing.T) {
r := rawJobData{
jobID: 42,
state: 4,
runUsage: &rawRunUsage{
maxMem: 512.5,
avgMem: 300.25,
usedMem: 410,
cpuTime: 1234,
peakCPUTime: 4,
peakCPUDuration: 30,
memEfficiency: 80.5,
avgCPUEfficiency: 62.25,
peakCPUEfficiency: 95,
peakCPU: 3.75,
},
}
info := rawJobToModel(r)
if info.MaxMem != 512.5 || info.AvgMem != 300.25 || info.UsedMem != 410 {
t.Errorf("memory: got max=%v avg=%v used=%v", info.MaxMem, info.AvgMem, info.UsedMem)
}
if info.CPUTime != 1234 {
t.Errorf("CPUTime = %d, want 1234", info.CPUTime)
}
if info.PeakCPU != 3.75 || info.PeakCPUTime != 4 || info.PeakCPUDuration != 30 {
t.Errorf("CPU peak: got peak=%v peak_time=%d duration=%v",
info.PeakCPU, info.PeakCPUTime, info.PeakCPUDuration)
}
if info.MemEfficiency != 80.5 || info.AvgCPUEfficiency != 62.25 || info.PeakCPUEfficiency != 95 {
t.Errorf("efficiency: got mem=%v avg_cpu=%v peak_cpu=%v",
info.MemEfficiency, info.AvgCPUEfficiency, info.PeakCPUEfficiency)
}
}
// bug-1736: a job with no run usage reported yet maps to zero values.
func TestRawJobToModel_NoRunUsage(t *testing.T) {
info := rawJobToModel(rawJobData{jobID: 42})
if info.CPUTime != 0 || info.PeakCPU != 0 || info.MemEfficiency != 0 || info.MaxMem != 0 {
t.Errorf("expected zero run usage, got %+v", info)
}
}
// bug-1736: the run-usage fields are JobInfo columns, so a REST query can
// select them (store rows are built from JobInfo's json tags).
func TestJobInfoExposesRunUsageColumns(t *testing.T) {
cols := map[string]bool{}
ty := reflect.TypeOf(models.JobInfo{})
for i := 0; i < ty.NumField(); i++ {
cols[strings.Split(ty.Field(i).Tag.Get("json"), ",")[0]] = true
}
for _, c := range []string{"max_mem", "avg_mem", "used_mem", "cpu_time",
"peak_cpu", "peak_cpu_time", "peak_cpu_duration", "mem_efficiency",
"avg_cpu_efficiency", "peak_cpu_efficiency"} {
if !cols[c] {
t.Errorf("JobInfo has no %q column", c)
}
}
}

66
src/frontend/apiserver/models/jobs.go
View file

@ -546,18 +546,29 @@ type JobInfo struct {
ReserveTime int64 `json:"reserve_time"`
Duration int `json:"duration"`
CPUTime int `json:"cpu_time"`
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
ExecUID int `json:"exec_uid"`
ExecHome string `json:"exec_home"`
ExecCWD string `json:"exec_cwd"`
ExecUser string `json:"exec_user"` //name change
ChargedSaap string `json:"charged_saap"`
RunTime int `json:"run_time"`
RunTimeEstimation int `json:"run_time_estimation"`
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
// Run usage (bug-1736), from the controller's accumulated job usage with
// LSF bjobs -l definitions; zero until the first usage report.
MaxMem float64 `json:"max_mem"` // MB
AvgMem float64 `json:"avg_mem"` // MB
UsedMem float64 `json:"used_mem"` // MB, latest
PeakCPU float64 `json:"peak_cpu"` // highest cores used at once
PeakCPUTime int `json:"peak_cpu_time"` // peak_cpu rounded
PeakCPUDuration float64 `json:"peak_cpu_duration"` // seconds, the interval that set the peak
MemEfficiency float64 `json:"mem_efficiency"` // percent of the rusage[mem] request
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // percent, cpu_time / elapsed / slots
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // percent, peak_cpu / slots
ExitStatus Jsonb `json:"exit_status"` //change type, JSONB as []byte
ExecHosts Jsonb `json:"exec_hosts"` //change type, JSONB as []byte
ExecUID int `json:"exec_uid"`
ExecHome string `json:"exec_home"`
ExecCWD string `json:"exec_cwd"`
ExecUser string `json:"exec_user"` //name change
ChargedSaap string `json:"charged_saap"`
RunTime int `json:"run_time"`
RunTimeEstimation int `json:"run_time_estimation"`
PendingReason Jsonb `json:"pending_reason"` //change type, JSONB as []byte
PIDInfo Jsonb `json:"pid_info"` // JSONB as []byte
JobRusage Jsonb `json:"job_rusage"` // JSONB as []byte
}
type JobExecHosts struct {
@ -598,25 +609,18 @@ type JobAdditionalFields struct {
FromHost string `json:"from_host"`
ExecHosts string `json:"exec_hosts"`
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
JobRusage Jsonb `json:"job_rusage"`
MaxMem float64 `json:"max_mem"`
AvgMem float64 `json:"avg_mem"`
UsedMem float64 `json:"used_mem"`
CPUTime int `json:"cpu_time"`
PeakCPUTime int `json:"peak_cpu_time"`
PeakCPUDuration float64 `json:"peak_cpu_duration"`
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage
AvgCPUEfficiency float64 `json:"avg_cpu_efficiency"` // Average CPU efficiency in percentage
PeakCPUEfficiency float64 `json:"peak_cpu_efficiency"` // Peak CPU efficiency in percentage
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
FirstHost string `json:"first_host"` // First host where the job ran
AllocSlot string `json:"alloc_slots"` // Allocated slot information
PIDInfo Jsonb `json:"pid_info"`
Pids []int `json:"pids"` // List of process IDs associated with the job
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
InteractiveJobURL string `json:"interactive_job_url"`
StartTime int64 `json:"start_time"`
EndTime int64 `json:"end_time"`
JobRusage Jsonb `json:"job_rusage"`
// max_mem/avg_mem/used_mem/cpu_time/peak_cpu*/..._efficiency are
// JobInfo columns now (bug-1736).
NumHosts int `json:"num_hosts"` // Number of hosts used by the job
FirstHost string `json:"first_host"` // First host where the job ran
AllocSlot string `json:"alloc_slots"` // Allocated slot information
PIDInfo Jsonb `json:"pid_info"`
Pids []int `json:"pids"` // List of process IDs associated with the job
Pgids []int `json:"pgids"` // List of process group IDs associated with the job
InteractiveJobURL string `json:"interactive_job_url"`
// CoreLimit/MemLimit: JobInfo has no equivalent field (these are
// submission-time resource limits, not job-state fields), but xcecli's
// JobFieldsMap accepts them as valid -o/-l display fields (bug-1674) --

68
src/frontend/xcecli/common/output/page_print_jobs.go
View file

@ -103,6 +103,10 @@ func (p *JobsPagePrinter) initColumnWidths(jobs *models.JobsResponse) {
width = len(fmt.Sprintf("%f", job.CoreLimit))
case models.HEADER_JOB_MEM_LIMIT:
width = len(fmt.Sprintf("%f", job.MemLimit))
default:
if v, ok := jobRunUsageColumn(format.name, job); ok {
width = len(v)
}
}
if !p.wideFormat && width > 15 {
@ -237,7 +241,11 @@ func (p *JobsPagePrinter) formatRows(jobs *models.JobsResponse, truncatFlag bool
case models.HEADER_JOB_MEM_LIMIT:
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, formatSizeString(job.MemLimit)), "-")
default:
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*v", width, "-"), "-")
v, ok := jobRunUsageColumn(format.name, job)
if !ok {
v = "-"
}
mainRow = appendOrDefault(mainRow, fmt.Sprintf("%"+align+"*s", width, v), "-")
}
}
rows = append(rows, mainRow)
@ -409,6 +417,9 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
}
PrintWrappedLine(w, endLine, 80, 18)
for _, l := range jobResourceUsageLines(job) {
fmt.Fprintln(w, l)
}
fmt.Println("")
// print the scheduling parameters table
fmt.Printf("SCHEDULING PARAMETERS\n")
@ -443,3 +454,58 @@ func printJobsVerboseOutput(jobs *models.JobsResponse, filter []string) {
}
}
}
// jobResourceUsageLines renders a started job's memory and CPU usage in the
// LSF bjobs -l shape (bug-1736): MAX/AVG MEM in Mbytes and MEM EFFICIENCY
// against the rusage[mem] request; CPU PEAK in cores with the interval that
// set it; average and peak CPU efficiency against the allocated slots. A job
// that has not started (PEND/PSUSP) has nothing to report.
func jobResourceUsageLines(job models.QueryJob) []string {
if job.JobStatus == "" || job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
return nil
}
return []string{
"",
" MEMORY USAGE:",
fmt.Sprintf(" MAX MEM: %.0f Mbytes; AVG MEM: %.0f Mbytes; MEM EFFICIENCY: %.2f%%",
job.MaxMem, job.AvgMem, job.MemEfficiency),
"",
" CPU USAGE:",
fmt.Sprintf(" CPU PEAK: %.2f ; CPU PEAK DURATION: %.0f second(s)",
job.PeakCPU, job.PeakCPUDuration),
fmt.Sprintf(" CPU AVERAGE EFFICIENCY: %.2f%% ; CPU PEAK EFFICIENCY: %.2f%%",
job.AvgCPUEfficiency, job.PeakCPUEfficiency),
}
}
// jobRunUsageColumn formats one run-usage column of `jobs list -o` (bug-1736),
// in the units of the -l view: memory in Mbytes, CPU peak in cores, durations
// in seconds, efficiencies in percent. ok is false for any other column. A
// job that has not started shows "-".
func jobRunUsageColumn(header string, job models.QueryJob) (string, bool) {
var v string
switch header {
case models.HEADER_JOB_CPU_TIME:
v = fmt.Sprintf("%d", job.CPUTime)
case models.HEADER_JOB_MAX_MEM:
v = fmt.Sprintf("%.0f", job.MaxMem)
case models.HEADER_JOB_AVG_MEM:
v = fmt.Sprintf("%.0f", job.AvgMem)
case models.HEADER_JOB_MEM_EFFICIENCY:
v = fmt.Sprintf("%.2f%%", job.MemEfficiency)
case models.HEADER_JOB_CPU_PEAK:
v = fmt.Sprintf("%.2f", job.PeakCPU)
case models.HEADER_JOB_CPU_PEAK_DURATION:
v = fmt.Sprintf("%.0f", job.PeakCPUDuration)
case models.HEADER_JOB_CPU_EFFICIENCY:
v = fmt.Sprintf("%.2f%%", job.AvgCPUEfficiency)
case models.HEADER_JOB_CPU_PEAK_EFFICIENCY:
v = fmt.Sprintf("%.2f%%", job.PeakCPUEfficiency)
default:
return "", false
}
if job.JobStatus == "PEND" || job.JobStatus == "PSUSP" {
v = "-"
}
return v, true
}

90
src/frontend/xcecli/common/output/page_print_jobs_test.go Normal file
View file

@ -0,0 +1,90 @@
package output
import (
"strings"
"testing"
"xcecli/models"
)
// bug-1736: `xcecli jobs list -l` shows the job's memory and CPU usage in the
// LSF bjobs -l shape once the job has started.
func TestJobResourceUsageLines_StartedJob(t *testing.T) {
job := models.QueryJob{
JobStatus: "RUN",
MaxMem: 512,
AvgMem: 300.4,
MemEfficiency: 80.5,
PeakCPU: 3.75,
PeakCPUDuration: 30,
AvgCPUEfficiency: 62.25,
PeakCPUEfficiency: 95,
}
got := strings.Join(jobResourceUsageLines(job), "\n")
for _, want := range []string{
"MEMORY USAGE:",
"MAX MEM: 512 Mbytes; AVG MEM: 300 Mbytes; MEM EFFICIENCY: 80.50%",
"CPU USAGE:",
"CPU PEAK: 3.75 ; CPU PEAK DURATION: 30 second(s)",
"CPU AVERAGE EFFICIENCY: 62.25% ; CPU PEAK EFFICIENCY: 95.00%",
} {
if !strings.Contains(got, want) {
t.Errorf("missing %q in:\n%s", want, got)
}
}
}
// A job that has not started has no usage to report.
func TestJobResourceUsageLines_PendingJob(t *testing.T) {
for _, st := range []string{"PEND", "PSUSP"} {
if lines := jobResourceUsageLines(models.QueryJob{JobStatus: st}); len(lines) != 0 {
t.Errorf("%s job: expected no usage lines, got %q", st, lines)
}
}
}
// bug-1736: the usage fields can be selected with -o and are requested by -l.
func TestJobFieldsMapIncludesRunUsage(t *testing.T) {
have := map[string]bool{}
for _, f := range models.JobFieldsMap {
have[f] = true
}
for _, f := range []string{"max_mem", "avg_mem", "mem_efficiency",
"peak_cpu", "peak_cpu_duration", "avg_cpu_efficiency",
"peak_cpu_efficiency"} {
if !have[f] {
t.Errorf("JobFieldsMap has no field %q", f)
}
}
}
// bug-1736: the run-usage columns render in `jobs list -o` (they printed "-").
func TestJobRunUsageColumn(t *testing.T) {
job := models.QueryJob{
JobStatus: "RUN", CPUTime: 1234, MaxMem: 512, AvgMem: 300.4,
MemEfficiency: 80.5, PeakCPU: 3.75, PeakCPUDuration: 30,
AvgCPUEfficiency: 62.25, PeakCPUEfficiency: 95,
}
cases := map[string]string{
models.HEADER_JOB_CPU_TIME: "1234",
models.HEADER_JOB_MAX_MEM: "512",
models.HEADER_JOB_AVG_MEM: "300",
models.HEADER_JOB_MEM_EFFICIENCY: "80.50%",
models.HEADER_JOB_CPU_PEAK: "3.75",
models.HEADER_JOB_CPU_PEAK_DURATION: "30",
models.HEADER_JOB_CPU_EFFICIENCY: "62.25%",
models.HEADER_JOB_CPU_PEAK_EFFICIENCY: "95.00%",
}
for h, want := range cases {
got, ok := jobRunUsageColumn(h, job)
if !ok || got != want {
t.Errorf("%s: got %q (ok=%v), want %q", h, got, ok, want)
}
}
if _, ok := jobRunUsageColumn(models.HEADER_JOB_USER, job); ok {
t.Errorf("USER is not a run-usage column")
}
// a job that has not started shows "-", not zeros
if got, _ := jobRunUsageColumn(models.HEADER_JOB_CPU_PEAK, models.QueryJob{JobStatus: "PEND"}); got != "-" {
t.Errorf("PEND job CPU_PEAK: got %q, want -", got)
}
}

64
src/frontend/xcecli/models/jobs.go
View file

@ -31,6 +31,14 @@ const (
HEADER_JOB_NUM_PROCESSORS = "NUM_PROCESSORS"
HEADER_JOB_INTERACTIVE_URL = "INTERACTIVE_URL"
HEADER_JOB_PENDING_REASON = "PENDING_REASON"
// run usage (bug-1736)
HEADER_JOB_MAX_MEM = "MAX_MEM"
HEADER_JOB_AVG_MEM = "AVG_MEM"
HEADER_JOB_MEM_EFFICIENCY = "MEM_EFFICIENCY"
HEADER_JOB_CPU_PEAK = "CPU_PEAK"
HEADER_JOB_CPU_PEAK_DURATION = "CPU_PEAK_DURATION"
HEADER_JOB_CPU_EFFICIENCY = "CPU_EFFICIENCY"
HEADER_JOB_CPU_PEAK_EFFICIENCY = "CPU_PEAK_EFFICIENCY"
// second line
HEADER_JOB_R15S = "r15s"
@ -120,6 +128,7 @@ type QueryJob struct {
AvgMem float64 `json:"avg_mem"`
UsedMem float64 `json:"used_mem"`
CPUTime int `json:"cpu_time"`
PeakCPU float64 `json:"peak_cpu"` // highest cores used at once (bug-1736)
PeakCPUTime int `json:"peak_cpu_time"`
PeakCPUDuration float64 `json:"peak_cpu_duration"`
MemEfficiency float64 `json:"mem_efficiency"` // Memory efficiency in percentage
@ -184,30 +193,37 @@ type QueryJob struct {
}
var JobFieldsMap = map[string]string{
HEADER_JOB_JOBID: "job_id",
HEADER_JOB_USER: "submit_user",
HEADER_JOB_STATUS: "job_status",
HEADER_JOB_QUEUE: "queue_name",
HEADER_JOB_COMMAND: "command",
HEADER_JOB_NAME: "job_name",
HEADER_JOB_SUBMIT_TIME: "submit_time",
HEADER_JOB_FROM_HOST: "from_host",
HEADER_JOB_EXEC_HOST: "exec_hosts",
HEADER_JOB_CORE_LIMIT: "core_limit",
HEADER_JOB_MEM_LIMIT: "mem_limit",
HEADER_JOB_PID: "pids",
HEADER_JOB_PGID: "pgids",
HEADER_JOB_PROJECT_NAME: "project_name",
HEADER_JOB_START_TIME: "start_time",
HEADER_JOB_END_TIME: "end_time",
HEADER_JOB_CWD: "cwd",
HEADER_JOB_CPU_TIME: "cpu_time",
HEADER_JOB_RESREQ: "resreq",
HEADER_JOB_DESCRIPTION: "job_description",
HEADER_JOB_LAUNCH_TYPE: "launch_type",
HEADER_JOB_NUM_PROCESSORS: "num_processors",
HEADER_JOB_INTERACTIVE_URL: "interactive_job_url",
HEADER_JOB_PENDING_REASON: "pending_reason",
HEADER_JOB_JOBID: "job_id",
HEADER_JOB_USER: "submit_user",
HEADER_JOB_STATUS: "job_status",
HEADER_JOB_QUEUE: "queue_name",
HEADER_JOB_COMMAND: "command",
HEADER_JOB_NAME: "job_name",
HEADER_JOB_SUBMIT_TIME: "submit_time",
HEADER_JOB_FROM_HOST: "from_host",
HEADER_JOB_EXEC_HOST: "exec_hosts",
HEADER_JOB_CORE_LIMIT: "core_limit",
HEADER_JOB_MEM_LIMIT: "mem_limit",
HEADER_JOB_PID: "pids",
HEADER_JOB_PGID: "pgids",
HEADER_JOB_PROJECT_NAME: "project_name",
HEADER_JOB_START_TIME: "start_time",
HEADER_JOB_END_TIME: "end_time",
HEADER_JOB_CWD: "cwd",
HEADER_JOB_CPU_TIME: "cpu_time",
HEADER_JOB_RESREQ: "resreq",
HEADER_JOB_DESCRIPTION: "job_description",
HEADER_JOB_LAUNCH_TYPE: "launch_type",
HEADER_JOB_NUM_PROCESSORS: "num_processors",
HEADER_JOB_INTERACTIVE_URL: "interactive_job_url",
HEADER_JOB_PENDING_REASON: "pending_reason",
HEADER_JOB_MAX_MEM: "max_mem",
HEADER_JOB_AVG_MEM: "avg_mem",
HEADER_JOB_MEM_EFFICIENCY: "mem_efficiency",
HEADER_JOB_CPU_PEAK: "peak_cpu",
HEADER_JOB_CPU_PEAK_DURATION: "peak_cpu_duration",
HEADER_JOB_CPU_EFFICIENCY: "avg_cpu_efficiency",
HEADER_JOB_CPU_PEAK_EFFICIENCY: "peak_cpu_efficiency",
//job_filter.HEADER_JOB_PRIORITY: "job_priority",
}