lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(controller): compute the LSF CPU peak and efficiency fields (bug-1736)

Browse source 
update_job_cpu_stats() (misc.c), run after each usage report:
- each host's cpu_rate = CPU-seconds gained / wall seconds since its
  previous report; the job's cores = the sum over hosts
- peak_cpu = highest job cores seen (peak_cpu_time = it rounded, the
  wire field is an integer); peak_cpu_duration = the interval that set it
- avg_cpu_efficiency = cpu_time / elapsed / slots, peak_cpu_efficiency =
  peak_cpu / slots, mem_efficiency = highest used_mem / rusage[mem]
  request (job-wide total, per task by default), all in percent
aggregate_job_runusage() no longer treats cumulative cpu_time as a peak.

New fields host sample_time/cpu_rate and job peak_cpu are appended to
the capnp structs (old WAL entries still decode) and copied by
merge_job_runusage(), so a promoted standby keeps the rate baselines.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-02 04:14:35 +00:00
parent 77b5102b3e
commit 6df644e6d5
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
8 changed files with 325 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/controller/testsuite/test_commit_barrier_reply.c
View file

@ -198,6 +198,11 @@ int update_pids(job_pidinfo_t **i, int n, int *p)
int update_pgids(job_pidinfo_t **i, int n, int *p)
{ (void)i; (void)n; (void)p; return 0; }
void aggregate_job_runusage(xce_job_runusage_t *r) { (void)r; }
void update_job_cpu_stats(xce_job_runusage_t *r, const char *h, int p,
int64_t n, int64_t st, int ns, double m)
{
(void)r; (void)h; (void)p; (void)n; (void)st; (void)ns; (void)m;
}
int on_scheduler_join(cluster_t *c) { (void)c; return 0; }
int add_apiserver(cluster_t *c, char *n, endpoint_t *e)
{ (void)c; (void)n; (void)e; return 0; }

152
src/controller/testsuite/test_host_runusage.c
View file

@ -273,33 +273,143 @@ static void test_aggregate_job_runusage_multi_host_sum_max_avg(void)
free_runusage(r);
}
static void test_aggregate_job_runusage_peak_cpu_time_is_running_max(void)
/* bug-1736: peak_cpu_time is now the LSF "CPU PEAK" (peak cores, set by
* update_job_cpu_stats()); aggregate_job_runusage() no longer touches it. */
static void test_aggregate_job_runusage_leaves_peak_cpu_alone(void)
{
int cpu1[] = { 50 };
double mem1[] = { 100.0 };
xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu1, mem1);
r->peak_cpu_time = 3;
aggregate_job_runusage(r);
TEST_ASSERT_EQUAL_INT_MESSAGE(50, r->peak_cpu_time,
"peak_cpu_time tracks the first report");
TEST_ASSERT_EQUAL_INT(50, r->cpu_time);
TEST_ASSERT_EQUAL_INT_MESSAGE(3, r->peak_cpu_time,
"cumulative cpu_time is not a peak");
/* Second report: a LOWER cpu_time (e.g. a merge/replay delivered an
* older report after a newer one) -- peak must NOT regress. */
r->host_runusage[0]->cpu_time = 20;
aggregate_job_runusage(r);
TEST_ASSERT_EQUAL_INT_MESSAGE(
50, r->peak_cpu_time,
"peak_cpu_time (bug-1736) is the running MAX across every report "
"this job has ever received, not just the latest one -- a lower "
"subsequent report must not regress it");
TEST_ASSERT_EQUAL_INT_MESSAGE(
20, r->cpu_time, "cpu_time itself IS the latest report, unlike peak");
free_runusage(r);
}
/* Third report: a NEW high -- peak must advance. */
r->host_runusage[0]->cpu_time = 90;
/* ------------------------------------------------------------------ */
/* Tests -- update_job_cpu_stats() (bug-1736, LSF definitions) */
/* ------------------------------------------------------------------ */
#define ASSERT_FLOAT_NEAR(expected, actual) \
TEST_ASSERT_TRUE_MESSAGE(fabs((double)(expected) - (double)(actual)) \
< 1e-3, \
#actual " != " #expected)
/* One report from `host`: set its cumulative cpu (and mem), re-aggregate
* and update the stats, as update_job_rusages() does. prev_cpu < 0 means
* the host had no earlier report. */
static void report(xce_job_runusage_t* r, int hostidx, int cpu, double mem,
int64_t now, int64_t start, int nslots, double req_mem)
{
xce_host_runusage_t* h = r->host_runusage[hostidx];
int prev_cpu = (h->sample_time > 0) ? h->cpu_time : -1;
h->cpu_time = cpu;
h->mem_usage = mem;
aggregate_job_runusage(r);
TEST_ASSERT_EQUAL_INT_MESSAGE(90, r->peak_cpu_time,
"peak_cpu_time advances on a new high");
update_job_cpu_stats(r, h->host_name, prev_cpu, now, start, nslots,
req_mem);
}
static xce_job_runusage_t* make_named_hosts(int n)
{
static const char* names[] = { "ha", "hb", "hc" };
xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r));
int i;
r->num_hosts = n;
r->host_runusage = (xce_host_runusage_t**)calloc(n, sizeof(*r->host_runusage));
for (i = 0; i < n; i++) {
r->host_runusage[i] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
r->host_runusage[i]->host_name = strdup(names[i]);
}
return r;
}
static void test_cpu_stats_rate_peak_and_efficiency(void)
{
xce_job_runusage_t* r = make_named_hosts(1);
/* first report: no interval yet, nothing to rate */
report(r, 0, 0, 100.0, 100, 100, 2, 0.0);
ASSERT_FLOAT_NEAR(0.0, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(100, (int)r->host_runusage[0]->sample_time);
/* +30 CPU-seconds in 30 s: 1 core busy */
report(r, 0, 30, 100.0, 130, 100, 2, 0.0);
ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(1, r->peak_cpu_time);
TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration);
ASSERT_FLOAT_NEAR(50.0, r->peak_cpu_efficiency); /* 1 core of 2 slots */
ASSERT_FLOAT_NEAR(50.0, r->avg_cpu_efficiency); /* 30 s / 30 s / 2 */
/* +30 in 60 s: 0.5 core -- the peak and its duration stay */
report(r, 0, 60, 100.0, 190, 100, 2, 0.0);
ASSERT_FLOAT_NEAR(0.5, r->host_runusage[0]->cpu_rate);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration);
ASSERT_FLOAT_NEAR(100.0 * 60 / 90 / 2, r->avg_cpu_efficiency);
free_runusage(r);
}
static void test_cpu_stats_sum_host_rates(void)
{
xce_job_runusage_t* r = make_named_hosts(2);
report(r, 0, 0, 10.0, 100, 100, 4, 0.0);
report(r, 1, 0, 10.0, 100, 100, 4, 0.0);
report(r, 0, 20, 10.0, 120, 100, 4, 0.0); /* host a: 1.0 core */
report(r, 1, 30, 10.0, 160, 100, 4, 0.0); /* host b: 0.5 core */
ASSERT_FLOAT_NEAR(1.5, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(2, r->peak_cpu_time); /* rounded */
TEST_ASSERT_EQUAL_INT(60, r->peak_cpu_duration);
ASSERT_FLOAT_NEAR(37.5, r->peak_cpu_efficiency); /* 1.5 / 4 */
free_runusage(r);
}
static void test_cpu_stats_mem_efficiency_is_peak(void)
{
xce_job_runusage_t* r = make_named_hosts(1);
report(r, 0, 0, 256.0, 100, 100, 1, 512.0);
ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency);
report(r, 0, 10, 128.0, 110, 100, 1, 512.0);
ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency); /* peak, not latest */
report(r, 0, 20, 384.0, 120, 100, 1, 512.0);
ASSERT_FLOAT_NEAR(75.0, r->mem_efficiency);
free_runusage(r);
}
static void test_cpu_stats_guards(void)
{
xce_job_runusage_t* r = make_named_hosts(1);
report(r, 0, 0, 64.0, 100, 100, 0, 0.0);
report(r, 0, 30, 64.0, 130, 100, 0, 0.0);
/* no slots, no memory request: no efficiencies */
ASSERT_FLOAT_NEAR(0.0, r->peak_cpu_efficiency);
ASSERT_FLOAT_NEAR(0.0, r->avg_cpu_efficiency);
ASSERT_FLOAT_NEAR(0.0, r->mem_efficiency);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
/* a replayed/out-of-order report (no later than the last one) and a
* CPU counter that went backwards leave the rate alone */
report(r, 0, 90, 64.0, 130, 100, 0, 0.0);
ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
report(r, 0, 10, 64.0, 140, 100, 0, 0.0);
ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
update_job_cpu_stats(NULL, "ha", 0, 1, 1, 1, 1.0); /* must not crash */
update_job_cpu_stats(r, "nosuch", 0, 200, 100, 1, 1.0);
free_runusage(r);
}
@ -333,7 +443,11 @@ int main(void)
RUN_TEST(test_update_host_runusage_parses_swap_and_task_count);
RUN_TEST(test_aggregate_job_runusage_single_host);
RUN_TEST(test_aggregate_job_runusage_multi_host_sum_max_avg);
RUN_TEST(test_aggregate_job_runusage_peak_cpu_time_is_running_max);
RUN_TEST(test_aggregate_job_runusage_leaves_peak_cpu_alone);
RUN_TEST(test_cpu_stats_rate_peak_and_efficiency);
RUN_TEST(test_cpu_stats_sum_host_rates);
RUN_TEST(test_cpu_stats_mem_efficiency_is_peak);
RUN_TEST(test_cpu_stats_guards);
RUN_TEST(test_aggregate_job_runusage_null_and_empty_are_safe);
return UNITY_END();

19
src/controller/testsuite/test_wal_job_rusage_fact.c
View file

@ -188,10 +188,12 @@ static job_t* wru_seed_job(uint64_t jobid, uint32_t state)
/* ── Real fixture data: 2 hosts' worth of accumulated CPU/mem usage ──────── */
static xce_host_runusage_t g_wru_host_a = {
.host_name = (char*)"wru-host-a", .cpu_time = 1200, .mem_usage = 512.5
.host_name = (char*)"wru-host-a", .cpu_time = 1200, .mem_usage = 512.5,
.sample_time = 1790000000, .cpu_rate = 1.25f /* bug-1736 */
};
static xce_host_runusage_t g_wru_host_b = {
.host_name = (char*)"wru-host-b", .cpu_time = 900, .mem_usage = 256.25
.host_name = (char*)"wru-host-b", .cpu_time = 900, .mem_usage = 256.25,
.sample_time = 1790000030, .cpu_rate = 0.5f
};
static xce_host_runusage_t* g_wru_host_ptrs[2];
@ -209,6 +211,7 @@ static xce_job_runusage_t wru_make_test_runusage(void)
r.mem_efficiency = 0.75f;
r.avg_cpu_efficiency = 0.6f;
r.peak_cpu_efficiency = 0.9f;
r.peak_cpu = 1.75f; /* bug-1736 */
r.num_hosts = 2;
g_wru_host_ptrs[0] = &g_wru_host_a;
@ -232,14 +235,19 @@ static void wru_assert_runusage_matches(xce_job_runusage_t* got)
TEST_ASSERT_EQUAL_FLOAT(0.75f, got->mem_efficiency);
TEST_ASSERT_EQUAL_FLOAT(0.6f, got->avg_cpu_efficiency);
TEST_ASSERT_EQUAL_FLOAT(0.9f, got->peak_cpu_efficiency);
TEST_ASSERT_EQUAL_FLOAT(1.75f, got->peak_cpu); /* bug-1736 */
TEST_ASSERT_EQUAL_INT(2, got->num_hosts);
TEST_ASSERT_NOT_NULL(got->host_runusage);
TEST_ASSERT_EQUAL_STRING("wru-host-a", got->host_runusage[0]->host_name);
TEST_ASSERT_EQUAL_INT(1200, got->host_runusage[0]->cpu_time);
TEST_ASSERT_EQUAL_FLOAT(512.5, got->host_runusage[0]->mem_usage);
TEST_ASSERT_TRUE(got->host_runusage[0]->sample_time == 1790000000);
TEST_ASSERT_EQUAL_FLOAT(1.25f, got->host_runusage[0]->cpu_rate);
TEST_ASSERT_EQUAL_STRING("wru-host-b", got->host_runusage[1]->host_name);
TEST_ASSERT_EQUAL_INT(900, got->host_runusage[1]->cpu_time);
TEST_ASSERT_EQUAL_FLOAT(256.25, got->host_runusage[1]->mem_usage);
TEST_ASSERT_TRUE(got->host_runusage[1]->sample_time == 1790000030);
TEST_ASSERT_EQUAL_FLOAT(0.5f, got->host_runusage[1]->cpu_rate);
}
/* Build a wal_jobstate_t header + trailing Cap'n Proto JobRunusage bytes --
@ -449,7 +457,7 @@ test_wru_apply_committed_entry_merges_without_freeing_live_host(void)
* this job would carry. */
xce_host_runusage_t update_host = {
.host_name = (char*)"wru-live-host", .cpu_time = 999,
.mem_usage = 888.0
.mem_usage = 888.0, .sample_time = 1790000100, .cpu_rate = 2.5f
};
xce_host_runusage_t new_host = {
.host_name = (char*)"wru-new-host", .cpu_time = 42, .mem_usage = 84.0
@ -460,6 +468,7 @@ test_wru_apply_committed_entry_merges_without_freeing_live_host(void)
memset(&r, 0, sizeof(r));
r.max_mem = 500.0;
r.cpu_time = 250;
r.peak_cpu = 3.25f; /* bug-1736 */
r.num_hosts = 2;
r.host_runusage = host_ptrs;
@ -478,6 +487,7 @@ test_wru_apply_committed_entry_merges_without_freeing_live_host(void)
TEST_ASSERT_NOT_NULL(got);
TEST_ASSERT_EQUAL_FLOAT(500.0, got->max_mem);
TEST_ASSERT_EQUAL_INT(250, got->cpu_time);
TEST_ASSERT_EQUAL_FLOAT(3.25f, got->peak_cpu); /* bug-1736 */
TEST_ASSERT_EQUAL_INT(2, got->num_hosts);
int live_idx = -1, found_new = 0;
@ -494,6 +504,9 @@ test_wru_apply_committed_entry_merges_without_freeing_live_host(void)
"malloc'd pointer)");
TEST_ASSERT_EQUAL_INT(999, h->cpu_time);
TEST_ASSERT_EQUAL_FLOAT(888.0, h->mem_usage);
/* bug-1736: a merged existing host keeps the rate baseline */
TEST_ASSERT_TRUE(h->sample_time == 1790000100);
TEST_ASSERT_EQUAL_FLOAT(2.5f, h->cpu_rate);
} else if (h->host_name != NULL &&
strcmp(h->host_name, "wru-new-host") == 0) {
found_new = 1;