lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(controller): compute the LSF CPU peak and efficiency fields (bug-1736)

Browse source 
update_job_cpu_stats() (misc.c), run after each usage report:
- each host's cpu_rate = CPU-seconds gained / wall seconds since its
  previous report; the job's cores = the sum over hosts
- peak_cpu = highest job cores seen (peak_cpu_time = it rounded, the
  wire field is an integer); peak_cpu_duration = the interval that set it
- avg_cpu_efficiency = cpu_time / elapsed / slots, peak_cpu_efficiency =
  peak_cpu / slots, mem_efficiency = highest used_mem / rusage[mem]
  request (job-wide total, per task by default), all in percent
aggregate_job_runusage() no longer treats cumulative cpu_time as a peak.

New fields host sample_time/cpu_rate and job peak_cpu are appended to
the capnp structs (old WAL entries still decode) and copied by
merge_job_runusage(), so a promoted standby keeps the rate baselines.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-02 04:14:35 +00:00
parent 77b5102b3e
commit 6df644e6d5
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
8 changed files with 325 additions and 50 deletions
Download patch file Download diff file Expand all files Collapse all files

152
src/controller/testsuite/test_host_runusage.c
View file

@ -273,33 +273,143 @@ static void test_aggregate_job_runusage_multi_host_sum_max_avg(void)
free_runusage(r);
}
static void test_aggregate_job_runusage_peak_cpu_time_is_running_max(void)
/* bug-1736: peak_cpu_time is now the LSF "CPU PEAK" (peak cores, set by
* update_job_cpu_stats()); aggregate_job_runusage() no longer touches it. */
static void test_aggregate_job_runusage_leaves_peak_cpu_alone(void)
{
int cpu1[] = { 50 };
double mem1[] = { 100.0 };
xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu1, mem1);
r->peak_cpu_time = 3;
aggregate_job_runusage(r);
TEST_ASSERT_EQUAL_INT_MESSAGE(50, r->peak_cpu_time,
"peak_cpu_time tracks the first report");
TEST_ASSERT_EQUAL_INT(50, r->cpu_time);
TEST_ASSERT_EQUAL_INT_MESSAGE(3, r->peak_cpu_time,
"cumulative cpu_time is not a peak");
/* Second report: a LOWER cpu_time (e.g. a merge/replay delivered an
* older report after a newer one) -- peak must NOT regress. */
r->host_runusage[0]->cpu_time = 20;
aggregate_job_runusage(r);
TEST_ASSERT_EQUAL_INT_MESSAGE(
50, r->peak_cpu_time,
"peak_cpu_time (bug-1736) is the running MAX across every report "
"this job has ever received, not just the latest one -- a lower "
"subsequent report must not regress it");
TEST_ASSERT_EQUAL_INT_MESSAGE(
20, r->cpu_time, "cpu_time itself IS the latest report, unlike peak");
free_runusage(r);
}
/* Third report: a NEW high -- peak must advance. */
r->host_runusage[0]->cpu_time = 90;
/* ------------------------------------------------------------------ */
/* Tests -- update_job_cpu_stats() (bug-1736, LSF definitions) */
/* ------------------------------------------------------------------ */
#define ASSERT_FLOAT_NEAR(expected, actual) \
TEST_ASSERT_TRUE_MESSAGE(fabs((double)(expected) - (double)(actual)) \
< 1e-3, \
#actual " != " #expected)
/* One report from `host`: set its cumulative cpu (and mem), re-aggregate
* and update the stats, as update_job_rusages() does. prev_cpu < 0 means
* the host had no earlier report. */
static void report(xce_job_runusage_t* r, int hostidx, int cpu, double mem,
int64_t now, int64_t start, int nslots, double req_mem)
{
xce_host_runusage_t* h = r->host_runusage[hostidx];
int prev_cpu = (h->sample_time > 0) ? h->cpu_time : -1;
h->cpu_time = cpu;
h->mem_usage = mem;
aggregate_job_runusage(r);
TEST_ASSERT_EQUAL_INT_MESSAGE(90, r->peak_cpu_time,
"peak_cpu_time advances on a new high");
update_job_cpu_stats(r, h->host_name, prev_cpu, now, start, nslots,
req_mem);
}
static xce_job_runusage_t* make_named_hosts(int n)
{
static const char* names[] = { "ha", "hb", "hc" };
xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r));
int i;
r->num_hosts = n;
r->host_runusage = (xce_host_runusage_t**)calloc(n, sizeof(*r->host_runusage));
for (i = 0; i < n; i++) {
r->host_runusage[i] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
r->host_runusage[i]->host_name = strdup(names[i]);
}
return r;
}
static void test_cpu_stats_rate_peak_and_efficiency(void)
{
xce_job_runusage_t* r = make_named_hosts(1);
/* first report: no interval yet, nothing to rate */
report(r, 0, 0, 100.0, 100, 100, 2, 0.0);
ASSERT_FLOAT_NEAR(0.0, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(100, (int)r->host_runusage[0]->sample_time);
/* +30 CPU-seconds in 30 s: 1 core busy */
report(r, 0, 30, 100.0, 130, 100, 2, 0.0);
ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(1, r->peak_cpu_time);
TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration);
ASSERT_FLOAT_NEAR(50.0, r->peak_cpu_efficiency); /* 1 core of 2 slots */
ASSERT_FLOAT_NEAR(50.0, r->avg_cpu_efficiency); /* 30 s / 30 s / 2 */
/* +30 in 60 s: 0.5 core -- the peak and its duration stay */
report(r, 0, 60, 100.0, 190, 100, 2, 0.0);
ASSERT_FLOAT_NEAR(0.5, r->host_runusage[0]->cpu_rate);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration);
ASSERT_FLOAT_NEAR(100.0 * 60 / 90 / 2, r->avg_cpu_efficiency);
free_runusage(r);
}
static void test_cpu_stats_sum_host_rates(void)
{
xce_job_runusage_t* r = make_named_hosts(2);
report(r, 0, 0, 10.0, 100, 100, 4, 0.0);
report(r, 1, 0, 10.0, 100, 100, 4, 0.0);
report(r, 0, 20, 10.0, 120, 100, 4, 0.0); /* host a: 1.0 core */
report(r, 1, 30, 10.0, 160, 100, 4, 0.0); /* host b: 0.5 core */
ASSERT_FLOAT_NEAR(1.5, r->peak_cpu);
TEST_ASSERT_EQUAL_INT(2, r->peak_cpu_time); /* rounded */
TEST_ASSERT_EQUAL_INT(60, r->peak_cpu_duration);
ASSERT_FLOAT_NEAR(37.5, r->peak_cpu_efficiency); /* 1.5 / 4 */
free_runusage(r);
}
static void test_cpu_stats_mem_efficiency_is_peak(void)
{
xce_job_runusage_t* r = make_named_hosts(1);
report(r, 0, 0, 256.0, 100, 100, 1, 512.0);
ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency);
report(r, 0, 10, 128.0, 110, 100, 1, 512.0);
ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency); /* peak, not latest */
report(r, 0, 20, 384.0, 120, 100, 1, 512.0);
ASSERT_FLOAT_NEAR(75.0, r->mem_efficiency);
free_runusage(r);
}
static void test_cpu_stats_guards(void)
{
xce_job_runusage_t* r = make_named_hosts(1);
report(r, 0, 0, 64.0, 100, 100, 0, 0.0);
report(r, 0, 30, 64.0, 130, 100, 0, 0.0);
/* no slots, no memory request: no efficiencies */
ASSERT_FLOAT_NEAR(0.0, r->peak_cpu_efficiency);
ASSERT_FLOAT_NEAR(0.0, r->avg_cpu_efficiency);
ASSERT_FLOAT_NEAR(0.0, r->mem_efficiency);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
/* a replayed/out-of-order report (no later than the last one) and a
* CPU counter that went backwards leave the rate alone */
report(r, 0, 90, 64.0, 130, 100, 0, 0.0);
ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
report(r, 0, 10, 64.0, 140, 100, 0, 0.0);
ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
update_job_cpu_stats(NULL, "ha", 0, 1, 1, 1, 1.0); /* must not crash */
update_job_cpu_stats(r, "nosuch", 0, 200, 100, 1, 1.0);
free_runusage(r);
}
@ -333,7 +443,11 @@ int main(void)
RUN_TEST(test_update_host_runusage_parses_swap_and_task_count);
RUN_TEST(test_aggregate_job_runusage_single_host);
RUN_TEST(test_aggregate_job_runusage_multi_host_sum_max_avg);
RUN_TEST(test_aggregate_job_runusage_peak_cpu_time_is_running_max);
RUN_TEST(test_aggregate_job_runusage_leaves_peak_cpu_alone);
RUN_TEST(test_cpu_stats_rate_peak_and_efficiency);
RUN_TEST(test_cpu_stats_sum_host_rates);
RUN_TEST(test_cpu_stats_mem_efficiency_is_peak);
RUN_TEST(test_cpu_stats_guards);
RUN_TEST(test_aggregate_job_runusage_null_and_empty_are_safe);
return UNITY_END();