/* test_host_runusage.c -- bug-1735/bug-1736 regression suite * * bug-1735: job memory usage always reads 0 in the scheduler's simulation * record file, despite the worker daemon genuinely measuring it. Two * independent, confirmed root causes in misc.c/job.c: * * 1. update_host_runusage() (misc.c) only ever matched resnames * "cputime"/"mem" -- but the REAL worker's rusage collector * (collect_job_rusages(), xcecd_cgroups.c) sends "cpu_time_sec"/ * "memory_rss" (see that file's res_names[] array). The two never * matched, so xce_host_runusage_t.cpu_time/mem_usage stayed at their * calloc()'d zero forever, for every real (non-simulated) job. * * 2. update_job_rusages() (job.c) had a standing FIXME ("update other * fields of job rusage later") -- xce_job_runusage_t's own job-level * max_mem/avg_mem/used_mem/cpu_time were never derived from the * per-host host_runusage[] entries update_host_runusage() populates, * so a multi-host job's aggregate usage stayed 0 even once (1) above * is fixed. * * bug-1736 (split from bug-1735, lower priority, display/reporting only): * two more gaps in the same area. * * 3. xce_host_runusage_t had NO field at all to receive the worker's * "memory_swap"/"tasks_current" resnames -- a complete dead end, not * a naming mismatch. Added mem_swap/tasks_current fields plus parsing. * * 4. xce_job_runusage_t.peak_cpu_time was never computed. Added as the * running MAX of aggregate_job_runusage()'s per-call cpu_time sum * across every call for a given job (not just the latest report). * * peak_cpu_duration/mem_efficiency/avg_cpu_efficiency/peak_cpu_efficiency * are DELIBERATELY still not covered -- see aggregate_job_runusage()'s own * doc comment (misc.c) for why: no unambiguous formula/definition exists * for these without a product decision, unlike peak_cpu_time. * * This file unit-tests all four fixes directly against the REAL misc.c * (linked standalone, mirroring resval_typeresolve_unit_test's/ * sched_worker_change_unit_test's isolation rationale -- misc.c's * update_host_runusage()/aggregate_job_runusage() are calloc/realloc/ * strcmp-only, no RTTI locking, no networking, so they link and run with * zero fake/stub scaffolding). get_ep_worker()/get_ep_role() (same file) * are untouched by this fix and unused by these tests, but still link * fine via controller_unit_test_deps' existing librtti_dep. */ #include #include #include #include #include "cluster.h" /* ------------------------------------------------------------------ */ /* Helpers */ /* ------------------------------------------------------------------ */ /* This project's shared unity_dep is built with double-precision support * excluded (TEST_ASSERT_EQUAL_DOUBLE compiles to an unconditional "Unity * Double Precision Disabled" failure) -- no other test file in this * codebase uses it either. Plain epsilon comparison in ordinary C avoids * Unity's own double formatting/compare path entirely. */ #define ASSERT_DOUBLE_EQ_MSG(expected, actual, msg) \ TEST_ASSERT_TRUE_MESSAGE(fabs((expected) - (actual)) < 1e-6, msg) #define ASSERT_DOUBLE_EQ(expected, actual) \ ASSERT_DOUBLE_EQ_MSG(expected, actual, #actual " != " #expected) static xce_rusage_t** make_usages(const char* name1, float val1, const char* name2, float val2) { xce_rusage_t** usages = (xce_rusage_t**)calloc(2, sizeof(xce_rusage_t*)); usages[0] = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t)); usages[0]->resname = strdup(name1); usages[0]->value = val1; usages[1] = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t)); usages[1]->resname = strdup(name2); usages[1]->value = val2; return usages; } static void free_usages(xce_rusage_t** usages) { free(usages[0]->resname); free(usages[0]); free(usages[1]->resname); free(usages[1]); free(usages); } static xce_rusage_t** make_usages4(const char* n1, float v1, const char* n2, float v2, const char* n3, float v3, const char* n4, float v4) { const char* names[4] = { n1, n2, n3, n4 }; float values[4] = { v1, v2, v3, v4 }; xce_rusage_t** usages = (xce_rusage_t**)calloc(4, sizeof(xce_rusage_t*)); int i; for (i = 0; i < 4; i++) { usages[i] = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t)); usages[i]->resname = strdup(names[i]); usages[i]->value = values[i]; } return usages; } static void free_usages4(xce_rusage_t** usages) { int i; for (i = 0; i < 4; i++) { free(usages[i]->resname); free(usages[i]); } free(usages); } static void free_runusage(xce_job_runusage_t* r) { int i; if (r == NULL) { return; } for (i = 0; i < r->num_hosts; i++) { free(r->host_runusage[i]->host_name); free(r->host_runusage[i]); } free(r->host_runusage); free(r); } /* ------------------------------------------------------------------ */ /* Tests -- update_host_runusage() resname alignment (root cause 1) */ /* ------------------------------------------------------------------ */ static void test_update_host_runusage_matches_real_worker_resnames(void) { xce_job_runusage_t* runusage = NULL; xce_rusage_t** usages = make_usages("cpu_time_sec", 42.0f, "memory_rss", 128.5f); int rc = update_host_runusage(&runusage, strdup("host-a"), 2, usages); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NOT_NULL(runusage); TEST_ASSERT_EQUAL_INT(1, runusage->num_hosts); TEST_ASSERT_EQUAL_INT_MESSAGE( 42, runusage->host_runusage[0]->cpu_time, "cpu_time must be set from the worker's real 'cpu_time_sec' resname " "(bug-1735 root cause 1) -- NOT the old, never-matching 'cputime'"); ASSERT_DOUBLE_EQ_MSG( 128.5, runusage->host_runusage[0]->mem_usage, "mem_usage must be set from the worker's real 'memory_rss' resname " "(bug-1735 root cause 1) -- NOT the old, never-matching 'mem'"); free_usages(usages); free_runusage(runusage); } static void test_update_host_runusage_updates_existing_host_in_place(void) { xce_job_runusage_t* runusage = NULL; xce_rusage_t** first = make_usages("cpu_time_sec", 10.0f, "memory_rss", 50.0f); xce_rusage_t** second = make_usages("cpu_time_sec", 20.0f, "memory_rss", 75.0f); update_host_runusage(&runusage, strdup("host-a"), 2, first); update_host_runusage(&runusage, strdup("host-a"), 2, second); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, runusage->num_hosts, "a second report for the SAME host must update in place, not " "append a duplicate host entry"); TEST_ASSERT_EQUAL_INT(20, runusage->host_runusage[0]->cpu_time); ASSERT_DOUBLE_EQ(75.0, runusage->host_runusage[0]->mem_usage); free_usages(first); free_usages(second); free_runusage(runusage); } /* ------------------------------------------------------------------ */ /* Tests -- bug-1736: memory_swap/tasks_current (root cause 3) */ /* ------------------------------------------------------------------ */ static void test_update_host_runusage_parses_swap_and_task_count(void) { xce_job_runusage_t* runusage = NULL; xce_rusage_t** usages = make_usages4( "cpu_time_sec", 5.0f, "memory_rss", 64.0f, "memory_swap", 12.5f, "tasks_current", 3.0f); int rc = update_host_runusage(&runusage, strdup("host-a"), 4, usages); TEST_ASSERT_EQUAL_INT(0, rc); ASSERT_DOUBLE_EQ_MSG( 12.5, runusage->host_runusage[0]->mem_swap, "mem_swap must be set from the worker's 'memory_swap' resname " "(bug-1736) -- previously had no field to land in at all"); TEST_ASSERT_EQUAL_INT_MESSAGE( 3, runusage->host_runusage[0]->tasks_current, "tasks_current must be set from the worker's 'tasks_current' " "resname (bug-1736) -- previously had no field to land in at all"); free_usages4(usages); free_runusage(runusage); } /* ------------------------------------------------------------------ */ /* Tests -- aggregate_job_runusage() (root cause 2) */ /* ------------------------------------------------------------------ */ static xce_job_runusage_t* make_runusage_with_hosts(int n, const int* cpu, const double* mem) { xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r)); int i; r->num_hosts = n; r->host_runusage = (xce_host_runusage_t**)calloc(n, sizeof(xce_host_runusage_t*)); for (i = 0; i < n; i++) { r->host_runusage[i] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t)); r->host_runusage[i]->host_name = strdup("h"); r->host_runusage[i]->cpu_time = cpu[i]; r->host_runusage[i]->mem_usage = mem[i]; } return r; } static void test_aggregate_job_runusage_single_host(void) { int cpu[] = { 30 }; double mem[] = { 256.0 }; xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu, mem); aggregate_job_runusage(r); ASSERT_DOUBLE_EQ(256.0, r->used_mem); ASSERT_DOUBLE_EQ(256.0, r->max_mem); ASSERT_DOUBLE_EQ(256.0, r->avg_mem); TEST_ASSERT_EQUAL_INT(30, r->cpu_time); free_runusage(r); } static void test_aggregate_job_runusage_multi_host_sum_max_avg(void) { int cpu[] = { 10, 20, 30 }; double mem[] = { 100.0, 300.0, 200.0 }; xce_job_runusage_t* r = make_runusage_with_hosts(3, cpu, mem); aggregate_job_runusage(r); ASSERT_DOUBLE_EQ_MSG( 600.0, r->used_mem, "used_mem is the SUM of every host's mem_usage (total footprint of " "a multi-host job), not any single host's value"); ASSERT_DOUBLE_EQ_MSG(300.0, r->max_mem, "max_mem is the MAX across hosts"); ASSERT_DOUBLE_EQ_MSG(200.0, r->avg_mem, "avg_mem is the AVERAGE across hosts"); TEST_ASSERT_EQUAL_INT_MESSAGE( 60, r->cpu_time, "job-level cpu_time is the SUM across hosts"); free_runusage(r); } /* bug-1736: peak_cpu_time is now the LSF "CPU PEAK" (peak cores, set by * update_job_cpu_stats()); aggregate_job_runusage() no longer touches it. */ static void test_aggregate_job_runusage_leaves_peak_cpu_alone(void) { int cpu1[] = { 50 }; double mem1[] = { 100.0 }; xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu1, mem1); r->peak_cpu_time = 3; aggregate_job_runusage(r); TEST_ASSERT_EQUAL_INT(50, r->cpu_time); TEST_ASSERT_EQUAL_INT_MESSAGE(3, r->peak_cpu_time, "cumulative cpu_time is not a peak"); free_runusage(r); } /* ------------------------------------------------------------------ */ /* Tests -- update_job_cpu_stats() (bug-1736, LSF definitions) */ /* ------------------------------------------------------------------ */ #define ASSERT_FLOAT_NEAR(expected, actual) \ TEST_ASSERT_TRUE_MESSAGE(fabs((double)(expected) - (double)(actual)) \ < 1e-3, \ #actual " != " #expected) /* One report from `host`: set its cumulative cpu (and mem), re-aggregate * and update the stats, as update_job_rusages() does. prev_cpu < 0 means * the host had no earlier report. */ static void report(xce_job_runusage_t* r, int hostidx, int cpu, double mem, int64_t now, int64_t start, int nslots, double req_mem) { xce_host_runusage_t* h = r->host_runusage[hostidx]; int prev_cpu = (h->sample_time > 0) ? h->cpu_time : -1; h->cpu_time = cpu; h->mem_usage = mem; aggregate_job_runusage(r); update_job_cpu_stats(r, h->host_name, prev_cpu, now, start, nslots, req_mem); } static xce_job_runusage_t* make_named_hosts(int n) { static const char* names[] = { "ha", "hb", "hc" }; xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r)); int i; r->num_hosts = n; r->host_runusage = (xce_host_runusage_t**)calloc(n, sizeof(*r->host_runusage)); for (i = 0; i < n; i++) { r->host_runusage[i] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t)); r->host_runusage[i]->host_name = strdup(names[i]); } return r; } static void test_cpu_stats_rate_peak_and_efficiency(void) { xce_job_runusage_t* r = make_named_hosts(1); /* first report: no interval yet, nothing to rate */ report(r, 0, 0, 100.0, 100, 100, 2, 0.0); ASSERT_FLOAT_NEAR(0.0, r->peak_cpu); TEST_ASSERT_EQUAL_INT(100, (int)r->host_runusage[0]->sample_time); /* +30 CPU-seconds in 30 s: 1 core busy */ report(r, 0, 30, 100.0, 130, 100, 2, 0.0); ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate); ASSERT_FLOAT_NEAR(1.0, r->peak_cpu); TEST_ASSERT_EQUAL_INT(1, r->peak_cpu_time); TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration); ASSERT_FLOAT_NEAR(50.0, r->peak_cpu_efficiency); /* 1 core of 2 slots */ ASSERT_FLOAT_NEAR(50.0, r->avg_cpu_efficiency); /* 30 s / 30 s / 2 */ /* +30 in 60 s: 0.5 core -- the peak and its duration stay */ report(r, 0, 60, 100.0, 190, 100, 2, 0.0); ASSERT_FLOAT_NEAR(0.5, r->host_runusage[0]->cpu_rate); ASSERT_FLOAT_NEAR(1.0, r->peak_cpu); TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration); ASSERT_FLOAT_NEAR(100.0 * 60 / 90 / 2, r->avg_cpu_efficiency); free_runusage(r); } static void test_cpu_stats_sum_host_rates(void) { xce_job_runusage_t* r = make_named_hosts(2); report(r, 0, 0, 10.0, 100, 100, 4, 0.0); report(r, 1, 0, 10.0, 100, 100, 4, 0.0); report(r, 0, 20, 10.0, 120, 100, 4, 0.0); /* host a: 1.0 core */ report(r, 1, 30, 10.0, 160, 100, 4, 0.0); /* host b: 0.5 core */ ASSERT_FLOAT_NEAR(1.5, r->peak_cpu); TEST_ASSERT_EQUAL_INT(2, r->peak_cpu_time); /* rounded */ TEST_ASSERT_EQUAL_INT(60, r->peak_cpu_duration); ASSERT_FLOAT_NEAR(37.5, r->peak_cpu_efficiency); /* 1.5 / 4 */ free_runusage(r); } static void test_cpu_stats_mem_efficiency_is_peak(void) { xce_job_runusage_t* r = make_named_hosts(1); report(r, 0, 0, 256.0, 100, 100, 1, 512.0); ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency); report(r, 0, 10, 128.0, 110, 100, 1, 512.0); ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency); /* peak, not latest */ report(r, 0, 20, 384.0, 120, 100, 1, 512.0); ASSERT_FLOAT_NEAR(75.0, r->mem_efficiency); free_runusage(r); } static void test_cpu_stats_guards(void) { xce_job_runusage_t* r = make_named_hosts(1); report(r, 0, 0, 64.0, 100, 100, 0, 0.0); report(r, 0, 30, 64.0, 130, 100, 0, 0.0); /* no slots, no memory request: no efficiencies */ ASSERT_FLOAT_NEAR(0.0, r->peak_cpu_efficiency); ASSERT_FLOAT_NEAR(0.0, r->avg_cpu_efficiency); ASSERT_FLOAT_NEAR(0.0, r->mem_efficiency); ASSERT_FLOAT_NEAR(1.0, r->peak_cpu); /* a replayed/out-of-order report (no later than the last one) and a * CPU counter that went backwards leave the rate alone */ report(r, 0, 90, 64.0, 130, 100, 0, 0.0); ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate); report(r, 0, 10, 64.0, 140, 100, 0, 0.0); ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate); ASSERT_FLOAT_NEAR(1.0, r->peak_cpu); update_job_cpu_stats(NULL, "ha", 0, 1, 1, 1, 1.0); /* must not crash */ update_job_cpu_stats(r, "nosuch", 0, 200, 100, 1, 1.0); free_runusage(r); } static void test_aggregate_job_runusage_null_and_empty_are_safe(void) { xce_job_runusage_t empty = { 0 }; aggregate_job_runusage(NULL); /* must not crash */ aggregate_job_runusage(&empty); /* num_hosts == 0, must not crash */ ASSERT_DOUBLE_EQ(0.0, empty.used_mem); } /* ------------------------------------------------------------------ */ /* Standalone Unity main (this binary is its own test executable -- see */ /* meson.build's host_runusage_unit_test) */ /* ------------------------------------------------------------------ */ void setUp(void) { return; } void tearDown(void) { return; } int main(void) { printf("\n>> bug-1735: host/job rusage resname + aggregation fix\n\n"); UNITY_BEGIN(); RUN_TEST(test_update_host_runusage_matches_real_worker_resnames); RUN_TEST(test_update_host_runusage_updates_existing_host_in_place); RUN_TEST(test_update_host_runusage_parses_swap_and_task_count); RUN_TEST(test_aggregate_job_runusage_single_host); RUN_TEST(test_aggregate_job_runusage_multi_host_sum_max_avg); RUN_TEST(test_aggregate_job_runusage_leaves_peak_cpu_alone); RUN_TEST(test_cpu_stats_rate_peak_and_efficiency); RUN_TEST(test_cpu_stats_sum_host_rates); RUN_TEST(test_cpu_stats_mem_efficiency_is_peak); RUN_TEST(test_cpu_stats_guards); RUN_TEST(test_aggregate_job_runusage_null_and_empty_are_safe); return UNITY_END(); }