myxce/src/controller/testsuite/test_host_runusage.c

/* test_host_runusage.c -- bug-1735/bug-1736 regression suite
 *
 * bug-1735: job memory usage always reads 0 in the scheduler's simulation
 * record file, despite the worker daemon genuinely measuring it. Two
 * independent, confirmed root causes in misc.c/job.c:
 *
 *   1. update_host_runusage() (misc.c) only ever matched resnames
 *      "cputime"/"mem" -- but the REAL worker's rusage collector
 *      (collect_job_rusages(), xcecd_cgroups.c) sends "cpu_time_sec"/
 *      "memory_rss" (see that file's res_names[] array). The two never
 *      matched, so xce_host_runusage_t.cpu_time/mem_usage stayed at their
 *      calloc()'d zero forever, for every real (non-simulated) job.
 *
 *   2. update_job_rusages() (job.c) had a standing FIXME ("update other
 *      fields of job rusage later") -- xce_job_runusage_t's own job-level
 *      max_mem/avg_mem/used_mem/cpu_time were never derived from the
 *      per-host host_runusage[] entries update_host_runusage() populates,
 *      so a multi-host job's aggregate usage stayed 0 even once (1) above
 *      is fixed.
 *
 * bug-1736 (split from bug-1735, lower priority, display/reporting only):
 * two more gaps in the same area.
 *
 *   3. xce_host_runusage_t had NO field at all to receive the worker's
 *      "memory_swap"/"tasks_current" resnames -- a complete dead end, not
 *      a naming mismatch. Added mem_swap/tasks_current fields plus parsing.
 *
 *   4. xce_job_runusage_t.peak_cpu_time was never computed. Added as the
 *      running MAX of aggregate_job_runusage()'s per-call cpu_time sum
 *      across every call for a given job (not just the latest report).
 *
 * peak_cpu_duration/mem_efficiency/avg_cpu_efficiency/peak_cpu_efficiency
 * are DELIBERATELY still not covered -- see aggregate_job_runusage()'s own
 * doc comment (misc.c) for why: no unambiguous formula/definition exists
 * for these without a product decision, unlike peak_cpu_time.
 *
 * This file unit-tests all four fixes directly against the REAL misc.c
 * (linked standalone, mirroring resval_typeresolve_unit_test's/
 * sched_worker_change_unit_test's isolation rationale -- misc.c's
 * update_host_runusage()/aggregate_job_runusage() are calloc/realloc/
 * strcmp-only, no RTTI locking, no networking, so they link and run with
 * zero fake/stub scaffolding). get_ep_worker()/get_ep_role() (same file)
 * are untouched by this fix and unused by these tests, but still link
 * fine via controller_unit_test_deps' existing librtti_dep.
 */
#include <math.h>
#include <stdlib.h>
#include <string.h>

#include <unity/unity.h>

#include "cluster.h"

/* ------------------------------------------------------------------ */
/* Helpers                                                             */
/* ------------------------------------------------------------------ */

/* This project's shared unity_dep is built with double-precision support
 * excluded (TEST_ASSERT_EQUAL_DOUBLE compiles to an unconditional "Unity
 * Double Precision Disabled" failure) -- no other test file in this
 * codebase uses it either. Plain epsilon comparison in ordinary C avoids
 * Unity's own double formatting/compare path entirely. */
#define ASSERT_DOUBLE_EQ_MSG(expected, actual, msg)                          \
    TEST_ASSERT_TRUE_MESSAGE(fabs((expected) - (actual)) < 1e-6, msg)
#define ASSERT_DOUBLE_EQ(expected, actual)                                   \
    ASSERT_DOUBLE_EQ_MSG(expected, actual, #actual " != " #expected)

static xce_rusage_t** make_usages(const char* name1, float val1,
                                  const char* name2, float val2)
{
    xce_rusage_t** usages = (xce_rusage_t**)calloc(2, sizeof(xce_rusage_t*));

    usages[0]          = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t));
    usages[0]->resname = strdup(name1);
    usages[0]->value   = val1;

    usages[1]          = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t));
    usages[1]->resname = strdup(name2);
    usages[1]->value   = val2;

    return usages;
}

static void free_usages(xce_rusage_t** usages)
{
    free(usages[0]->resname);
    free(usages[0]);
    free(usages[1]->resname);
    free(usages[1]);
    free(usages);
}

static xce_rusage_t** make_usages4(const char* n1, float v1, const char* n2,
                                   float v2, const char* n3, float v3,
                                   const char* n4, float v4)
{
    const char*    names[4]  = { n1, n2, n3, n4 };
    float          values[4] = { v1, v2, v3, v4 };
    xce_rusage_t** usages    = (xce_rusage_t**)calloc(4, sizeof(xce_rusage_t*));
    int            i;

    for (i = 0; i < 4; i++) {
        usages[i]          = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t));
        usages[i]->resname = strdup(names[i]);
        usages[i]->value   = values[i];
    }

    return usages;
}

static void free_usages4(xce_rusage_t** usages)
{
    int i;

    for (i = 0; i < 4; i++) {
        free(usages[i]->resname);
        free(usages[i]);
    }
    free(usages);
}

static void free_runusage(xce_job_runusage_t* r)
{
    int i;

    if (r == NULL) {
        return;
    }

    for (i = 0; i < r->num_hosts; i++) {
        free(r->host_runusage[i]->host_name);
        free(r->host_runusage[i]);
    }
    free(r->host_runusage);
    free(r);
}

/* ------------------------------------------------------------------ */
/* Tests -- update_host_runusage() resname alignment (root cause 1)   */
/* ------------------------------------------------------------------ */

static void
test_update_host_runusage_matches_real_worker_resnames(void)
{
    xce_job_runusage_t* runusage = NULL;
    xce_rusage_t**       usages
        = make_usages("cpu_time_sec", 42.0f, "memory_rss", 128.5f);

    int rc = update_host_runusage(&runusage, strdup("host-a"), 2, usages);

    TEST_ASSERT_EQUAL_INT(0, rc);
    TEST_ASSERT_NOT_NULL(runusage);
    TEST_ASSERT_EQUAL_INT(1, runusage->num_hosts);
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        42, runusage->host_runusage[0]->cpu_time,
        "cpu_time must be set from the worker's real 'cpu_time_sec' resname "
        "(bug-1735 root cause 1) -- NOT the old, never-matching 'cputime'");
    ASSERT_DOUBLE_EQ_MSG(
        128.5, runusage->host_runusage[0]->mem_usage,
        "mem_usage must be set from the worker's real 'memory_rss' resname "
        "(bug-1735 root cause 1) -- NOT the old, never-matching 'mem'");

    free_usages(usages);
    free_runusage(runusage);
}

static void test_update_host_runusage_updates_existing_host_in_place(void)
{
    xce_job_runusage_t* runusage = NULL;
    xce_rusage_t** first  = make_usages("cpu_time_sec", 10.0f, "memory_rss", 50.0f);
    xce_rusage_t** second = make_usages("cpu_time_sec", 20.0f, "memory_rss", 75.0f);

    update_host_runusage(&runusage, strdup("host-a"), 2, first);
    update_host_runusage(&runusage, strdup("host-a"), 2, second);

    TEST_ASSERT_EQUAL_INT_MESSAGE(
        1, runusage->num_hosts,
        "a second report for the SAME host must update in place, not "
        "append a duplicate host entry");
    TEST_ASSERT_EQUAL_INT(20, runusage->host_runusage[0]->cpu_time);
    ASSERT_DOUBLE_EQ(75.0, runusage->host_runusage[0]->mem_usage);

    free_usages(first);
    free_usages(second);
    free_runusage(runusage);
}

/* ------------------------------------------------------------------ */
/* Tests -- bug-1736: memory_swap/tasks_current (root cause 3)         */
/* ------------------------------------------------------------------ */

static void test_update_host_runusage_parses_swap_and_task_count(void)
{
    xce_job_runusage_t* runusage = NULL;
    xce_rusage_t**       usages  = make_usages4(
        "cpu_time_sec", 5.0f, "memory_rss", 64.0f, "memory_swap", 12.5f,
        "tasks_current", 3.0f);

    int rc = update_host_runusage(&runusage, strdup("host-a"), 4, usages);

    TEST_ASSERT_EQUAL_INT(0, rc);
    ASSERT_DOUBLE_EQ_MSG(
        12.5, runusage->host_runusage[0]->mem_swap,
        "mem_swap must be set from the worker's 'memory_swap' resname "
        "(bug-1736) -- previously had no field to land in at all");
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        3, runusage->host_runusage[0]->tasks_current,
        "tasks_current must be set from the worker's 'tasks_current' "
        "resname (bug-1736) -- previously had no field to land in at all");

    free_usages4(usages);
    free_runusage(runusage);
}

/* ------------------------------------------------------------------ */
/* Tests -- aggregate_job_runusage() (root cause 2)                    */
/* ------------------------------------------------------------------ */

static xce_job_runusage_t* make_runusage_with_hosts(int n, const int* cpu,
                                                     const double* mem)
{
    xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r));
    int                  i;

    r->num_hosts     = n;
    r->host_runusage
        = (xce_host_runusage_t**)calloc(n, sizeof(xce_host_runusage_t*));

    for (i = 0; i < n; i++) {
        r->host_runusage[i]
            = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
        r->host_runusage[i]->host_name = strdup("h");
        r->host_runusage[i]->cpu_time  = cpu[i];
        r->host_runusage[i]->mem_usage = mem[i];
    }

    return r;
}

static void test_aggregate_job_runusage_single_host(void)
{
    int    cpu[] = { 30 };
    double mem[] = { 256.0 };
    xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu, mem);

    aggregate_job_runusage(r);

    ASSERT_DOUBLE_EQ(256.0, r->used_mem);
    ASSERT_DOUBLE_EQ(256.0, r->max_mem);
    ASSERT_DOUBLE_EQ(256.0, r->avg_mem);
    TEST_ASSERT_EQUAL_INT(30, r->cpu_time);

    free_runusage(r);
}

static void test_aggregate_job_runusage_multi_host_sum_max_avg(void)
{
    int    cpu[] = { 10, 20, 30 };
    double mem[] = { 100.0, 300.0, 200.0 };
    xce_job_runusage_t* r = make_runusage_with_hosts(3, cpu, mem);

    aggregate_job_runusage(r);

    ASSERT_DOUBLE_EQ_MSG(
        600.0, r->used_mem,
        "used_mem is the SUM of every host's mem_usage (total footprint of "
        "a multi-host job), not any single host's value");
    ASSERT_DOUBLE_EQ_MSG(300.0, r->max_mem, "max_mem is the MAX across hosts");
    ASSERT_DOUBLE_EQ_MSG(200.0, r->avg_mem, "avg_mem is the AVERAGE across hosts");
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        60, r->cpu_time, "job-level cpu_time is the SUM across hosts");

    free_runusage(r);
}

/* bug-1736: peak_cpu_time is now the LSF "CPU PEAK" (peak cores, set by
 * update_job_cpu_stats()); aggregate_job_runusage() no longer touches it. */
static void test_aggregate_job_runusage_leaves_peak_cpu_alone(void)
{
    int    cpu1[] = { 50 };
    double mem1[] = { 100.0 };
    xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu1, mem1);

    r->peak_cpu_time = 3;
    aggregate_job_runusage(r);
    TEST_ASSERT_EQUAL_INT(50, r->cpu_time);
    TEST_ASSERT_EQUAL_INT_MESSAGE(3, r->peak_cpu_time,
                                  "cumulative cpu_time is not a peak");

    free_runusage(r);
}

/* ------------------------------------------------------------------ */
/* Tests -- update_job_cpu_stats() (bug-1736, LSF definitions)         */
/* ------------------------------------------------------------------ */

#define ASSERT_FLOAT_NEAR(expected, actual)                                  \
    TEST_ASSERT_TRUE_MESSAGE(fabs((double)(expected) - (double)(actual))    \
                                 < 1e-3,                                     \
                             #actual " != " #expected)

/* One report from `host`: set its cumulative cpu (and mem), re-aggregate
 * and update the stats, as update_job_rusages() does. prev_cpu < 0 means
 * the host had no earlier report. */
static void report(xce_job_runusage_t* r, int hostidx, int cpu, double mem,
                   int64_t now, int64_t start, int nslots, double req_mem)
{
    xce_host_runusage_t* h        = r->host_runusage[hostidx];
    int                  prev_cpu = (h->sample_time > 0) ? h->cpu_time : -1;

    h->cpu_time  = cpu;
    h->mem_usage = mem;
    aggregate_job_runusage(r);
    update_job_cpu_stats(r, h->host_name, prev_cpu, now, start, nslots,
                         req_mem);
}

static xce_job_runusage_t* make_named_hosts(int n)
{
    static const char* names[] = { "ha", "hb", "hc" };
    xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r));
    int                  i;

    r->num_hosts     = n;
    r->host_runusage = (xce_host_runusage_t**)calloc(n, sizeof(*r->host_runusage));
    for (i = 0; i < n; i++) {
        r->host_runusage[i] = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
        r->host_runusage[i]->host_name = strdup(names[i]);
    }
    return r;
}

static void test_cpu_stats_rate_peak_and_efficiency(void)
{
    xce_job_runusage_t* r = make_named_hosts(1);

    /* first report: no interval yet, nothing to rate */
    report(r, 0, 0, 100.0, 100, 100, 2, 0.0);
    ASSERT_FLOAT_NEAR(0.0, r->peak_cpu);
    TEST_ASSERT_EQUAL_INT(100, (int)r->host_runusage[0]->sample_time);

    /* +30 CPU-seconds in 30 s: 1 core busy */
    report(r, 0, 30, 100.0, 130, 100, 2, 0.0);
    ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
    ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
    TEST_ASSERT_EQUAL_INT(1, r->peak_cpu_time);
    TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration);
    ASSERT_FLOAT_NEAR(50.0, r->peak_cpu_efficiency); /* 1 core of 2 slots */
    ASSERT_FLOAT_NEAR(50.0, r->avg_cpu_efficiency);  /* 30 s / 30 s / 2 */

    /* +30 in 60 s: 0.5 core -- the peak and its duration stay */
    report(r, 0, 60, 100.0, 190, 100, 2, 0.0);
    ASSERT_FLOAT_NEAR(0.5, r->host_runusage[0]->cpu_rate);
    ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);
    TEST_ASSERT_EQUAL_INT(30, r->peak_cpu_duration);
    ASSERT_FLOAT_NEAR(100.0 * 60 / 90 / 2, r->avg_cpu_efficiency);

    free_runusage(r);
}

static void test_cpu_stats_sum_host_rates(void)
{
    xce_job_runusage_t* r = make_named_hosts(2);

    report(r, 0, 0, 10.0, 100, 100, 4, 0.0);
    report(r, 1, 0, 10.0, 100, 100, 4, 0.0);
    report(r, 0, 20, 10.0, 120, 100, 4, 0.0); /* host a: 1.0 core */
    report(r, 1, 30, 10.0, 160, 100, 4, 0.0); /* host b: 0.5 core */
    ASSERT_FLOAT_NEAR(1.5, r->peak_cpu);
    TEST_ASSERT_EQUAL_INT(2, r->peak_cpu_time); /* rounded */
    TEST_ASSERT_EQUAL_INT(60, r->peak_cpu_duration);
    ASSERT_FLOAT_NEAR(37.5, r->peak_cpu_efficiency); /* 1.5 / 4 */

    free_runusage(r);
}

static void test_cpu_stats_mem_efficiency_is_peak(void)
{
    xce_job_runusage_t* r = make_named_hosts(1);

    report(r, 0, 0, 256.0, 100, 100, 1, 512.0);
    ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency);
    report(r, 0, 10, 128.0, 110, 100, 1, 512.0);
    ASSERT_FLOAT_NEAR(50.0, r->mem_efficiency); /* peak, not latest */
    report(r, 0, 20, 384.0, 120, 100, 1, 512.0);
    ASSERT_FLOAT_NEAR(75.0, r->mem_efficiency);

    free_runusage(r);
}

static void test_cpu_stats_guards(void)
{
    xce_job_runusage_t* r = make_named_hosts(1);

    report(r, 0, 0, 64.0, 100, 100, 0, 0.0);
    report(r, 0, 30, 64.0, 130, 100, 0, 0.0);
    /* no slots, no memory request: no efficiencies */
    ASSERT_FLOAT_NEAR(0.0, r->peak_cpu_efficiency);
    ASSERT_FLOAT_NEAR(0.0, r->avg_cpu_efficiency);
    ASSERT_FLOAT_NEAR(0.0, r->mem_efficiency);
    ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);

    /* a replayed/out-of-order report (no later than the last one) and a
     * CPU counter that went backwards leave the rate alone */
    report(r, 0, 90, 64.0, 130, 100, 0, 0.0);
    ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
    report(r, 0, 10, 64.0, 140, 100, 0, 0.0);
    ASSERT_FLOAT_NEAR(1.0, r->host_runusage[0]->cpu_rate);
    ASSERT_FLOAT_NEAR(1.0, r->peak_cpu);

    update_job_cpu_stats(NULL, "ha", 0, 1, 1, 1, 1.0); /* must not crash */
    update_job_cpu_stats(r, "nosuch", 0, 200, 100, 1, 1.0);

    free_runusage(r);
}

static void test_aggregate_job_runusage_null_and_empty_are_safe(void)
{
    xce_job_runusage_t empty = { 0 };

    aggregate_job_runusage(NULL); /* must not crash */
    aggregate_job_runusage(&empty); /* num_hosts == 0, must not crash */

    ASSERT_DOUBLE_EQ(0.0, empty.used_mem);
}

/* ------------------------------------------------------------------ */
/* Standalone Unity main (this binary is its own test executable -- see */
/* meson.build's host_runusage_unit_test)                               */
/* ------------------------------------------------------------------ */

void setUp(void) { return; }
void tearDown(void) { return; }

int main(void)
{
    printf("\n>> bug-1735: host/job rusage resname + aggregation fix\n\n");

    UNITY_BEGIN();

    RUN_TEST(test_update_host_runusage_matches_real_worker_resnames);
    RUN_TEST(test_update_host_runusage_updates_existing_host_in_place);
    RUN_TEST(test_update_host_runusage_parses_swap_and_task_count);
    RUN_TEST(test_aggregate_job_runusage_single_host);
    RUN_TEST(test_aggregate_job_runusage_multi_host_sum_max_avg);
    RUN_TEST(test_aggregate_job_runusage_leaves_peak_cpu_alone);
    RUN_TEST(test_cpu_stats_rate_peak_and_efficiency);
    RUN_TEST(test_cpu_stats_sum_host_rates);
    RUN_TEST(test_cpu_stats_mem_efficiency_is_peak);
    RUN_TEST(test_cpu_stats_guards);
    RUN_TEST(test_aggregate_job_runusage_null_and_empty_are_safe);

    return UNITY_END();
}