myxce/src/scheduler/testsuite/test_scheduler_dispatch_commit.c

/*
 * test_scheduler_dispatch_commit.c - scheduler-dispatch-commit spec, task
 * 2.1 (Req 1.1, 1.2, 1.3): sched_commit_decision()'s DISPATCH branch
 * transitions a job to JOB_STAT_RUN in the same operation that already
 * commits job->allocs/job->accounts, so sched_account_jobs()'s existing
 * JOB_IS_RUN() gate (scheduler.c, unmodified by this task) picks up a
 * dispatched job's resource consumption starting the same round it was
 * dispatched, and continues to include it regardless of how many rounds
 * elapse before worker confirmation arrives (bug-1624).
 *
 * Links the REAL production scheduler_srcs (same "whole-scheduler_srcs,
 * minus main.c" technique test_scheduler_connect.c/
 * test_scheduler_backpressure.c already establish in meson.build for
 * reaching scheduler.c's non-static entry points without white-box
 * #including a ~1400-line file -- no existing test in this directory does
 * that, for exactly the link-surface reason test_resource_accounting.c's
 * own header comment documents). Unlike those two files, this one never
 * calls sched_init()/sched_fini() (no controller-join, no picker/allocator
 * threads), so it needs none of their `-Wl,--wrap=xcecli_*` interposition:
 * sched_commit_decision() itself only touches sched->core and the (empty,
 * in this fixture) plugin-hook chain, never sched->apictx.
 *
 * Extended for task 5.1 (Req 1.1, 1.2, 1.3, 2.1, 2.3 -- bug-1624's
 * end-to-end regression test): task 2.1's own single-round-boundary test
 * above proves the core mechanism; the two tests added at the bottom of
 * this file extend that proof to (a) GAP_ROUNDS=5 consecutive round
 * boundaries rather than just one, and (b) an explicit value comparison
 * between a "confirmed same round" scenario and a "confirmed after several
 * rounds" scenario, rather than only comparing each to a hardcoded
 * constant -- see run_dispatch_then_n_rounds()'s own comment for why no
 * live controller connection or simulated schedStartJob notify event is
 * needed to make that comparison meaningful: the controller's own role in
 * this flow (transitioning to RUN, notifying) is already independently
 * covered by tasks 3.1-4.1's controller-side suites
 * (test_dispatch_commit.c, test_dispatch_fail_terminal.c,
 * test_dispatch_fail_retry.c, test_jobdispatch_no_double_notify.c); nothing
 * scheduler-side that determines resInUse ever reads a confirmation event
 * at all, only job->state, which task 2.1's fix already sets synchronously
 * at dispatch-commit.
 */
#include <errno.h>
#include <stdlib.h>
#include <string.h>

#include <unity/unity.h>

#include "preason.h"
#include "scheduler.h"
#include "xcelimit.h"

#define WORKER_NAME ((char*)"worker1")
#define JOB_ID      4242ULL
#define RES_NAME    "slots"
#define WANT_VALUE  4.0f
#define GAP_ROUNDS  5

static xce_sched_worker_t* add_worker(xce_sched_core_t* core, char* name)
{
    xce_sched_worker_t* w = (xce_sched_worker_t*)calloc(1, sizeof(*w));
    int                  rc;

    TEST_ASSERT_NOT_NULL(w);

    w->name = strdup(name);
    TEST_ASSERT_NOT_NULL(w->name);

    rc = zarray_init(&(w->resvals), 8, NULL);
    TEST_ASSERT_EQUAL_INT(0, rc);
    rc = zarray_init(&(w->resInUse), 8, NULL);
    TEST_ASSERT_EQUAL_INT(0, rc);

    rc = sched_add_worker(core, w);
    TEST_ASSERT_TRUE(rc >= 0);

    return w;
}

static xce_sched_job_t* add_pending_job(xce_sched_core_t* core, uint64_t jobid)
{
    xce_sched_job_t* job = (xce_sched_job_t*)calloc(1, sizeof(*job));
    int              rc;

    TEST_ASSERT_NOT_NULL(job);

    job->jobid = jobid;
    job->state = JOB_STAT_PEND;

    rc = sched_add_job(core, job);
    TEST_ASSERT_TRUE(rc >= 0);

    return job;
}

/* Builds the wire-shaped xce_job_alloc_t a DISPATCH decision carries -- same
 * field shape every other fixture in this directory building one by hand
 * uses (e.g. test_sim_resource_contention.c's own make_dispatch_alloc()). */
static xce_job_alloc_t* make_dispatch_alloc(uint64_t jobid, char* host,
                                            float value)
{
    xce_alloc_t*       res    = (xce_alloc_t*)calloc(1, sizeof(*res));
    xce_host_alloc_t*  h      = (xce_host_alloc_t*)calloc(1, sizeof(*h));
    xce_host_alloc_t** hosts  = (xce_host_alloc_t**)calloc(1, sizeof(*hosts));
    xce_alloc_t**      allocs = (xce_alloc_t**)calloc(1, sizeof(*allocs));
    xce_job_alloc_t*   j      = (xce_job_alloc_t*)calloc(1, sizeof(*j));

    TEST_ASSERT_NOT_NULL(res);
    TEST_ASSERT_NOT_NULL(h);
    TEST_ASSERT_NOT_NULL(hosts);
    TEST_ASSERT_NOT_NULL(allocs);
    TEST_ASSERT_NOT_NULL(j);

    res->resname  = strdup(RES_NAME);
    res->reserve  = strdup("run");
    res->instid   = 1;
    res->value    = value;
    res->from     = 0;
    res->duration = 0;

    allocs[0] = res;

    h->host    = strdup(host);
    h->nallocs = 1;
    h->allocs  = allocs;

    hosts[0] = h;

    j->jobid         = jobid;
    j->nhosts        = 1;
    j->allocs        = hosts;
    j->nres_shared   = 0;
    j->allocs_shared = NULL;

    return j;
}

void setUp(void) { return; }
void tearDown(void) { return; }

/*
 * Requirement 1.1: committing a DISPATCH decision transitions the job to
 * JOB_STAT_RUN in the SAME operation that commits job->allocs/job->accounts
 * -- the one new line this task adds to sched_commit_decision(), alongside
 * the pre-existing allocs/accounts assignment it must not disturb.
 */
static void test_dispatch_commit_transitions_job_to_run(void)
{
    xce_scheduler_t     sched;
    xce_sched_job_t*    job;
    xce_job_alloc_t*    alloc;
    xce_decision_t*     decision;
    int                 rc;

    memset(&sched, 0, sizeof(sched));
    rc = xce_sched_core_init(&(sched.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    add_worker(&(sched.core), WORKER_NAME);
    job = add_pending_job(&(sched.core), JOB_ID);
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_PEND, job->state);

    alloc    = make_dispatch_alloc(JOB_ID, WORKER_NAME, WANT_VALUE);
    decision = make_dispatch_decision(alloc);
    TEST_ASSERT_NOT_NULL(decision);

    rc = sched_commit_decision(&sched, decision);
    TEST_ASSERT_TRUE(rc >= 0);

    /* the new behavior this task adds */
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);

    /* the pre-existing behavior this task must not disturb */
    TEST_ASSERT_EQUAL_PTR(alloc, job->allocs);
    TEST_ASSERT_NOT_NULL(job->accounts);
    TEST_ASSERT_EQUAL_INT(1, job->accounts->nhosts);
    TEST_ASSERT_EQUAL_STRING(WORKER_NAME, job->accounts->accounts[0]->host);
}

/*
 * Requirement 1.2 / 1.3 (bug-1624's actual regression test): the
 * scheduler's existing sched_account_jobs()/JOB_IS_RUN() gate -- unmodified
 * by this task -- naturally includes a dispatched job's resource
 * consumption starting the SAME round it was dispatched, and continues to
 * include it across a simulated round boundary (resInUse unconditionally
 * zeroed, exactly as xce_core_begin_session() does every round) with no
 * worker confirmation event in between. Before this task, job->state would
 * still read JOB_STAT_PEND at this point, so sched_account_jobs()'s
 * !JOB_IS_RUN() branch would skip the job and the final assertion below
 * would see 0.0 instead of WANT_VALUE.
 */
static void test_dispatch_commit_survives_round_boundary_before_confirmation(void)
{
    xce_scheduler_t     sched;
    xce_sched_worker_t* w;
    xce_sched_job_t*    job;
    xce_job_alloc_t*    alloc;
    xce_decision_t*     decision;
    xce_resource_t*     vres = NULL;
    int                 rc;
    int                 residx;
    float               vals[1];
    float               zero[1] = { 0.0f };

    memset(&sched, 0, sizeof(sched));
    rc = xce_sched_core_init(&(sched.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    w   = add_worker(&(sched.core), WORKER_NAME);
    job = add_pending_job(&(sched.core), JOB_ID);

    /* Register RES_NAME up front, the way real resource config-load does
     * before any job is ever charged, rather than relying on
     * sched_update_res_accounts()'s own lazy first-use registration path --
     * out of scope for what this test is proving. */
    residx = add_vres(&vres, sched.core.resmgr, RES_NAME, 0);
    TEST_ASSERT_TRUE(residx >= 0);

    alloc    = make_dispatch_alloc(JOB_ID, WORKER_NAME, WANT_VALUE);
    decision = make_dispatch_decision(alloc);
    TEST_ASSERT_NOT_NULL(decision);

    rc = sched_commit_decision(&sched, decision);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);
    TEST_ASSERT_NOT_NULL(job->accounts);
    TEST_ASSERT_EQUAL_INT(residx, job->accounts->accounts[0]->accounts[0]->residx);

    get_resvals(&(w->resInUse), 1, &residx, vals);
    TEST_ASSERT_EQUAL_FLOAT(WANT_VALUE, vals[0]);

    /* simulate the next round boundary */
    put_resvals(&(w->resInUse), 1, &residx, zero);
    get_resvals(&(w->resInUse), 1, &residx, vals);
    TEST_ASSERT_EQUAL_FLOAT(0.0f, vals[0]);

    rc = sched_account_jobs(&(sched.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    get_resvals(&(w->resInUse), 1, &residx, vals);
    TEST_ASSERT_EQUAL_FLOAT(WANT_VALUE, vals[0]);
}

/*
 * Negative-space guard: this task's new line lives strictly inside the
 * XCE_DECISION_DISPATCH branch. A PEND decision must not touch job->state
 * at all (still JOB_STAT_PEND) -- proves the fix is scoped to DISPATCH, not
 * an unconditional job->state = JOB_STAT_RUN hoisted above the branch.
 */
static void test_pend_decision_does_not_touch_job_state(void)
{
    xce_scheduler_t   sched;
    xce_sched_job_t*  job;
    xce_pendreason_t* reason;
    xce_decision_t*   decision;
    int               rc;

    memset(&sched, 0, sizeof(sched));
    rc = xce_sched_core_init(&(sched.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    job = add_pending_job(&(sched.core), JOB_ID);

    reason = make_pendreason(JOB_ID);
    TEST_ASSERT_NOT_NULL(reason);
    decision = make_pend_decision(reason);
    TEST_ASSERT_NOT_NULL(decision);

    rc = sched_commit_decision(&sched, decision);
    TEST_ASSERT_TRUE(rc >= 0);

    TEST_ASSERT_EQUAL_UINT(JOB_STAT_PEND, job->state);
    TEST_ASSERT_NULL(job->allocs);
    TEST_ASSERT_NULL(job->accounts);
}

/*
 * Task 5.1 (bug-1624's own regression test, "several rounds" language):
 * extends the single-round-boundary proof above
 * (test_dispatch_commit_survives_round_boundary_before_confirmation, task
 * 2.1) to GAP_ROUNDS consecutive round boundaries -- zeroing resInUse and
 * calling sched_account_jobs() GAP_ROUNDS times in a row, exactly as
 * GAP_ROUNDS real driver rounds would via xce_core_begin_session(), with no
 * worker-confirmation event injected anywhere in the loop -- and asserts
 * resInUse is restored to WANT_VALUE after EVERY one of them, not merely
 * that it eventually comes back after one round. Before task 2.1/3.1's fix,
 * job->state would still read JOB_STAT_PEND at commit time, so this would
 * already fail at round 1 (the same way the pre-existing single-round test
 * would); this test additionally proves the fix holds continuously across
 * a realistic multi-round confirmation gap, not just a single round
 * boundary.
 */
static void test_dispatch_commit_survives_several_rounds_before_confirmation(void)
{
    xce_scheduler_t     sched;
    xce_sched_worker_t* w;
    xce_sched_job_t*    job;
    xce_job_alloc_t*    alloc;
    xce_decision_t*     decision;
    xce_resource_t*     vres = NULL;
    int                 rc;
    int                 residx;
    float               val;
    float               zero[1] = { 0.0f };
    int                 round;

    memset(&sched, 0, sizeof(sched));
    rc = xce_sched_core_init(&(sched.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    w   = add_worker(&(sched.core), WORKER_NAME);
    job = add_pending_job(&(sched.core), JOB_ID);

    residx = add_vres(&vres, sched.core.resmgr, RES_NAME, 0);
    TEST_ASSERT_TRUE(residx >= 0);

    alloc    = make_dispatch_alloc(JOB_ID, WORKER_NAME, WANT_VALUE);
    decision = make_dispatch_decision(alloc);
    TEST_ASSERT_NOT_NULL(decision);

    rc = sched_commit_decision(&sched, decision);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);

    get_resvals(&(w->resInUse), 1, &residx, &val);
    TEST_ASSERT_EQUAL_FLOAT(WANT_VALUE, val);

    for (round = 0; round < GAP_ROUNDS; round++) {
        /* still no schedStartJob/confirmation event anywhere in this loop
         * -- job->accounts/job->state were set once, at dispatch commit,
         * and nothing else re-arms them between rounds. */
        put_resvals(&(w->resInUse), 1, &residx, zero);
        get_resvals(&(w->resInUse), 1, &residx, &val);
        TEST_ASSERT_EQUAL_FLOAT(0.0f, val);

        rc = sched_account_jobs(&(sched.core));
        TEST_ASSERT_EQUAL_INT(0, rc);

        get_resvals(&(w->resInUse), 1, &residx, &val);
        TEST_ASSERT_EQUAL_FLOAT(WANT_VALUE, val);
    }
}

/*
 * Shared fixture for the comparison test below: builds a fresh core +
 * worker + dispatched job (same shape as the tests above), then simulates
 * `nrounds` successive round boundaries -- each zeroing resInUse exactly as
 * xce_core_begin_session() does every round, then re-deriving it via
 * sched_account_jobs() -- with NO worker-confirmation event injected at any
 * point, and returns the worker's final resInUse value for RES_NAME.
 *
 * `nrounds == 1` stands in for "the worker's confirmation would have
 * arrived the very same round dispatch was committed": since resInUse for
 * a RUN job is fully determined by sched_account_jobs()'s JOB_IS_RUN()
 * gate (unmodified by this feature) and job->state is already RUN
 * synchronously at dispatch-commit (Requirement 1.1), a same-round
 * confirmation event applying JOB_STAT_RUN again is a confirmed no-op
 * against an already-RUN job (design.md, "Scheduler Dispatch Commit"
 * Responsibilities & Constraints) -- it changes nothing that
 * sched_account_jobs() reads. So the very next round's accounting pass
 * already reflects exactly what a same-round confirmation would have
 * produced; there is nothing further for an explicit confirmation event to
 * contribute to this value, which is why simulating that event is not
 * needed here to make the comparison meaningful.
 *
 * `nrounds > 1` simulates however many additional rounds elapse with still
 * no confirmation in between. Per Requirement 1.2, the two must be
 * identical -- asserted explicitly by
 * test_dispatch_commit_multi_round_gap_matches_same_round_confirmation
 * below, not merely by separately comparing each against a hardcoded
 * constant.
 */
static float run_dispatch_then_n_rounds(int nrounds)
{
    xce_scheduler_t     sched;
    xce_sched_worker_t* w;
    xce_sched_job_t*    job;
    xce_job_alloc_t*    alloc;
    xce_decision_t*     decision;
    xce_resource_t*     vres = NULL;
    int                 rc;
    int                 residx;
    float               val;
    float               zero[1] = { 0.0f };
    int                 i;

    memset(&sched, 0, sizeof(sched));
    rc = xce_sched_core_init(&(sched.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    w   = add_worker(&(sched.core), WORKER_NAME);
    job = add_pending_job(&(sched.core), JOB_ID);

    residx = add_vres(&vres, sched.core.resmgr, RES_NAME, 0);
    TEST_ASSERT_TRUE(residx >= 0);

    alloc    = make_dispatch_alloc(JOB_ID, WORKER_NAME, WANT_VALUE);
    decision = make_dispatch_decision(alloc);
    TEST_ASSERT_NOT_NULL(decision);

    rc = sched_commit_decision(&sched, decision);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);

    for (i = 0; i < nrounds; i++) {
        put_resvals(&(w->resInUse), 1, &residx, zero);
        rc = sched_account_jobs(&(sched.core));
        TEST_ASSERT_EQUAL_INT(0, rc);
    }

    get_resvals(&(w->resInUse), 1, &residx, &val);
    return val;
}

/*
 * Task 5.1's explicit "matching what a same-round confirmation would have
 * produced" language: directly compares the resInUse value produced after
 * only ONE round boundary since dispatch (run_dispatch_then_n_rounds(1) --
 * see its own comment for why this stands in for "confirmed the same
 * round") against the value produced after GAP_ROUNDS round boundaries
 * with still no confirmation in between. Per Requirement 1.2, a job's
 * contribution to resInUse must not depend on how many rounds elapse
 * before confirmation -- both values must be identical, and both must
 * equal WANT_VALUE. This is deliberately a value comparison against a
 * second, independently-computed scenario, not merely two separate
 * comparisons against the same hardcoded constant.
 */
static void test_dispatch_commit_multi_round_gap_matches_same_round_confirmation(void)
{
    float same_round_val;
    float multi_round_gap_val;

    same_round_val      = run_dispatch_then_n_rounds(1);
    multi_round_gap_val = run_dispatch_then_n_rounds(GAP_ROUNDS);

    TEST_ASSERT_EQUAL_FLOAT(WANT_VALUE, same_round_val);
    TEST_ASSERT_EQUAL_FLOAT(WANT_VALUE, multi_round_gap_val);
    TEST_ASSERT_EQUAL_FLOAT(same_round_val, multi_round_gap_val);
}

int main(void)
{
    UNITY_BEGIN();
    RUN_TEST(test_dispatch_commit_transitions_job_to_run);
    RUN_TEST(test_dispatch_commit_survives_round_boundary_before_confirmation);
    RUN_TEST(test_dispatch_commit_survives_several_rounds_before_confirmation);
    RUN_TEST(test_dispatch_commit_multi_round_gap_matches_same_round_confirmation);
    RUN_TEST(test_pend_decision_does_not_touch_job_state);
    return UNITY_END();
}