myxce/src/scheduler/testsuite/test_redispatch.c

/*
 * test_redispatch.c - white-box regression tests for bug-1498: the scheduler
 * re-dispatched a job it had already dispatched (running job killed by a
 * duplicate dispatch).
 *
 * Root cause: sched_job_is_ready() was a stub returning 1 unconditionally, so
 * a job lingering in the fcfs pendJobs queue was always elected; and a
 * dispatched job was never marked non-PEND, so fcfs_update_job re-enqueued it
 * on every job update (the controller defers its schedStartJob notify).
 *
 * Strategy: pull the REAL scheduler.c into this TU (same white-box pattern as
 * test_job_update.c pulling schedobj.c), stub the scheduler-internal externs
 * defined in other .c files not linked here, and drive:
 *   - sched_job_is_ready: PEND electable; RUN/EXIT/DONE/suspended/NULL not.
 *   - sched_commit_decision(XCE_DECISION_DISPATCH): marks the job RUN so it
 *     leaves the PEND set and is no longer electable.
 */
#define UNIT_TEST 1
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdatomic.h>

#include <unity/unity.h>

#include "dataq.h"
#include "kvmgr.h"
#include "mylock.h"
#include "scheduler.h"

/* Pull the real implementation under test. */
#include "../src/scheduler.c"

/* ── Stubs for scheduler.c externs defined in other .c files ─────────────── */

/* plugin.c — plugin dispatch table & lifecycle (not linked here). */
int  plugin_job_update(void* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; }
int  plugin_queue_update(void* s, xce_sched_queue_t* q) { (void)s; (void)q; return 0; }
int  plugin_worker_update(void* s, xce_sched_worker_t* w) { (void)s; (void)w; return 0; }
int  plugin_job_state_change(void* s, xce_sched_job_t* j, uint32_t st)
{
    (void)s; (void)j; (void)st; return 0;
}
int plugin_job_rusage_update(void* s, xce_sched_job_t* j, xce_job_rusages_t* r)
{
    (void)s; (void)j; (void)r; return 0;
}
int plugin_commit_decision(void* s, xce_decision_t* d) { (void)s; (void)d; return 0; }
int plugin_begin_session(void* s) { (void)s; return 0; }
int plugin_end_session(void* s) { (void)s; return 0; }
int plugin_config_update(void* s, xce_conf4sched_t* c) { (void)s; (void)c; return 0; }
int plugin_init(void* s) { (void)s; return 0; }
void plugin_unregister(void* s) { (void)s; }
int plugin_job_election(void* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; }

/* schedobj.c / resource.c / client helpers — stubbed for these tests. */
void          kill_sched_job(void* arg) { (void)arg; }
void          kill_sched_queue(void* arg) { (void)arg; }
void          kill_sched_worker(void* arg) { (void)arg; }
void          kill_sched_user(void* arg) { (void)arg; }
void          kill_sched_project(void* arg) { (void)arg; }
int  sched_commit_job_alloc(xce_scheduler_t* s, xce_sched_job_t* j)
{
    (void)s; (void)j; return 0;
}
int  sched_charge_account(xce_scheduler_t* s, xce_job_account_t* a)
{
    (void)s; (void)a; return 0;
}
int  sched_uncharge_account(xce_scheduler_t* s, xce_job_account_t* a)
{
    (void)s; (void)a; return 0;
}
/* Recorded for the JOB_RUSAGE routing test below; stores the report on the
 * matching job (in-place), -ENOENT when unknown — mirrors schedobj.c. */
static xce_job_rusages_t* g_last_rusage = NULL;
int  sched_apply_job_rusage(xce_scheduler_t* s, xce_job_rusages_t* r)
{
    xce_sched_job_t* job;
    if (r == NULL) {
        return -EINVAL;
    }
    g_last_rusage = r;
    job = sched_get_job(s, r->jobid);
    if (job == NULL) {
        return -ENOENT;
    }
    job->rusages = r;
    return 0;
}
/* Recorded for the routing test below; mimics the real schedobj.c semantics
 * (in-place state update of the existing job, -ENOENT when unknown). */
static xce_job_t* g_last_update_fields_job = NULL;
int  sched_update_existing_job(xce_scheduler_t* s, xce_job_t* j)
{
    xce_sched_job_t* job;
    if (j == NULL) {
        return -EINVAL;
    }
    g_last_update_fields_job = j;
    job = sched_get_job(s, j->jobid);
    if (job == NULL) {
        return -ENOENT;
    }
    job->state = j->state;
    return 0;
}
int  sched_update_existing_queue(xce_scheduler_t* s, xce_queue_t* q)
{
    (void)s; (void)q; return -ENOENT;
}

/* schedobj.c / decision.c / plugins — additional externs scheduler.c
 * references (defined in .c files not linked here). */
int  sched_ctx_init(xce_sched_ctx_t* ctx, void* arg, xce_sched_obj_t* obj)
{
    (void)ctx; (void)arg; (void)obj; return 0;
}
void sched_ctx_fini(xce_sched_ctx_t* ctx) { (void)ctx; }
void sched_ctx_set_primary(xce_sched_ctx_t* ctx, xce_primary_resreq_t* p)
{
    (void)ctx; (void)p;
}
xce_decision_t* make_pend_decision(xce_pendreason_t* r) { (void)r; return NULL; }
xce_decision_t* make_dispatch_decision(xce_job_alloc_t* a) { (void)a; return NULL; }
int register_fcfs(void* s) { (void)s; return 0; }
int register_limit(void* s) { (void)s; return 0; }
int register_seqjob_allocator(void* s) { (void)s; return 0; }
int register_parjob_allocator(void* s) { (void)s; return 0; }
void kill_sched_resval(xce_val_t* v) { (void)v; }
xce_sched_job_t*   make_sched_job(void* a, xce_job_t* j) { (void)a; (void)j; return NULL; }
xce_sched_queue_t* make_sched_queue(void* a, xce_queue_t* q) { (void)a; (void)q; return NULL; }
xce_sched_worker_t* make_sched_worker(void* a, xce_worker_t* w) { (void)a; (void)w; return NULL; }
xce_sched_user_t*   make_sched_user(void* a, char* n) { (void)a; (void)n; return NULL; }
xce_sched_project_t* make_sched_project(void* a, char* n) { (void)a; (void)n; return NULL; }
void sched_worker_release_host_ctx(xce_sched_worker_t* w) { (void)w; }

void setUp(void) { return; }
void tearDown(void) { return; }

/* Note: sched_get_job / sched_add_job / sched_get_queue / ... are defined by
 * scheduler.c itself (real kvmgr-backed) and come in via the #include — no
 * stubs needed. They operate on sched->jobs etc., which fixture_init creates. */

/* ── Fixture ─────────────────────────────────────────────────────────────── */

static void fixture_init(xce_scheduler_t* sched)
{
    kvmgr_params_t jobs_params = { .key_type   = KVMGR_KEY_INT,
                                   .nslots     = 59,
                                   .delta      = 16,
                                   .datakiller = kill_sched_job };

    memset(sched, 0, sizeof(*sched));
    resreq_init(1);
    xstring_table_init(1024 * 16);

    sched->jobs = makeInstance(&xkvmgr_type, &jobs_params);

    /* pend_updates double-buffer + lock for driving sched_update() */
    mylock_init(&(sched->lock));
    sched->pend_updates[0] = (dataq_t*)calloc(1, sizeof(dataq_t));
    dataq_init(sched->pend_updates[0], NULL);
    sched->pend_updates[1] = (dataq_t*)calloc(1, sizeof(dataq_t));
    dataq_init(sched->pend_updates[1], NULL);
}

static xce_sched_job_t* seed_job(xce_scheduler_t* sched, uint64_t jobid,
                                 uint32_t state)
{
    xce_sched_job_t* j = (xce_sched_job_t*)calloc(1, sizeof(*j));
    TEST_ASSERT_NOT_NULL(j);
    j->jobid = jobid;
    j->state = state;
    sched_add_job(sched, j);
    return j;
}

/* ── sched_job_is_ready ──────────────────────────────────────────────────── */

static void test_ready_null_not_ready(void)
{
    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(NULL));
}

static void test_ready_pend_is_ready(void)
{
    xce_sched_job_t j = { .state = JOB_STAT_PEND };
    TEST_ASSERT_EQUAL_INT(1, sched_job_is_ready(&j));
}

static void test_ready_run_not_ready(void)
{
    xce_sched_job_t j = { .state = JOB_STAT_RUN };
    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
}

static void test_ready_exit_not_ready(void)
{
    xce_sched_job_t j = { .state = JOB_STAT_EXIT };
    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
}

static void test_ready_done_not_ready(void)
{
    xce_sched_job_t j = { .state = JOB_STAT_DONE };
    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
}

static void test_ready_suspended_not_ready(void)
{
    xce_sched_job_t j = { .state = JOB_STAT_PEND | JOB_STAT_USUSP };
    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j));
}

/* ── sched_commit_decision dispatch marks the job RUN ────────────────────── */

static void test_dispatch_commit_marks_job_run(void)
{
    xce_scheduler_t  sched;
    xce_sched_job_t* job;
    xce_job_alloc_t  alloc;
    xce_decision_t   decision;

    fixture_init(&sched);

    job = seed_job(&sched, 42, JOB_STAT_PEND);

    memset(&alloc, 0, sizeof(alloc));
    alloc.jobid = 42;

    memset(&decision, 0, sizeof(decision));
    decision.op      = XCE_DECISION_DISPATCH;
    decision.u.alloc = &alloc;

    TEST_ASSERT_EQUAL_INT(0, sched_commit_decision(&sched, &decision));

    /* The dispatched job leaves PEND: state is RUN and it is not electable. */
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);
    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(job));
}

static void test_dispatch_commit_null_alloc_unchanged(void)
{
    xce_scheduler_t  sched;
    xce_sched_job_t* job;
    xce_decision_t   decision;

    fixture_init(&sched);
    job = seed_job(&sched, 43, JOB_STAT_PEND);

    memset(&decision, 0, sizeof(decision));
    decision.op      = XCE_DECISION_PEND;
    decision.u.reason = NULL;

    TEST_ASSERT_EQUAL_INT(0, sched_commit_decision(&sched, &decision));
    /* Non-dispatch decision must not touch the job's state. */
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_PEND, job->state);
}

/* ── G-8 dequeue routing (bug-765 remediation; research.md follow-up) ────── */

/* The notify single-job update path: the handler enqueues
 * XCE_SCHED_OP_UPDATE_JOB_FIELDS, and sched_update() must dispatch it to
 * sched_update_existing_job() (in-place), NOT to the bulk CREATE path
 * (sched_update_jobs → make_sched_job → key_add EEXIST-drop). We drive the
 * real dequeue loop with a call-recording stub for sched_update_existing_job
 * and assert it is reached and the existing job is mutated in place. */
static void test_update_fields_routes_to_inplace_path(void)
{
    xce_scheduler_t sched;
    fixture_init(&sched);

    xce_sched_job_t* job = seed_job(&sched, 42, JOB_STAT_PEND);
    xce_sched_obj_t* so  = (xce_sched_obj_t*)calloc(1, sizeof(*so));
    job->schedobj        = so; /* live runtime state must survive */

    g_last_update_fields_job = NULL;
    xce_sched_update_request_t* req =
        (xce_sched_update_request_t*)calloc(1, sizeof(*req));
    xce_job_t* upd = (xce_job_t*)calloc(1, sizeof(*upd));
    upd->jobid     = 42;
    upd->state     = JOB_STAT_RUN;
    req->op                    = XCE_SCHED_OP_UPDATE_JOB_FIELDS;
    req->u.jobfieldsupdate.job = upd;
    dataq_enq(sched.pend_updates[0], &(req->link));

    TEST_ASSERT_EQUAL_INT(0, sched_update(&sched));

    /* routed to sched_update_existing_job (not the bulk CREATE path) */
    TEST_ASSERT_EQUAL_PTR(upd, g_last_update_fields_job);
    /* existing job mutated in place, runtime state kept */
    TEST_ASSERT_EQUAL_PTR(job, sched_get_job(&sched, 42));
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);
    TEST_ASSERT_EQUAL_PTR(so, job->schedobj);
}

/* The bulk dataset path (XCE_SCHED_OP_UPDATE_JOB) routes to sched_update_jobs
 * (CREATE) → sched_add_job → key_add. For an ALREADY-existing jobid, key_add
 * must EEXIST-drop the fresh object rather than replacing the live one — the
 * existing job keeps its identity and runtime state (no corruption from a
 * re-pushed dataset row). We drive the REAL sched_add_job (the exact call the
 * bulk path makes after make_sched_job) with a fresh duplicate object. */
static void test_bulk_update_existing_job_no_damage(void)
{
    xce_scheduler_t sched;
    fixture_init(&sched);

    xce_sched_job_t* job = seed_job(&sched, 7, JOB_STAT_RUN);
    xce_sched_obj_t* so  = (xce_sched_obj_t*)calloc(1, sizeof(*so));
    job->schedobj        = so;

    /* A fresh object for the same jobid carrying STALE state — what
     * make_sched_job would build from a re-pushed dataset row. */
    xce_sched_job_t* fresh = (xce_sched_job_t*)calloc(1, sizeof(*fresh));
    fresh->jobid           = 7;
    fresh->state           = JOB_STAT_PEND;
    fresh->schedobj        = (xce_sched_obj_t*)calloc(1, sizeof(xce_sched_obj_t));

    int rc = sched_add_job(&sched, fresh);
    TEST_ASSERT(rc != 0); /* key_add EEXIST — the stale duplicate is dropped */

    /* the live job is untouched: identity, state, runtime state all preserved */
    TEST_ASSERT_EQUAL_PTR(job, sched_get_job(&sched, 7));
    TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state);
    TEST_ASSERT_EQUAL_PTR(so, job->schedobj);
}

/* ── G-5 dequeue routing: JOB_RUSAGE → sched_apply_job_rusage ───────────── */

/* The rusage notify handler enqueues XCE_SCHED_OP_JOB_RUSAGE; sched_update()
 * must dispatch it to sched_apply_job_rusage(), which stores the report on
 * the matching job (bug-765 G-5 remediation). We drive the real dequeue loop
 * with the recording stub and assert routing + storage. */
static void test_rusage_routes_to_apply_on_job(void)
{
    xce_scheduler_t sched;
    fixture_init(&sched);

    xce_sched_job_t* job = seed_job(&sched, 7, JOB_STAT_RUN);

    g_last_rusage = NULL;
    xce_sched_update_request_t* req =
        (xce_sched_update_request_t*)calloc(1, sizeof(*req));
    xce_job_rusages_t* ru = (xce_job_rusages_t*)calloc(1, sizeof(*ru));
    ru->jobid             = 7;
    req->op               = XCE_SCHED_OP_JOB_RUSAGE;
    req->u.job_rusage.rusages = ru;
    dataq_enq(sched.pend_updates[0], &(req->link));

    TEST_ASSERT_EQUAL_INT(0, sched_update(&sched));

    /* routed to sched_apply_job_rusage and stored on the job */
    TEST_ASSERT_EQUAL_PTR(ru, g_last_rusage);
    TEST_ASSERT_EQUAL_PTR(ru, job->rusages);
}

/* ── Runner ──────────────────────────────────────────────────────────────── */

int main(int argc, char* argv[])
{
    (void)argc; (void)argv;
    UNITY_BEGIN();
    RUN_TEST(test_ready_null_not_ready, 0);
    RUN_TEST(test_ready_pend_is_ready, 0);
    RUN_TEST(test_ready_run_not_ready, 0);
    RUN_TEST(test_ready_exit_not_ready, 0);
    RUN_TEST(test_ready_done_not_ready, 0);
    RUN_TEST(test_ready_suspended_not_ready, 0);
    RUN_TEST(test_dispatch_commit_marks_job_run, 0);
    RUN_TEST(test_dispatch_commit_null_alloc_unchanged, 0);
    RUN_TEST(test_update_fields_routes_to_inplace_path, 0);
    RUN_TEST(test_bulk_update_existing_job_no_damage, 0);
    RUN_TEST(test_rusage_routes_to_apply_on_job, 0);
    return UNITY_END();
}