/* * test_redispatch.c - white-box regression tests for bug-1498: the scheduler * re-dispatched a job it had already dispatched (running job killed by a * duplicate dispatch). * * Root cause: sched_job_is_ready() was a stub returning 1 unconditionally, so * a job lingering in the fcfs pendJobs queue was always elected; and a * dispatched job was never marked non-PEND, so fcfs_update_job re-enqueued it * on every job update (the controller defers its schedStartJob notify). * * Strategy: pull the REAL scheduler.c into this TU (same white-box pattern as * test_job_update.c pulling schedobj.c), stub the scheduler-internal externs * defined in other .c files not linked here, and drive: * - sched_job_is_ready: PEND electable; RUN/EXIT/DONE/suspended/NULL not. * - sched_commit_decision(XCE_DECISION_DISPATCH): marks the job RUN so it * leaves the PEND set and is no longer electable. */ #define UNIT_TEST 1 #include #include #include #include #include #include #include "dataq.h" #include "kvmgr.h" #include "mylock.h" #include "scheduler.h" /* Pull the real implementation under test. */ #include "../src/scheduler.c" /* ── Stubs for scheduler.c externs defined in other .c files ─────────────── */ /* plugin.c — plugin dispatch table & lifecycle (not linked here). */ int plugin_job_update(void* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; } int plugin_queue_update(void* s, xce_sched_queue_t* q) { (void)s; (void)q; return 0; } int plugin_worker_update(void* s, xce_sched_worker_t* w) { (void)s; (void)w; return 0; } int plugin_job_state_change(void* s, xce_sched_job_t* j, uint32_t st) { (void)s; (void)j; (void)st; return 0; } int plugin_job_rusage_update(void* s, xce_sched_job_t* j, xce_job_rusages_t* r) { (void)s; (void)j; (void)r; return 0; } int plugin_commit_decision(void* s, xce_decision_t* d) { (void)s; (void)d; return 0; } int plugin_begin_session(void* s) { (void)s; return 0; } int plugin_end_session(void* s) { (void)s; return 0; } int plugin_config_update(void* s, xce_conf4sched_t* c) { (void)s; (void)c; return 0; } int plugin_init(void* s) { (void)s; return 0; } void plugin_unregister(void* s) { (void)s; } int plugin_job_election(void* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; } /* schedobj.c / resource.c / client helpers — stubbed for these tests. */ void kill_sched_job(void* arg) { (void)arg; } void kill_sched_queue(void* arg) { (void)arg; } void kill_sched_worker(void* arg) { (void)arg; } void kill_sched_user(void* arg) { (void)arg; } void kill_sched_project(void* arg) { (void)arg; } int sched_commit_job_alloc(xce_scheduler_t* s, xce_sched_job_t* j) { (void)s; (void)j; return 0; } int sched_charge_account(xce_scheduler_t* s, xce_job_account_t* a) { (void)s; (void)a; return 0; } int sched_uncharge_account(xce_scheduler_t* s, xce_job_account_t* a) { (void)s; (void)a; return 0; } /* Recorded for the JOB_RUSAGE routing test below; stores the report on the * matching job (in-place), -ENOENT when unknown — mirrors schedobj.c. */ static xce_job_rusages_t* g_last_rusage = NULL; int sched_apply_job_rusage(xce_scheduler_t* s, xce_job_rusages_t* r) { xce_sched_job_t* job; if (r == NULL) { return -EINVAL; } g_last_rusage = r; job = sched_get_job(s, r->jobid); if (job == NULL) { return -ENOENT; } job->rusages = r; return 0; } /* Recorded for the routing test below; mimics the real schedobj.c semantics * (in-place state update of the existing job, -ENOENT when unknown). */ static xce_job_t* g_last_update_fields_job = NULL; int sched_update_existing_job(xce_scheduler_t* s, xce_job_t* j) { xce_sched_job_t* job; if (j == NULL) { return -EINVAL; } g_last_update_fields_job = j; job = sched_get_job(s, j->jobid); if (job == NULL) { return -ENOENT; } job->state = j->state; return 0; } int sched_update_existing_queue(xce_scheduler_t* s, xce_queue_t* q) { (void)s; (void)q; return -ENOENT; } /* schedobj.c / decision.c / plugins — additional externs scheduler.c * references (defined in .c files not linked here). */ int sched_ctx_init(xce_sched_ctx_t* ctx, void* arg, xce_sched_obj_t* obj) { (void)ctx; (void)arg; (void)obj; return 0; } void sched_ctx_fini(xce_sched_ctx_t* ctx) { (void)ctx; } void sched_ctx_set_primary(xce_sched_ctx_t* ctx, xce_primary_resreq_t* p) { (void)ctx; (void)p; } xce_decision_t* make_pend_decision(xce_pendreason_t* r) { (void)r; return NULL; } xce_decision_t* make_dispatch_decision(xce_job_alloc_t* a) { (void)a; return NULL; } int register_fcfs(void* s) { (void)s; return 0; } int register_limit(void* s) { (void)s; return 0; } int register_seqjob_allocator(void* s) { (void)s; return 0; } int register_parjob_allocator(void* s) { (void)s; return 0; } void kill_sched_resval(xce_val_t* v) { (void)v; } xce_sched_job_t* make_sched_job(void* a, xce_job_t* j) { (void)a; (void)j; return NULL; } xce_sched_queue_t* make_sched_queue(void* a, xce_queue_t* q) { (void)a; (void)q; return NULL; } xce_sched_worker_t* make_sched_worker(void* a, xce_worker_t* w) { (void)a; (void)w; return NULL; } xce_sched_user_t* make_sched_user(void* a, char* n) { (void)a; (void)n; return NULL; } xce_sched_project_t* make_sched_project(void* a, char* n) { (void)a; (void)n; return NULL; } void sched_worker_release_host_ctx(xce_sched_worker_t* w) { (void)w; } void setUp(void) { return; } void tearDown(void) { return; } /* Note: sched_get_job / sched_add_job / sched_get_queue / ... are defined by * scheduler.c itself (real kvmgr-backed) and come in via the #include — no * stubs needed. They operate on sched->jobs etc., which fixture_init creates. */ /* ── Fixture ─────────────────────────────────────────────────────────────── */ static void fixture_init(xce_scheduler_t* sched) { kvmgr_params_t jobs_params = { .key_type = KVMGR_KEY_INT, .nslots = 59, .delta = 16, .datakiller = kill_sched_job }; memset(sched, 0, sizeof(*sched)); resreq_init(1); xstring_table_init(1024 * 16); sched->jobs = makeInstance(&xkvmgr_type, &jobs_params); /* pend_updates double-buffer + lock for driving sched_update() */ mylock_init(&(sched->lock)); sched->pend_updates[0] = (dataq_t*)calloc(1, sizeof(dataq_t)); dataq_init(sched->pend_updates[0], NULL); sched->pend_updates[1] = (dataq_t*)calloc(1, sizeof(dataq_t)); dataq_init(sched->pend_updates[1], NULL); } static xce_sched_job_t* seed_job(xce_scheduler_t* sched, uint64_t jobid, uint32_t state) { xce_sched_job_t* j = (xce_sched_job_t*)calloc(1, sizeof(*j)); TEST_ASSERT_NOT_NULL(j); j->jobid = jobid; j->state = state; sched_add_job(sched, j); return j; } /* ── sched_job_is_ready ──────────────────────────────────────────────────── */ static void test_ready_null_not_ready(void) { TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(NULL)); } static void test_ready_pend_is_ready(void) { xce_sched_job_t j = { .state = JOB_STAT_PEND }; TEST_ASSERT_EQUAL_INT(1, sched_job_is_ready(&j)); } static void test_ready_run_not_ready(void) { xce_sched_job_t j = { .state = JOB_STAT_RUN }; TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j)); } static void test_ready_exit_not_ready(void) { xce_sched_job_t j = { .state = JOB_STAT_EXIT }; TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j)); } static void test_ready_done_not_ready(void) { xce_sched_job_t j = { .state = JOB_STAT_DONE }; TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j)); } static void test_ready_suspended_not_ready(void) { xce_sched_job_t j = { .state = JOB_STAT_PEND | JOB_STAT_USUSP }; TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(&j)); } /* ── sched_commit_decision dispatch marks the job RUN ────────────────────── */ static void test_dispatch_commit_marks_job_run(void) { xce_scheduler_t sched; xce_sched_job_t* job; xce_job_alloc_t alloc; xce_decision_t decision; fixture_init(&sched); job = seed_job(&sched, 42, JOB_STAT_PEND); memset(&alloc, 0, sizeof(alloc)); alloc.jobid = 42; memset(&decision, 0, sizeof(decision)); decision.op = XCE_DECISION_DISPATCH; decision.u.alloc = &alloc; TEST_ASSERT_EQUAL_INT(0, sched_commit_decision(&sched, &decision)); /* The dispatched job leaves PEND: state is RUN and it is not electable. */ TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state); TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(job)); } static void test_dispatch_commit_null_alloc_unchanged(void) { xce_scheduler_t sched; xce_sched_job_t* job; xce_decision_t decision; fixture_init(&sched); job = seed_job(&sched, 43, JOB_STAT_PEND); memset(&decision, 0, sizeof(decision)); decision.op = XCE_DECISION_PEND; decision.u.reason = NULL; TEST_ASSERT_EQUAL_INT(0, sched_commit_decision(&sched, &decision)); /* Non-dispatch decision must not touch the job's state. */ TEST_ASSERT_EQUAL_UINT(JOB_STAT_PEND, job->state); } /* ── G-8 dequeue routing (bug-765 remediation; research.md follow-up) ────── */ /* The notify single-job update path: the handler enqueues * XCE_SCHED_OP_UPDATE_JOB_FIELDS, and sched_update() must dispatch it to * sched_update_existing_job() (in-place), NOT to the bulk CREATE path * (sched_update_jobs → make_sched_job → key_add EEXIST-drop). We drive the * real dequeue loop with a call-recording stub for sched_update_existing_job * and assert it is reached and the existing job is mutated in place. */ static void test_update_fields_routes_to_inplace_path(void) { xce_scheduler_t sched; fixture_init(&sched); xce_sched_job_t* job = seed_job(&sched, 42, JOB_STAT_PEND); xce_sched_obj_t* so = (xce_sched_obj_t*)calloc(1, sizeof(*so)); job->schedobj = so; /* live runtime state must survive */ g_last_update_fields_job = NULL; xce_sched_update_request_t* req = (xce_sched_update_request_t*)calloc(1, sizeof(*req)); xce_job_t* upd = (xce_job_t*)calloc(1, sizeof(*upd)); upd->jobid = 42; upd->state = JOB_STAT_RUN; req->op = XCE_SCHED_OP_UPDATE_JOB_FIELDS; req->u.jobfieldsupdate.job = upd; dataq_enq(sched.pend_updates[0], &(req->link)); TEST_ASSERT_EQUAL_INT(0, sched_update(&sched)); /* routed to sched_update_existing_job (not the bulk CREATE path) */ TEST_ASSERT_EQUAL_PTR(upd, g_last_update_fields_job); /* existing job mutated in place, runtime state kept */ TEST_ASSERT_EQUAL_PTR(job, sched_get_job(&sched, 42)); TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state); TEST_ASSERT_EQUAL_PTR(so, job->schedobj); } /* The bulk dataset path (XCE_SCHED_OP_UPDATE_JOB) routes to sched_update_jobs * (CREATE) → sched_add_job → key_add. For an ALREADY-existing jobid, key_add * must EEXIST-drop the fresh object rather than replacing the live one — the * existing job keeps its identity and runtime state (no corruption from a * re-pushed dataset row). We drive the REAL sched_add_job (the exact call the * bulk path makes after make_sched_job) with a fresh duplicate object. */ static void test_bulk_update_existing_job_no_damage(void) { xce_scheduler_t sched; fixture_init(&sched); xce_sched_job_t* job = seed_job(&sched, 7, JOB_STAT_RUN); xce_sched_obj_t* so = (xce_sched_obj_t*)calloc(1, sizeof(*so)); job->schedobj = so; /* A fresh object for the same jobid carrying STALE state — what * make_sched_job would build from a re-pushed dataset row. */ xce_sched_job_t* fresh = (xce_sched_job_t*)calloc(1, sizeof(*fresh)); fresh->jobid = 7; fresh->state = JOB_STAT_PEND; fresh->schedobj = (xce_sched_obj_t*)calloc(1, sizeof(xce_sched_obj_t)); int rc = sched_add_job(&sched, fresh); TEST_ASSERT(rc != 0); /* key_add EEXIST — the stale duplicate is dropped */ /* the live job is untouched: identity, state, runtime state all preserved */ TEST_ASSERT_EQUAL_PTR(job, sched_get_job(&sched, 7)); TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, job->state); TEST_ASSERT_EQUAL_PTR(so, job->schedobj); } /* ── G-5 dequeue routing: JOB_RUSAGE → sched_apply_job_rusage ───────────── */ /* The rusage notify handler enqueues XCE_SCHED_OP_JOB_RUSAGE; sched_update() * must dispatch it to sched_apply_job_rusage(), which stores the report on * the matching job (bug-765 G-5 remediation). We drive the real dequeue loop * with the recording stub and assert routing + storage. */ static void test_rusage_routes_to_apply_on_job(void) { xce_scheduler_t sched; fixture_init(&sched); xce_sched_job_t* job = seed_job(&sched, 7, JOB_STAT_RUN); g_last_rusage = NULL; xce_sched_update_request_t* req = (xce_sched_update_request_t*)calloc(1, sizeof(*req)); xce_job_rusages_t* ru = (xce_job_rusages_t*)calloc(1, sizeof(*ru)); ru->jobid = 7; req->op = XCE_SCHED_OP_JOB_RUSAGE; req->u.job_rusage.rusages = ru; dataq_enq(sched.pend_updates[0], &(req->link)); TEST_ASSERT_EQUAL_INT(0, sched_update(&sched)); /* routed to sched_apply_job_rusage and stored on the job */ TEST_ASSERT_EQUAL_PTR(ru, g_last_rusage); TEST_ASSERT_EQUAL_PTR(ru, job->rusages); } /* ── Runner ──────────────────────────────────────────────────────────────── */ int main(int argc, char* argv[]) { (void)argc; (void)argv; UNITY_BEGIN(); RUN_TEST(test_ready_null_not_ready, 0); RUN_TEST(test_ready_pend_is_ready, 0); RUN_TEST(test_ready_run_not_ready, 0); RUN_TEST(test_ready_exit_not_ready, 0); RUN_TEST(test_ready_done_not_ready, 0); RUN_TEST(test_ready_suspended_not_ready, 0); RUN_TEST(test_dispatch_commit_marks_job_run, 0); RUN_TEST(test_dispatch_commit_null_alloc_unchanged, 0); RUN_TEST(test_update_fields_routes_to_inplace_path, 0); RUN_TEST(test_bulk_update_existing_job_no_damage, 0); RUN_TEST(test_rusage_routes_to_apply_on_job, 0); return UNITY_END(); }