/* * test_snapshot_load_allocator_regression.c - scheduler-snapshot-job- * allocation spec, task 4.1 (Req 2 AC 2, AC 4): the direct regression test * proving a PENDING job restored from a snapshot reaches the scheduler's * REAL allocator entry point -- sched_make_allocation() (scheduler.c) -- * and is evaluated without crashing or producing undefined behavior. This * is the exact bug-1569-class failure shape task 1.2 already fixed * (job->schedobj construction for restored jobs, snapshot_load.c's * snapshot_build_schedobj()); this file is the test that would have * caught it. * * ── Why this is a NEW file, not an extension of test_snapshot_save_load.c * ──────────────────────────────────────────────────────────────────── * * sched_make_allocation() (and its callees sched_make_primary_allocation/ * sched_make_compound_allocation/sched_make_alter_allocation) are `static` * to scheduler.c. Reaching them from a test requires either (a) white-box * #include-ing scheduler.c into the test's own TU, or (b) linking the real, * separately-compiled scheduler_srcs (scheduler.c included) and driving them * indirectly through sched_init()'s real picker/allocator threads. * * test_snapshot_save_load.c already white-box #includes core.c/ * core_clock.c/core_events.c/core_session.c/decision.c/resource.c/ * snapshot.c/snapshot_load.c/snapshot_wire.c into ONE translation unit * (its own header comment documents the exact stub set this combination * already needs: kill_sched_* family, plugin_unregister, * sched_worker_release_host_ctx, sched_resolve_job_picker, * plugin_prepare_allocation, sched_ctx_get_uncomit_host). Adding * `#include "../src/scheduler.c"` on top would pull in scheduler.c's own, * much larger dependency closure -- client.h's xcecli_* family (real * network I/O calls sched_init()/sched_publish_dispatch()/ * sched_publish_pend_reason() make), sel_register_alloc_thread(), * plugin.c's plugin registration machinery, and every plugins/ file * (fcfs.c, builtin_fcfs.c, seqjob_allocator.c, parjob_allocator.c, * limit.c) that scheduler.c's sched_init() resolves against -- on top of * an already-large existing stub surface, for a single-TU combination no * other file in this directory attempts. This is exactly the "link-surface * reason" test_resource_accounting.c's own header comment (and * test_scheduler_connect.c's, verbatim) already flags as ruling this * approach out for scheduler.c specifically: "scheduler.c's own dependency * surface is large enough that whole-file inclusion was already ruled out * by this codebase's own precedent... object-linking the real * scheduler_srcs sidesteps that without needing dozens of hand-written * stubs." * * This file instead follows the established, lower-risk precedent already * used by test_scheduler_connect.c / test_scheduler_backpressure.c / * test_scheduler_dispatch_commit.c: link the REAL, whole scheduler_srcs * list (everything sched_init() and xce_sched_core_save_snapshot()/ * xce_sched_core_load_snapshot() need, scheduler.c included, each file * compiled once as its own separate translation unit -- see meson.build) * and call sched_init()'s real, public entry point. `-Wl,--wrap=` doubles * for xcecli_start()/xcecli_join() (identical, trivial always-succeed * doubles to test_scheduler_backpressure.c's own) let this file's one real * sched_init() call succeed with no live controller, so the real * sched_job_picker()/sched_allocator() threads (scheduler.c) actually * start. * * ── How the restored job reaches sched_make_allocation() ─────────────── * * A small, independent SOURCE xce_sched_core_t is built directly (not * through sched_init(), which would start a second, unwanted pair of * picker/allocator threads) containing exactly one PENDING job whose * resource request is "type == any" -- the same universal default every * job without an explicit resreq gets (schedobj.c's make_sched_job(), * snapshot_load.c's own NULLSTR() fallback), guaranteed parseable without * any resource types pre-registered. The REAL xce_sched_core_save_snapshot() * (snapshot.c) serializes it to a sched_snapshot_data_t; the REAL * xce_sched_core_load_snapshot() (snapshot_load.c, task 1.2's fix) then * restores it into the live sched.core -- this is the exact call under * test, and the only place job->schedobj gets built for a restored job. * * The restored job's id is then fed directly into sched.core.jbuf via * cbuf_tryput() -- the same "bypass the dormant picker thread, drive the * real allocator thread directly" technique test_scheduler_backpressure.c * already established (sched_go() is never called here, so * sched_job_picker() stays parked on cbuf_get(&core.wbuf) and never * contends with this direct jbuf feed). The real, live sched_allocator() * thread (started by sched_init() above) drains jbuf, calls * plugin_job_election() then sched_make_allocation(sched, job->schedobj) * -- the unconditional job->schedobj dereference this feature's task 1.2 * fixed (previously NULL for every restored job, bug-1569's exact failure * shape). "type == any" parses into an XCE_RESREQ_ALTER-tagged schedobj * (one compound child, one primary inside it), so this genuinely exercises * sched_make_allocation() -> sched_make_alter_allocation() -> * sched_make_compound_allocation() -> sched_make_primary_allocation(), * each dereferencing `obj`/`obj->u.*` in turn -- the same unconditional * dereference chain that was unconditionally NULL for every restored job * before task 1.2. No worker exists in sched.core, so every alternative * pends; sched_make_alter_allocation()'s own pre-existing, documented gap * (scheduler.c: "FIXME: merge pend reason from all children") never * propagates a child's pend reason up to the top-level obj->reason, so * sched_allocator() WARNs and produces no decision for this specific job * -- an unrelated, pre-existing gap in ALTER-tag reason merging, not a * crash and not this test's concern. This test therefore asserts the * restored job's own progress by confirming jbuf drains (proving * sched_make_allocation() was entered and returned control), then proves * the allocator thread's continued, uncorrupted health afterward by * running an independent CONTROL job through to a real decision. * * If job->schedobj were still NULL here (task 1.2 not applied), * sched_make_allocation() -> sched_make_alter_allocation() would * immediately SIGSEGV on its first `obj->u.alters.count` dereference of a * NULL obj (scheduler.c) -- under this project's ASAN build, that aborts * the whole process before this test's own assertions ever run, which is * exactly the class of failure this test is written to catch (Req 2.2, * 2.4). This does NOT cover a job whose resreq itself fails to parse on * load (design.md's own documented, separate, pre-existing boundary) -- * this test's resreq ("type == any") always parses. */ #include #include #include #include #include #include #include "client.h" #include "core.h" #include "decision.h" #include "preason.h" #include "resreq.h" #include "scheduler.h" #include "xce/cluster.h" #include "xstring.h" extern int scheduler_req_handler(xce_request_t*, void*); #define JOBID ((uint64_t)424242ULL) #define CONTROL_JOBID ((uint64_t)909090ULL) /* ════════════════════════════════════════════════════════════════════════ * __wrap_xcecli_start / __wrap_xcecli_join -- trivial always-succeed * doubles, identical rationale/shape to * test_scheduler_backpressure.c's own: this file's one real sched_init() * call must succeed with no live controller so the real picker/allocator * threads actually start. * ════════════════════════════════════════════════════════════════════════ */ int __wrap_xcecli_start(xcecli_ctx_t c) { (void)c; return 0; } int __wrap_xcecli_join(xcecli_ctx_t c, int role, hostattrs_t* attrs) { (void)c; (void)role; (void)attrs; return 0; } void setUp(void) { return; } void tearDown(void) { return; } /* build_source_core_with_pending_job - an independent SOURCE core (never * passed through sched_init(), so no extra picker/allocator threads start * for it) containing exactly one PENDING job with a resreq that parses * successfully. Never torn down -- same "fixtures here are never torn * down" convention as every other white-box/real-scheduler_srcs test in * this directory (e.g. test_core_full_cycle.c's own header comment). */ static void build_source_core_with_pending_job(xce_sched_core_t* core_src) { xce_sched_queue_t* q; xce_sched_job_t* job; char* errmsg = NULL; int rc; rc = xce_sched_core_init(core_src); TEST_ASSERT_TRUE_MESSAGE(rc >= 0, "fixture: xce_sched_core_init() must " "succeed for the source core"); q = (xce_sched_queue_t*)calloc(1, sizeof(*q)); TEST_ASSERT_NOT_NULL(q); q->name = makeString("normal"); rc = sched_add_queue(core_src, q); TEST_ASSERT_TRUE_MESSAGE(rc >= 0, "fixture: sched_add_queue() must " "succeed on the source core"); job = (xce_sched_job_t*)calloc(1, sizeof(*job)); TEST_ASSERT_NOT_NULL(job); job->jobid = JOBID; job->state = JOB_STAT_PEND; job->queue = sched_get_queue(core_src, "normal"); job->user = sched_get_user(core_src, "rshen"); job->project = sched_get_project(core_src, "default"); TEST_ASSERT_NOT_NULL_MESSAGE(job->queue, "fixture: source queue lookup"); TEST_ASSERT_NOT_NULL_MESSAGE(job->user, "fixture: source user " "get-or-create"); TEST_ASSERT_NOT_NULL_MESSAGE(job->project, "fixture: source project " "get-or-create"); /* "type == any" -- the same universal default resreq schedobj.c's * make_sched_job() and snapshot_load.c's own NULLSTR() fallback both * use when no explicit resreq is given; parses successfully with no * resource types registered on core_src->resmgr. This is the "resource * request that parses successfully" this task's own text requires -- * the pre-existing, out-of-boundary resreq-parse-FAILURE path * (design.md's Known Limitation) is deliberately NOT what this test * exercises. */ job->resreq = (xce_resreq_t*)calloc(1, sizeof(*(job->resreq))); TEST_ASSERT_NOT_NULL(job->resreq); rc = parse_resreq(job->resreq, "type == any", core_src->resmgr, &errmsg); TEST_ASSERT_TRUE_MESSAGE(rc >= 0, "fixture: parse_resreq('type == any') " "must succeed"); /* job->schedobj is deliberately left NULL here: the SOURCE core's own * schedobj is irrelevant to save (build_wire_job(), snapshot_wire.c, * never reads job->schedobj) -- this test's entire point is that the * LOAD side reconstructs it from scratch via snapshot_build_schedobj() * (task 1.2's fix), not that it was ever copied across. */ rc = sched_add_job(core_src, job); TEST_ASSERT_TRUE_MESSAGE(rc >= 0, "fixture: sched_add_job() must " "succeed on the source core"); } /* build_pend_only_job - a hand-built, independent CONTROL job, structurally * identical to test_scheduler_backpressure.c's own helper of the same name: * tag=XCE_RESREQ_COMPOUND with an EMPTY `primaries` list, so * sched_make_compound_allocation()'s for-loop (scheduler.c) has zero * iterations and returns rc=0 ("pend") without ever touching a worker, a * resource, or an allocator plugin -- a guaranteed, allocator-plugin- * independent PEND outcome, used here only to prove the real * sched_allocator() thread is still alive and correctly cycling AFTER it * has processed the restored job under test (see that file's own header * comment for the full rationale of this construction). */ static xce_sched_job_t* build_pend_only_job(uint64_t jobid) { xce_sched_job_t* job = (xce_sched_job_t*)calloc(1, sizeof(*job)); xce_sched_obj_t* schedobj = (xce_sched_obj_t*)calloc(1, sizeof(*schedobj)); xce_compound_schedobj_t* compound = (xce_compound_schedobj_t*)calloc(1, sizeof(*compound)); xce_compound_resreq_t* resreq = (xce_compound_resreq_t*)calloc(1, sizeof(*resreq)); xlist_init(&(resreq->primaries), 1); compound->owner = schedobj; compound->resreq = resreq; compound->reason = NULL; schedobj->tag = XCE_RESREQ_COMPOUND; schedobj->u.compound = compound; schedobj->reason = make_pendreason(jobid); job->jobid = jobid; job->state = JOB_STAT_PEND; job->schedobj = schedobj; return job; } /* * Req 2 AC 2 / AC 4: a pending job restored from a snapshot, whose * resource request parses successfully, is evaluated for resource * allocation via the scheduler's REAL, normal allocation-evaluation path * (sched_make_allocation(), reached through the real, live * sched_allocator() thread) without crashing or producing undefined * behavior. */ static void test_restored_pending_job_reaches_allocator_without_crashing(void) { xce_client_config_t config; xce_cluster_config_t cluster_cfg; xce_scheduler_t sched; xce_sched_core_t core_src; sched_snapshot_data_t wire_data; xce_sched_job_t* restored; xce_decision_t* decision = NULL; uint64_t jobid = JOBID; int rc; int waited_ms; int got_decision = 0; memset(&config, 0, sizeof(config)); config.nmasters = 1; config.masters[0] = "127.0.0.1"; config.master_ports[0] = 19998; config.from = "sched_snapshot_alloc_regression_test"; config.key = "keepsecret$"; memset(&cluster_cfg, 0, sizeof(cluster_cfg)); cluster_cfg.candidates_count = 1; strncpy(cluster_cfg.candidates[0].hostname, "127.0.0.1", XCE_MAXHOSTNAME_LEN - 1); cluster_cfg.candidates[0].port = 19998; strncpy(cluster_cfg.identity, "sched_snapshot_alloc_regression_test", sizeof(cluster_cfg.identity) - 1); memset(&sched, 0, sizeof(sched)); rc = sched_init(&sched, &config, &cluster_cfg, scheduler_req_handler); TEST_ASSERT_TRUE_MESSAGE( rc >= 0, "fixture: sched_init() must succeed (xcecli_start()/" "xcecli_join() wrapped to succeed with no I/O) so the " "real picker/allocator threads actually start"); build_source_core_with_pending_job(&core_src); memset(&wire_data, 0, sizeof(wire_data)); rc = xce_sched_core_save_snapshot(&core_src, &wire_data); TEST_ASSERT_TRUE_MESSAGE( rc >= 0, "fixture: xce_sched_core_save_snapshot() must succeed"); /* ── The actual load under test ────────────────────────────────── * xce_sched_core_load_snapshot() (snapshot_load.c) reconstructs * job->schedobj for this restored PENDING job via * snapshot_build_schedobj() -- task 1.2's fix. sched.core is freshly * initialized by sched_init() above (no jobs/queues of its own yet), * matching core.h's documented load precondition. */ rc = xce_sched_core_load_snapshot(&(sched.core), &wire_data); TEST_ASSERT_TRUE_MESSAGE( rc >= 0, "xce_sched_core_load_snapshot() must succeed"); restored = sched_get_job(&(sched.core), JOBID); TEST_ASSERT_NOT_NULL_MESSAGE( restored, "restored job must be present in the live scheduler " "core after load"); TEST_ASSERT_EQUAL_UINT(JOB_STAT_PEND, restored->state); TEST_ASSERT_NOT_NULL_MESSAGE( restored->schedobj, "job->schedobj must be non-NULL after load (task 1.2's fix) -- a " "NULL schedobj here would be bug-1569's exact failure shape the " "instant it reaches sched_make_allocation() below"); /* ── Drive the restored job through the REAL allocator ─────────── * Feed jbuf directly (bypassing only the DORMANT picker thread's own * job-selection step -- sched_go() is never called, so * sched_job_picker() stays parked on cbuf_get(&core.wbuf) and never * touches jbuf itself, the same established technique * test_scheduler_backpressure.c already uses). The allocator side is * fully real: the live sched_allocator() thread (already running, * started for real by sched_init() above) drains jbuf and calls the * real, static sched_make_allocation(sched, job->schedobj). */ rc = cbuf_tryput(&(sched.core.jbuf), &jobid); TEST_ASSERT_TRUE_MESSAGE( rc >= 0, "fixture: jbuf must have room -- it starts empty since " "the dormant picker thread never touches it " "(sched_go() is never called in this test)"); /* Bounded, non-blocking poll -- never a real blocking call, so a * regression that wedges instead of crashing fails loudly with a clear * message rather than hanging the whole test binary (same discipline * test_scheduler_backpressure.c's own header comment establishes) -- * for jbuf to drain back to 0. cbuf mechanics guarantee only * sched_allocator()'s own single-consumer cbuf_get() can ever remove * this jobid from jbuf, so jbuf.len reaching 0 is direct proof that * call fired and returned with THIS job, meaning the thread proceeded * into sched_get_job()/plugin_job_election()/sched_make_allocation() * on it (all unconditional, in sequence) rather than being stuck * before ever dequeuing it. * * This job's own resreq ("type == any") parses into an * XCE_RESREQ_ALTER-tagged schedobj (parse_resreq_primary() always * stamps maxTasks=1 on the inner primary, but the top-level tag here * is ALTER, not a bare PRIMARY): with zero workers in sched.core, * every alternative pends, and sched_make_alter_allocation()'s own * pre-existing, documented gap (scheduler.c: "FIXME: merge pend * reason from all children") never propagates any child's pend * reason up to the top-level obj->reason -- so sched_allocator() * WARNs ("job pending but pend reason is NULL") and produces no * decision for THIS job, by design of that unrelated, pre-existing * gap, not a crash. This test therefore does not assert on a decision * for the restored job itself -- only that it was fully drained and * processed -- and instead proves the allocator thread's continued * health afterward via an independent control job below. */ for (waited_ms = 0; waited_ms < 2000; waited_ms += 5) { struct timespec ts = { 0, 5 * 1000 * 1000 }; if (sched.core.jbuf.len == 0) { break; } nanosleep(&ts, NULL); } TEST_ASSERT_EQUAL_INT_MESSAGE( 0, sched.core.jbuf.len, "jbuf must drain back to 0 within 2s -- a timeout here (rather " "than a crash) would mean the allocator thread died or wedged " "evaluating the restored job's schedobj, itself evidence of the " "bug this test guards against"); /* ── Prove the allocator thread is still alive and healthy AFTER * processing the restored job ──────────────────────────────────── * If job->schedobj's reconstruction (task 1.2's fix) had produced a * malformed/corrupting scheduling representation instead of merely * "pends with no reason" (e.g. a partially-initialized union member * or a bad xlist), the symptom would not necessarily be an immediate * crash -- ASAN's redzone checks catch out-of-bounds writes/UAFs * synchronously, but a logically-corrupt-but-in-bounds structure * could instead wedge or corrupt state that only manifests on the * NEXT allocation attempt. Feeding a second, entirely independent, * well-formed job through the same live allocator thread and * confirming it reaches a normal PEND decision proves the thread * returned cleanly from processing the restored job and is not * wedged or left in a corrupted state. */ { xce_sched_job_t* control_job = build_pend_only_job(CONTROL_JOBID); rc = sched_add_job(&(sched.core), control_job); TEST_ASSERT_TRUE(rc >= 0); rc = cbuf_tryput(&(sched.core.jbuf), &(control_job->jobid)); TEST_ASSERT_TRUE_MESSAGE( rc >= 0, "fixture: jbuf must have room for the control job -- " "the restored job was already drained above"); } for (waited_ms = 0; waited_ms < 2000; waited_ms += 5) { struct timespec ts = { 0, 5 * 1000 * 1000 }; if (cbuf_tryget(&(sched.core.decisions), &decision) >= 0) { got_decision = 1; break; } nanosleep(&ts, NULL); } TEST_ASSERT_TRUE_MESSAGE( got_decision, "sched_allocator() must produce a decision for the CONTROL job " "within 2s after processing the restored job -- a timeout here " "would mean the thread died or wedged as a delayed effect of " "evaluating the restored job's schedobj"); TEST_ASSERT_NOT_NULL(decision); TEST_ASSERT_EQUAL_INT(XCE_DECISION_PEND, decision->op); TEST_ASSERT_NOT_NULL(decision->u.reason); TEST_ASSERT_EQUAL_UINT64(CONTROL_JOBID, decision->u.reason->jobid); /* Reaching this line at all -- under this project's ASAN build, with * no heap-buffer-overflow/UAF/SEGV report having aborted the process * first -- combined with the control job's own successful, ordinary * decision above, is the proof: job->schedobj was a genuinely valid, * non-NULL, well-formed scheduling representation all the way through * sched_make_allocation()'s real dispatch on it, and processing it * left the allocator thread and sched.core in a healthy, uncorrupted * state (Req 2.2, 2.4). This is the direct regression test for * bug-1569's failure shape. */ } int main(void) { UNITY_BEGIN(); RUN_TEST(test_restored_pending_job_reaches_allocator_without_crashing); return UNITY_END(); }