/* * test_sim_reconcile_unaccounted.c - contract test for * sim_reconcile_unaccounted_jobs() (sim_loop.c, scheduler-sim-decision- * parity spec, tasks 2.2-2.4, "Recorded-Decision Reconciliation" component, * design.md; Requirement 1.1, 1.2, 3.1, 3.2, 3.3, 4.1, 4.2). Also covers * scheduler-sim-reconcile-terminal-guard task 1.1 (RED phase, Requirement * 1.1, 1.2, 1.3; bug-1764): the two tests at the very bottom of this file * (test_reconcile_never_reallocates_a_{done,exit}_job_despite_stray_ * dispatch_entry) prove a job driven to JOB_STAT_DONE/JOB_STAT_EXIT via * xce_core_push_events() must never have its allocs/accounts resurrected by * a later round's stray/duplicate recorded DISPATCH entry -- both are * EXPECTED TO FAIL against today's sim_reconcile_unaccounted_jobs(), whose * trigger (job->accounts == NULL alone) has no job->state check yet (task * 1.2 adds it). test_reconcile_captures_non_run_state_job_when_dispatch_ * recorded() above already serves as this feature's non-terminal-case * regression guard (Requirement 1.2) and needed no duplication. * Also covers scheduler-sim-eager-reconciliation task 2.1 (Requirement 1.2, 1.3, 1.4): * resource-in-use accounting via the REAL sched_account_jobs() (scheduler.c, * unchanged by that spec, verbatim-copied in below -- see this file's own * "White-box #include set" paragraph) correctly defers to a captured job's * REAL running state, not its (now possibly much earlier) capture time. * * Scope: the once-per-round trigger (RUN-state job with job->accounts == * NULL), the jobid-based search through this round's * decoded_session->decisions, the "not found" path -- leave the job * unaccounted, report usage unavailable (== job->accounts staying NULL, * which sched_account_jobs() already tolerates safely), never treat this as * an error, and stay safe to re-check next round -- AND (task 2.3/2.4) a * FOUND match's allocation reconstruction (ownership transfer, array * compaction) plus feeding the reconstructed decision through the same * three-call reporting channel (sched_record_session_add_decision(), * sim_print_decision(), xce_repl_check_decision()) a freshly-drained * decision uses, and that an already-accounted job produces NO duplicate * report through this mechanism (Requirement 3.3). * * **Correction (post-review, scheduler-sim-repl-verbosity task 4.2 * remediation):** the reporting channel's console-print leg is now * sim_print_decision_gate(sim, d), not a bare sim_print_decision(d) -- * review found this file's own reconciliation call site had been missed by * task 4.2's original single-call-site scope (bug-1610). The two tests at * the bottom of this file (test_reconcile_continue_mode_with_repl_state_ * suppresses_bare_decision_line / test_reconcile_step_mode_with_repl_ * state_still_prints_bare_decision_line) pin that gate's contract at THIS * call site specifically, using the same repl_state_ensure()-based * direct-call fixture technique test_repl_round_rollup.c's own fixtures * establish. test_reconcile_reports_reconstructed_decision_through_ * existing_channel above (task 2.4's original test) deliberately leaves * sim->repl_state NULL and continues to assert the bare line IS present -- * empirically confirmed unaffected by this remediation, since a NULL * repl_state falls through the gate's own NULL-safety default ("never * load-bearing, degrade to print") exactly like every other REPL hook in * this codebase. * * sim_reconcile_unaccounted_jobs() is `static` and not yet called from * anywhere in production code (task 3 wires it into * sched_sim_driver_run()'s round-boundary sequence) -- called directly here * instead, standalone, per design.md's own Testing Strategy. * * White-box #include set and stub set: originally IDENTICAL to * test_sim_event_loop.c's own (same rationale -- sim_loop.c's compiled TU * unconditionally references every one of these as externs, whether or not * any single test here calls sched_sim_driver_run() itself). This file * never calls sched_sim_driver_run()/spawns the picker thread, so (unlike * test_sim_event_loop.c) no `static` storage workaround is needed for its * own `xce_sched_sim_driver_t` fixtures. Task 2.1 (scheduler-sim-eager- * reconciliation) additionally #includes ../src/resource.c and replaces the * PREVIOUS no-op sched_account_jobs() stub with a REAL reimplementation * (body copied verbatim from scheduler.c, same precedent test_sim_ * contention_parity.c's own copy documents -- scheduler.c itself is never * white-box #included in this directory, too large a link-time dependency * surface, see test_resource_accounting.c's own header comment) -- this * file's own task 2.1 test is the first test here to actually call that * function, and a no-op stub would defeat its entire premise. Pulling in * resource.c makes sched_worker_release_host_ctx() a REAL (non-stub) symbol * too (previously a local no-op here) and adds one new link-satisfying-only * stub, sched_ctx_get_uncomit_host() -- see the stub definitions below for * both. * * Fixture technique: xce_sched_core_init() plus core_build_job()/ * sched_add_job() (core_events.c/core.c, both white-box #included) to place * a job directly into sim.core.jobs -- same technique * write_header_and_snapshot_with_pending_job() (test_sim_event_loop.c) * already establishes for baking a job straight into a core without a real * `.replay` file. `decoded_session` (replay_session_t) and its * `decisions[]` (replay_decision_t*) are hand-built directly in memory * (calloc), not round-tripped through capnp -- sim_reconcile_unaccounted_ * jobs() only ever reads these plain heap structures, so no encode/decode * step is needed to exercise it. Fixtures are not torn down (matches this * whole suite's own "detect_leaks=0" convention, meson.build). */ #include #include #include #include #include #include #include #include #include "controller.capnp.h" #include "kvmgr.h" #include "resobj.h" #include "scheduler.h" #include "xstring.h" /* Pull in the real functions under test into this TU (white-box #include, * same combined set test_sim_event_loop.c's own header comment documents * and justifies, symbol by symbol). */ #include "../src/core.c" #include "../src/core_clock.c" #include "../src/core_events.c" #include "../src/core_session.c" #include "../src/decision.c" #include "../src/plugin.c" #include "../src/plugins/builtin_fcfs.c" #include "../src/recording.c" #include "../src/recording_session.c" #include "../src/repl.c" #include "../src/repl_fmt.c" #include "../src/repl_rollup.c" #include "../src/repl_break.c" #include "../src/repl_info.c" #include "../src/repl_params.c" #include "../src/resource.c" #include "../src/sim.c" #include "../src/sim_loop.c" #include "../src/sim_picker.c" /* bug-1567: sim_loop.c calls sim_bootstrap_picker() (sim_loop.c-cap sibling) as an extern -- undefined reference at link time otherwise */ #include "../src/sim_record.c" #include "../src/sim_verify.c" #include "../src/snapshot.c" #include "../src/snapshot_load.c" #include "../src/snapshot_wire.c" void kill_sched_job(void* arg) { (void)arg; } void kill_sched_queue(void* arg) { (void)arg; } void kill_sched_worker(void* arg) { (void)arg; } void kill_sched_user(void* arg) { (void)arg; } void kill_sched_project(void* arg) { (void)arg; } void kill_sched_resval(xce_val_t* val) { (void)val; } /* sched_worker_release_host_ctx() is now a REAL (non-stub) symbol pulled in * from resource.c (task 2.1 addition, below) -- no local stand-in needed or * permitted (a local definition here would collide at link time). Unused by * any test in this file (no worker is ever released), listed only because * resource.c defines it -- same precedent test_resource_accounting.c's own * header comment documents. */ /* sched_fill_limit_requests() (plugin_prepare_allocation()'s, plugin.c, the * "limit" plugin category, linker dependency) is likewise now a REAL * (non-stub) symbol pulled in from resource.c -- no local stand-in needed * or permitted (a local definition here would collide at link time, same * as sched_worker_release_host_ctx() above). Never invoked by any test in * this file (no real allocation is ever driven), listed only because * resource.c defines it. */ /* resource.c's sched_try_alloc_on_host_()'s own link-time dependency -- * never actually invoked (this file never drives real allocation, only * calls sched_account_jobs()/sched_charge_account() directly). Same * link-satisfying-only stub test_sim_contention_parity.c's own copy * establishes needing once resource.c is white-box #included. */ int sched_ctx_get_uncomit_host(xvec_t** v, xce_sched_ctx_t* c, int i) { (void)v; (void)c; (void)i; return 0; } /* Task 2.1 addition (scheduler-sim-eager-reconciliation, Req 1.2/1.3): a * REAL (not no-op) reimplementation of scheduler.c's own sched_account_jobs() * (scheduler.c:1258, scheduler-sim-decision-parity task 1.1's relocated * xce_sched_core_t* signature) -- body copied verbatim, same precedent * test_sim_contention_parity.c's own copy of this function documents * (scheduler.c itself is never white-box #included in this directory -- * ~1400 lines, too large a link-time dependency surface, see * test_resource_accounting.c's own header comment). This file's task 2.1 * test exists specifically to observe what THIS function does to * w->resInUse for a captured-but-not-yet-RUN job vs. the same job once it * transitions to RUN -- a no-op stub would defeat that test's entire * premise, exactly like test_sim_contention_parity.c's own rationale for * needing the real body. This supersedes the file's previous no-op stand-in * (task 1.1 and earlier tests here never called this function themselves, * so the no-op was harmless for them; task 2.1 is the first test in this * file to call it). */ int sched_account_jobs(xce_sched_core_t* core) { iterator_t it; int rc = 0; rc = iter_make(&it, core->jobs); if (rc < 0) { return rc; } for (;;) { xce_sched_job_t* job = NULL; rc = iter_next(&it, (void**)&job); if ((rc < 0) || (rc == RTTI_ITER_NO_DATA)) { break; } if (!JOB_IS_RUN(job->state)) { continue; } if (job->accounts != NULL) { sched_charge_account(core, job->accounts); } } if (rc == RTTI_ITER_NO_DATA) { rc = 0; } iter_destroy(&it); return rc; } /* Real reimplementation of schedobj.c's sched_resolve_job_picker -- same * copy every white-box test in this directory that does not link * schedobj.c already establishes. */ int sched_resolve_job_picker(xce_sched_core_t* core, xce_sched_plugin_t** out) { xce_sched_plugin_t* plugin = NULL; int rc = 0; if ((core == NULL) || (out == NULL)) { return -EINVAL; } *out = NULL; if (core->job_pick_plugin == NULL) { return -EINVAL; } lock(core->plugins, RTTI_LOCK_READ); rc = key_find(core->plugins, core->job_pick_plugin, (void**)&plugin); unlock(core->plugins); if (rc <= 0) { return (rc < 0) ? rc : -ENOENT; } if ((plugin == NULL) || (plugin->picker_iface == NULL) || (plugin->picker_iface->next_job == NULL) || (plugin->picker_iface->begin_session == NULL) || (plugin->picker_iface->end_session == NULL) || (plugin->picker_iface->on_dispatch == NULL) || (plugin->picker_iface->on_pend == NULL)) { return -EINVAL; } *out = plugin; return 0; } void setUp(void) { return; } void tearDown(void) { return; } /* ---------------------------------------------------------------------- */ /* Fixture helpers */ /* ---------------------------------------------------------------------- */ static void fixture_init(xce_sched_sim_driver_t* sim) { int rc; memset(sim, 0, sizeof(*sim)); rc = xce_sched_core_init(&(sim->core)); TEST_ASSERT_EQUAL_INT(0, rc); } /* Bakes one job directly into `core->jobs` (no `.replay` file, no capnp -- * same core_build_job()+sched_add_job() technique test_sim_event_loop.c's * own write_header_and_snapshot_with_pending_job() establishes). Passing a * non-NULL `allocs` reproduces the "already accounted" job shape (mirrors * core_build_job()'s own `if (j->allocs != NULL) { ...; job->accounts = * make_job_account(job->allocs); }` branch, core_events.c) -- pass NULL for * the ordinary "unaccounted RUN-state job" shape this file's central tests * target. */ static xce_sched_job_t* add_job(xce_sched_core_t* core, uint64_t jobid, unsigned int state, xce_job_alloc_t* allocs) { xce_queue_t wire_queue; xce_sched_queue_t* q; job_submit_data_t submit; xce_job_t wire_job; xce_sched_job_t* job; int queue_rc; memset(&wire_queue, 0, sizeof(wire_queue)); wire_queue.queue = (char*)"normal"; wire_queue.status = QUEUE_STAT_OPEN; wire_queue.priority = 1; q = core_build_queue(&wire_queue); TEST_ASSERT_NOT_NULL(q); /* Tolerate -EEXIST: a test placing MULTIPLE jobs on the same core (task * 1.2's own multi-job isolation test is the first here to do so) calls * add_job() more than once against the same `core`, and the "normal" * queue this helper (re-)builds every call is already registered from * the first call -- that is expected, not a fixture bug, so only a * genuine failure (neither success nor EEXIST) is a hard assertion * failure. The freshly-built-but-unused `q` on the EEXIST path is * simply leaked, matching this whole suite's own "fixtures are not * torn down, detect_leaks=0" convention documented in this file's own * header comment. */ queue_rc = sched_add_queue(core, q); TEST_ASSERT_TRUE_MESSAGE( (queue_rc >= 0) || (queue_rc == -EEXIST), "sched_add_queue must succeed, or fail with -EEXIST when this " "fixture core's \"normal\" queue was already registered by an " "earlier add_job() call against the same core"); memset(&submit, 0, sizeof(submit)); submit.queue = (char*)"normal"; submit.user = (char*)"alice"; submit.mintasks = 1; submit.maxtasks = 1; memset(&wire_job, 0, sizeof(wire_job)); wire_job.jobid = jobid; wire_job.state = state; wire_job.submit = &submit; wire_job.allocs = allocs; job = core_build_job(core, &wire_job); TEST_ASSERT_NOT_NULL(job); TEST_ASSERT_GREATER_OR_EQUAL(0, sched_add_job(core, job)); return job; } static replay_decision_t* make_dispatch_entry(uint64_t jobid) { xce_job_alloc_t* alloc = make_job_alloc(jobid, 0, NULL); sched_dispatch_decision_t* dispatch = (sched_dispatch_decision_t*)calloc(1, sizeof(*dispatch)); replay_decision_t* rd = (replay_decision_t*)calloc(1, sizeof(*rd)); dispatch->alloc = alloc; rd->body_which = ReplayDecision_body_dispatch; rd->body.dispatch = dispatch; return rd; } static replay_decision_t* make_pend_entry(uint64_t jobid) { sched_pend_decision_t* pend = (sched_pend_decision_t*)calloc(1, sizeof(*pend)); replay_decision_t* rd = (replay_decision_t*)calloc(1, sizeof(*rd)); pend->jobid = jobid; pend->reason = NULL; rd->body_which = ReplayDecision_body_pend; rd->body.pend = pend; return rd; } static replay_session_t* make_decoded_session(uint64_t begin_round, replay_decision_t* entry) { replay_session_t* s = (replay_session_t*)calloc(1, sizeof(*s)); s->begin_round = begin_round; s->summary = NULL; if (entry == NULL) { s->decisions = NULL; s->n_decisions = 0; return s; } s->decisions = (replay_decision_t**)calloc(1, sizeof(*(s->decisions))); s->decisions[0] = entry; s->n_decisions = 1; return s; } /* task 2.3 fixture helper: a real, one-host/one-resource xce_job_alloc_t -- * same field shape sim_verify_job_alloc_eq() already validates (design.md's * Allowed Dependencies), built via the real xcealloc.c production helpers * (make_alloc/make_allocs/make_host_alloc/make_host_allocs/make_job_alloc), * not a hand-rolled struct literal -- so a reconciled job's `job->accounts` * shape can be checked against real, non-trivial resource values. */ static xce_job_alloc_t* make_test_job_alloc_with_host(uint64_t jobid, const char* host, const char* resname, float value) { xce_alloc_t* a = make_alloc((char*)resname, NULL, 0, value, 0); xce_alloc_t** allocs = make_allocs(1, a); xce_host_alloc_t* h = make_host_alloc((char*)host, 1, allocs); xce_host_alloc_t** hallocs = make_host_allocs(1, h); return make_job_alloc(jobid, 1, hallocs); } /* task 2.3 fixture helper: wraps an already-built xce_job_alloc_t (as * opposed to make_dispatch_entry() above, which builds its own empty * (nhosts=0) alloc) into a DISPATCH-shaped replay_decision_t. */ static replay_decision_t* make_dispatch_entry_from_alloc(xce_job_alloc_t* alloc) { sched_dispatch_decision_t* dispatch = (sched_dispatch_decision_t*)calloc(1, sizeof(*dispatch)); replay_decision_t* rd = (replay_decision_t*)calloc(1, sizeof(*rd)); dispatch->alloc = alloc; rd->body_which = ReplayDecision_body_dispatch; rd->body.dispatch = dispatch; return rd; } /* Task 2.1 fixture helpers (scheduler-sim-eager-reconciliation, Req * 1.2/1.3): register a real, consumable "slots" resource and add a real * worker owning a finite amount of it -- same technique * test_sim_contention_parity.c's own register_slots_resource()/add_worker() * establish (that file's own header comment documents why the resource must * be pre-registered: reconciliation's real make_job_account()/ * make_res_account() path always produces residx == -1, lazily resolved by * sched_update_res_accounts()'s own key_find(core->resmgr, ...) against * whatever the resource was registered as -- pre-registering here means * this fixture's own worker resvals/resInUse setup and the reconciled * job's eventual account both resolve to the SAME index). */ static int register_slots_resource(xce_sched_core_t* core) { xce_resource_t* res_template = (xce_resource_t*)calloc(1, sizeof(*res_template)); xce_resource_t* registered; TEST_ASSERT_NOT_NULL(res_template); res_template->name = strdup("slots"); res_template->valtype = XCE_VAL_NUMERIC; res_template->ordtype = XCE_ORD_DECR; res_template->flags = XCE_RESF_LIMIT | XCE_RESF_CONSUMABLE | XCE_RESF_RELEASE | XCE_RESF_DEFAULT_HOST; registered = makeResource(core->resmgr, res_template); TEST_ASSERT_NOT_NULL(registered); return registered->idx; } /* Adds a real xce_sched_worker_t owning `capacity` units of `residx`, added * directly via sched_add_worker() -- sched_add_worker() itself does not * initialize resvals/resInUse, callers own it (same convention every * worker-adding fixture in this directory establishes). resInUse starts * genuinely empty (zarray_init only, nothing put()) -- Requirement 1.2's * own "still zero" checkpoint below depends on this starting state, not on * an explicit zero being written. */ static xce_sched_worker_t* add_worker(xce_sched_core_t* core, const char* name, int residx, float capacity) { xce_sched_worker_t* w; int rc; float val[1] = { capacity }; w = (xce_sched_worker_t*)calloc(1, sizeof(*w)); TEST_ASSERT_NOT_NULL(w); w->name = strdup(name); TEST_ASSERT_NOT_NULL(w->name); rc = zarray_init(&(w->resvals), 8, NULL); TEST_ASSERT_EQUAL_INT(0, rc); rc = zarray_init(&(w->resInUse), 8, NULL); TEST_ASSERT_EQUAL_INT(0, rc); rc = put_resvals(&(w->resvals), 1, &residx, val); TEST_ASSERT_TRUE(rc >= 0); rc = sched_add_worker(core, w); TEST_ASSERT_TRUE(rc >= 0); return w; } /* ---------------------------------------------------------------------- */ /* Test 1 (the task's own primary Observable): a RUN-state job with no */ /* known allocation and no matching recorded decision in this round's */ /* window is left unaccounted (usage unavailable), and the call returns */ /* normally -- no error, nothing fabricated (Requirement 4.1, 4.2). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_leaves_run_job_unaccounted_when_no_recorded_decision(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; replay_session_t* decoded; fixture_init(&sim); /* RUN-state, no known allocation -- e.g. a job dispatched before the * snapshot this (hypothetical) replay started from, exactly the * scenario this task's own Observable Completion text describes. */ job = add_job(&(sim.core), 7001ULL, JOB_STAT_RUN, NULL); TEST_ASSERT_NULL(job->accounts); TEST_ASSERT_NULL(job->allocs); /* This round's decoded recorded-decision data has nothing for this * jobid at all -- the replay's own window never captured it. */ decoded = make_decoded_session(1000ULL, NULL); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_NULL_MESSAGE( job->accounts, "no matching recorded decision -- job->accounts must stay NULL " "(usage unavailable), never fabricated"); TEST_ASSERT_NULL_MESSAGE(job->allocs, "job->allocs must likewise stay " "NULL -- nothing to reconstruct " "from"); printf("test_reconcile_leaves_run_job_unaccounted_when_no_recorded_" "decision: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 2: a job left unresolved this round is safe to re-check again on */ /* a later round, with no adverse effect (task's own explicit text). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_is_safe_to_recheck_on_a_later_round(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; replay_session_t* decoded_round1; replay_session_t* decoded_round2; fixture_init(&sim); job = add_job(&(sim.core), 7002ULL, JOB_STAT_RUN, NULL); decoded_round1 = make_decoded_session(1000ULL, NULL); sim_reconcile_unaccounted_jobs(&sim, decoded_round1, NULL); TEST_ASSERT_NULL(job->accounts); /* Round 2: still no matching recorded decision -- calling again must * not error, crash, or otherwise behave differently just because this * is a repeat check of the same job. */ decoded_round2 = make_decoded_session(2000ULL, NULL); sim_reconcile_unaccounted_jobs(&sim, decoded_round2, NULL); TEST_ASSERT_NULL_MESSAGE( job->accounts, "re-checking an unresolved job on a later round must remain safe " "and leave it unaccounted, with no adverse effect"); printf( "test_reconcile_is_safe_to_recheck_on_a_later_round: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 3: a job that already has known accounts (either reconciled */ /* earlier or freshly dispatched by the live path) is never re-touched -- */ /* the trigger condition (job->accounts == NULL) must gate correctly even */ /* when a matching recorded DISPATCH entry for the same jobid IS present */ /* this round. Extended (task 2.4) to also prove Requirement 3.3's "report */ /* only the fresh one" guarantee at the decision-reporting layer, not just */ /* at the job->accounts layer: neither the recording channel nor the */ /* step/continue console output may see a duplicate report for this job. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_skips_job_that_already_has_accounts(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_account_t* accounts_before; replay_session_t* decoded; sched_record_session_ctx_t* record_ctx; char tmp[] = "/tmp/xce_simreconcile_dedup_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* captured; fixture_init(&sim); job = add_job(&(sim.core), 7003ULL, JOB_STAT_RUN, make_job_alloc(7003ULL, 0, NULL)); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "fixture setup: job must already have accounts"); accounts_before = job->accounts; decoded = make_decoded_session(1000ULL, make_dispatch_entry(7003ULL)); record_ctx = sched_record_session_ctx_new(1000ULL); TEST_ASSERT_NOT_NULL(record_ctx); /* Same dup()/dup2() stdout-capture technique test_sim_verify.c's own * capture_stdout_around()/run_sim_and_capture_stdout() establish, * inlined here (this call takes more than the zero args that helper's * function-pointer shape allows). */ TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE(saved_stdout >= 0); TEST_ASSERT_TRUE(dup2(fd, STDOUT_FILENO) >= 0); sim_reconcile_unaccounted_jobs(&sim, decoded, record_ctx); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); captured = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(captured); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, captured, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); TEST_ASSERT_EQUAL_PTR_MESSAGE( accounts_before, job->accounts, "an already-accounted job must never be re-touched, even when a " "matching recorded decision exists this round (Requirement 3.3's " "double-application guard)"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, decoded->n_decisions, "an already-accounted job's matching entry must not be consumed/" "compacted either -- the trigger never fires for it at all"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, record_ctx->n, "Requirement 3.3: an already-accounted job (fresh live decision or " "earlier reconciliation) must produce NO recorded-decision report " "from this mechanism -- reporting only the fresh one, never a " "duplicate"); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "[sim] DISPATCH"), "Requirement 3.3: an already-accounted job must produce NO " "step/continue console output from this mechanism either"); free(captured); printf("test_reconcile_skips_job_that_already_has_accounts: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 4: a recorded PEND entry sharing the same jobid must never be */ /* mistaken for a match -- only a DISPATCH entry can legitimately account */ /* for a RUN-state job (design.md's Implementation Notes). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_ignores_non_dispatch_entry_with_matching_jobid( void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; replay_session_t* decoded; fixture_init(&sim); job = add_job(&(sim.core), 7004ULL, JOB_STAT_RUN, NULL); decoded = make_decoded_session(1000ULL, make_pend_entry(7004ULL)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_NULL_MESSAGE( job->accounts, "a recorded PEND entry must never be treated as a DISPATCH match, " "even when its jobid matches -- job stays unaccounted"); printf("test_reconcile_ignores_non_dispatch_entry_with_matching_jobid:" " PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 5 (scheduler-sim-eager-reconciliation task 1.1, replaces the old */ /* test_reconcile_skips_non_run_state_job): the trigger is now */ /* state-independent -- a job whose recorded DISPATCH decision is found */ /* this round gets its allocation captured regardless of whether it has */ /* reached RUN state yet. This is the inverse of the old (buggy) expected */ /* behavior: previously a PEND job with a matching recorded DISPATCH entry */ /* stayed uncaptured; it must now be captured just like an already-RUN */ /* job (Requirement 1.1, 3.1). The dominant real-world case (dispatch and */ /* its RUN-state confirmation recorded many rounds apart) depends on this. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_captures_non_run_state_job_when_dispatch_recorded(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* jalloc; replay_session_t* decoded; fixture_init(&sim); /* PEND-state, no known allocation yet -- the realistic case: this * job's recorded DISPATCH decision has been found in this round's * replay stream, but its RUN-state confirmation has not been replayed * yet (may land many rounds later). */ job = add_job(&(sim.core), 7005ULL, JOB_STAT_PEND, NULL); TEST_ASSERT_NULL(job->accounts); TEST_ASSERT_NULL(job->allocs); jalloc = make_test_job_alloc_with_host(7005ULL, "worker1", "slots", 2.0f); decoded = make_decoded_session(1000ULL, make_dispatch_entry_from_alloc(jalloc)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc, job->allocs, "a PEND-state job's recorded DISPATCH decision must now be " "captured (ownership TAKEOVER, identical pointer) just like an " "already-RUN job -- capture eligibility no longer depends on " "job->state"); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "job->accounts must be built (make_job_account()) from the " "captured allocation even though the job has not reached RUN yet"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, decoded->n_decisions, "the now-fully-consumed recorded entry must be removed from " "decoded_session->decisions, exactly as for an already-RUN job"); printf("test_reconcile_captures_non_run_state_job_when_dispatch_" "recorded: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 7 (task 2.3): a matching recorded DISPATCH entry is reconstructed */ /* into job->allocs (ownership TAKEOVER -- same pointer, not a copy) and */ /* job->accounts (built from it), and the consumed entry is removed from */ /* decoded_session (Requirement 1.1, 1.2, 3.2). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_reconstructs_allocs_and_accounts_from_matching_dispatch(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* jalloc; replay_session_t* decoded; fixture_init(&sim); job = add_job(&(sim.core), 7010ULL, JOB_STAT_RUN, NULL); TEST_ASSERT_NULL(job->accounts); TEST_ASSERT_NULL(job->allocs); jalloc = make_test_job_alloc_with_host(7010ULL, "worker1", "slots", 2.0f); decoded = make_decoded_session(1000ULL, make_dispatch_entry_from_alloc(jalloc)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc, job->allocs, "job->allocs must take OWNERSHIP of the recorded entry's alloc " "(identical pointer), not a duplicate copy -- same pattern " "scheduler.c's sched_commit_decision() uses"); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "job->accounts must be built (make_job_account()) from the " "reconstructed job->allocs"); TEST_ASSERT_EQUAL_INT(1, job->accounts->nhosts); TEST_ASSERT_EQUAL_STRING("worker1", job->accounts->accounts[0]->host); TEST_ASSERT_EQUAL_FLOAT(2.0f, job->accounts->accounts[0]->accounts[0]->reserved); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, decoded->n_decisions, "the now-fully-consumed recorded entry must be removed from " "decoded_session->decisions"); printf("test_reconcile_reconstructs_allocs_and_accounts_from_matching_" "dispatch: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 8 (task 2.3): compaction is SHIFT-DOWN, not swap-with-last -- the */ /* remaining recorded entries must keep their ORIGINAL RELATIVE ORDER, per */ /* design.md's Implementation Notes (a later --sim-verify positional walk */ /* depends on this). A 4-entry fixture with the match in the middle */ /* distinguishes the two strategies: swap-with-last would relocate the */ /* last entry (C) into the removed slot, producing [A, C, B] instead of the */ /* correct [A, B, C]. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_compacts_via_shift_down_preserving_order(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; replay_session_t* decoded; fixture_init(&sim); job = add_job(&(sim.core), 7011ULL, JOB_STAT_RUN, NULL); decoded = (replay_session_t*)calloc(1, sizeof(*decoded)); decoded->begin_round = 1000ULL; decoded->summary = NULL; decoded->decisions = (replay_decision_t**)calloc(4, sizeof(*(decoded->decisions))); decoded->decisions[0] = make_pend_entry(9001ULL); /* A */ decoded->decisions[1] = make_dispatch_entry(7011ULL); /* matched */ decoded->decisions[2] = make_pend_entry(9002ULL); /* B */ decoded->decisions[3] = make_pend_entry(9003ULL); /* C */ decoded->n_decisions = 4; sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_NOT_NULL(job->accounts); TEST_ASSERT_EQUAL_INT_MESSAGE( 3, decoded->n_decisions, "exactly one entry (the matched DISPATCH) must be consumed"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(9001ULL, decoded->decisions[0]->body.pend->jobid, "entry A must stay first"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 9002ULL, decoded->decisions[1]->body.pend->jobid, "entry B must be second (shift-down) -- swap-with-last would put C " "here instead"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(9003ULL, decoded->decisions[2]->body.pend->jobid, "entry C must be third, untouched"); printf("test_reconcile_compacts_via_shift_down_preserving_order: " "PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 9 (task 2.3, the task's own PRIMARY correctness bar): reconciling a */ /* job's allocation from a recorded DISPATCH entry, then releasing that */ /* job's allocation through its NORMAL end-of-life path -- */ /* XCE_CORE_EVENT_JOB_STATE_CHANGE (core_events.c) -- is clean: no */ /* double-free, no use-after-free. This directly exercises the */ /* move-then-null-then-free ownership handling design.md's Implementation */ /* Notes require: `entry->body.dispatch->alloc` must be nulled out BEFORE */ /* free_ReplayDecision(entry) frees the rest of the entry, or that free */ /* would free the exact memory job->allocs now owns, and this exact */ /* end-of-life release would then double-free it. Meaningful evidence for */ /* this only under the project's ASAN build (see meson.build/tasks.md), */ /* not as a plain functional assertion. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_then_release_via_job_state_change_is_asan_clean(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; replay_session_t* decoded; xce_core_event_t ev; int push_rc; fixture_init(&sim); job = add_job(&(sim.core), 7012ULL, JOB_STAT_RUN, NULL); decoded = make_decoded_session( 1000ULL, make_dispatch_entry_from_alloc(make_test_job_alloc_with_host( 7012ULL, "worker1", "slots", 4.0f))); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_NOT_NULL(job->accounts); TEST_ASSERT_NOT_NULL(job->allocs); TEST_ASSERT_EQUAL_INT(0, decoded->n_decisions); memset(&ev, 0, sizeof(ev)); ev.type = XCE_CORE_EVENT_JOB_STATE_CHANGE; ev.u.job_state_change.jobid = 7012ULL; ev.u.job_state_change.old_state = JOB_STAT_RUN; ev.u.job_state_change.new_state = JOB_STAT_DONE; push_rc = xce_core_push_events(&(sim.core), &ev, 1); TEST_ASSERT_EQUAL_INT(0, push_rc); TEST_ASSERT_NULL_MESSAGE( job->accounts, "released cleanly via the normal end-of-life path"); TEST_ASSERT_NULL_MESSAGE( job->allocs, "released cleanly via the normal end-of-life path"); /* Also exercise decoded_session's own round-end cleanup path * (free_ReplaySession_ptr(), the same call sched_sim_driver_run() makes * every round) now that n_decisions correctly reflects only the * remaining, still-owned entries (here: none) -- ASAN-clean under a * leak-detecting run (this suite's registered meson test uses * detect_leaks=0 per its own established fixture-teardown convention; * this call is exercised here regardless, and was additionally verified * with ASAN_OPTIONS=detect_leaks=1 against the standalone binary as * part of this task's own validation). */ free_ReplaySession_ptr(&decoded); printf("test_reconcile_then_release_via_job_state_change_is_asan_clean: " "PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 10 (task 2.4, the task's own PRIMARY Observable): a matching */ /* recorded DISPATCH entry, once reconstructed (task 2.3), is fed through */ /* the exact same three-call decision-reporting channel a freshly-drained */ /* decision uses -- sched_record_session_add_decision() (proven via a real */ /* record_ctx's own accumulated decisions[]/n/n_dispatched fields, exactly */ /* the same struct sched_sim_driver_run()'s drain loop populates) and */ /* sim_print_decision() (proven via captured stdout -- the literal */ /* mechanism by which a decision "shows... in step/continue output", per */ /* this task's own Observable Completion text). xce_repl_check_decision() */ /* is also called (not independently observable here without a live REPL */ /* breakpoint fixture -- repl.c/repl_break.c/repl_info.c are unchanged and */ /* out of this task's boundary per design.md) but must not crash, which */ /* this test's mere completion already proves for both sim->interactive */ /* on and off. Preserves the original decision's type (DISPATCH) and */ /* resource-allocation detail (Requirement 3.2): the SAME job->allocs */ /* pointer the reconciliation step already reconstructed is what reaches */ /* both the recording channel and the console output. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_reports_reconstructed_decision_through_existing_channel(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* jalloc; replay_session_t* decoded; sched_record_session_ctx_t* record_ctx; char tmp[] = "/tmp/xce_simreconcile_report_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* captured; fixture_init(&sim); job = add_job(&(sim.core), 7020ULL, JOB_STAT_RUN, NULL); TEST_ASSERT_NULL(job->accounts); jalloc = make_test_job_alloc_with_host(7020ULL, "worker9", "slots", 3.0f); decoded = make_decoded_session(1000ULL, make_dispatch_entry_from_alloc(jalloc)); record_ctx = sched_record_session_ctx_new(1000ULL); TEST_ASSERT_NOT_NULL(record_ctx); /* Exercise the REPL hook too (sim->interactive on) -- must remain a * safe no-op with no repl_state allocated yet (repl.h's own "never * load-bearing" precondition), proving xce_repl_check_decision() is * actually reached and does not crash regardless of interactive mode. */ sim.interactive = true; TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE(saved_stdout >= 0); TEST_ASSERT_TRUE(dup2(fd, STDOUT_FILENO) >= 0); sim_reconcile_unaccounted_jobs(&sim, decoded, record_ctx); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); captured = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(captured); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, captured, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); /* The reconciliation itself (task 2.3) still happened. */ TEST_ASSERT_EQUAL_PTR(jalloc, job->allocs); TEST_ASSERT_NOT_NULL(job->accounts); /* sched_record_session_add_decision() reporting (Requirement 3.1). */ TEST_ASSERT_EQUAL_INT_MESSAGE( 1, record_ctx->n, "the reconstructed DISPATCH decision must be reported through the " "recording channel exactly once"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, record_ctx->n_dispatched, "must be counted as a DISPATCH"); TEST_ASSERT_EQUAL_INT_MESSAGE( ReplayDecision_body_dispatch, record_ctx->decisions[0]->body_which, "must be recorded with the DISPATCH body shape"); TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc, record_ctx->decisions[0]->body.dispatch->alloc, "Requirement 3.2: the recorded entry must carry the SAME " "resource-allocation detail (identical pointer, borrowed -- same " "convention recording_session.c's own header comment documents " "for a freshly-drained decision) the reconciliation step " "reconstructed, not a copy or a different value"); /* sim_print_decision() console output (Requirement 3.1 -- this IS the * "shows... in step/continue output" mechanism). */ TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "[sim] DISPATCH jobid="), "the reconstructed decision must be printed through the same " "step/continue console output a freshly-computed decision uses"); free(captured); printf("test_reconcile_reports_reconstructed_decision_through_existing_" "channel: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 6: NULL `sim`/`decoded_session` is a safe no-op (design.md's */ /* Service Interface preconditions -- "never load-bearing" convention). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_null_args_are_a_safe_noop(void) { xce_sched_sim_driver_t sim; replay_session_t* decoded; fixture_init(&sim); decoded = make_decoded_session(1000ULL, NULL); sim_reconcile_unaccounted_jobs(NULL, decoded, NULL); sim_reconcile_unaccounted_jobs(&sim, NULL, NULL); sim_reconcile_unaccounted_jobs(NULL, NULL, NULL); /* Reaching this line without a crash IS the assertion. */ TEST_ASSERT_TRUE(1); printf("test_reconcile_null_args_are_a_safe_noop: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 12 (scheduler-sim-eager-reconciliation task 2.1, Req 1.2, 1.3, */ /* 1.4): resource-in-use accounting must defer to the job's REAL running */ /* state, not its capture time. task 1.1 widened sim_reconcile_unaccounted_ */ /* jobs()'s trigger so a job's allocation is captured (job->allocs/ */ /* job->accounts set) as soon as its recorded DISPATCH decision is found, */ /* regardless of job->state -- this test proves the OTHER half of that */ /* design's correctness: sched_account_jobs() (scheduler.c, UNCHANGED, */ /* real body pulled in above) still gates charging on JOB_IS_RUN(state), */ /* so a captured-but-still-PEND job contributes nothing to w->resInUse */ /* (Req 1.2) until it genuinely transitions to RUN, at which point the */ /* SAME captured allocation (never re-captured, never re-typed) is charged */ /* (Req 1.3). Req 1.4 ("matches normal-mode's equivalent-point reporting") */ /* is satisfied by construction: this test drives the exact same */ /* sched_account_jobs() normal mode's own driver.c calls at the identical */ /* round-boundary position (design.md's Architecture Integration note) -- */ /* no separate normal-mode comparison fixture is needed to prove a shared */ /* function behaves consistently with itself. */ /* ---------------------------------------------------------------------- */ static void test_captured_pend_job_excluded_from_resinuse_until_run(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_sched_worker_t* w; xce_job_alloc_t* jalloc; replay_session_t* decoded; int residx; float got; int rc; fixture_init(&sim); residx = register_slots_resource(&(sim.core)); w = add_worker(&(sim.core), "worker1", residx, 10.0f); /* PEND-state job, no known allocation yet -- exactly task 1.1's own * widened trigger shape: its recorded DISPATCH decision has been found * this round, but its RUN-state confirmation has not been replayed * yet. */ job = add_job(&(sim.core), 7030ULL, JOB_STAT_PEND, NULL); TEST_ASSERT_NULL(job->accounts); TEST_ASSERT_NULL(job->allocs); jalloc = make_test_job_alloc_with_host(7030ULL, "worker1", "slots", 4.0f); decoded = make_decoded_session(1000ULL, make_dispatch_entry_from_alloc(jalloc)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc, job->allocs, "sanity: the PEND job's recorded DISPATCH decision must be captured " "(task 1.1's own widened gate) before this test's own checkpoints " "mean anything"); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "sanity: job->accounts must be built from the " "captured allocation even though the job is still " "PEND"); TEST_ASSERT_TRUE_MESSAGE( JOB_STAT_PEND == job->state, "sanity: the job must still genuinely be PEND at this checkpoint -- " "otherwise the next assertion (resInUse == 0) would be trivially " "true for the wrong reason"); /* Checkpoint 1 (Requirement 1.2): recompute resInUse from scratch via * the REAL sched_account_jobs() -- a captured-but-PEND job must * contribute nothing. */ rc = sched_account_jobs(&(sim.core)); TEST_ASSERT_EQUAL_INT(0, rc); got = -1.0f; rc = get_resvals(&(w->resInUse), 1, &residx, &got); TEST_ASSERT_EQUAL_INT(1, rc); TEST_ASSERT_EQUAL_FLOAT_MESSAGE( 0.0f, got, "Requirement 1.2: a job whose allocation was captured early (while " "still PEND) must contribute ZERO to its worker's resInUse until it " "genuinely transitions to RUN -- sched_account_jobs()'s own " "JOB_IS_RUN gate (scheduler.c, unchanged by this feature) is the " "sole mechanism deferring this, and must still be effective now " "that capture happens earlier than before"); /* Transition the SAME job to RUN -- direct field write, same white-box * technique test_sim_contention_parity.c's own capacity_aware_ * allocator_thread_fn() uses to model a worker-confirmed state change * without driving the full plugin_job_state_change() machinery (out of * this file's own boundary -- scheduler.c is not white-box included). */ job->state = JOB_STAT_RUN; /* Checkpoint 2 (Requirement 1.3): recompute resInUse again -- the SAME * captured allocation (jalloc, never re-captured -- sim_reconcile_ * unaccounted_jobs() is not called a second time here, proving this * checkpoint's nonzero value comes from the EARLIER capture, not a * fresh one) must now be charged. */ rc = sched_account_jobs(&(sim.core)); TEST_ASSERT_EQUAL_INT(0, rc); got = -1.0f; rc = get_resvals(&(w->resInUse), 1, &residx, &got); TEST_ASSERT_EQUAL_INT(1, rc); TEST_ASSERT_EQUAL_FLOAT_MESSAGE( 4.0f, got, "Requirement 1.3: once the captured-early job genuinely transitions " "to RUN, its worker's resInUse must reflect its allocation from " "that point forward -- using the EXACT allocation amount (4.0) " "captured earlier by sim_reconcile_unaccounted_jobs(), not a " "freshly re-captured or re-typed value"); printf( "test_captured_pend_job_excluded_from_resinuse_until_run: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 13 (scheduler-sim-eager-reconciliation task 1.2, Requirement 3.2): */ /* a job with job->accounts ALREADY set is never re-searched or */ /* re-captured on a later round's call, regardless of its state (PEND */ /* here, deliberately different from test_reconcile_skips_job_that_ */ /* already_has_accounts' RUN-state fixture above -- the state genuinely */ /* does not matter, only job->accounts != NULL does) -- even when a */ /* decoded session that round happens to also contain a matching-jobid */ /* DISPATCH entry. Proves the state-widened gate (task 1.1) did not */ /* weaken the pre-existing, untouched job->accounts != NULL guard */ /* (research.md's "Does removing the RUN-state gate risk fabricating or */ /* double-capturing?" log entry: this guard was never touched by task */ /* 1.1, so this is regression-proof coverage for an existing guarantee, */ /* not new-behavior coverage). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_still_skips_already_captured_job_regardless_of_state(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_account_t* accounts_before; xce_job_alloc_t* allocs_before; replay_session_t* decoded_round1; replay_session_t* decoded_round2; fixture_init(&sim); /* PEND-state, already captured (mirrors add_job()'s own "already * accounted" fixture shape, same technique * test_reconcile_skips_job_that_already_has_accounts() uses -- here * deliberately with a non-RUN state). */ job = add_job(&(sim.core), 7040ULL, JOB_STAT_PEND, make_job_alloc(7040ULL, 0, NULL)); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "fixture setup: job must already have accounts"); accounts_before = job->accounts; allocs_before = job->allocs; /* Round 1: nothing to do with this job either way -- establishes the * "later round" framing the next call demonstrates. */ decoded_round1 = make_decoded_session(4000ULL, NULL); sim_reconcile_unaccounted_jobs(&sim, decoded_round1, NULL); TEST_ASSERT_EQUAL_PTR(accounts_before, job->accounts); /* Round 2 ("a later round"): a recorded DISPATCH entry for THIS job's * own jobid is present. If the job->accounts != NULL guard were ever * bypassed under the widened, state-independent trigger, this would be * re-captured. It must not be. */ decoded_round2 = make_decoded_session(5000ULL, make_dispatch_entry(7040ULL)); sim_reconcile_unaccounted_jobs(&sim, decoded_round2, NULL); TEST_ASSERT_EQUAL_PTR_MESSAGE( accounts_before, job->accounts, "an already-captured PEND-state job must never be re-captured on a " "later round, even with a matching-jobid recorded DISPATCH entry " "present -- job->accounts != NULL remains the sole gate under the " "widened, state-independent trigger"); TEST_ASSERT_EQUAL_PTR_MESSAGE( allocs_before, job->allocs, "job->allocs likewise must stay exactly what it already was -- " "never swapped for the later round's entry"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, decoded_round2->n_decisions, "the matching entry must not be consumed either -- the trigger " "never fires for an already-captured job at all"); printf("test_reconcile_still_skips_already_captured_job_regardless_of_" "state: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 14 (scheduler-sim-eager-reconciliation task 1.2, Requirement 4.4): */ /* a JOB_STAT_PEND job with NO matching recorded decision this round stays */ /* uncaptured (job->accounts stays NULL) after one call, and remains */ /* eligible -- demonstrated (per this task's own design.md guidance) by */ /* calling again with a NOW-matching decoded session and confirming */ /* capture succeeds then. Mirrors test_reconcile_is_safe_to_recheck_on_a_ */ /* later_round()'s "safe to retry" idiom above, but (unlike that sibling */ /* test, whose second round also has no match) this is the genuinely new */ /* behavior task 1.1 enables: a PEND job now gets multiple real chances */ /* across rounds and can actually succeed on a later one. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_no_match_leaves_pend_job_uncaptured_and_retries(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* jalloc; replay_session_t* decoded_round1; replay_session_t* decoded_round2; fixture_init(&sim); /* PEND-state, no known allocation, no recorded decision available yet * -- the realistic in-flight case task 1.1's widened gate exists for. */ job = add_job(&(sim.core), 7041ULL, JOB_STAT_PEND, NULL); TEST_ASSERT_NULL(job->accounts); TEST_ASSERT_NULL(job->allocs); decoded_round1 = make_decoded_session(4100ULL, NULL); sim_reconcile_unaccounted_jobs(&sim, decoded_round1, NULL); TEST_ASSERT_NULL_MESSAGE( job->accounts, "no matching recorded decision this round -- a PEND job must stay " "uncaptured (usage unavailable), never fabricated"); TEST_ASSERT_NULL_MESSAGE(job->allocs, "job->allocs likewise stays NULL"); /* Round 2: the job's recorded DISPATCH decision now shows up. The job * remains eligible (was never given up on) -- capture succeeds this * time, proving the previous failed attempt left no "give up" state * behind. */ jalloc = make_test_job_alloc_with_host(7041ULL, "workerB", "slots", 1.0f); decoded_round2 = make_decoded_session( 4200ULL, make_dispatch_entry_from_alloc(jalloc)); sim_reconcile_unaccounted_jobs(&sim, decoded_round2, NULL); TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc, job->allocs, "a job that stayed uncaptured with no match must remain fully " "eligible on a later round -- capture succeeds as soon as a " "matching recorded decision appears"); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "job->accounts must be built once the retry finds a " "match"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, decoded_round2->n_decisions, "the now-matched entry must be consumed exactly like any other " "capture"); printf("test_reconcile_no_match_leaves_pend_job_uncaptured_and_retries: " "PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 15 (scheduler-sim-eager-reconciliation task 1.2, Requirement 1.1, */ /* 3.1 under concurrent load): several simultaneously outstanding jobs, */ /* each with a DIFFERENT jobid, spread across three scenarios a single */ /* reconcile call must resolve with zero cross-job interference: */ /* - job_matched: PEND, its own DISPATCH entry IS this round. */ /* - job_no_match: PEND, no recorded entry at all this round. */ /* - job_already: captured on a hypothetical EARLIER round already */ /* (job->accounts/job->allocs preset directly via add_job()'s own */ /* "already accounted" shape, same technique */ /* test_reconcile_skips_job_that_already_has_accounts() and Test 13 */ /* above use) -- AND, to make the "no accidental overwrite" claim */ /* non-trivial, a matching-jobid DISPATCH entry for it is ALSO */ /* present this round. */ /* Asserts each job's own job->accounts reflects ONLY that job's own */ /* recorded decision, with no cross-job data or overwrite (the single-job */ /* tests above cannot distinguish "correct" from "correct by coincidence, */ /* only one candidate available", per design.md's own Testing Strategy). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_multiple_jobs_captured_independently_across_different_rounds( void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job_matched; xce_sched_job_t* job_no_match; xce_sched_job_t* job_already; xce_job_account_t* already_accounts_before; xce_job_alloc_t* already_allocs_before; xce_job_alloc_t* jalloc_matched; xce_job_alloc_t* jalloc_already_new; replay_session_t* decoded; fixture_init(&sim); job_matched = add_job(&(sim.core), 7050ULL, JOB_STAT_PEND, NULL); job_no_match = add_job(&(sim.core), 7051ULL, JOB_STAT_PEND, NULL); job_already = add_job(&(sim.core), 7052ULL, JOB_STAT_RUN, make_job_alloc(7052ULL, 0, NULL)); TEST_ASSERT_NOT_NULL_MESSAGE( job_already->accounts, "fixture setup: job_already must already have accounts, " "simulating capture on an earlier round"); already_accounts_before = job_already->accounts; already_allocs_before = job_already->allocs; jalloc_matched = make_test_job_alloc_with_host(7050ULL, "workerA", "slots", 5.0f); jalloc_already_new = make_test_job_alloc_with_host( 7052ULL, "worker-should-not-apply", "slots", 99.0f); decoded = (replay_session_t*)calloc(1, sizeof(*decoded)); decoded->begin_round = 6000ULL; decoded->summary = NULL; decoded->decisions = (replay_decision_t**)calloc(2, sizeof(*(decoded->decisions))); decoded->decisions[0] = make_dispatch_entry_from_alloc(jalloc_matched); decoded->decisions[1] = make_dispatch_entry_from_alloc(jalloc_already_new); decoded->n_decisions = 2; /* job_no_match (7051) intentionally has no entry in this round's * decoded stream at all. */ sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); /* job_matched: captured, and reflects EXACTLY its own jobid's data. */ TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc_matched, job_matched->allocs, "job_matched must capture its own recorded DISPATCH entry"); TEST_ASSERT_NOT_NULL(job_matched->accounts); TEST_ASSERT_EQUAL_STRING_MESSAGE( "workerA", job_matched->accounts->accounts[0]->host, "job_matched's captured account must reflect its OWN entry's " "host, not another job's"); TEST_ASSERT_EQUAL_FLOAT_MESSAGE( 5.0f, job_matched->accounts->accounts[0]->accounts[0]->reserved, "job_matched's captured account must reflect its OWN entry's " "reserved value, not another job's"); /* job_no_match: stays uncaptured -- no entry existed for it. */ TEST_ASSERT_NULL_MESSAGE( job_no_match->accounts, "job_no_match has no matching recorded entry this round -- must " "stay uncaptured"); TEST_ASSERT_NULL(job_no_match->allocs); /* job_already: completely unchanged, despite a matching-jobid DISPATCH * entry for it being present this round -- proving no accidental * overwrite/cross-contamination from the wider per-round scan. */ TEST_ASSERT_EQUAL_PTR_MESSAGE( already_accounts_before, job_already->accounts, "job_already's accounts must be the IDENTICAL pointer from before " "this call -- never overwritten, even with a matching entry " "present this round"); TEST_ASSERT_EQUAL_PTR_MESSAGE( already_allocs_before, job_already->allocs, "job_already's allocs must likewise be the IDENTICAL pointer from " "before this call"); /* job_already's own matching entry was never even searched for (its * job->accounts != NULL guard skips it before the search begins), so * it must remain fully intact and unconsumed in decoded_session -- only * job_matched's entry was consumed. */ TEST_ASSERT_EQUAL_INT_MESSAGE( 1, decoded->n_decisions, "exactly one entry (job_matched's) must be consumed -- " "job_already's entry is left untouched since that job was never " "eligible to be searched for at all"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 7052ULL, decoded->decisions[0]->body.dispatch->alloc->jobid, "the remaining entry must be job_already's own, untouched, entry"); TEST_ASSERT_EQUAL_PTR_MESSAGE( jalloc_already_new, decoded->decisions[0]->body.dispatch->alloc, "job_already's entry's alloc must be the SAME pointer it started " "with -- never taken over, since that job was already captured"); printf("test_reconcile_multiple_jobs_captured_independently_across_" "different_rounds: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 16 (scheduler-sim-repl-verbosity task 4.2 remediation, post-review: */ /* review found this file's own second production call site to */ /* sim_print_decision() -- inside sim_reconcile_unaccounted_jobs() itself */ /* -- was never gated by sim_print_decision_gate() (sim_loop.c), the ONE */ /* other call site task 4.2 originally wired. Since this reconciliation */ /* path is a PRIMARY source of DISPATCH decisions surfacing in replay mode */ /* (most live-drain-loop activity is PEND), a round where it fires used to */ /* show BOTH this bare "[sim] DISPATCH jobid=..." line AND the Round */ /* Rollup's summary counting the same dispatch -- the exact bug-1610 */ /* duplication this whole task exists to eliminate, just via this second */ /* call site. This test proves the fix: with sim->interactive == true AND */ /* a properly-allocated sim->repl_state in XCE_REPL_MODE_CONTINUE, the bare */ /* line from THIS mechanism must no longer appear -- the Round Rollup's own */ /* summary becomes the sole source of decision output here too, matching */ /* design.md's Requirement 2.3 ("the round summary is the sole source of */ /* decision output") for real this time. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_continue_mode_with_repl_state_suppresses_bare_decision_line( void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* jalloc; replay_session_t* decoded; sched_record_session_ctx_t* record_ctx; char tmp[] = "/tmp/xce_simreconcile_gate_continue_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* captured; fixture_init(&sim); job = add_job(&(sim.core), 7060ULL, JOB_STAT_RUN, NULL); TEST_ASSERT_NULL(job->accounts); jalloc = make_test_job_alloc_with_host(7060ULL, "worker9", "slots", 3.0f); decoded = make_decoded_session(1000ULL, make_dispatch_entry_from_alloc(jalloc)); record_ctx = sched_record_session_ctx_new(1000ULL); TEST_ASSERT_NOT_NULL(record_ctx); /* The gate-relevant fixture shape (unlike test_reconcile_reports_ * reconstructed_decision_through_existing_channel above, which * deliberately leaves repl_state NULL to prove the OTHER half of the * gate's NULL-safety contract): a genuinely allocated repl_state in * XCE_REPL_MODE_CONTINUE, the same repl_state_ensure() technique * test_repl_round_rollup.c's own fixtures use for a direct-call (no * real REPL command loop) test. */ sim.interactive = true; sim.repl_state = repl_state_ensure(&sim); TEST_ASSERT_NOT_NULL(sim.repl_state); sim.repl_state->mode = XCE_REPL_MODE_CONTINUE; TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE(saved_stdout >= 0); TEST_ASSERT_TRUE(dup2(fd, STDOUT_FILENO) >= 0); sim_reconcile_unaccounted_jobs(&sim, decoded, record_ctx); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); captured = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(captured); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, captured, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); /* The reconciliation itself (task 2.3) still happened -- this gate only * ever affects the console print, nothing else in the reporting chain. */ TEST_ASSERT_EQUAL_PTR(jalloc, job->allocs); TEST_ASSERT_NOT_NULL(job->accounts); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, record_ctx->n, "the recording channel (sched_record_session_add_decision()) must " "still report the reconstructed decision -- only the console print " "is gated, not the recording"); /* THE ASSERTION THIS TEST EXISTS TO PROVE: the bare per-decision line * must NOT appear via this call site while sim->interactive && * sim->repl_state->mode == XCE_REPL_MODE_CONTINUE (bug-1610, this * remediation's own scope). */ TEST_ASSERT_NULL_MESSAGE( strstr(captured, "[sim] DISPATCH jobid="), "under continue mode with a genuinely allocated repl_state, " "sim_reconcile_unaccounted_jobs()'s own reconstructed-decision " "report must NOT print the bare per-decision line -- the Round " "Rollup's own summary is the sole source of decision output here " "too, closing bug-1610's duplication gap at this second call site"); free(captured); free(sim.repl_state); printf( "test_reconcile_continue_mode_with_repl_state_suppresses_bare_" "decision_line: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 17 (regression-safety half of Test 16): under `step` mode with a */ /* genuinely allocated repl_state, the bare per-decision line from THIS */ /* same call site must still appear exactly as before -- only `continue` */ /* mode is gated (Requirement 5.1's "step mode ... unaffected" guarantee, */ /* now proven for this second call site too, not just the drain loop's). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_step_mode_with_repl_state_still_prints_bare_decision_line( void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* jalloc; replay_session_t* decoded; char tmp[] = "/tmp/xce_simreconcile_gate_step_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* captured; fixture_init(&sim); job = add_job(&(sim.core), 7061ULL, JOB_STAT_RUN, NULL); jalloc = make_test_job_alloc_with_host(7061ULL, "worker9", "slots", 3.0f); decoded = make_decoded_session(1000ULL, make_dispatch_entry_from_alloc(jalloc)); sim.interactive = true; sim.repl_state = repl_state_ensure(&sim); TEST_ASSERT_NOT_NULL(sim.repl_state); /* XCE_REPL_MODE_STEP == 0, already repl_state_ensure()'s calloc() * default -- set explicitly regardless, so this test's intent reads * clearly and does not silently depend on the enum's numeric value. */ sim.repl_state->mode = XCE_REPL_MODE_STEP; TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE(saved_stdout >= 0); TEST_ASSERT_TRUE(dup2(fd, STDOUT_FILENO) >= 0); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); captured = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(captured); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, captured, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); TEST_ASSERT_EQUAL_PTR(jalloc, job->allocs); TEST_ASSERT_NOT_NULL(job->accounts); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "[sim] DISPATCH jobid="), "while single-stepping (repl_state allocated, mode == " "XCE_REPL_MODE_STEP), sim_reconcile_unaccounted_jobs()'s own " "reconstructed-decision report must still print the bare " "per-decision line exactly as before -- only continue mode is " "gated"); free(captured); free(sim.repl_state); printf( "test_reconcile_step_mode_with_repl_state_still_prints_bare_" "decision_line: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 18 (scheduler-sim-reconcile-terminal-guard task 1.1, RED phase, */ /* Requirement 1.1, 1.3; bug-1764): a job driven all the way to */ /* JOB_STAT_DONE via xce_core_push_events() -- whose normal end-of-life */ /* cleanup (core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE case) correctly */ /* frees job->allocs/job->accounts, leaving job->accounts == NULL -- must */ /* NEVER have its allocation resurrected by sim_reconcile_unaccounted_ */ /* jobs(), even when this round's decoded_session carries a stray/ */ /* duplicate recorded DISPATCH entry for that same jobid (the live-observed */ /* bug-1764 shape: an array job's retried/leftover dispatch attempt still */ /* present in the replay stream after the job has already completed). */ /* Today's trigger check (sim_loop.c:690) is `job->accounts != NULL` alone, */ /* with NO check on job->state -- it cannot distinguish "not yet allocated" */ /* from "already completed and released", so THIS TEST IS EXPECTED TO FAIL */ /* until task 1.2 adds the terminal-state exclusion */ /* (JOB_IS_DONE()/JOB_IS_EXIT()). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_never_reallocates_a_done_job_despite_stray_dispatch_entry(void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* stray_alloc; replay_session_t* decoded; xce_core_event_t ev; int push_rc; fixture_init(&sim); /* RUN-state job that already has a real allocation -- the "actively * holding resources" shape a terminal transition must release, same * "already accounted" fixture shape add_job()'s own header comment and * test_reconcile_skips_job_that_already_has_accounts() above establish. */ job = add_job(&(sim.core), 7070ULL, JOB_STAT_RUN, make_job_alloc(7070ULL, 0, NULL)); TEST_ASSERT_NOT_NULL_MESSAGE( job->allocs, "fixture setup: job must start with a real allocation"); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "fixture setup: job must start with real accounts"); /* Drive the job to JOB_STAT_DONE through the SAME production event path * bug-1764's root cause depends on -- xce_core_push_events()'s * XCE_CORE_EVENT_JOB_STATE_CHANGE case (core_events.c), same technique * test_reconcile_then_release_via_job_state_change_is_asan_clean() above * already establishes. */ memset(&ev, 0, sizeof(ev)); ev.type = XCE_CORE_EVENT_JOB_STATE_CHANGE; ev.u.job_state_change.jobid = 7070ULL; ev.u.job_state_change.old_state = JOB_STAT_RUN; ev.u.job_state_change.new_state = JOB_STAT_DONE; push_rc = xce_core_push_events(&(sim.core), &ev, 1); TEST_ASSERT_EQUAL_INT(0, push_rc); /* Precondition check (NOT this test's main assertion): confirm the * existing end-of-life cleanup genuinely nulled allocs/accounts before * blaming reconciliation for anything below. */ TEST_ASSERT_TRUE_MESSAGE(JOB_IS_DONE(job->state), "precondition: job must genuinely be DONE " "before this test's real assertion means " "anything"); TEST_ASSERT_NULL_MESSAGE( job->allocs, "precondition: xce_core_push_events()'s normal " "end-of-life cleanup must null job->allocs on a " "terminal transition"); TEST_ASSERT_NULL_MESSAGE( job->accounts, "precondition: xce_core_push_events()'s normal " "end-of-life cleanup must null job->accounts on a " "terminal transition"); /* A later replay round's decoded_session still carries a stray/ * duplicate DISPATCH entry for this now-terminal job's jobid -- e.g. a * retried dispatch attempt for one element of an array job (bug-1764's * live-observed shape). */ stray_alloc = make_test_job_alloc_with_host(7070ULL, "worker-stray", "slots", 9.0f); decoded = make_decoded_session( 2000ULL, make_dispatch_entry_from_alloc(stray_alloc)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); /* THE ASSERTION THIS TEST EXISTS TO PROVE (expected to FAIL today): a * DONE job's allocation must never be resurrected, no matter what the * replay stream still contains for it. */ TEST_ASSERT_NULL_MESSAGE( job->allocs, "expected allocs NULL after reconciling a terminal (DONE) job, but " "allocs was populated -- today's trigger (job->accounts == NULL " "alone) cannot distinguish 'not yet allocated' from 'already " "completed and released' (bug-1764)"); TEST_ASSERT_NULL_MESSAGE( job->accounts, "expected accounts NULL after reconciling a terminal (DONE) job, " "but accounts was populated -- same root cause as the allocs " "assertion above"); printf("test_reconcile_never_reallocates_a_done_job_despite_stray_" "dispatch_entry: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Test 19 (scheduler-sim-reconcile-terminal-guard task 1.1, RED phase, */ /* Requirement 1.1, 1.3; bug-1764): identical scenario to Test 18 above but */ /* with JOB_STAT_EXIT instead of JOB_STAT_DONE -- both terminal states must */ /* be covered, not just one (Requirement 1.3's "treat a job's terminal */ /* state as permanent" applies to EXIT exactly as much as DONE). Also */ /* EXPECTED TO FAIL today, for the identical reason as Test 18. */ /* ---------------------------------------------------------------------- */ static void test_reconcile_never_reallocates_an_exit_job_despite_stray_dispatch_entry( void) { xce_sched_sim_driver_t sim; xce_sched_job_t* job; xce_job_alloc_t* stray_alloc; replay_session_t* decoded; xce_core_event_t ev; int push_rc; fixture_init(&sim); job = add_job(&(sim.core), 7071ULL, JOB_STAT_RUN, make_job_alloc(7071ULL, 0, NULL)); TEST_ASSERT_NOT_NULL_MESSAGE( job->allocs, "fixture setup: job must start with a real allocation"); TEST_ASSERT_NOT_NULL_MESSAGE( job->accounts, "fixture setup: job must start with real accounts"); memset(&ev, 0, sizeof(ev)); ev.type = XCE_CORE_EVENT_JOB_STATE_CHANGE; ev.u.job_state_change.jobid = 7071ULL; ev.u.job_state_change.old_state = JOB_STAT_RUN; ev.u.job_state_change.new_state = JOB_STAT_EXIT; push_rc = xce_core_push_events(&(sim.core), &ev, 1); TEST_ASSERT_EQUAL_INT(0, push_rc); TEST_ASSERT_TRUE_MESSAGE(JOB_IS_EXIT(job->state), "precondition: job must genuinely be EXIT " "before this test's real assertion means " "anything"); TEST_ASSERT_NULL_MESSAGE( job->allocs, "precondition: xce_core_push_events()'s normal " "end-of-life cleanup must null job->allocs on a " "terminal transition"); TEST_ASSERT_NULL_MESSAGE( job->accounts, "precondition: xce_core_push_events()'s normal " "end-of-life cleanup must null job->accounts on a " "terminal transition"); stray_alloc = make_test_job_alloc_with_host(7071ULL, "worker-stray-exit", "slots", 9.0f); decoded = make_decoded_session( 2100ULL, make_dispatch_entry_from_alloc(stray_alloc)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); TEST_ASSERT_NULL_MESSAGE( job->allocs, "expected allocs NULL after reconciling a terminal (EXIT) job, but " "allocs was populated -- today's trigger (job->accounts == NULL " "alone) cannot distinguish 'not yet allocated' from 'already " "completed and released' (bug-1764)"); TEST_ASSERT_NULL_MESSAGE( job->accounts, "expected accounts NULL after reconciling a terminal (EXIT) job, " "but accounts was populated -- same root cause as the allocs " "assertion above"); printf("test_reconcile_never_reallocates_an_exit_job_despite_stray_" "dispatch_entry: PASS\n"); } /* ---------------------------------------------------------------------- */ /* REPL-capture helper (scheduler-sim-reconcile-terminal-guard task 2.2): */ /* duplicated from test_repl_resource_table.c's/test_repl_breakpoints_ */ /* inspection.c's own identical capture_dispatch()/capture_stdout_around() */ /* helpers, per this file family's established "duplicate rather than share */ /* via a new header" convention (design.md's File Structure Plan; those two */ /* files' own header comments document the same choice). This file already */ /* white-box #includes the full repl.c/repl_fmt.c/repl_rollup.c/repl_break. */ /* c/repl_info.c/repl_params.c stack every OTHER test here needs for */ /* sim_reconcile_unaccounted_jobs()'s own xce_repl_check_decision() call -- */ /* only the stdout-redirect capture helpers themselves are new here, not a */ /* new #include. */ /* ---------------------------------------------------------------------- */ static char* capture_stdout_around(void (*fn)(void)) { char tmp[] = "/tmp/xce_simreconcile_replcap_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* buf; TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0, "dup(STDOUT_FILENO) must " "succeed"); TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0, "dup2 redirect of STDOUT_FILENO must succeed"); fn(); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); buf = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(buf); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, buf, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); unlink(tmp); return buf; } static xce_sched_sim_driver_t* g_dispatch_sim; static const char* g_dispatch_line; static void run_dispatch_for_capture(void) { (void)repl_dispatch(g_dispatch_sim, g_dispatch_line); } static char* capture_dispatch(xce_sched_sim_driver_t* sim, const char* line) { g_dispatch_sim = sim; g_dispatch_line = line; return capture_stdout_around(run_dispatch_for_capture); } /* ---------------------------------------------------------------------- */ /* Test 20 (scheduler-sim-reconcile-terminal-guard task 2.2, Requirement */ /* 2.1, 2.2 -- FINAL task of this spec): REPL-visible confirmation of tasks */ /* 1.1/1.2's fix, mirroring bug-1764's live-observed shape -- an array job */ /* where some elements are DONE/EXIT while others remain running -- at the */ /* `info worker`/`info summary` display level Requirement 2 actually cares */ /* about, not just at the job->allocs/job->accounts level Tests 18/19 above */ /* already pin. */ /* */ /* Three sibling jobs, all dispatched onto the SAME worker ("workerarr"): */ /* - jobid 7080: driven to JOB_STAT_DONE via xce_core_push_events() */ /* (XCE_CORE_EVENT_JOB_STATE_CHANGE, the exact production path Test 18 */ /* uses), which correctly frees its real starting allocation. */ /* - jobids 7081/7082: remain genuinely JOB_STAT_RUN with their own real, */ /* distinct allocations to the same worker throughout. */ /* A later round's decoded_session then carries a stray/duplicate DISPATCH */ /* entry for jobid 7080 -- bug-1764's own live-observed cause (a retried/ */ /* leftover dispatch attempt for one array element still present in the */ /* replay stream after that element already completed) -- built with the */ /* exact same make_test_job_alloc_with_host()/make_dispatch_entry_from_ */ /* alloc()/make_decoded_session() helpers Tests 18/19 use. */ /* sim_reconcile_unaccounted_jobs() is called directly, the same convention */ /* design.md's Testing Strategy establishes and every other test in this */ /* file follows (the function is `static`, not driven through a full */ /* round-boundary path by any test in this suite). */ /* */ /* "info worker workerarr" and "info summary" are then captured through */ /* capture_dispatch() (above). Requirement 2.1: the completed job's id must */ /* be ABSENT from the assigned-jobs listing while both still-running jobs' */ /* ids remain present. Requirement 2.2 (display consistency): info */ /* summary's own running count -- computed purely from job->state via */ /* JOB_IS_RUN(), entirely independent of job->allocs/job->accounts -- must */ /* agree with exactly those two still-running jobs, proving the assigned- */ /* jobs listing and the running count tell the SAME story once the guard is */ /* in place (bug-1764's actual symptom was these two views disagreeing: */ /* the assigned-jobs listing showed 3 while running was correctly only 2). */ /* ---------------------------------------------------------------------- */ static void test_reconcile_info_worker_and_summary_hide_completed_job_after_stray_dispatch( void) { xce_sched_sim_driver_t sim; xce_sched_worker_t* w; xce_sched_job_t* job_done; xce_sched_job_t* job_run1; xce_sched_job_t* job_run2; xce_job_alloc_t* alloc_done_initial; xce_job_alloc_t* alloc_run1; xce_job_alloc_t* alloc_run2; xce_job_alloc_t* stray_alloc; replay_session_t* decoded; xce_core_event_t ev; int residx; int push_rc; char* captured_worker; char* captured_summary; char done_jobid_str[64]; char run1_jobid_str[64]; char run2_jobid_str[64]; fixture_init(&sim); residx = register_slots_resource(&(sim.core)); w = add_worker(&(sim.core), "workerarr", residx, 20.0f); TEST_ASSERT_NOT_NULL(w); /* jobid 7080 starts genuinely RUN with a real allocation -- the * "actively holding resources" shape a terminal transition must * release, same fixture shape Test 18/19 above establish. */ alloc_done_initial = make_test_job_alloc_with_host(7080ULL, "workerarr", "slots", 3.0f); job_done = add_job(&(sim.core), 7080ULL, JOB_STAT_RUN, alloc_done_initial); TEST_ASSERT_NOT_NULL_MESSAGE( job_done->allocs, "fixture setup: job_done must start with a real allocation"); TEST_ASSERT_NOT_NULL_MESSAGE( job_done->accounts, "fixture setup: job_done must start with real accounts"); /* jobids 7081/7082: the "some elements remain running" half of the * array-job scenario -- genuinely RUN, each with its own real, distinct * allocation to the SAME worker, untouched for the rest of this test. */ alloc_run1 = make_test_job_alloc_with_host(7081ULL, "workerarr", "slots", 2.0f); job_run1 = add_job(&(sim.core), 7081ULL, JOB_STAT_RUN, alloc_run1); TEST_ASSERT_NOT_NULL(job_run1->allocs); alloc_run2 = make_test_job_alloc_with_host(7082ULL, "workerarr", "slots", 4.0f); job_run2 = add_job(&(sim.core), 7082ULL, JOB_STAT_RUN, alloc_run2); TEST_ASSERT_NOT_NULL(job_run2->allocs); /* Drive jobid 7080 to JOB_STAT_DONE through the SAME production event * path bug-1764's root cause depends on (Test 18's own technique). */ memset(&ev, 0, sizeof(ev)); ev.type = XCE_CORE_EVENT_JOB_STATE_CHANGE; ev.u.job_state_change.jobid = 7080ULL; ev.u.job_state_change.old_state = JOB_STAT_RUN; ev.u.job_state_change.new_state = JOB_STAT_DONE; push_rc = xce_core_push_events(&(sim.core), &ev, 1); TEST_ASSERT_EQUAL_INT(0, push_rc); TEST_ASSERT_TRUE_MESSAGE(JOB_IS_DONE(job_done->state), "precondition: job_done must genuinely be DONE " "before this test's real assertions mean " "anything"); TEST_ASSERT_NULL_MESSAGE( job_done->allocs, "precondition: normal end-of-life cleanup must " "null job_done->allocs on its terminal " "transition"); TEST_ASSERT_NULL_MESSAGE( job_done->accounts, "precondition: normal end-of-life cleanup must " "null job_done->accounts on its terminal " "transition"); /* A later replay round's decoded_session still carries a stray/ * duplicate DISPATCH entry for jobid 7080 -- bug-1764's live-observed * shape (a retried dispatch attempt for one array element still present * in the replay stream after that element already completed). */ stray_alloc = make_test_job_alloc_with_host(7080ULL, "workerarr", "slots", 9.0f); decoded = make_decoded_session( 9000ULL, make_dispatch_entry_from_alloc(stray_alloc)); sim_reconcile_unaccounted_jobs(&sim, decoded, NULL); /* Sanity re-check (the accounting-level half of this task's own * Observable, already covered by Tests 18/19 -- repeated here only to * localize a failure before trusting the REPL-level assertions below). */ TEST_ASSERT_NULL_MESSAGE( job_done->allocs, "job_done's allocation must still be NULL after reconciling this " "round's stray DISPATCH entry for its own jobid"); TEST_ASSERT_NULL_MESSAGE(job_done->accounts, "job_done's accounts must likewise still be " "NULL"); /* ---- The REPL-visible assertions this task actually exists for. ---- */ fmt_jobid(done_jobid_str, sizeof(done_jobid_str), 7080ULL); fmt_jobid(run1_jobid_str, sizeof(run1_jobid_str), 7081ULL); fmt_jobid(run2_jobid_str, sizeof(run2_jobid_str), 7082ULL); captured_worker = capture_dispatch(&sim, "info worker workerarr"); TEST_ASSERT_NULL_MESSAGE( strstr(captured_worker, done_jobid_str), "Requirement 2.1: the completed job must be ABSENT from 'info " "worker's assigned-jobs listing, even though this round's " "decoded_session carried a stray DISPATCH entry for its jobid " "(bug-1764)"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured_worker, run1_jobid_str), "the first still-running job must be present in the assigned-jobs " "listing"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured_worker, run2_jobid_str), "the second still-running job must be present in the assigned-jobs " "listing"); free(captured_worker); captured_summary = capture_dispatch(&sim, "info summary"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured_summary, "3 jobs (0 pending, 2 running)"), "Requirement 2.2: 'info summary's own running count (computed " "purely from job->state, independent of job->allocs/accounts) must " "agree with the assigned-jobs listing above -- exactly the two " "still-running jobs, never inflated by the completed one"); free(captured_summary); printf( "test_reconcile_info_worker_and_summary_hide_completed_job_after_" "stray_dispatch: PASS\n"); } /* * bug-1759/bug-1779: a reconciled (replay-reconstructed) dispatch must * remove the job from the picker's OWN candidate bookkeeping (fcfs_queue_t. * pendJobs), exactly like a LIVE dispatch does -- sched_allocator() * (scheduler.c) always calls job_picker->picker_iface->on_dispatch() before * building the decision, but sim_reconcile_unaccounted_jobs() (this file's * subject) never did. Left unfixed, sim_picker_thread_fn() (sim_loop.c) -- * a separate, always-running thread that walks next_job() every round and * unconditionally PENDs whatever it is still offered (it predates this * reconciliation mechanism and was never taught about it) -- walks right * past the "already reconciled" job on the very next round and emits a * bogus PEND decision for a job that has already been dispatched. This is * the live-observed root cause of both bug-1759 (50-job batch, 22 * already-dispatched jobs also received a PEND) and bug-1779 (10-element * array job, 5 already-dispatched elements also received a PEND). * * Fixture: sim_bootstrap_picker() (sim_picker.c, bug-1567) is the same * retroactive registration a real replay's initial snapshot bootstrap * performs -- register builtin_fcfs, then replay the existing queue/job as * picker candidates, so `job` genuinely sits in the picker's pendJobs * exactly as it would after a live JOBS_PUSH, without needing a full * xce_core_push_events() event round trip. */ static void test_reconcile_removes_reconciled_job_from_picker_candidates(void) { xce_sched_sim_driver_t sim; replay_decision_t* dispatch_entry; replay_session_t* decoded_session; xce_sched_job_t* job; xce_sched_plugin_t* picker = NULL; xce_sched_job_t* offered; int rc; fixture_init(&sim); job = add_job(&(sim.core), 9101ULL, JOB_STAT_PEND, NULL); TEST_ASSERT_NOT_NULL(job); sim_bootstrap_picker(&sim); rc = sched_resolve_job_picker(&(sim.core), &picker); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NOT_NULL(picker); /* Fixture sanity: the job is genuinely a live picker candidate before * reconciliation runs. */ picker->picker_iface->begin_session(picker); offered = picker->picker_iface->next_job(picker); picker->picker_iface->end_session(picker); TEST_ASSERT_NOT_NULL_MESSAGE( offered, "fixture sanity: job must be a picker candidate before " "reconciliation runs"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 9101ULL, offered->jobid, "fixture sanity: the offered candidate must be this test's own job"); dispatch_entry = make_dispatch_entry(9101ULL); decoded_session = make_decoded_session(0, dispatch_entry); sim_reconcile_unaccounted_jobs(&sim, decoded_session, NULL); TEST_ASSERT_NOT_NULL_MESSAGE( job->allocs, "reconciliation must still populate job->allocs exactly as before " "this fix"); picker->picker_iface->begin_session(picker); offered = picker->picker_iface->next_job(picker); picker->picker_iface->end_session(picker); TEST_ASSERT_NULL_MESSAGE( offered, "bug-1759/bug-1779: a job reconciled (dispatched) via replayed " "decision data must be removed from the picker's candidate set, " "exactly like a live dispatch (sched_allocator()'s on_dispatch() " "call, scheduler.c) -- otherwise sim_picker_thread_fn()'s " "unconditional-PEND thread re-offers and re-PENDs it on a later " "round even though it has already been dispatched"); printf("test_reconcile_removes_reconciled_job_from_picker_candidates: " "PASS\n"); } int main(void) { UNITY_BEGIN(); RUN_TEST( test_reconcile_leaves_run_job_unaccounted_when_no_recorded_decision); RUN_TEST(test_reconcile_is_safe_to_recheck_on_a_later_round); RUN_TEST(test_reconcile_skips_job_that_already_has_accounts); RUN_TEST(test_reconcile_ignores_non_dispatch_entry_with_matching_jobid); RUN_TEST( test_reconcile_captures_non_run_state_job_when_dispatch_recorded); RUN_TEST( test_reconcile_reconstructs_allocs_and_accounts_from_matching_dispatch); RUN_TEST(test_reconcile_compacts_via_shift_down_preserving_order); RUN_TEST(test_reconcile_then_release_via_job_state_change_is_asan_clean); RUN_TEST( test_reconcile_reports_reconstructed_decision_through_existing_channel); RUN_TEST(test_reconcile_null_args_are_a_safe_noop); RUN_TEST(test_captured_pend_job_excluded_from_resinuse_until_run); RUN_TEST( test_reconcile_still_skips_already_captured_job_regardless_of_state); RUN_TEST(test_reconcile_no_match_leaves_pend_job_uncaptured_and_retries); RUN_TEST( test_reconcile_multiple_jobs_captured_independently_across_different_rounds); RUN_TEST( test_reconcile_continue_mode_with_repl_state_suppresses_bare_decision_line); RUN_TEST( test_reconcile_step_mode_with_repl_state_still_prints_bare_decision_line); RUN_TEST( test_reconcile_never_reallocates_a_done_job_despite_stray_dispatch_entry); RUN_TEST( test_reconcile_never_reallocates_an_exit_job_despite_stray_dispatch_entry); RUN_TEST( test_reconcile_info_worker_and_summary_hide_completed_job_after_stray_dispatch); RUN_TEST(test_reconcile_removes_reconciled_job_from_picker_candidates); return UNITY_END(); }