/* * test_sim_verify.c - contract test for decision verification * (scheduler-core spec, task 11.4, Req 5 AC 15/18). * * Observable under test (task 11.4's own text): * - replaying a fixture file with verification enabled against its own * previously-recorded output reports zero mismatches. * - replaying against a deliberately-altered copy reports the exact * mismatch. * * Two granularities, same split test_sim_event_loop.c's own header comment * already establishes for task 11.2: * * Unit-level (white-box): calls sim_decision_eq()/sim_log_mismatch() * (sim_verify.c, task 11.4's own sibling file to sim_loop.c) directly * against hand-built replay_decision_t/xce_decision_t pairs -- DISPATCH * match/mismatch (jobid, host, resource value, from), PEND match/mismatch * (jobid, main_reason), a body_which/op type mismatch, and PREEMPT's * documented always-false path -- each mismatch also asserting the * first-difference description (libxce-alloc-preason-equality * Requirement 5.3). This is the only way to exercise DISPATCH * comparison at all: sim_loop.c's real picker thread * (sim_picker_thread_fn) only ever emits PEND decisions (its own doc * comment: "conservatively PENDs every candidate ... rather than * fabricate a DISPATCH decision with no real worker/resource assignment * behind it") -- real resource-matching allocation belongs to the * not-yet-designed scheduler-allocation spec. sim_decision_eq() must * still compare DISPATCH decisions correctly (Req 5 AC 15 does not * exempt them), so that comparator is proven directly here rather than * left unreachable. * * Integration-level: drives the full sched_sim_driver_run() (this task's * other Observable) against a fixture built from a REAL * QUEUE_UPDATE+JOBS_PUSH event pair (one queue, one JOB_STAT_PEND job) * feeding the REAL builtin_fcfs picker -- since core.c's own * sched_job_is_ready()/sched_queue_is_ready() are unconditional `return * 1` stubs (task 2.4's own documented, still-live scope note), any queue/ * job pair is immediately "ready", so this always produces exactly one * live PEND decision via make_pendreason() (preason.c), whose * xce_job_preason_t is always freshly calloc()'d (main_reason=0, no host/ * res reasons) -- a fully deterministic, reproducible target to compare * against. `--sim-verify`'s ERROR()-logged mismatch report is observed by * redirecting STDOUT_FILENO around the sched_sim_driver_run() call, the * same dup()/dup2() capture technique * test_apply_correctness_verification.c's own * test_apply_divergent_loud_fence_log_omits_false_peer_hwm_clause * establishes (that file's own doc comment explains WHY this works: with * tlog never tlog_init()'d in this test process -- true here too, nothing * in this TU calls it -- every ERROR()/WARN() falls through tlog's * early-print path, a raw write() to STDOUT_FILENO). * * Fixture strategy: same production-code-reuse technique test_sim_startup.c/ * test_sim_event_loop.c already establish for the header+snapshot prefix * (sched_record_start() against a small real xce_scheduler_t fixture) PLUS * hand-constructed ReplayEvent/ReplaySession messages appended afterward via * the real encode_ReplayEvent_ptr()/encode_ReplaySession_ptr()/ * write_capnp_message() codec, same technique test_sim_event_loop.c's own * append_jobstatechange_event()/append_replay_session() and * test_e2e_capnp.c's test_dispatch_decision_round_trip()/ * test_pend_decision_round_trip() already prove works. * * White-box #include of the same TU set test_sim_event_loop.c uses (see * that file's own header comment, including task 11.5's * recording_session.c/sim_record.c AND task 12.1's repl.c -- all required at * link time by sched_sim_driver_run()'s/repl.h's unconditional externs * regardless of whether --sim-output/--interactive is ever set in a given * test) PLUS ../src/sim_verify.c (the code under test). */ #include #include #include #include #include #include #include #include #include #include #include "controller.capnp.h" #include "kvmgr.h" #include "preason.h" #include "scheduler.h" #include "xstring.h" /* Pull in the real functions under test into this TU (white-box #include, * same convention as test_sim_event_loop.c/test_sim_startup.c). */ #include "../src/core.c" #include "../src/core_clock.c" #include "../src/core_events.c" #include "../src/core_session.c" #include "../src/decision.c" #include "../src/plugin.c" #include "../src/plugins/builtin_fcfs.c" #include "../src/recording.c" #include "../src/recording_session.c" #include "../src/repl.c" #include "../src/repl_fmt.c" #include "../src/repl_rollup.c" #include "../src/repl_break.c" #include "../src/repl_info.c" #include "../src/repl_params.c" #include "../src/sim.c" #include "../src/sim_loop.c" #include "../src/sim_picker.c" /* bug-1567: sched_sim_driver_run() now calls sim_bootstrap_picker() (sim_loop.c-cap sibling) as an extern -- undefined reference at link time otherwise */ #include "../src/sim_limit.c" /* bug-1825: sched_sim_driver_run() now also calls sim_bootstrap_limit_plugin() (sim_loop.c-cap sibling, sim_picker.c's own bug-1567 precedent) as an extern -- undefined reference at link time otherwise */ #include "../src/sim_record.c" #include "../src/sim_verify.c" #include "../src/snapshot.c" #include "../src/snapshot_load.c" #include "../src/snapshot_wire.c" void kill_sched_job(void* arg) { (void)arg; } void kill_sched_queue(void* arg) { (void)arg; } void kill_sched_worker(void* arg) { (void)arg; } void kill_sched_user(void* arg) { (void)arg; } void kill_sched_project(void* arg) { (void)arg; } void kill_sched_resval(xce_val_t* val) { (void)val; } /* repl_info.c (scheduler-sim-diagnostics spec, task 2.1) now calls * limit_get_configured_limits() (plugins/limit.c, not part of this TU) to * resolve `info limit`'s name/all argument -- link-satisfying stub only, * same rationale as kill_sched_job()/kill_sched_resval() above; this * file's own fixtures never register the limit plugin, so NULL * (limit_get_configured_limits()'s own documented "plugin absent" return) * is also the behaviorally correct answer here, not just a link * placeholder. repl_info_limit() itself is not yet wired into * repl_cmd_info() (task 4), so no test in this file can reach it anyway. */ var limit_get_configured_limits(xce_sched_core_t* core) { (void)core; return NULL; } /* scheduler-sim-diagnostics spec, task 3.1: repl_info.c now also calls * limit_get_usage_instances() (plugins/limit.c, not part of this TU) to * read a limit's live usage instances -- identical link-satisfying stub, * same rationale as limit_get_configured_limits() immediately above; this * file's own fixtures never register the limit plugin, so NULL * (limit_get_usage_instances()'s own documented "plugin absent" return) is * also the behaviorally correct answer here, not just a link placeholder. */ var limit_get_usage_instances(xce_sched_core_t* core) { (void)core; return NULL; } /* scheduler-sim-diagnostics spec, task 6 (bug-1825): sim_loop.c's * sched_sim_driver_run() now also calls sim_bootstrap_limit_plugin() * (sim_limit.c, pulled into this TU via the #include above), which in * turn calls limit_new_plugin() (plugins/limit.c, not part of this TU) -- * identical link-satisfying stub rationale as limit_get_configured_ * limits()/limit_get_usage_instances() immediately above. Returning NULL * here is also the behaviorally correct answer, not just a link * placeholder: sim_bootstrap_limit_plugin() already handles a NULL * limit_new_plugin() return gracefully (logs an ERROR, registers nothing, * returns) -- the exact same "out of memory" path sim_bootstrap_picker() * (sim_picker.c) takes for a NULL builtin_fcfs_new_plugin() -- so this * file's fixtures simply end up with no limit plugin registered, same * observable state as before this task. */ xce_sched_plugin_t* limit_new_plugin(void) { return NULL; } /* limit_seed_config() (plugins/limit.c, not part of this TU) -- bug-1825 * remediation round 2's new honestly-typed config-seed entry point. * sim_bootstrap_limit_plugin() (sim_limit.c) only calls it after * plugin_register_core() has already succeeded against a real, non-NULL * limit_new_plugin() plugin -- unreachable in this file since the * limit_new_plugin() stub immediately above always returns NULL, but the * symbol must still resolve at link time. Same link-satisfying-stub * rationale as limit_new_plugin() immediately above. */ int limit_seed_config(xce_sched_plugin_t* plugin, xce_sched_core_t* core, xce_conf4sched_t* config) { (void)plugin; (void)core; (void)config; return 0; } /* resource-accounting.md's reconciliation (resource.c, not linked here -- * this file only #includes core_events.c). core_events.c's * XCE_CORE_EVENT_JOB_RUSAGE handler calls this whenever job->accounts != * NULL; link-satisfying stub only, same rationale as every other stub in * this block (see test_job_update.c's identical stub). */ int sched_reconcile_job_resource_usage(xce_sched_core_t* c, xce_job_account_t* a, xce_job_rusages_t* r) { (void)c; (void)a; (void)r; return 0; } /* core_reset_workers()'s (core_session.c) linker dependency -- same stub * test_sim_event_loop.c/test_core_full_cycle.c already establish; never * invoked (no worker is ever added here). */ void sched_worker_release_host_ctx(xce_sched_worker_t* w) { (void)w; } /* plugin_prepare_allocation()'s (plugin.c, the "limit" plugin category) * linker dependency -- never invoked (this file never drives real * allocation). */ int sched_fill_limit_requests(xce_sched_alloc_req_t* alloc_req, xlist_t* limits, xlist_t* rusages, int nhosts, int nslots) { (void)alloc_req; (void)limits; (void)rusages; (void)nhosts; (void)nslots; return 0; } /* sched_sim_driver_run()'s (sim_loop.c, scheduler-sim-decision-parity task * 2.1) new round-boundary call to the relocated sched_account_jobs() * (core.h) -- a no-op stand-in here since this file does not link * scheduler.c/resource.c, same rationale test_sim_equivalence.c's own copy * documents. */ int sched_account_jobs(xce_sched_core_t* core) { (void)core; return 0; } /* bug-1836 (scheduler-driver-equivalence fix, Recorded-Decision * Reconciliation capacity gate): link-satisfying stubs, mirroring this * file's existing sched_account_jobs() no-op precedent immediately above -- * sim_loop.c's sim_reconcile_unaccounted_jobs() now also calls * sched_check_host_satisfy()/sched_check_shared_satisfy() (resource.c) to * re-validate a recorded decision against live capacity, and * sched_charge_account() (resource.c) to keep that check's own view of * capacity up to date across multiple candidates in the same round. This * file never exercises real resource contention (same rationale as the * existing sched_account_jobs()/sched_ctx_get_uncomit_host() no-ops), so * the satisfy checks trivially always succeed and the charge call is a * no-op. */ int sched_check_host_satisfy(xce_sched_core_t* core, xce_sched_worker_t* w, xvec_t* uncomitted, xvec_idval_t* wanted, xvec_idval_t* unmeet) { (void)core; (void)w; (void)uncomitted; (void)wanted; (void)unmeet; return 1; } int sched_check_shared_satisfy(xce_sched_core_t* core, xvec_t* uncomitted, xvec_idval_t* wanted, xvec_idval_t* unmeet) { (void)core; (void)uncomitted; (void)wanted; (void)unmeet; return 1; } int sched_charge_account(xce_sched_core_t* core, xce_job_account_t* acct) { (void)core; (void)acct; return 0; } int sched_uncharge_account(xce_sched_core_t* core, xce_job_account_t* acct) { (void)core; (void)acct; return 0; } /* Real reimplementation of schedobj.c's sched_resolve_job_picker -- same * copy test_sim_event_loop.c/test_sim_startup.c/test_core_full_cycle.c each * carry (schedobj.c is not linked here: it would need plugin_register(), * which requires a genuine xce_scheduler_t* neither this file's fixtures * nor sim_loop.c's own core-scoped registration path have, Req 5 AC 17). */ int sched_resolve_job_picker(xce_sched_core_t* core, xce_sched_plugin_t** out) { xce_sched_plugin_t* plugin = NULL; int rc = 0; if ((core == NULL) || (out == NULL)) { return -EINVAL; } *out = NULL; if (core->job_pick_plugin == NULL) { return -EINVAL; } lock(core->plugins, RTTI_LOCK_READ); rc = key_find(core->plugins, core->job_pick_plugin, (void**)&plugin); unlock(core->plugins); if (rc <= 0) { return (rc < 0) ? rc : -ENOENT; } if ((plugin == NULL) || (plugin->picker_iface == NULL) || (plugin->picker_iface->next_job == NULL) || (plugin->picker_iface->begin_session == NULL) || (plugin->picker_iface->end_session == NULL) || (plugin->picker_iface->on_dispatch == NULL) || (plugin->picker_iface->on_pend == NULL)) { return -EINVAL; } *out = plugin; return 0; } /* scheduler-sim-live-allocation spec, task 4.1: sim_loop.c's real * sched_sim_driver_run() (white-box #include'd into this TU above) now * spawns sched_job_picker()/sched_allocator() (sched_election.c) directly * instead of its own, now-removed sim_picker_thread_fn() -- an undefined * reference at link time in this white-box TU otherwise, since * sched_election.c's own sched_allocator() calls the REAL * sched_make_allocation()/sched_commit_decision() (scheduler.c), neither * part of this file's own hand-picked white-box #include set (this file's * own header comment lists that set). This file's own test(s) do not * exercise genuine resource-matching allocation, so this link-satisfying * pair faithfully reproduces the exact PEND-only contract the REMOVED * sim_picker_thread_fn() itself used to provide -- split across the SAME * two-thread jbuf hand-off the real production pair now uses (one * producer, one consumer, matching the project's jbuf capacity-1 * rendezvous invariant) instead of the old single fused thread -- so this * file's own existing assertions are unaffected by this task's production * change. Genuine resource-based dispatch through the real, unmodified * sched_job_picker()/sched_allocator() pair is proved separately by * test_sim_live_allocation.c (task 4.1's own dedicated fixture, which * links the real scheduler_srcs instead of white-boxing individual files, * precisely so it can reach the real allocation chain this stub * deliberately does not). */ void* sched_job_picker(void* arg) { xce_sched_core_t* core = (xce_sched_core_t*)arg; xce_sched_plugin_t* picker = NULL; for (;;) { int go = 0; uint64_t jobid = 0; cbuf_get(&(core->wbuf), &go); if (sched_resolve_job_picker(core, &picker) < 0) { jobid = 0; cbuf_put(&(core->jbuf), &jobid); continue; } picker->picker_iface->begin_session(picker); for (;;) { xce_sched_job_t* job = picker->picker_iface->next_job(picker); if (job == NULL) { break; } jobid = job->jobid; cbuf_put(&(core->jbuf), &jobid); } jobid = 0; cbuf_put(&(core->jbuf), &jobid); picker->picker_iface->end_session(picker); } return NULL; } void* sched_allocator(void* arg) { xce_sched_core_t* core = (xce_sched_core_t*)arg; xce_sched_plugin_t* picker = NULL; for (;;) { uint64_t jobid = 0; xce_decision_t* decision = NULL; xce_sched_job_t* job; cbuf_get(&(core->jbuf), &jobid); if (jobid == 0) { decision = NULL; cbuf_put(&(core->decisions), &decision); continue; } if (jobid == XCE_SCHED_RECONCILE_WAKE) { /* bug-1837: this file's stand-in sched_allocator() must service * the same Recorded-Decision Reconciliation relay protocol the * real sched_election.c implementation does (see that file's * matching branch for the full rationale) -- this file's own * sched_sim_driver_run() calls set sim->allocator_thread_live, * so sim_reconcile_unaccounted_jobs() routes on_dispatch() * through jbuf/reconcile_req/reconcile_ack exactly as * production does; without this branch, a reconciliation * success here would hang forever waiting on an ack this stub * never sends. */ uint64_t reconcile_jobid = 0; int reconcile_rc = 0; cbuf_get(&(core->reconcile_req), &reconcile_jobid); (void)sched_resolve_job_picker(core, &picker); if (picker != NULL) { reconcile_rc = picker->picker_iface->on_dispatch( picker, reconcile_jobid); } cbuf_put(&(core->reconcile_ack), &reconcile_rc); continue; } job = sched_get_job(core, jobid); if (job == NULL) { continue; } (void)sched_resolve_job_picker(core, &picker); if (picker != NULL) { picker->picker_iface->on_pend(picker, job->jobid); } decision = make_pend_decision(make_pendreason(job->jobid)); cbuf_put(&(core->decisions), &decision); } return NULL; } /* scheduler-sim-live-allocation spec, task 4.1 (bug-1829 fix): sim_loop.c's * real sched_sim_driver_run() (white-box #include'd into this TU above) now * also calls sim_bootstrap_seqjob_allocator()/sim_bootstrap_parjob_ * allocator() (sim_alloc.c) -- undefined references at link time in this * white-box TU otherwise, since neither symbol is part of this file's own * hand-picked white-box #include set. This file's own test(s) exercise the * link-satisfying sched_job_picker()/sched_allocator() stub pair above * (which never reaches real primary allocation), so a true no-op is the * correct, behavior-preserving stand-in here -- the real bootstrap logic is * exercised separately by test_sim_live_allocation.c (task 4.1's own * dedicated fixture, which links the real scheduler_srcs). */ void sim_bootstrap_seqjob_allocator(xce_sched_sim_driver_t* sim) { (void)sim; } void sim_bootstrap_parjob_allocator(xce_sched_sim_driver_t* sim) { (void)sim; } void setUp(void) { return; } void tearDown(void) { return; } /* ---------------------------------------------------------------------- */ /* Unit-level fixture helpers: hand-built xce_job_alloc_t trees. */ /* ---------------------------------------------------------------------- */ /* SESSION_FROM - the `from` a DISPATCH allocation decided in a session that * began at `begin_ns` (epoch nanoseconds) carries: the session start in * whole seconds, floor division (libxce-alloc-preason-equality Decision 8; * scheduler.c's sched_make_primary_alloc_() stamps it, and in simulation * mode the session starts at the recorded beginRound). Hand-built recorded * DISPATCH fixtures replayed through sched_sim_driver_run() below derive * their `from` from the same begin_round they are recorded under. */ #define SESSION_FROM(begin_ns) ((uint64_t)(begin_ns) / 1000000000ULL) /* The recorded round-1 beginRound those DISPATCH fixtures are appended * under (logical nanoseconds). */ #define ROUND1_BEGIN_NS 2000ULL static xce_alloc_t* make_test_alloc(const char* resname, const char* reserve, uint64_t instid, double value, uint64_t from, int64_t duration) { xce_alloc_t* a = (xce_alloc_t*)calloc(1, sizeof(*a)); TEST_ASSERT_NOT_NULL(a); a->resname = strdup(resname); a->reserve = strdup(reserve); a->instid = instid; a->value = value; a->from = from; a->duration = duration; return a; } static xce_host_alloc_t* make_test_host_alloc(const char* host, xce_alloc_t** allocs, int nallocs) { xce_host_alloc_t* h = (xce_host_alloc_t*)calloc(1, sizeof(*h)); TEST_ASSERT_NOT_NULL(h); h->host = strdup(host); h->nallocs = nallocs; h->allocs = allocs; return h; } static xce_job_alloc_t* make_test_job_alloc(uint64_t jobid, xce_host_alloc_t** hosts, int nhosts) { xce_job_alloc_t* j = (xce_job_alloc_t*)calloc(1, sizeof(*j)); TEST_ASSERT_NOT_NULL(j); j->jobid = jobid; j->nhosts = nhosts; j->allocs = hosts; j->nres_shared = 0; j->allocs_shared = NULL; return j; } static replay_decision_t make_replay_dispatch_decision(xce_job_alloc_t* alloc) { replay_decision_t d; sched_dispatch_decision_t* dd = (sched_dispatch_decision_t*)calloc(1, sizeof(*dd)); TEST_ASSERT_NOT_NULL(dd); dd->alloc = alloc; memset(&d, 0, sizeof(d)); d.body_which = ReplayDecision_body_dispatch; d.body.dispatch = dd; return d; } static replay_decision_t make_replay_pend_decision(uint64_t jobid, int main_reason) { replay_decision_t d; sched_pend_decision_t* pd = (sched_pend_decision_t*)calloc(1, sizeof(*pd)); xce_job_preason_t* reason = (xce_job_preason_t*)calloc(1, sizeof(*reason)); TEST_ASSERT_NOT_NULL(pd); TEST_ASSERT_NOT_NULL(reason); reason->main_reason = main_reason; pd->jobid = jobid; pd->reason = reason; memset(&d, 0, sizeof(d)); d.body_which = ReplayDecision_body_pend; d.body.pend = pd; return d; } /* ---------------------------------------------------------------------- */ /* Unit-level tests: sim_decision_eq() directly. */ /* ---------------------------------------------------------------------- */ /* Asserts that sim_decision_eq()'s description (libxce-alloc-preason- * equality Requirement 5.3) was produced and contains `needle` -- the name * of the differing field and both of its values -- then frees it. */ static void assert_why_names(char* why, const char* needle) { char msg[512]; TEST_ASSERT_NOT_NULL_MESSAGE(why, "a mismatch must produce a " "description of the difference"); snprintf(msg, sizeof(msg), "description \"%s\" must contain \"%s\"", why, needle); TEST_ASSERT_NOT_NULL_MESSAGE(strstr(why, needle), msg); free(why); } static void test_dispatch_decisions_with_identical_content_match(void) { char* why; xce_alloc_t* res_a[1] = { make_test_alloc("gpu", "resv1", 1, 2.0, 100, 1800) }; xce_host_alloc_t* host_a[1] = { make_test_host_alloc("hostA", res_a, 1) }; xce_job_alloc_t* expected_alloc = make_test_job_alloc(42, host_a, 1); xce_alloc_t* res_b[1] = { make_test_alloc("gpu", "resv1", 1, 2.0, 100, 1800) }; xce_host_alloc_t* host_b[1] = { make_test_host_alloc("hostA", res_b, 1) }; xce_job_alloc_t* actual_alloc = make_test_job_alloc(42, host_b, 1); replay_decision_t expected = make_replay_dispatch_decision(expected_alloc); xce_decision_t* actual = make_dispatch_decision(actual_alloc); /* Two structurally-identical, but never pointer-equal, allocation * trees: proves the comparison is real structural equality, not a * pointer/memcmp shortcut (this file's own header comment). */ TEST_ASSERT_NOT_EQUAL(expected_alloc, actual_alloc); TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL)); why = (char*)"stale"; TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, &why)); TEST_ASSERT_NULL_MESSAGE(why, "an equal comparison must leave no " "description"); printf("test_dispatch_decisions_with_identical_content_match: PASS\n"); } static void test_dispatch_decisions_differing_jobid_mismatch(void) { xce_alloc_t* res[1] = { make_test_alloc("gpu", "r", 1, 1.0, 0, 0) }; xce_host_alloc_t* host[1] = { make_test_host_alloc("hostA", res, 1) }; xce_job_alloc_t* expected_alloc = make_test_job_alloc(42, host, 1); xce_alloc_t* res2[1] = { make_test_alloc("gpu", "r", 1, 1.0, 0, 0) }; xce_host_alloc_t* host2[1] = { make_test_host_alloc("hostA", res2, 1) }; xce_job_alloc_t* actual_alloc = make_test_job_alloc(43, host2, 1); replay_decision_t expected = make_replay_dispatch_decision(expected_alloc); xce_decision_t* actual = make_dispatch_decision(actual_alloc); char ja[64]; char jb[64]; char needle[160]; char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, NULL)); TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why)); fmt_jobid(ja, sizeof(ja), 42ULL); fmt_jobid(jb, sizeof(jb), 43ULL); snprintf(needle, sizeof(needle), "jobid: %s != %s", ja, jb); assert_why_names(why, needle); printf("test_dispatch_decisions_differing_jobid_mismatch: PASS\n"); } /* * test_dispatch_decisions_differing_from_timestamp_mismatch - * libxce-alloc-preason-equality Requirement 5.1 (inverts the former * bug-1838-followup "from still matches" lock): `from` used to be excluded * because make_alloc() stamped it with time(NULL), a real-OS-clock value * that differed between a recording and its replay. Every allocation a * session decides now carries that session's start (Requirement 4.1, * scheduler.c's sched_make_primary_alloc_()), and a replayed session starts * at its recorded beginRound, so the two sides agree for a faithful replay * and `from` is compared (by libxce's job_alloc_eq()). A `from` difference * is therefore a real divergence: it must be reported, and the description * must name `from` with both values. */ static void test_dispatch_decisions_differing_from_timestamp_mismatch(void) { xce_alloc_t* res_a[1] = { make_test_alloc("gpu", "resv1", 1, 2.0, 1790000000ULL, 1800) }; xce_host_alloc_t* host_a[1] = { make_test_host_alloc("hostA", res_a, 1) }; xce_job_alloc_t* expected_alloc = make_test_job_alloc(42, host_a, 1); xce_alloc_t* res_b[1] = { make_test_alloc("gpu", "resv1", 1, 2.0, 1ULL, 1800) }; xce_host_alloc_t* host_b[1] = { make_test_host_alloc("hostA", res_b, 1) }; xce_job_alloc_t* actual_alloc = make_test_job_alloc(42, host_b, 1); replay_decision_t expected = make_replay_dispatch_decision(expected_alloc); xce_decision_t* actual = make_dispatch_decision(actual_alloc); char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why)); assert_why_names(why, "allocs[0](host=hostA).allocs[0](resname=gpu)." "from: 1790000000 != 1"); printf("test_dispatch_decisions_differing_from_timestamp_mismatch: " "PASS\n"); } static void test_pend_decisions_with_identical_content_match(void) { replay_decision_t expected = make_replay_pend_decision(9001ULL, 3); xce_pendreason_t* reason = make_pendreason(9001ULL); xce_decision_t* actual; char* why; reason->reasons->main_reason = 3; actual = make_pend_decision(reason); why = (char*)"stale"; TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, &why)); TEST_ASSERT_NULL(why); printf("test_pend_decisions_with_identical_content_match: PASS\n"); } static void test_pend_decisions_differing_jobid_mismatch(void) { replay_decision_t expected = make_replay_pend_decision(9001ULL, 0); xce_decision_t* actual = make_pend_decision(make_pendreason(9002ULL)); char ja[64]; char jb[64]; char needle[160]; char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why)); fmt_jobid(ja, sizeof(ja), 9001ULL); fmt_jobid(jb, sizeof(jb), 9002ULL); snprintf(needle, sizeof(needle), "jobid: %s != %s", ja, jb); assert_why_names(why, needle); printf("test_pend_decisions_differing_jobid_mismatch: PASS\n"); } static void test_pend_decisions_differing_main_reason_mismatch(void) { replay_decision_t expected = make_replay_pend_decision(9001ULL, 3); xce_decision_t* actual = make_pend_decision(make_pendreason(9001ULL)); /* actual's main_reason stays 0 (make_pendreason()'s own fresh-calloc * default) -- differs from expected's 3. */ char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why)); assert_why_names(why, "main_reason: 3 != 0"); printf("test_pend_decisions_differing_main_reason_mismatch: PASS\n"); } /* * test_pend_decisions_differing_reason_detail_still_match - bug-1845 * regression: the recorded (expected) side represents a live cache-hit * PEND (core.h's documented "simulation isolation" -- sim/replay never * registers core->unsatisfiable_lookup_cb, so it can never itself produce * this shape, only ever receive it from a recorded normal-mode run) -- * main_reason set, zero host/resource reason detail * (sched_pend_from_cached_reason_(), scheduler.c, never adds any). The * live (actual) side represents replay's own full per-host computation for * the SAME job/round -- same main_reason, but with real host- and * resource-based reason detail attached (sched_try_alloc_on_host_(), * resource.c). Same semantic outcome (PEND, same main reason) at two * different, architecturally-legitimate detail levels -- must still * compare equal. */ static void test_pend_decisions_differing_reason_detail_still_match(void) { replay_decision_t expected = make_replay_pend_decision(9001ULL, 0); xce_pendreason_t* reason = make_pendreason(9001ULL); xce_host_reason_t* hreasons[1]; xce_res_reason_t* rreasons[2]; xce_decision_t* actual; hreasons[0] = (xce_host_reason_t*)calloc(1, sizeof(xce_host_reason_t)); hreasons[0]->host = strdup("workerA"); hreasons[0]->reason = 1; rreasons[0] = (xce_res_reason_t*)calloc(1, sizeof(xce_res_reason_t)); rreasons[0]->resource = strdup("mem"); rreasons[0]->host = strdup("workerA"); rreasons[0]->reason = 2; rreasons[1] = (xce_res_reason_t*)calloc(1, sizeof(xce_res_reason_t)); rreasons[1]->resource = strdup("slots"); rreasons[1]->host = strdup("workerA"); rreasons[1]->reason = 2; reason->reasons->main_reason = 0; reason->reasons->nhost_reasons = 1; reason->reasons->host_based_reasons = hreasons; reason->reasons->nres_reasons = 2; reason->reasons->res_based_reasons = rreasons; actual = make_pend_decision(reason); TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL)); printf("test_pend_decisions_differing_reason_detail_still_match: PASS\n"); } static void test_decision_kind_mismatch_reported(void) { replay_decision_t expected = make_replay_pend_decision(1ULL, 0); xce_job_alloc_t* alloc = make_test_job_alloc(1, NULL, 0); xce_decision_t* actual = make_dispatch_decision(alloc); /* Recorded PEND vs. live DISPATCH for the SAME jobid: still a * mismatch -- op/body_which disagreement is reported, not ignored. * The description names the kinds, recorded side first. */ char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why)); assert_why_names(why, "kind: PEND != DISPATCH"); printf("test_decision_kind_mismatch_reported: PASS\n"); } static void test_preempt_decision_always_reports_mismatch(void) { xce_preempt_t preempt; xce_decision_t actual; replay_decision_t expected; memset(&preempt, 0, sizeof(preempt)); preempt.jobid = 55ULL; memset(&actual, 0, sizeof(actual)); actual.op = XCE_DECISION_PREEMPT; actual.u.preempt = &preempt; /* Even a recorded entry that ALSO claims PREEMPT for the same jobid * still reports a mismatch -- recording_session.c's own documented * "no live path produces this decision type today" makes any PREEMPT * comparison unreachable-in-practice and therefore untrustworthy. */ memset(&expected, 0, sizeof(expected)); expected.body_which = ReplayDecision_body_preempt; char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, &actual, &why)); assert_why_names(why, "PREEMPT comparison not supported"); printf("test_preempt_decision_always_reports_mismatch: PASS\n"); } static void test_decision_eq_defends_against_null_inputs(void) { xce_decision_t* actual = make_pend_decision(make_pendreason(1ULL)); replay_decision_t expected = make_replay_pend_decision(1ULL, 0); char* why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual, NULL)); TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL, NULL)); TEST_ASSERT_FALSE(sim_decision_eq(NULL, NULL, NULL)); /* A missing side is named: which of the two has no decision. */ TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual, &why)); assert_why_names(why, "no recorded decision at this index"); why = NULL; TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL, &why)); assert_why_names(why, "no live decision"); printf("test_decision_eq_defends_against_null_inputs: PASS\n"); } /* ---------------------------------------------------------------------- */ /* Unit-level test: sim_log_mismatch()'s captured output content. */ /* ---------------------------------------------------------------------- */ static char* capture_stdout_around(void (*fn)(void)) { char tmp[] = "/tmp/xce_simverify_logcap_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* buf; TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0, "dup(STDOUT_FILENO) must " "succeed"); TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0, "dup2 redirect of STDOUT_FILENO must succeed"); fn(); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); buf = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(buf); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, buf, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); unlink(tmp); return buf; } static void log_one_mismatch_for_capture(void) { replay_decision_t expected = make_replay_pend_decision(9001ULL, 3); xce_decision_t* actual = make_pend_decision(make_pendreason(9001ULL)); sim_log_mismatch(12345ULL, 2, &expected, actual, "main_reason: 3 != 0"); } static void log_one_mismatch_without_why_for_capture(void) { replay_decision_t expected = make_replay_pend_decision(9001ULL, 3); xce_decision_t* actual = make_pend_decision(make_pendreason(9001ULL)); sim_log_mismatch(12345ULL, 2, &expected, actual, NULL); } static void test_log_mismatch_reports_logical_time_and_decision_detail(void) { char jobid_marker[80]; char jobid_buf[64]; char* buf = capture_stdout_around(log_one_mismatch_for_capture); /* bug-1591: sim_verify_describe_{replay_,}decision() now render jobids * via fmt_jobid(), not raw %llu -- this marker must match that same * human-readable form. */ fmt_jobid(jobid_buf, sizeof(jobid_buf), 9001ULL); snprintf(jobid_marker, sizeof(jobid_marker), "jobid=%s", jobid_buf); TEST_ASSERT_NOT_NULL_MESSAGE(strstr(buf, "MISMATCH"), "log_mismatch must report a MISMATCH"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(buf, "begin_round=12345"), "log_mismatch must report the session's exact logical time"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(buf, "decision_index=2"), "log_mismatch must report the decision's position within the " "session"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(buf, jobid_marker), "log_mismatch must report actionable decision detail (jobid)"); free(buf); printf("test_log_mismatch_reports_logical_time_and_decision_detail: " "PASS\n"); } /* * Requirement 5.3: the mismatch line keeps its logical-time and decision * context and appends the difference description it is given; with no * description (only when allocating it failed) the line is logged without * one, unchanged otherwise. */ static void test_log_mismatch_appends_provided_description(void) { char* buf = capture_stdout_around(log_one_mismatch_for_capture); char* nodiff; TEST_ASSERT_NOT_NULL(strstr(buf, "begin_round=12345")); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(buf, " -- diff: main_reason: 3 != 0"), "the mismatch line must append the provided description"); free(buf); nodiff = capture_stdout_around(log_one_mismatch_without_why_for_capture); TEST_ASSERT_NOT_NULL(strstr(nodiff, "MISMATCH")); TEST_ASSERT_NOT_NULL(strstr(nodiff, "decision_index=2")); TEST_ASSERT_NULL_MESSAGE(strstr(nodiff, "-- diff:"), "with no description the line carries no " "diff clause"); free(nodiff); printf("test_log_mismatch_appends_provided_description: PASS\n"); } /* Every line of `captured` that reports a MISMATCH also carries a * ` -- diff: ` clause (Requirement 5.3: each mismatch names its * difference). Returns the number of MISMATCH lines seen. */ static int assert_every_mismatch_line_has_diff(const char* captured) { const char* line = captured; int n = 0; while ((line != NULL) && (*line != '\0')) { const char* eol = strchr(line, '\n'); size_t len = (eol != NULL) ? (size_t)(eol - line) : strlen(line); char* one = strndup(line, len); TEST_ASSERT_NOT_NULL(one); if (strstr(one, "MISMATCH") != NULL) { n++; TEST_ASSERT_NOT_NULL_MESSAGE(strstr(one, " -- diff: "), one); } free(one); line = (eol != NULL) ? (eol + 1) : NULL; } return n; } /* ---------------------------------------------------------------------- */ /* Integration-level fixture helpers: a real .replay stream driving the */ /* REAL builtin_fcfs picker to a genuine live PEND decision. */ /* ---------------------------------------------------------------------- */ static void make_tmp_dir(char* out, size_t out_len, const char* suffix) { char tmpl[PATH_MAX]; snprintf(tmpl, sizeof(tmpl), "/tmp/sched_simverify%s_XXXXXX", suffix); TEST_ASSERT_NOT_NULL(mkdtemp(tmpl)); TEST_ASSERT_TRUE(strlen(tmpl) < out_len); strcpy(out, tmpl); } /* Builds a small, real xce_scheduler_t fixture purely to drive * sched_record_start() and produce a REAL, valid `.replay` header+snapshot * prefix (same technique test_sim_startup.c/test_sim_event_loop.c's own * build_fixture_scheduler() establish) -- but, unlike * test_sim_event_loop.c's copy, deliberately records schedPolicy= * "builtin_fcfs" (a string literal, never freed -- this throwaway fixture * scheduler is never torn down, matching this whole suite's documented * accepted-leak convention) so sim_open_and_validate_header() carries * "builtin_fcfs" into sim->core.job_pick_plugin and * sim_register_default_picker() (sim_loop.c) registers the REAL * builtin_fcfs plugin -- this file's own Observable needs the real picker, * not a stub, to prove sim_decision_eq() against a genuinely-produced live * decision. No picker needs to be key_add'ed into THIS throwaway * scheduler's own core.plugins: xce_sched_core_save_snapshot() does not * require a resolvable picker (Req 5 AC 21: "Picker state is NOT stored in * snapshot"). */ /* bug-1836 fix: sim_reconcile_unaccounted_jobs()'s new capacity gate calls * sched_get_worker() for every host named in a reconciled job's recorded * allocation, and treats "host not found" as a capacity miss -- this file's * own sched_check_host_satisfy() stub (link-satisfying only, resource.c is * not white-box #included here) always reports "satisfied" once reached, * but a bare, unregistered host name is now rejected BEFORE that stub is * even called. This helper registers a bare worker (no real resource * values needed -- the stub never reads them) so tests whose fixtures name * "hostA"/"hostC" in a recorded DISPATCH decision resolve correctly. */ static void add_bare_worker(xce_sched_core_t* core, const char* name) { xce_sched_worker_t* w = (xce_sched_worker_t*)calloc(1, sizeof(*w)); TEST_ASSERT_NOT_NULL(w); w->name = strdup(name); TEST_ASSERT_NOT_NULL(w->name); TEST_ASSERT_EQUAL_INT(0, zarray_init(&(w->resvals), 8, NULL)); TEST_ASSERT_EQUAL_INT(0, zarray_init(&(w->resInUse), 8, NULL)); TEST_ASSERT_TRUE(sched_add_worker(core, w) >= 0); } static void build_fixture_scheduler(xce_scheduler_t* sched, char* record_dir) { int rc; memset(sched, 0, sizeof(*sched)); mylock_init(&(sched->record_lock)); rc = xce_sched_core_init(&(sched->core)); TEST_ASSERT_EQUAL_INT(0, rc); sched->core.job_pick_plugin = (char*)"builtin_fcfs"; sched->record_dir = record_dir; /* bug-1836 fix: registered once here (captured into every fixture's * initial snapshot) rather than per-test, since several tests in this * file build a recorded DISPATCH decision naming "hostA"/"hostC" -- * harmless for tests that never reference these hosts. */ add_bare_worker(&(sched->core), "hostA"); add_bare_worker(&(sched->core), "hostC"); } /* Same "leave the file open, skip sched_record_stop()'s own endSession * marker" technique test_sim_event_loop.c's own write_header_and_snapshot() * establishes -- see that function's doc comment for the full rationale. */ static FILE* write_header_and_snapshot(char* out_path, size_t out_path_len, const char* suffix) { char record_dir[PATH_MAX]; xce_scheduler_t sched; int rc; make_tmp_dir(record_dir, sizeof(record_dir), suffix); build_fixture_scheduler(&sched, record_dir); rc = sched_record_start(&sched, "fixture.replay"); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_TRUE(strlen(sched.record_file_path) < out_path_len); strcpy(out_path, sched.record_file_path); return sched.record_file; } /* Appends one ReplayEvent{queueUpdate} message -- creates `queue_name` in * the core (core_events.c's core_upsert_queue(), fires * on_queue_update(picker, q) against the now-registered real builtin_fcfs). * status/priority are set but never gate readiness: core.c's own * sched_queue_is_ready() is an unconditional `return 1` stub (task 2.4's * still-live documented scope note) -- any value here works. */ static void append_queue_update_event(FILE* f, uint64_t ts, const char* queue_name) { xce_queue_t queue; replay_event_t src; struct capn c; struct capn_segment* cs; ReplayEvent_ptr p; memset(&queue, 0, sizeof(queue)); queue.queue = (char*)queue_name; queue.status = QUEUE_STAT_OPEN; queue.priority = 1; memset(&src, 0, sizeof(src)); src.ts = ts; src.body_which = ReplayEvent_body_queueUpdate; src.body.queue_update = &queue; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplayEvent_ptr(cs, &p, &src); capn_setp(capn_root(&c), 0, p.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplayEvent{jobsPush} message with exactly one * JOB_STAT_PEND job assigned to `queue_name` -- MUST be appended AFTER * append_queue_update_event() for the same queue name within the same * session's accumulated batch: core_build_job() (core_events.c) looks the * queue up via sched_get_queue() with no get-or-create fallback (unlike * user/project), so the queue must already exist. Drives * core_events.c's real JOBS_PUSH handling: sched_add_job() + * on_job_push(picker, job) against the real builtin_fcfs -- which (since * sched_job_is_ready()/sched_queue_is_ready() are unconditional `return 1` * stubs) makes this job an immediate next_job() candidate, so the picker * thread PENDs it for real via make_pend_decision(make_pendreason(jobid)) * (sim_loop.c's own sim_picker_thread_fn). */ static void append_jobs_push_event(FILE* f, uint64_t ts, uint64_t jobid, const char* queue_name) { job_submit_data_t submit; xce_job_t job; xce_job_t* jobs[1]; xce_jobs_data_t jobs_data; replay_event_t src; struct capn c; struct capn_segment* cs; ReplayEvent_ptr p; memset(&submit, 0, sizeof(submit)); submit.queue = (char*)queue_name; submit.user = (char*)"alice"; submit.mintasks = 1; submit.maxtasks = 1; memset(&job, 0, sizeof(job)); job.jobid = jobid; job.state = JOB_STAT_PEND; job.submit = &submit; jobs[0] = &job; memset(&jobs_data, 0, sizeof(jobs_data)); jobs_data.n_total = 1; jobs_data.count = 1; jobs_data.jobs = jobs; memset(&src, 0, sizeof(src)); src.ts = ts; src.body_which = ReplayEvent_body_jobsPush; src.body.jobs_push = &jobs_data; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplayEvent_ptr(cs, &p, &src); capn_setp(capn_root(&c), 0, p.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends the .replay stream's whole-stream termination marker * (ReplayEvent{endSession}, Req 5 AC 20) -- same technique * test_sim_event_loop.c's own append_end_of_stream_marker() establishes. */ static void append_end_of_stream_marker(FILE* f) { replay_event_t src; struct capn c; struct capn_segment* cs; ReplayEvent_ptr p; memset(&src, 0, sizeof(src)); src.body_which = ReplayEvent_body_endSession; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplayEvent_ptr(cs, &p, &src); capn_setp(capn_root(&c), 0, p.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplaySession carrying exactly one recorded PendDecision * (jobid, main_reason) -- the "recorded output" side of this file's own * Observable, built via the real encode_ReplaySession_ptr() codec * (recording_session.c's own real encode call site, not a hand-rolled * byte layout). */ static void append_replay_session_with_pend(FILE* f, uint64_t begin_round, uint64_t jobid, int main_reason) { xce_job_preason_t reason; sched_pend_decision_t pend; replay_decision_t decision; replay_decision_t* decisions[1]; replay_session_t session; struct capn c; struct capn_segment* cs; ReplaySession_ptr sp; memset(&reason, 0, sizeof(reason)); reason.main_reason = main_reason; memset(&pend, 0, sizeof(pend)); pend.jobid = jobid; pend.reason = &reason; memset(&decision, 0, sizeof(decision)); decision.ts = begin_round; decision.body_which = ReplayDecision_body_pend; decision.body.pend = &pend; decisions[0] = &decision; memset(&session, 0, sizeof(session)); session.begin_round = begin_round; session.decisions = decisions; session.n_decisions = 1; session.summary = NULL; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplaySession_ptr(cs, &sp, &session); capn_setp(capn_root(&c), 0, sp.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplayEvent{jobStateChange} message -- scheduler-sim- * decision-parity spec, task 3 (Req 1.3/1.4, 2.1/2.2, 5.2): this is the * mechanism (core_events.c's XCE_CORE_EVENT_JOB_STATE_CHANGE handling) * that flips a job to RUN state during replay independently of any * decision -- it does NOT set job->accounts/job->allocs (core_events.c * only clears them when leaving RUN state), which is exactly what leaves * a RUN-state job with job->accounts == NULL for Recorded-Decision * Reconciliation (sim_reconcile_unaccounted_jobs(), sim_loop.c) to pick * up. Same encode/write technique append_queue_update_event() above * establishes. */ static void append_job_state_change_event(FILE* f, uint64_t ts, uint64_t jobid, uint32_t old_state, uint32_t new_state) { job_state_change_event_t jsc; replay_event_t src; struct capn c; struct capn_segment* cs; ReplayEvent_ptr p; memset(&jsc, 0, sizeof(jsc)); jsc.jobid = jobid; jsc.old_state = old_state; jsc.new_state = new_state; jsc.exit_code = 0; memset(&src, 0, sizeof(src)); src.ts = ts; src.body_which = ReplayEvent_body_jobStateChange; src.body.job_state_change = &jsc; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplayEvent_ptr(cs, &p, &src); capn_setp(capn_root(&c), 0, p.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplaySession carrying TWO recorded decisions -- a DISPATCH * for `dispatch_alloc`'s jobid and a PEND for `pend_jobid` -- scheduler- * sim-decision-parity spec, task 3 (Req 5.2). Reuses * make_replay_dispatch_decision()/make_replay_pend_decision() (this file's * own unit-level fixture helpers, above) rather than hand-rolling the * decision shape a second time. Mirrors append_replay_session_with_pend()'s * own encode_ReplaySession_ptr() technique, just with two decisions * instead of one. */ static void append_replay_session_with_dispatch_and_pend( FILE* f, uint64_t begin_round, xce_job_alloc_t* dispatch_alloc, uint64_t pend_jobid, int pend_main_reason) { replay_decision_t d_dispatch; replay_decision_t d_pend; replay_decision_t* decisions[2]; replay_session_t session; struct capn c; struct capn_segment* cs; ReplaySession_ptr sp; d_dispatch = make_replay_dispatch_decision(dispatch_alloc); d_dispatch.ts = begin_round; d_pend = make_replay_pend_decision(pend_jobid, pend_main_reason); d_pend.ts = begin_round; decisions[0] = &d_dispatch; decisions[1] = &d_pend; memset(&session, 0, sizeof(session)); session.begin_round = begin_round; session.decisions = decisions; session.n_decisions = 2; session.summary = NULL; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplaySession_ptr(cs, &sp, &session); capn_setp(capn_root(&c), 0, sp.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplaySession carrying a SINGLE recorded DISPATCH decision, * with no accompanying PEND -- bug-1759/bug-1779 correction (scheduler-sim- * decision-parity): sim_reconcile_unaccounted_jobs() now calls the picker's * on_dispatch() the moment it reconstructs a job's allocation, retiring it * from the picker's own pendJobs immediately -- so the live picker no * longer independently re-offers/re-PENDs a job in the SAME round its * DISPATCH is reconciled (previously append_replay_session_with_dispatch_ * and_pend(), just below, modeled that now-corrected stale-PEND echo). */ static void append_replay_session_with_dispatch(FILE* f, uint64_t begin_round, xce_job_alloc_t* dispatch_alloc) { replay_decision_t d_dispatch; replay_decision_t* decisions[1]; replay_session_t session; struct capn c; struct capn_segment* cs; ReplaySession_ptr sp; d_dispatch = make_replay_dispatch_decision(dispatch_alloc); d_dispatch.ts = begin_round; decisions[0] = &d_dispatch; memset(&session, 0, sizeof(session)); session.begin_round = begin_round; session.decisions = decisions; session.n_decisions = 1; session.summary = NULL; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplaySession_ptr(cs, &sp, &session); capn_setp(capn_root(&c), 0, sp.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplaySession carrying THREE recorded decisions -- a DISPATCH * for `dispatch_alloc_a`'s jobid, a PEND for `pend_jobid`, and a DISPATCH * for `dispatch_alloc_c`'s jobid (in that order: the still-PEND job's own * recorded PEND sits BETWEEN the two DISPATCH entries) -- scheduler-sim- * decision-parity spec, task 4.2 regression coverage (Req 5.1, 5.2): proves * sim_reconcile_unaccounted_jobs()'s memmove-based compaction correctly * consumes TWO, non-adjacent entries from the same round's * decoded_session->decisions (task 3's own * append_replay_session_with_dispatch_and_pend() above only ever needed to * consume a single, leading entry), leaving the middle PEND entry correctly * repositioned for the positional --sim-verify walk. Reuses * make_replay_dispatch_decision()/make_replay_pend_decision() the same way * append_replay_session_with_dispatch_and_pend() does. */ static void append_replay_session_with_two_dispatches_and_pend( FILE* f, uint64_t begin_round, xce_job_alloc_t* dispatch_alloc_a, uint64_t pend_jobid, int pend_main_reason, xce_job_alloc_t* dispatch_alloc_c) { replay_decision_t d_dispatch_a; replay_decision_t d_pend; replay_decision_t d_dispatch_c; replay_decision_t* decisions[3]; replay_session_t session; struct capn c; struct capn_segment* cs; ReplaySession_ptr sp; d_dispatch_a = make_replay_dispatch_decision(dispatch_alloc_a); d_dispatch_a.ts = begin_round; d_pend = make_replay_pend_decision(pend_jobid, pend_main_reason); d_pend.ts = begin_round; d_dispatch_c = make_replay_dispatch_decision(dispatch_alloc_c); d_dispatch_c.ts = begin_round; decisions[0] = &d_dispatch_a; decisions[1] = &d_pend; decisions[2] = &d_dispatch_c; memset(&session, 0, sizeof(session)); session.begin_round = begin_round; session.decisions = decisions; session.n_decisions = 3; session.summary = NULL; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplaySession_ptr(cs, &sp, &session); capn_setp(capn_root(&c), 0, sp.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplaySession carrying TWO recorded PendDecisions -- scheduler- * sim-decision-parity spec, task 4.2 regression coverage: a still-genuinely- * pending job is re-offered (and re-PENDed) by the real picker EVERY round * it remains unresolved, not just the round it first arrives in * (builtin_fcfs.c's own pendJobs discipline: "A non-pending job is left * removed and not reinserted" -- the converse holds too, a job that stays * PEND is never removed from its queue's pendJobs, so next_job() keeps * offering it every subsequent round). Used for this file's own second * round, where the still-PEND job from round 1 recurs alongside a fresh * job. */ static void append_replay_session_with_two_pends(FILE* f, uint64_t begin_round, uint64_t jobid1, int main_reason1, uint64_t jobid2, int main_reason2) { replay_decision_t d1; replay_decision_t d2; replay_decision_t* decisions[2]; replay_session_t session; struct capn c; struct capn_segment* cs; ReplaySession_ptr sp; d1 = make_replay_pend_decision(jobid1, main_reason1); d1.ts = begin_round; d2 = make_replay_pend_decision(jobid2, main_reason2); d2.ts = begin_round; decisions[0] = &d1; decisions[1] = &d2; memset(&session, 0, sizeof(session)); session.begin_round = begin_round; session.decisions = decisions; session.n_decisions = 2; session.summary = NULL; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplaySession_ptr(cs, &sp, &session); capn_setp(capn_root(&c), 0, sp.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Appends one ReplaySession carrying ZERO recorded decisions -- the same * shape a genuinely idle round produces in a real recording * (recording_session.c's own sched_record_session_finalize() writes * decisions=NULL/n_decisions=0 whenever a round's ctx->n stays 0, i.e. no * sched_record_session_add_decision() call happened that round). Used by * scheduler-sim-eager-reconciliation task 3.2's round-separated-dispatch * test below for the round a job's JOB_STATE_CHANGE-to-RUN confirmation * removes it from the live picker's own pendJobs before that round's * picker session ever runs, leaving nothing to record. */ static void append_empty_replay_session(FILE* f, uint64_t begin_round) { replay_session_t session; struct capn c; struct capn_segment* cs; ReplaySession_ptr sp; memset(&session, 0, sizeof(session)); session.begin_round = begin_round; session.decisions = NULL; session.n_decisions = 0; session.summary = NULL; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_ReplaySession_ptr(cs, &sp, &session); capn_setp(capn_root(&c), 0, sp.p); TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); capn_free(&c); } /* Runs sched_sim_driver_run() against `path` with --sim-verify set to * `verify`, capturing everything written to stdout during the call (the * same dup()/dup2() technique capture_stdout_around() above uses, inlined * here since sched_sim_driver_run() needs live arguments, not a bare * void(void) callback). */ static char* run_sim_and_capture_stdout(const char* path, bool verify, int* out_rc) { sched_sim_args_t args; xce_sched_sim_driver_t sim; char tmp[] = "/tmp/xce_simverify_run_XXXXXX"; int fd = mkstemp(tmp); int saved_stdout; off_t len; char* buf; TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "mkstemp for stdout capture must " "succeed"); fflush(stdout); saved_stdout = dup(STDOUT_FILENO); TEST_ASSERT_TRUE_MESSAGE(saved_stdout >= 0, "dup(STDOUT_FILENO) must " "succeed"); TEST_ASSERT_TRUE_MESSAGE(dup2(fd, STDOUT_FILENO) >= 0, "dup2 redirect of STDOUT_FILENO must succeed"); memset(&args, 0, sizeof(args)); args.replay_path = path; args.sim_start_ts = SCHED_SIM_START_TS_LATEST; args.verify = verify; *out_rc = sched_sim_driver_run(&sim, &args); fflush(stdout); dup2(saved_stdout, STDOUT_FILENO); close(saved_stdout); len = lseek(fd, 0, SEEK_CUR); TEST_ASSERT_TRUE(len >= 0); buf = (char*)calloc(1, (size_t)len + 1); TEST_ASSERT_NOT_NULL(buf); TEST_ASSERT_TRUE(lseek(fd, 0, SEEK_SET) == 0); if (len > 0) { ssize_t nread = read(fd, buf, (size_t)len); TEST_ASSERT_TRUE(nread == len); } close(fd); unlink(tmp); return buf; } /* ---------------------------------------------------------------------- */ /* Integration-level tests: full sched_sim_driver_run(), --sim-verify. */ /* ---------------------------------------------------------------------- */ /* * Observable (a): replaying a fixture against its own previously-recorded * output (here: the exact PEND decision the real picker deterministically * produces -- main_reason=0, make_pendreason()'s own fresh-calloc default) * with verification enabled reports ZERO mismatches. */ static void test_verify_against_self_reports_zero_mismatches(void) { char path[PATH_MAX]; FILE* f; int rc; char* captured; f = write_header_and_snapshot(path, sizeof(path), "_selfmatch"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); append_jobs_push_event(f, 1500ULL, 777ULL, "normal"); append_replay_session_with_pend(f, 2000ULL, 777ULL, 0); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, true, &rc); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "MISMATCH"), "verifying a fixture against its own previously-recorded decision " "must report zero mismatches"); free(captured); printf("test_verify_against_self_reports_zero_mismatches: PASS\n"); } /* * bug-1632/bug-1763: sched_sim_driver_run() must not return with its picker * and allocator threads still running. Nearly every caller (this file's * run_sim_and_capture_stdout() among them) runs the driver on a stack-local * xce_sched_sim_driver_t and never calls sched_sim_driver_fini(); threads * left running kept a pointer into that dead frame, and ASAN caught one * reading it (stack-use-after-return in cbuf_get). */ static void test_run_returns_with_its_threads_stopped(void) { char path[PATH_MAX]; FILE* f; sched_sim_args_t args; xce_sched_sim_driver_t sim; int rc; f = write_header_and_snapshot(path, sizeof(path), "_threads"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); append_jobs_push_event(f, 1500ULL, 777ULL, "normal"); append_replay_session_with_pend(f, 2000ULL, 777ULL, 0); append_end_of_stream_marker(f); fclose(f); memset(&args, 0, sizeof(args)); args.replay_path = path; args.sim_start_ts = SCHED_SIM_START_TS_LATEST; rc = sched_sim_driver_run(&sim, &args); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_FALSE_MESSAGE(sim.allocator_thread_live, "bug-1632: sched_sim_driver_run() returned with its picker and " "allocator threads still running"); sched_sim_driver_fini(&sim); printf("test_run_returns_with_its_threads_stopped: PASS\n"); } /* * Observable (b): replaying against a deliberately-altered copy (recorded * main_reason flipped to 99, which the real picker thread can never * produce -- make_pendreason() always calloc()s a fresh, all-zero * xce_job_preason_t) reports the EXACT mismatch: MISMATCH is logged, * naming the specific job. */ static void test_verify_against_altered_copy_reports_exact_mismatch(void) { char path[PATH_MAX]; char jobid_marker[80]; char jobid_buf[64]; FILE* f; int rc; char* captured; f = write_header_and_snapshot(path, sizeof(path), "_altered"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); append_jobs_push_event(f, 1500ULL, 777ULL, "normal"); append_replay_session_with_pend(f, 2000ULL, 777ULL, 99); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, true, &rc); /* bug-1591: sim_verify_describe_{replay_,}decision() now render * jobids via fmt_jobid(), not raw %llu -- this marker must match that * same human-readable form. */ fmt_jobid(jobid_buf, sizeof(jobid_buf), 777ULL); snprintf(jobid_marker, sizeof(jobid_marker), "jobid=%s", jobid_buf); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "MISMATCH"), "verifying against a deliberately-altered recorded decision must " "report a mismatch"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, jobid_marker), "the mismatch report must name the specific job"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, " -- diff: main_reason: 99 != 0"), "the mismatch report must name the differing field and both values"); TEST_ASSERT_EQUAL_INT(1, assert_every_mismatch_line_has_diff(captured)); free(captured); printf("test_verify_against_altered_copy_reports_exact_mismatch: " "PASS\n"); } /* * A recorded decision left over after the live stream ends (recorded * PEND(777), PEND(778); only job 777 was ever pushed, so the live picker * emits only PEND(777)) is reported as a mismatch whose diff says which * side is missing: "no live decision" (Requirement 5.3). */ static void test_verify_leftover_recorded_decision_names_missing_side(void) { char path[PATH_MAX]; FILE* f; int rc; char* captured; f = write_header_and_snapshot(path, sizeof(path), "_leftover"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); append_jobs_push_event(f, 1500ULL, 777ULL, "normal"); append_replay_session_with_two_pends(f, 2000ULL, 777ULL, 0, 778ULL, 0); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, true, &rc); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "decision_index=1"), "the left-over recorded decision must be reported at its index"); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, " -- diff: no live decision"), "a left-over recorded decision's mismatch must name the missing " "live side"); TEST_ASSERT_EQUAL_INT(1, assert_every_mismatch_line_has_diff(captured)); free(captured); printf("test_verify_leftover_recorded_decision_names_missing_side: " "PASS\n"); } /* * sim->verify == false must skip verification entirely, even against the * SAME deliberately-altered fixture the previous test proves DOES mismatch * when verification is on -- proves the --sim-verify gate itself, not just * the comparator. */ static void test_verify_disabled_reports_no_mismatches_even_when_altered(void) { char path[PATH_MAX]; FILE* f; int rc; char* captured; f = write_header_and_snapshot(path, sizeof(path), "_verifyoff"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); append_jobs_push_event(f, 1500ULL, 777ULL, "normal"); append_replay_session_with_pend(f, 2000ULL, 777ULL, 99); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, false, &rc); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "MISMATCH"), "sim->verify == false must never report a mismatch, even against a " "fixture the verify-enabled test proves mismatches"); free(captured); printf("test_verify_disabled_reports_no_mismatches_even_when_altered: " "PASS\n"); } /* * test_verify_with_already_resolved_job_reports_no_mismatch -- scheduler- * sim-decision-parity spec, task 3 (Req 5.2): --sim-verify against a * recording containing an already-resolved job (job A: pushed PEND, then * flipped to RUN by a JOB_STATE_CHANGE event in the SAME round's event * batch, with a matching recorded DISPATCH decision this round -- the * Recorded-Decision Reconciliation trigger shape) must report ZERO * mismatches, alongside a genuinely-live-pended job (job B) whose own * recorded PEND decision the real picker independently reproduces. * * Before task 3's round-boundary wiring, sim_reconcile_unaccounted_jobs() * was never called, so job A's RUN state was never reconciled: the live * picker never independently re-decides an already-RUN job (it only ever * offers PEND-state jobs), so the live-drained decision stream has only * job B's PEND -- one entry -- while decoded_session->decisions still * carries BOTH the DISPATCH(A) and PEND(B) entries (neither ever * consumed). The positional verify walk then finds job A's recorded * DISPATCH left over with no corresponding live decision once the live * stream ends, and reports it as a mismatch (expected=DISPATCH(A), * actual=NULL) -- a false positive entirely attributable to this * feature's own gap, not a real behavioral divergence. After task 3's * fix, reconciliation consumes and removes job A's DISPATCH entry from * decoded_session BEFORE this verify loop ever runs, so only job B's PEND * remains to compare against the live stream's own single PEND -- zero * mismatches. */ static void test_verify_with_already_resolved_job_reports_no_mismatch(void) { char path[PATH_MAX]; FILE* f; int rc; char* captured; xce_alloc_t* res[1]; xce_host_alloc_t* host[1]; xce_job_alloc_t* alloc; f = write_header_and_snapshot(path, sizeof(path), "_reconciled"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); /* Job A (501): already resolved before this round's live picker ever * considers it -- pushed PEND, then flipped to RUN in the SAME event * batch, so it never reaches next_job() this round. Job B (502): * genuinely still PEND -- the real picker independently re-pends it. */ append_jobs_push_event(f, 1500ULL, 501ULL, "normal"); append_jobs_push_event(f, 1500ULL, 502ULL, "normal"); append_job_state_change_event(f, 1800ULL, 501ULL, JOB_STAT_PEND, JOB_STAT_RUN); res[0] = make_test_alloc("slots", "run", 1, 4.0, SESSION_FROM(ROUND1_BEGIN_NS), 0); host[0] = make_test_host_alloc("hostA", res, 1); alloc = make_test_job_alloc(501ULL, host, 1); append_replay_session_with_dispatch_and_pend(f, ROUND1_BEGIN_NS, alloc, 502ULL, 0); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, true, &rc); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "MISMATCH"), "reconciling an already-resolved job's recorded DISPATCH decision " "must not produce a false --sim-verify mismatch (Requirement 5.2)"); free(captured); printf("test_verify_with_already_resolved_job_reports_no_mismatch: " "PASS\n"); } /* * test_verify_with_multiple_already_resolved_jobs_across_rounds_reports_no_mismatch * -- scheduler-sim-decision-parity spec, task 4.2 (Req 5.1, 5.2): extends * task 3's own single-already-resolved-job proof * (test_verify_with_already_resolved_job_reports_no_mismatch above) to a * MULTI-job, MULTI-round scenario: * * Round 1: TWO already-resolved jobs (A=501, C=503) plus ONE genuinely * still-PEND job (B=502). The round's recorded decisions are * [DISPATCH(A), PEND(B), DISPATCH(C)] -- the still-pending job's own * recorded PEND sits BETWEEN the two DISPATCH entries reconciliation * must consume, exercising the memmove-based compaction's non-adjacent- * removal path (task 3's own test only ever consumed a single, leading * entry). * * Round 2: one ordinary, freshly-arriving job (D=504), no reconciliation * needed at all -- proves the PRIOR round's compaction has no * cross-round effect (design.md's Implementation Notes: "decoded_session * is decoded fresh once per round ... no cross-round effect") and that * --sim-verify's positional walk resumes correctly at di=0 for the new * round. * * Zero mismatches across both rounds is the Observable. */ static void test_verify_with_multiple_already_resolved_jobs_across_rounds_reports_no_mismatch( void) { char path[PATH_MAX]; FILE* f; int rc; char* captured; xce_alloc_t* res_a[1]; xce_host_alloc_t* host_a[1]; xce_job_alloc_t* alloc_a; xce_alloc_t* res_c[1]; xce_host_alloc_t* host_c[1]; xce_job_alloc_t* alloc_c; f = write_header_and_snapshot(path, sizeof(path), "_reconciled_multi"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); /* Jobs A (501) and C (503): both already resolved before this round's * live picker ever considers them. Job B (502): genuinely still PEND * -- the real picker independently re-pends it. */ append_jobs_push_event(f, 1500ULL, 501ULL, "normal"); append_jobs_push_event(f, 1500ULL, 502ULL, "normal"); append_jobs_push_event(f, 1500ULL, 503ULL, "normal"); append_job_state_change_event(f, 1800ULL, 501ULL, JOB_STAT_PEND, JOB_STAT_RUN); append_job_state_change_event(f, 1801ULL, 503ULL, JOB_STAT_PEND, JOB_STAT_RUN); res_a[0] = make_test_alloc("slots", "run", 1, 4.0, SESSION_FROM(ROUND1_BEGIN_NS), 0); host_a[0] = make_test_host_alloc("hostA", res_a, 1); alloc_a = make_test_job_alloc(501ULL, host_a, 1); res_c[0] = make_test_alloc("slots", "run", 1, 2.0, SESSION_FROM(ROUND1_BEGIN_NS), 0); host_c[0] = make_test_host_alloc("hostC", res_c, 1); alloc_c = make_test_job_alloc(503ULL, host_c, 1); /* Recorded order: DISPATCH(A), PEND(B), DISPATCH(C) -- B's entry is * sandwiched between the two entries reconciliation consumes. */ append_replay_session_with_two_dispatches_and_pend(f, ROUND1_BEGIN_NS, alloc_a, 502ULL, 0, alloc_c); /* Round 2: job B (502) is still genuinely PEND (never dispatched, never * state-changed) so builtin_fcfs's pendJobs discipline re-offers it to * the live picker again this round -- it is NOT an already-resolved * job, so reconciliation has nothing to do with it; it simply recurs * as an ordinary live PEND, same as job D (504), the fresh arrival. * Recorded order [502, 504] matches queue_time ordering (502 queued at * ts=1500, before 504's ts=2500) -- no already-resolved job is in the * mix at all this round, proving no cross-round bleed from round 1's * own compaction. */ append_jobs_push_event(f, 2500ULL, 504ULL, "normal"); append_replay_session_with_two_pends(f, 3000ULL, 502ULL, 0, 504ULL, 0); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, true, &rc); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NULL_MESSAGE( strstr(captured, "MISMATCH"), "reconciling TWO already-resolved jobs in the same round (with a " "genuinely-pending job's own recorded decision sandwiched between " "them), followed by an ordinary second round with no reconciliation " "needed, must not produce a false --sim-verify mismatch anywhere " "(Requirement 5.1, 5.2)"); free(captured); printf("test_verify_with_multiple_already_resolved_jobs_across_rounds_" "reports_no_mismatch: PASS\n"); } /* * test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_mismatch -- * RENAMED and INVERTED (2026-09-21, bug-1838, scheduler-driver-equivalence): * this test used to be test_sim_verify_no_false_mismatch_across_round_ * separated_dispatch, proving scheduler-sim-eager-reconciliation's Req 4.2 * ("a dispatch decision and its RUN-state confirmation recorded several * rounds apart must not produce a false --sim-verify mismatch") via * sim_reconcile_unaccounted_jobs() force-applying job 601's recorded round-1 * DISPATCH decision while it was still PEND (the gate was state-independent * at the time). * * bug-1838 narrowed that gate back to JOB_IS_RUN-only (see sim_loop.c's own * updated header comment on sim_reconcile_unaccounted_jobs() for the full * rationale: scheduler-sim-live-allocation, completed after this spec, gave * sim mode a genuine capacity-checked live picker/allocator path, making * reconciliation's PEND-inclusive widening redundant for an ordinary * cross-round gap -- the live path now independently re-derives the SAME * decision the recording made). Job 601 here is freshly PEND at round 1, so * it is no longer a reconciliation candidate at all -- but THIS FILE's own * sched_job_picker()/sched_allocator() pair (see this file's header comment, * "Genuine resource-based dispatch... is proved separately by * test_sim_live_allocation.c") is a deliberate PEND-only stub, incapable of * ever producing a live DISPATCH. With neither mechanism able to dispatch * job 601 anymore, comparing the live stream's PEND(601) against the * recorded DISPATCH(601) now correctly reports a MISMATCH -- not a false * one: this fixture's own environment genuinely cannot reproduce what the * recording claims happened, and --sim-verify is supposed to say so. * * The genuine, mismatch-free proof of Req 4.2 (a real dispatch-then-later- * confirmation gap causing no false mismatch) now lives in * test_sim_live_allocation.c's test_sim_verify_no_false_mismatch_after_ * live_dispatch_and_later_confirmation, which links the real scheduler_srcs * and can genuinely dispatch job 601 the way this file's stub never could * -- see that test's own header comment for the full mirrored rationale. * This test is kept, inverted, as a regression lock on the NEW behavior: * reconciliation must not "help" a PEND job here anymore, even when a * matching recorded DISPATCH entry exists. */ static void test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_mismatch(void) { char path[PATH_MAX]; FILE* f; int rc; char* captured; xce_alloc_t* res[1]; xce_host_alloc_t* host[1]; xce_job_alloc_t* alloc; f = write_header_and_snapshot(path, sizeof(path), "_roundsep"); TEST_ASSERT_NOT_NULL(f); append_queue_update_event(f, 1000ULL, "normal"); append_jobs_push_event(f, 1500ULL, 601ULL, "normal"); res[0] = make_test_alloc("slots", "run", 1, 4.0, SESSION_FROM(ROUND1_BEGIN_NS), 0); host[0] = make_test_host_alloc("hostA", res, 1); alloc = make_test_job_alloc(601ULL, host, 1); /* Round 1: job 601 pushed PEND this same round; recorded decisions = * [DISPATCH(601)]. bug-1838: sim_reconcile_unaccounted_jobs() is now * JOB_IS_RUN-only, so it does not touch job 601 (still PEND) at all -- * this file's own PEND-only sched_allocator() stub (see this file's * header comment) is the only thing that CAN decide it, and it always * PENDs. The live stream therefore produces PEND(601) against a * recorded DISPATCH(601) -- a genuine, correctly-reported mismatch. */ append_replay_session_with_dispatch(f, ROUND1_BEGIN_NS, alloc); /* Round 2: job 601 is STILL a picker candidate (nothing retired it -- * reconciliation never touched it, and the stub allocator's on_pend() * path doesn't retire candidates either), so the live stream produces * ANOTHER PEND(601) here against this round's empty recorded * decisions -- also a genuine mismatch, not asserted on individually * (this test only checks that at least one mismatch is reported). */ append_empty_replay_session(f, 3000ULL); append_end_of_stream_marker(f); fclose(f); captured = run_sim_and_capture_stdout(path, true, &rc); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_NOT_NULL_MESSAGE( strstr(captured, "MISMATCH"), "a PEND-state job's recorded DISPATCH decision must no longer be " "force-applied by reconciliation (bug-1838) -- this file's " "PEND-only live stub cannot reproduce it either, so --sim-verify " "must correctly report a real mismatch here, not silently agree " "with a decision nothing in this environment actually made"); /* Round 1: recorded DISPATCH vs live PEND; round 2: a live PEND with * no recorded entry. Both lines name their difference. */ TEST_ASSERT_NOT_NULL(strstr(captured, " -- diff: kind: DISPATCH != PEND")); TEST_ASSERT_NOT_NULL( strstr(captured, " -- diff: no recorded decision at this index")); TEST_ASSERT_TRUE(assert_every_mismatch_line_has_diff(captured) >= 2); free(captured); printf("test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_" "mismatch: PASS\n"); } int main(void) { UNITY_BEGIN(); RUN_TEST(test_dispatch_decisions_with_identical_content_match); RUN_TEST(test_dispatch_decisions_differing_jobid_mismatch); RUN_TEST(test_dispatch_decisions_differing_from_timestamp_mismatch); RUN_TEST(test_pend_decisions_with_identical_content_match); RUN_TEST(test_pend_decisions_differing_jobid_mismatch); RUN_TEST(test_pend_decisions_differing_main_reason_mismatch); RUN_TEST(test_pend_decisions_differing_reason_detail_still_match); RUN_TEST(test_decision_kind_mismatch_reported); RUN_TEST(test_preempt_decision_always_reports_mismatch); RUN_TEST(test_decision_eq_defends_against_null_inputs); RUN_TEST(test_log_mismatch_reports_logical_time_and_decision_detail); RUN_TEST(test_log_mismatch_appends_provided_description); RUN_TEST(test_verify_against_self_reports_zero_mismatches); RUN_TEST(test_verify_against_altered_copy_reports_exact_mismatch); RUN_TEST(test_verify_leftover_recorded_decision_names_missing_side); RUN_TEST(test_verify_disabled_reports_no_mismatches_even_when_altered); RUN_TEST(test_verify_with_already_resolved_job_reports_no_mismatch); RUN_TEST( test_verify_with_multiple_already_resolved_jobs_across_rounds_reports_no_mismatch); RUN_TEST( test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_mismatch); RUN_TEST(test_run_returns_with_its_threads_stopped); return UNITY_END(); }