From 4e57bde6937dea7059ae6e0cdb21790b58ae67ad Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 29 Sep 2026 06:48:38 +0000 Subject: [PATCH] feat(libxce-alloc-preason-equality): replay reproduces the reservation start time (task 5.3) Add test_sim_replay_reproduces_recorded_alloc_from: a recorded DISPATCH at an epoch-scale beginRound with a sub-second part replays at the default simulation settings with no --sim-verify mismatch, and the live allocation's from equals beginRound in whole seconds. Share the dispatch fixture setup of the existing tests through two helpers. Live: 2-round HA soak recordings (11 DISPATCH decisions) replayed with --sim-verify and no --sim-speed report 0 mismatches. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../libxce-alloc-preason-equality/tasks.md | 5 +- .../testsuite/test_sim_live_allocation.c | 280 ++++++++++-------- 2 files changed, 166 insertions(+), 119 deletions(-) diff --git a/.kiro/specs/libxce-alloc-preason-equality/tasks.md b/.kiro/specs/libxce-alloc-preason-equality/tasks.md index e604088a..a970ec19 100644 --- a/.kiro/specs/libxce-alloc-preason-equality/tasks.md +++ b/.kiro/specs/libxce-alloc-preason-equality/tasks.md @@ -68,7 +68,7 @@ Test-first throughout (red → green → refactor). Every sub-task leaves the bu - _Requirements: 5.1, 5.2, 5.3, 5.4, 6.1, 6.4_ - _Depends: 2.1, 3.1, 4.1, 4.2_ -- [ ] 5. Validation +- [x] 5. Validation - [x] 5.1 Remove duplicated coverage from the verification unit tests - Remove the dispatch allocation-level tests that only re-prove the allocation comparisons now owned by libxce; keep the pending-decision, decision-level and integration tests, the inverted reservation-start-time test from 4.3, and the description-text tests from 4.3. - Done when the verification test file no longer duplicates libxce coverage and still passes. @@ -78,7 +78,7 @@ Test-first throughout (red → green → refactor). Every sub-task leaves the bu - Confirm no source still names any of the five removed private comparators (the string, allocation, host-allocation, job-allocation and pending-reason ones); the decision-level comparison keeps its name. - Done when every suite passes and the search finds no match. - _Requirements: 5.4, 6.6, 6.7_ -- [ ] 5.3 Replay reproduces the reservation start time (after spec scheduler-sim-discrete-clock) +- [x] 5.3 Replay reproduces the reservation start time (after spec scheduler-sim-discrete-clock) - Add a test that a replayed allocation's reservation start time equals the recorded one at the default simulation settings. - Record a session on a local deployment and replay it with verification at the default settings. - Done when the test passes and the live replay reports zero mismatches; then mark bug-1881 fixed through the buglog skill with the fix commit. @@ -92,3 +92,4 @@ Test-first throughout (red → green → refactor). Every sub-task leaves the bu - 4.2 left the `from`-excluded comments at `test_sim_verify.c` ~639-681 for 4.3, which inverts the test they document. - `sim_verify.c` builds descriptions with `vsnprintf` + `malloc`, not `vasprintf`: about 25 scheduler tests `#include "../src/sim_verify.c"` after `` without `_GNU_SOURCE`. Same ownership contract as libxce. - 5.2: `xce-build.sh all` stops at the xceadmin live-HA e2e test (`test_live_config_push_against_primary_reaches_success_path`, versionTag 0); it uses controller binaries from the shared default build dirs, not this build. The spec needs only lib/scheduler/controller: built with `--incremental` per component, then `meson test --no-suite xceadmin` gave 116/116, 0 Unity :FAIL lines, 0 sanitizer reports. +- 5.3: `test_sim_replay_reproduces_recorded_alloc_from` (test_sim_live_allocation.c) uses an epoch-scale beginRound with a 0.75 s fraction; the older `begin_round = 2000` fixtures cannot tell a correct `from` from 0 or from sub-second drift (a +0.5 s clock mutant in sim_loop.c fails only the new test). Standalone `xce_scheduler --sim` exits 1 under ASAN from exit-time LeakSanitizer reports (snapshot decode), so live replays need `ASAN_OPTIONS=detect_leaks=0`, as the HA soak's replay gate already sets. Live proof: 2-round HA soak (/home/claude/tmp/xce-alloceq-53), 3 clean recordings, 11 recorded DISPATCH decisions, replayed with `--sim-verify` and no `--sim-speed`: 0 mismatches. diff --git a/src/scheduler/testsuite/test_sim_live_allocation.c b/src/scheduler/testsuite/test_sim_live_allocation.c index e2d630fd..44e4761f 100644 --- a/src/scheduler/testsuite/test_sim_live_allocation.c +++ b/src/scheduler/testsuite/test_sim_live_allocation.c @@ -514,6 +514,102 @@ static FILE* write_header_and_snapshot(char* out_path, size_t out_path_len) return sched.record_file; } +/* append_dispatchable_job_events - the four push events every test below + * shares, stamped ts, ts+1, ts+2, ts+3: a "slots" resource, + * one worker with WORKER_SLOTS free, queue QUEUE_NAME, and job TEST_JOBID + * (1 task, TEST_RESREQ: "rusage[mem=1]" plus the auto-injected 1 slot per + * task) -- enough for the real picker/allocator to DISPATCH the job at the + * next session. */ +static void append_dispatchable_job_events(FILE* f, uint64_t ts) +{ + xce_resource_t* slots_res; + xce_resource_t* res_ptrs[1]; + xce_worker_t* worker_ptrs[1]; + xce_job_t* job_ptrs[1]; + + slots_res = (xce_resource_t*)calloc(1, sizeof(*slots_res)); + TEST_ASSERT_NOT_NULL(slots_res); + slots_res->name = strdup("slots"); + res_ptrs[0] = slots_res; + append_resourcespush_event(f, ts, res_ptrs, 1); + + worker_ptrs[0] = make_wire_worker(WORKER_NAME, WORKER_SLOTS); + append_workerspush_event(f, ts + 1, worker_ptrs, 1); + + append_queueupdate_event(f, ts + 2, make_wire_queue(QUEUE_NAME, 1)); + + job_ptrs[0] = make_push_job(TEST_JOBID, TEST_RESREQ, (char*)"alice", + (char*)QUEUE_NAME, (char*)"proj1", 1, 1); + append_jobspush_event(f, ts + 3, job_ptrs, 1); +} + +/* append_replay_session_with_dispatch - a ReplaySession at `begin_round` + * whose one recorded decision is the DISPATCH the real allocator chain + * produces for append_dispatchable_job_events()'s job, as a normal-mode + * recording would carry it: one host allocation naming WORKER_NAME, built + * the way test_sim_contention_parity.c's make_dispatch_alloc() fixtures + * are. job_alloc_eq() (xcealloc.h, via sim_verify.c's + * sim_decision_eq()) compares xce_alloc_t field-by-field, excluding only + * instid -- so duration and from must match EXACTLY what the real + * seqjob_allocator()->sched_make_allocs()->make_alloc() chain plus + * sched_make_primary_alloc_()'s session stamp (scheduler.c) produce: + * - duration=-1: seqjob_allocator.c's own sched_make_allocs() call + * passes a NULL duration array ("FIXME: no duration now"), so + * sched_make_allocs() (scheduler.c) always defaults every entry's + * duration to -1 today. + * - from=begin_round / 10^9: every allocation is stamped with its + * deciding session's start in whole seconds (libxce-alloc-preason- + * equality Decision 8), and the normal-mode recorder takes beginRound + * from that same reading. make_alloc()'s own time(NULL) default is + * therefore overwritten below. + * instid=1 is kept realistic (sched_get_id() returns core->nextid++, and + * this is the first allocation made against a fresh core) even though the + * comparison ignores it. */ +static void append_replay_session_with_dispatch(FILE* f, uint64_t begin_round) +{ + xce_alloc_t* a = make_alloc((char*)"slots", NULL, 1, 1.0, -1); + xce_alloc_t** allocs; + xce_host_alloc_t* h; + xce_host_alloc_t** hallocs; + xce_job_alloc_t* alloc; + sched_dispatch_decision_t dispatch; + replay_decision_t d_dispatch; + replay_decision_t* decisions[1]; + replay_session_t session; + struct capn c; + struct capn_segment* cs; + ReplaySession_ptr sp; + + a->from = begin_round / 1000000000ULL; + allocs = make_allocs(1, a); + h = make_host_alloc((char*)WORKER_NAME, 1, allocs); + hallocs = make_host_allocs(1, h); + alloc = make_job_alloc(TEST_JOBID, 1, hallocs); + + memset(&dispatch, 0, sizeof(dispatch)); + dispatch.alloc = alloc; + + memset(&d_dispatch, 0, sizeof(d_dispatch)); + d_dispatch.body_which = ReplayDecision_body_dispatch; + d_dispatch.body.dispatch = &dispatch; + d_dispatch.ts = begin_round; + + decisions[0] = &d_dispatch; + + memset(&session, 0, sizeof(session)); + session.begin_round = begin_round; + session.decisions = decisions; + session.n_decisions = 1; + session.summary = NULL; + + capn_init_malloc(&c); + cs = capn_root(&c).seg; + encode_ReplaySession_ptr(cs, &sp, &session); + capn_setp(capn_root(&c), 0, sp.p); + TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); + capn_free(&c); +} + /* ---------------------------------------------------------------------- */ /* The sim driver under test -- static storage, see this file's own header */ /* comment ("bug-1598/1632/1763 fixture-safety requirement") for why. */ @@ -543,32 +639,13 @@ test_sim_driver_run_dispatches_a_job_with_available_resources(void) { char path[PATH_MAX]; FILE* f; - xce_resource_t* slots_res; - xce_resource_t* res_ptrs[1]; - xce_worker_t* worker_ptrs[1]; - xce_queue_t* queue; - xce_job_t* job_ptrs[1]; sched_sim_args_t args; xce_sched_job_t* sjob; int rc; f = write_header_and_snapshot(path, sizeof(path)); - slots_res = (xce_resource_t*)calloc(1, sizeof(*slots_res)); - TEST_ASSERT_NOT_NULL(slots_res); - slots_res->name = strdup("slots"); - res_ptrs[0] = slots_res; - append_resourcespush_event(f, 1001ULL, res_ptrs, 1); - - worker_ptrs[0] = make_wire_worker(WORKER_NAME, WORKER_SLOTS); - append_workerspush_event(f, 1002ULL, worker_ptrs, 1); - - queue = make_wire_queue(QUEUE_NAME, 1); - append_queueupdate_event(f, 1003ULL, queue); - - job_ptrs[0] = make_push_job(TEST_JOBID, TEST_RESREQ, (char*)"alice", - (char*)QUEUE_NAME, (char*)"proj1", 1, 1); - append_jobspush_event(f, 1004ULL, job_ptrs, 1); + append_dispatchable_job_events(f, 1001ULL); /* Empty decisions -- see this file's own header comment for why this * is load-bearing, not incidental. */ @@ -656,110 +733,23 @@ test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation( { char path[PATH_MAX]; FILE* f; - xce_resource_t* slots_res; - xce_resource_t* res_ptrs[1]; - xce_worker_t* worker_ptrs[1]; - xce_queue_t* queue; - xce_job_t* job_ptrs[1]; xce_sched_job_t* sjob; char* captured; int rc; f = write_header_and_snapshot(path, sizeof(path)); - slots_res = (xce_resource_t*)calloc(1, sizeof(*slots_res)); - TEST_ASSERT_NOT_NULL(slots_res); - slots_res->name = strdup("slots"); - res_ptrs[0] = slots_res; - append_resourcespush_event(f, 1001ULL, res_ptrs, 1); + append_dispatchable_job_events(f, 1001ULL); - worker_ptrs[0] = make_wire_worker(WORKER_NAME, WORKER_SLOTS); - append_workerspush_event(f, 1002ULL, worker_ptrs, 1); - - queue = make_wire_queue(QUEUE_NAME, 1); - append_queueupdate_event(f, 1003ULL, queue); - - job_ptrs[0] = make_push_job(TEST_JOBID, TEST_RESREQ, (char*)"alice", - (char*)QUEUE_NAME, (char*)"proj1", 1, 1); - append_jobspush_event(f, 1004ULL, job_ptrs, 1); - - /* Round 1: empty recorded decisions -- --sim-verify compares the live - * picker/allocator's OWN freshly-drained DISPATCH decision against - * this round's decoded_session, which carries nothing recorded at all. - * A live decision with no corresponding recorded entry is ITSELF - * reported as a mismatch by sim_loop.c's own drain loop (expected=NULL) - * -- so, unlike test_sim_driver_run_dispatches_a_job_with_available_ - * resources above, THIS test must record the SAME DISPATCH the live - * path is about to independently produce, proving they agree, not - * merely that the live path ran. Built the same way test_sim_ - * contention_parity.c's own make_dispatch_alloc()-based fixtures do: - * one host allocation naming WORKER_NAME, wanting exactly what a - * 1-task job's TEST_RESREQ ("rusage[mem=1]", auto-injected 1 slot per - * task) actually requests. - * - * job_alloc_eq() (xcealloc.h, via sim_verify.c's sim_decision_eq()) - * compares xce_alloc_t field-by-field, excluding only instid -- so - * duration and from must match EXACTLY what the real - * seqjob_allocator()->sched_make_allocs()->make_alloc() chain plus - * sched_make_primary_alloc_()'s session stamp (scheduler.c) will - * independently produce for this exact fixture, not placeholders: - * - duration=-1: seqjob_allocator.c's own sched_make_allocs() call - * passes a NULL duration array ("FIXME: no duration now"), so - * sched_make_allocs() (scheduler.c) always defaults every entry's - * duration to -1 today. - * - from=ROUND1_BEGIN_NS / 10^9: every allocation is stamped with - * its deciding session's start in whole seconds (libxce-alloc- - * preason-equality Decision 8), and in simulation mode the session - * starts at the recorded beginRound. make_alloc()'s own time(NULL) - * default is therefore overwritten below. - * instid=1 is kept realistic (sched_get_id() returns core->nextid++, - * and this is the first allocation made against a fresh core) even - * though the comparison ignores it. */ - { - const uint64_t ROUND1_BEGIN_NS = 2000ULL; - xce_alloc_t* a = make_alloc((char*)"slots", NULL, 1, 1.0, - -1); - xce_alloc_t** allocs; - xce_host_alloc_t* h; - xce_host_alloc_t** hallocs; - xce_job_alloc_t* alloc; - sched_dispatch_decision_t dispatch; - replay_decision_t d_dispatch; - replay_decision_t* decisions[1]; - replay_session_t session; - struct capn c; - struct capn_segment* cs; - ReplaySession_ptr sp; - - a->from = ROUND1_BEGIN_NS / 1000000000ULL; - allocs = make_allocs(1, a); - h = make_host_alloc((char*)WORKER_NAME, 1, allocs); - hallocs = make_host_allocs(1, h); - alloc = make_job_alloc(TEST_JOBID, 1, hallocs); - - memset(&dispatch, 0, sizeof(dispatch)); - dispatch.alloc = alloc; - - memset(&d_dispatch, 0, sizeof(d_dispatch)); - d_dispatch.body_which = ReplayDecision_body_dispatch; - d_dispatch.body.dispatch = &dispatch; - d_dispatch.ts = ROUND1_BEGIN_NS; - - decisions[0] = &d_dispatch; - - memset(&session, 0, sizeof(session)); - session.begin_round = ROUND1_BEGIN_NS; - session.decisions = decisions; - session.n_decisions = 1; - session.summary = NULL; - - capn_init_malloc(&c); - cs = capn_root(&c).seg; - encode_ReplaySession_ptr(cs, &sp, &session); - capn_setp(capn_root(&c), 0, sp.p); - TEST_ASSERT_EQUAL_INT(0, write_capnp_message(f, &c)); - capn_free(&c); - } + /* Round 1: --sim-verify compares the live picker/allocator's OWN + * freshly-drained DISPATCH decision against this round's recorded + * decisions. A live decision with no corresponding recorded entry is + * ITSELF reported as a mismatch by sim_loop.c's own drain loop + * (expected=NULL) -- so, unlike test_sim_driver_run_dispatches_a_job_ + * with_available_resources above, THIS test must record the SAME + * DISPATCH the live path is about to independently produce, proving + * they agree, not merely that the live path ran. */ + append_replay_session_with_dispatch(f, 2000ULL); /* Round 2: job's separately-recorded RUN confirmation, several rounds * (here: one) after its own round-1 DISPATCH -- a redundant, safe @@ -797,11 +787,67 @@ test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation( free(captured); } +/* + * libxce-alloc-preason-equality task 5.3 (Requirement 4.3, 5.4): at the + * default simulation settings (no --sim-speed; the discrete clock of spec + * scheduler-sim-discrete-clock), a replayed allocation's `from` equals the + * recorded one. The recorded session starts at a realistic epoch-scale + * beginRound with a non-zero sub-second part (the second embedded in + * bug-1880's array job id, plus 0.75 s), so a replayed session start that drifts past beginRound, a + * stamp read from a different clock, or a stamp taken from the simulator's + * session-less wrapper core (0) all yield a different whole second. + * Checked both ways: --sim-verify's job_alloc_eq() reports no mismatch, and + * the live job's own allocation carries beginRound / 10^9 directly. + * Own driver instance (g_sim3) for the reason g_sim2's comment gives. + */ +#define REPLAY_FROM_BEGIN_NS ((uint64_t)1790578108750000000ULL) + +static xce_sched_sim_driver_t g_sim3; + +static void test_sim_replay_reproduces_recorded_alloc_from(void) +{ + char path[PATH_MAX]; + FILE* f; + xce_sched_job_t* sjob; + xce_alloc_t* live; + char* captured; + int rc; + + f = write_header_and_snapshot(path, sizeof(path)); + append_dispatchable_job_events(f, REPLAY_FROM_BEGIN_NS - 4000000ULL); + append_replay_session_with_dispatch(f, REPLAY_FROM_BEGIN_NS); + append_end_of_stream_marker(f); + fclose(f); + + captured = run_sim_and_capture_stdout(&g_sim3, path, true, &rc); + + TEST_ASSERT_EQUAL_INT(0, rc); + TEST_ASSERT_NULL_MESSAGE( + strstr(captured, "MISMATCH"), + "replaying a recorded DISPATCH at the default settings must " + "reproduce every allocation field, `from` included"); + + sjob = sched_get_job(&(g_sim3.core), TEST_JOBID); + TEST_ASSERT_NOT_NULL(sjob); + TEST_ASSERT_EQUAL_UINT(JOB_STAT_RUN, sjob->state); + TEST_ASSERT_NOT_NULL(sjob->allocs); + TEST_ASSERT_EQUAL_INT(1, sjob->allocs->nhosts); + TEST_ASSERT_EQUAL_INT(1, sjob->allocs->allocs[0]->nallocs); + live = sjob->allocs->allocs[0]->allocs[0]; + TEST_ASSERT_EQUAL_UINT64_MESSAGE( + REPLAY_FROM_BEGIN_NS / 1000000000ULL, live->from, + "the replayed allocation's `from` must equal the recorded session " + "start in whole seconds"); + + free(captured); +} + int main(void) { UNITY_BEGIN(); RUN_TEST(test_sim_driver_run_dispatches_a_job_with_available_resources); RUN_TEST( test_sim_verify_no_false_mismatch_after_live_dispatch_and_later_confirmation); + RUN_TEST(test_sim_replay_reproduces_recorded_alloc_from); return UNITY_END(); }