fix(scheduler): 3 --sim-verify false positives + snapshot resreq_fp gap (bug-1844/1845/1772)
/xce-audit-invariants scheduler dimension-1/4 audit, prompted by a fresh user-captured .replay file that still showed massive --sim-verify mismatch volume after bug-1843's fix. Root-caused via temporary diagnostic instrumentation against the real capture (reverted, never committed) rather than trusting the surface symptom. bug-1844: sim_verify_alloc_eq() (sim_verify.c) compared `instid`, a bare per-process monotonic counter (scheduler.c's sched_get_id(), core->nextid) stamped at allocation-build time -- not semantic content. The original recording's counter and a fresh replay's counter have no reason to reach the same value for "the same" dispatch. Live-verified: every DISPATCH mismatch showed identical value/duration/resname/reserve, differing only in instid (e.g. 826 vs 1). Dropped from the comparison, same treatment as bug-1842's `from` exclusion. bug-1845: sim_verify_job_preason_eq() required PEND decisions' full host/resource reason-detail lists to match. core.h documents "simulation isolation" as deliberate design: sim/replay never registers core->unsatisfiable_lookup_cb, so it can never hit the unsatisfiable-cache main.c's normal-mode branch registers. A live cache hit (sched_pend_from_cached_reason_()) records only main_reason with zero detail; replay always falls through to the full per-host computation and always produces detail. Same semantic outcome (PEND, same main_reason) at two architecturally-legitimate detail levels. Relaxed the comparator to main_reason only. Separately verified the cache itself is sound (key is safe within a round since capacity only decreases and the cache clears every round) -- ruling it out as the cause of a second, still-open divergence (bug-1846: one job genuinely dispatches one round early in replay vs. the original recording -- a suspected resource-accounting parity gap, not yet root-caused, logged for follow-up). bug-1772 (previously deferred, incorrectly tagged "fixed"): re-found independently by a dimension-4 snapshot-field-completeness audit subagent. snapshot_build_primary_schedobj() (snapshot_load.c) never computed resreq_fp, unlike its declared model core_build_primary_schedobj() (core_events.c) -- every snapshot-restored job got a zeroed fingerprint, which alloc_cache.c's cache key folds in directly alongside user/queue/project, so two distinct restored jobs sharing those three fields could silently share an incorrect unsatisfiable-cache verdict. Added the same 4-field computation the live path uses. TDD throughout: each fix has a dedicated regression test, and each was verified via genuine red (revert fix, confirm the new test fails) then green (restore, full 85/85 scheduler suite). Also ran three parallel dimension audits (core I/O isolation, picker/allocator jbuf rendezvous) -- both clean, no violations found. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
parent
5ee329b1b8
commit
fcf119b4ac
6 changed files with 309 additions and 50 deletions
|
|
@ -1245,6 +1245,74 @@ test_snapshot_load_wires_real_task_count_into_resreq_primary(void)
|
|||
TEST_ASSERT_EQUAL_INT(6, restored->resreq->u.primary->maxTasks);
|
||||
}
|
||||
|
||||
/*
|
||||
* bug-1772 fix (xce-audit-invariants scheduler, dimension 4): a job
|
||||
* restored from a snapshot must get a genuinely-computed
|
||||
* schedobj->u.primary->resreq_fp, not the all-zero calloc() default.
|
||||
* snapshot_build_primary_schedobj() (snapshot_load.c) is documented as a
|
||||
* field-for-field port of core_build_primary_schedobj() (core_events.c) --
|
||||
* it had ported every field except this one. Without the fix, EVERY
|
||||
* restored job's resreq_fp is {0,0,0,0}; alloc_cache.c's build_key_()
|
||||
* folds resreq_fp directly into the unsatisfiable-cache key, so two
|
||||
* distinct restored jobs sharing a user/queue/project would silently
|
||||
* collapse onto the same cache entry.
|
||||
*/
|
||||
static void
|
||||
test_snapshot_load_computes_resreq_fingerprint_for_restored_primary(void)
|
||||
{
|
||||
xce_sched_core_t core;
|
||||
sched_snapshot_data_t in;
|
||||
xce_jobs_data_t jobs_data;
|
||||
xce_queue_t wire_queue;
|
||||
xce_queue_t* queues[1];
|
||||
xce_job_t* job;
|
||||
xce_job_t* jobs[1];
|
||||
xce_sched_job_t* restored;
|
||||
int rc;
|
||||
|
||||
memset(&g_stub_state_a, 0, sizeof(g_stub_state_a));
|
||||
fixture_init(&core, &g_stub_plugin_a);
|
||||
|
||||
memset(&wire_queue, 0, sizeof(wire_queue));
|
||||
wire_queue.queue = "normal";
|
||||
wire_queue.priority = 1;
|
||||
wire_queue.status = QUEUE_STAT_OPEN;
|
||||
queues[0] = &wire_queue;
|
||||
|
||||
job = make_job_with_resreq(602, JOB_STAT_PEND, "normal", "alice", "proj1",
|
||||
"type == any");
|
||||
job->submit->mintasks = 4;
|
||||
job->submit->maxtasks = 6;
|
||||
jobs[0] = job;
|
||||
|
||||
memset(&jobs_data, 0, sizeof(jobs_data));
|
||||
jobs_data.count = 1;
|
||||
jobs_data.jobs = jobs;
|
||||
|
||||
memset(&in, 0, sizeof(in));
|
||||
in.jobs = &jobs_data;
|
||||
in.queues = queues;
|
||||
in.n_queues = 1;
|
||||
|
||||
rc = xce_sched_core_load_snapshot(&core, &in);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
|
||||
restored = sched_get_job(&core, 602);
|
||||
TEST_ASSERT_NOT_NULL(restored);
|
||||
TEST_ASSERT_NOT_NULL(restored->schedobj);
|
||||
TEST_ASSERT_EQUAL_INT(XCE_RESREQ_PRIMARY, restored->schedobj->tag);
|
||||
TEST_ASSERT_NOT_NULL(restored->schedobj->u.primary);
|
||||
|
||||
/* The fix under test: resreq_fp must reflect the restored primary's own
|
||||
* real (task 2.2-wired) task count -- not {0,0,0,0}. */
|
||||
TEST_ASSERT_EQUAL_UINT32(
|
||||
4, restored->schedobj->u.primary->resreq_fp.min_tasks);
|
||||
TEST_ASSERT_EQUAL_UINT32(
|
||||
6, restored->schedobj->u.primary->resreq_fp.max_tasks);
|
||||
TEST_ASSERT_EQUAL_UINT32(
|
||||
0, restored->schedobj->u.primary->resreq_fp.select_node_id);
|
||||
}
|
||||
|
||||
/* Regression-safety test (NOT a new feature): a job whose resreq fails to
|
||||
* parse on load is a pre-existing, out-of-boundary gap (design.md "Known
|
||||
* limitation") that task 1.2 must NOT try to fix -- it must only make sure
|
||||
|
|
@ -2545,6 +2613,7 @@ int main(void)
|
|||
RUN_TEST(
|
||||
test_snapshot_load_builds_schedobj_for_primary_compound_and_alter_resreq);
|
||||
RUN_TEST(test_snapshot_load_wires_real_task_count_into_resreq_primary);
|
||||
RUN_TEST(test_snapshot_load_computes_resreq_fingerprint_for_restored_primary);
|
||||
RUN_TEST(
|
||||
test_snapshot_load_leaves_schedobj_null_when_resreq_fails_to_parse);
|
||||
RUN_TEST(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue