lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(libxce-alloc-preason-equality): sim verify uses public job_alloc_eq and logs the first difference (task 4.3)

Browse source 
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-28 10:58:41 +00:00
parent 69112c2d22
commit 24b057189f
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
4 changed files with 415 additions and 217 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/scheduler/src/sim_loop.c
View file

@ -138,10 +138,10 @@ extern int msg_root_has_shape(struct capn* c, capn_ptr* out, size_t want_words,
* comment for the full rationale.
*/
extern int sim_decision_eq(const replay_decision_t* expected,
const xce_decision_t* actual);
const xce_decision_t* actual, char** why);
extern void sim_log_mismatch(uint64_t begin_round, int decision_index,
const replay_decision_t* expected,
const xce_decision_t* actual);
const xce_decision_t* actual, const char* why);
/*
* Shared with sim_record.c (task 11.5, Req 5 AC 19) -- same "extern
@ -1611,9 +1611,14 @@ int sched_sim_driver_run(xce_sched_sim_driver_t* sim,
? decoded_session->decisions[di]
: NULL;
if (!sim_decision_eq(expected, d)) {
sim_log_mismatch(begin_round, di, expected, d);
char* why = NULL;
/* Requirement 5.3: the mismatch line names the first
* difference; this loop owns and frees it. */
if (!sim_decision_eq(expected, d, &why)) {
sim_log_mismatch(begin_round, di, expected, d, why);
}
free(why);
di++;
}
/* Req 5 AC 19 (task 11.5): "append a ReplayDecision to the
@ -1635,7 +1640,8 @@ int sched_sim_driver_run(xce_sched_sim_driver_t* sim,
for (j = di; j < decoded_session->n_decisions; j++) {
sim_log_mismatch(begin_round, j,
decoded_session->decisions[j], NULL);
decoded_session->decisions[j], NULL,
"no live decision");
}
}

352
src/scheduler/src/sim_verify.c
View file

@ -1,10 +1,13 @@
#include <stdarg.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "controller.capnp.h"
#include "decision.h"
#include "utils.h"
#include "xce/types.h"
#include "xcealloc.h"
/*
* sim_verify.c - deterministic replay verification (scheduler-core spec,
@ -62,164 +65,71 @@
* make_pend_decision() calls (decision.c), the other from
* decode_ReplaySession_ptr()'s deep-copy-from-capnp-arena walk
* (controller.capnp.c). Pointer or memcmp equality would therefore always
* report a false mismatch; every comparator below instead walks both trees
* field-by-field, matching the exact xce_job_alloc_t/xce_job_preason_t
* shapes recording_session.c's own encode side (the inverse direction)
* already documents.
*/
/*
* sim_verify_str_eq - NULL-safe string equality. Both trees under
* comparison route every string through STRING_DUP()/makeString()-style
* heap copies on their own independent path (decode_* on one side,
* make_pend_decision()/the picker's own construction on the other), so a
* NULL/non-NULL string field is itself real signal, not noise -- (a==b==
* NULL) counts as equal, exactly one of the two being NULL never does.
*/
static int sim_verify_str_eq(const char* a, const char* b)
{
if (a == b) {
return 1;
}
if ((a == NULL) || (b == NULL)) {
return 0;
}
return strcmp(a, b) == 0;
}
/*
* sim_verify_alloc_eq - xce_alloc_t (one resource allocation record, e.g.
* one GPU/memory grant) field-by-field equality. `value` is compared with
* plain `==`: both sides are exact copies (decode/construction, no
* arithmetic in between) of the same double, so bit-exact comparison is
* correct here, not the usual floating-point-tolerance concern.
* report a false mismatch.
*
* `from` is DELIBERATELY excluded (bug-1838-followup): make_alloc()
* (xcealloc.c) unconditionally stamps it with time(NULL), the real OS wall
* clock, not core.c's virtualized sim clock. `expected->from` is whatever
* real second the ORIGINAL recording happened to run in; `actual->from` is
* whatever real second THIS replay happens to run in -- two independent
* real-world timestamps with no reason to agree, even though the decision
* they're attached to is semantically identical (same job, same host, same
* resource, same amount). Comparing them made every genuinely-correct live
* DISPATCH decision report a false --sim-verify MISMATCH whenever a replay
* runs so much as one wall-clock second away from its own recording --
* i.e. essentially always, for any replay of a real, non-synthetic
* recording. `from` is reservation bookkeeping (when this grant started, for
* expiry/accounting purposes), not part of the decision's own identity --
* unlike `duration`, which IS a deterministic, request-derived value and
* stays part of this comparison.
* libxce-alloc-preason-equality (Requirements 5.1-5.3, 6.1): DISPATCH
* payloads are compared with libxce's public job_alloc_eq() (xcealloc.h),
* the single shared allocation comparison. It compares every allocation
* field except `instid` (a per-process counter, bug-1844) -- including
* `from`, which every allocation now carries as its deciding session's
* start (Requirement 4.1), so a faithful replay reproduces it exactly. PEND
* payloads keep the simulator-local main-reason-only check below
* (sim_verify_pend_main_reason_eq(), bug-1845), not libxce's full-match
* job_preason_eq(). Each comparison can describe its first difference; the
* caller hands that description to sim_log_mismatch().
*/
/*
* sim_verify_why - *why = newly allocated formatted description, or NULL if
* formatting or the allocation fails. No-op when why is NULL. Same contract
* as libxce's equality functions. Built with vsnprintf() + malloc() rather
* than vasprintf(): this file is also white-box #include'd into test TUs
* whose <stdio.h> is already included without _GNU_SOURCE.
*/
static void sim_verify_why(char** why, const char* fmt, ...)
__attribute__((format(printf, 2, 3)));
static void sim_verify_why(char** why, const char* fmt, ...)
{
va_list ap;
char* s = NULL;
int n;
if (why == NULL) {
return;
}
*why = NULL;
va_start(ap, fmt);
n = vsnprintf(NULL, 0, fmt, ap);
va_end(ap);
if (n < 0) {
return;
}
s = (char*)malloc((size_t)n + 1);
if (s == NULL) {
return;
}
va_start(ap, fmt);
if (vsnprintf(s, (size_t)n + 1, fmt, ap) < 0) {
free(s);
s = NULL;
}
va_end(ap);
*why = s;
}
/*
* sim_verify_pend_main_reason_eq - the simulator's PEND comparison:
* main_reason ONLY (Requirement 5.2). Deliberately NOT libxce's full-match
* job_preason_eq(), which answers a different question ("are these two
* pending-reason objects identical") than this one ("is this the same
* replay outcome").
*
* `instid` is ALSO deliberately excluded (bug-1844, same class of issue as
* `from` above): stamped from `sched->core.nextid++` (scheduler.c's
* sched_get_id()) at allocation-build time -- a bare per-process monotonic
* counter, not a semantic property of what got allocated. It counts how
* many allocation instances THIS core has built over its own entire
* execution history, so the ORIGINAL recording's counter (which may have
* been running, and building allocations, for far longer before this
* particular capture window even started) and a fresh replay's own counter
* (starting from whatever its loaded snapshot's nextid happens to be) have
* no reason to reach the same value for "the same" dispatch, even when
* every other field -- value, duration, resname, reserve, host -- matches
* exactly. Live-verified: a real captured .replay file showed every single
* DISPATCH's `instid` differing (e.g. recorded=826 vs replay=1, both
* otherwise byte-identical) while zero other fields ever differed.
*/
static int sim_verify_alloc_eq(const xce_alloc_t* a, const xce_alloc_t* b)
{
if (a == b) {
return 1;
}
if ((a == NULL) || (b == NULL)) {
return 0;
}
return (a->value == b->value) && (a->duration == b->duration)
&& sim_verify_str_eq(a->resname, b->resname)
&& sim_verify_str_eq(a->reserve, b->reserve);
}
/*
* sim_verify_host_alloc_eq - xce_host_alloc_t equality: same host name,
* same count and (in-order) content of per-resource xce_alloc_t entries.
* Order-sensitive comparison is correct here: both the recorded and the
* live tree are built by the SAME deterministic scheduling round replaying
* against the SAME core state (Req 5's whole simulation-mode premise), so
* the host/resource allocation arrays are expected to come out in the same
* order, not just the same multiset.
*/
static int sim_verify_host_alloc_eq(const xce_host_alloc_t* a,
const xce_host_alloc_t* b)
{
int i;
if (a == b) {
return 1;
}
if ((a == NULL) || (b == NULL)) {
return 0;
}
if (a->nallocs != b->nallocs) {
return 0;
}
if (!sim_verify_str_eq(a->host, b->host)) {
return 0;
}
for (i = 0; i < a->nallocs; i++) {
if (!sim_verify_alloc_eq(a->allocs[i], b->allocs[i])) {
return 0;
}
}
return 1;
}
/*
* sim_verify_job_alloc_eq - xce_job_alloc_t (a DISPATCH decision's full
* payload) equality: jobid, per-host allocation list (xce_host_alloc_t,
* via sim_verify_host_alloc_eq above), and the shared-resource allocation
* list (xce_alloc_t, via sim_verify_alloc_eq).
*/
static int sim_verify_job_alloc_eq(const xce_job_alloc_t* a,
const xce_job_alloc_t* b)
{
int i;
if (a == b) {
return 1;
}
if ((a == NULL) || (b == NULL)) {
return 0;
}
if ((a->jobid != b->jobid) || (a->nhosts != b->nhosts)
|| (a->nres_shared != b->nres_shared)) {
return 0;
}
for (i = 0; i < a->nhosts; i++) {
if (!sim_verify_host_alloc_eq(a->allocs[i], b->allocs[i])) {
return 0;
}
}
for (i = 0; i < a->nres_shared; i++) {
if (!sim_verify_alloc_eq(a->allocs_shared[i], b->allocs_shared[i])) {
return 0;
}
}
return 1;
}
/*
* sim_verify_job_preason_eq - xce_job_preason_t (a PEND decision's reason
* payload) equality: main_reason ONLY.
*
* bug-1845 fix: this used to also require the full host-based/resource-
* based reason detail lists to match (added as an optional "cheap to add"
* deepening of the check, never proven necessary by a dedicated test).
* That is wrong: core.h documents, as a deliberate "simulation isolation"
* bug-1845: core.h documents, as a deliberate "simulation isolation"
* design decision, that "a simulation-mode session never touches the real
* cache implementation at all" -- core->unsatisfiable_lookup_cb/
* unsatisfiable_populate_cb are registered only by main.c's normal-mode
@ -240,17 +150,56 @@ static int sim_verify_job_alloc_eq(const xce_job_alloc_t* a,
* computation) showed {main_reason=0, nhost_reasons=0, nres_reasons=2} --
* main_reason matched, detail did not, and never architecturally can.
*/
static int sim_verify_job_preason_eq(const xce_job_preason_t* a,
const xce_job_preason_t* b)
static int sim_verify_pend_main_reason_eq(const xce_job_preason_t* a,
const xce_job_preason_t* b,
char** why)
{
if (a == b) {
return 1;
}
if ((a == NULL) || (b == NULL)) {
sim_verify_why(why, "reason: %s != %s",
(a != NULL) ? "non-NULL" : "NULL",
(b != NULL) ? "non-NULL" : "NULL");
return 0;
}
if (a->main_reason != b->main_reason) {
sim_verify_why(why, "main_reason: %d != %d", a->main_reason,
b->main_reason);
return 0;
}
return a->main_reason == b->main_reason;
return 1;
}
/* sim_verify_kind_name - the decision kind a recorded body_which names. */
static const char* sim_verify_kind_name(int body_which)
{
switch (body_which) {
case ReplayDecision_body_dispatch:
return "DISPATCH";
case ReplayDecision_body_pend:
return "PEND";
case ReplayDecision_body_preempt:
return "PREEMPT";
default:
return "UNKNOWN";
}
}
/* sim_verify_op_name - the decision kind a live op names. */
static const char* sim_verify_op_name(int op)
{
switch (op) {
case XCE_DECISION_DISPATCH:
return "DISPATCH";
case XCE_DECISION_PEND:
return "PEND";
case XCE_DECISION_PREEMPT:
return "PREEMPT";
default:
return "UNKNOWN";
}
}
/*
@ -263,33 +212,75 @@ static int sim_verify_job_preason_eq(const xce_job_preason_t* a,
* body_which to name the SAME decision kind before comparing payloads --
* an op/body_which mismatch (e.g. recorded PEND, live DISPATCH) is itself
* reported as a mismatch, not silently ignored.
*
* `why` (optional, libxce-alloc-preason-equality Requirement 5.3): same
* ownership contract as libxce's job_alloc_eq(). When non-NULL, *why is
* set to NULL on a match, or to a newly allocated description of the first
* difference on a mismatch (NULL if allocating it fails), recorded side
* first: job_alloc_eq()'s path for DISPATCH, `jobid: A != B` or
* `main_reason: A != B` for PEND, `kind: A != B` for a kind mismatch,
* `no recorded decision at this index`/`no live decision` for a missing
* side, `PREEMPT comparison not supported` for PREEMPT. The caller frees.
*/
int sim_decision_eq(const replay_decision_t* expected,
const xce_decision_t* actual)
const xce_decision_t* actual, char** why)
{
if ((expected == NULL) || (actual == NULL)) {
if (why != NULL) {
*why = NULL;
}
if (expected == NULL) {
sim_verify_why(why, "no recorded decision at this index");
return 0;
}
if (actual == NULL) {
sim_verify_why(why, "no live decision");
return 0;
}
if ((actual->op == XCE_DECISION_DISPATCH)
|| (actual->op == XCE_DECISION_PEND)) {
int want = (actual->op == XCE_DECISION_DISPATCH)
? ReplayDecision_body_dispatch
: ReplayDecision_body_pend;
if ((int)expected->body_which != want) {
sim_verify_why(why, "kind: %s != %s",
sim_verify_kind_name(expected->body_which),
sim_verify_op_name(actual->op));
return 0;
}
}
switch (actual->op) {
case XCE_DECISION_DISPATCH:
if (expected->body_which != ReplayDecision_body_dispatch) {
if (expected->body.dispatch == NULL) {
sim_verify_why(why, "dispatch: NULL != non-NULL");
return 0;
}
return (expected->body.dispatch != NULL)
&& sim_verify_job_alloc_eq(expected->body.dispatch->alloc,
actual->u.alloc);
return job_alloc_eq(expected->body.dispatch->alloc, actual->u.alloc,
why);
case XCE_DECISION_PEND:
if (expected->body_which != ReplayDecision_body_pend) {
return 0;
}
if ((expected->body.pend == NULL) || (actual->u.reason == NULL)) {
sim_verify_why(why, "pend: %s != %s",
(expected->body.pend != NULL) ? "non-NULL" : "NULL",
(actual->u.reason != NULL) ? "non-NULL" : "NULL");
return 0;
}
return (expected->body.pend->jobid == actual->u.reason->jobid)
&& sim_verify_job_preason_eq(expected->body.pend->reason,
actual->u.reason->reasons);
if (expected->body.pend->jobid != actual->u.reason->jobid) {
if (why != NULL) {
char ja[64];
char jb[64];
fmt_jobid(ja, sizeof(ja), expected->body.pend->jobid);
fmt_jobid(jb, sizeof(jb), actual->u.reason->jobid);
sim_verify_why(why, "jobid: %s != %s", ja, jb);
}
return 0;
}
return sim_verify_pend_main_reason_eq(expected->body.pend->reason,
actual->u.reason->reasons, why);
case XCE_DECISION_PREEMPT:
/* recording_session.c's own documented reason (see its
@ -313,9 +304,13 @@ int sim_decision_eq(const replay_decision_t* expected,
"as a mismatch",
__func__, jobidStr);
}
sim_verify_why(why, "PREEMPT comparison not supported");
return 0;
default:
sim_verify_why(why, "kind: %s != %s",
sim_verify_kind_name(expected->body_which),
sim_verify_op_name(actual->op));
return 0;
}
}
@ -453,10 +448,17 @@ static void sim_verify_describe_decision(const xce_decision_t* d, char* buf,
* or `actual` may be NULL (a length mismatch between the recorded and live
* decision counts for this session -- sim_loop.c's own drain loop reports
* that case through this same function rather than a separate one).
*
* `why` (libxce-alloc-preason-equality Requirement 5.3, bug-1881): the
* first-difference description, appended as ` -- diff: <why>` when
* non-NULL; NULL (only when allocating the description failed) leaves the
* line otherwise unchanged. This function does not compare again and does
* not free `why`: sim_loop.c, which calls sim_decision_eq() right before
* it, owns and frees the string.
*/
void sim_log_mismatch(uint64_t begin_round, int decision_index,
const replay_decision_t* expected,
const xce_decision_t* actual)
const xce_decision_t* actual, const char* why)
{
char expected_desc[SIM_VERIFY_DESC_BUF_SZ];
char actual_desc[SIM_VERIFY_DESC_BUF_SZ];
@ -467,7 +469,9 @@ void sim_log_mismatch(uint64_t begin_round, int decision_index,
sim_verify_describe_decision(actual, actual_desc, sizeof(actual_desc));
ERROR("%s decision verification MISMATCH at begin_round=%llu "
"decision_index=%d expected_ts=%llu -- expected: %s -- actual: %s",
"decision_index=%d expected_ts=%llu -- expected: %s -- actual: "
"%s%s%s",
__func__, (unsigned long long)begin_round, decision_index,
(unsigned long long)expected_ts, expected_desc, actual_desc);
(unsigned long long)expected_ts, expected_desc, actual_desc,
(why != NULL) ? " -- diff: " : "", (why != NULL) ? why : "");
}

259
src/scheduler/testsuite/test_sim_verify.c
View file

@ -14,9 +14,11 @@
* Unit-level (white-box): calls sim_decision_eq()/sim_log_mismatch()
* (sim_verify.c, task 11.4's own sibling file to sim_loop.c) directly
* against hand-built replay_decision_t/xce_decision_t pairs -- DISPATCH
* match/mismatch (jobid, host, resource value), PEND match/mismatch
* match/mismatch (jobid, host, resource value, from), PEND match/mismatch
* (jobid, main_reason), a body_which/op type mismatch, and PREEMPT's
* documented always-false path. This is the only way to exercise DISPATCH
* documented always-false path -- each mismatch also asserting the
* first-difference description (libxce-alloc-preason-equality
* Requirement 5.3). This is the only way to exercise DISPATCH
* comparison at all: sim_loop.c's real picker thread
* (sim_picker_thread_fn) only ever emits PEND decisions (its own doc
* comment: "conservatively PENDs every candidate ... rather than
@ -558,8 +560,24 @@ static replay_decision_t make_replay_pend_decision(uint64_t jobid,
/* Unit-level tests: sim_decision_eq() directly. */
/* ---------------------------------------------------------------------- */
/* Asserts that sim_decision_eq()'s description (libxce-alloc-preason-
* equality Requirement 5.3) was produced and contains `needle` -- the name
* of the differing field and both of its values -- then frees it. */
static void assert_why_names(char* why, const char* needle)
{
char msg[512];
TEST_ASSERT_NOT_NULL_MESSAGE(why, "a mismatch must produce a "
"description of the difference");
snprintf(msg, sizeof(msg), "description \"%s\" must contain \"%s\"",
why, needle);
TEST_ASSERT_NOT_NULL_MESSAGE(strstr(why, needle), msg);
free(why);
}
static void test_dispatch_decisions_with_identical_content_match(void)
{
char* why;
xce_alloc_t* res_a[1]
= { make_test_alloc("gpu", "resv1", 1, 2.0, 100, 1800) };
xce_host_alloc_t* host_a[1] = { make_test_host_alloc("hostA", res_a, 1) };
@ -577,7 +595,11 @@ static void test_dispatch_decisions_with_identical_content_match(void)
* trees: proves the comparison is real structural equality, not a
* pointer/memcmp shortcut (this file's own header comment). */
TEST_ASSERT_NOT_EQUAL(expected_alloc, actual_alloc);
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL));
why = (char*)"stale";
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, &why));
TEST_ASSERT_NULL_MESSAGE(why, "an equal comparison must leave no "
"description");
printf("test_dispatch_decisions_with_identical_content_match: PASS\n");
}
@ -595,7 +617,17 @@ static void test_dispatch_decisions_differing_jobid_mismatch(void)
replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
xce_decision_t* actual = make_dispatch_decision(actual_alloc);
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
char ja[64];
char jb[64];
char needle[160];
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, NULL));
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
fmt_jobid(ja, sizeof(ja), 42ULL);
fmt_jobid(jb, sizeof(jb), 43ULL);
snprintf(needle, sizeof(needle), "jobid: %s != %s", ja, jb);
assert_why_names(why, needle);
printf("test_dispatch_decisions_differing_jobid_mismatch: PASS\n");
}
@ -613,7 +645,11 @@ static void test_dispatch_decisions_differing_host_mismatch(void)
replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
xce_decision_t* actual = make_dispatch_decision(actual_alloc);
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
assert_why_names(why, "allocs[0](host=hostA).host: \"hostA\" != "
"\"hostB\"");
printf("test_dispatch_decisions_differing_host_mismatch: PASS\n");
}
@ -631,27 +667,30 @@ static void test_dispatch_decisions_differing_resource_value_mismatch(void)
replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
xce_decision_t* actual = make_dispatch_decision(actual_alloc);
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
assert_why_names(why, "allocs[0](host=hostA).allocs[0](resname=gpu)."
"value: 1 != 2");
printf("test_dispatch_decisions_differing_resource_value_mismatch: "
"PASS\n");
}
/*
* test_dispatch_decisions_differing_from_timestamp_still_match -
* bug-1838-followup regression: `from` (xce_alloc_t's wall-clock reservation-
* start stamp, set unconditionally by make_alloc() via time(NULL) --
* xcealloc.c) is real-OS-clock-derived, not part of a decision's semantic
* identity. `expected`'s `from` is whatever second the ORIGINAL recording
* ran in; `actual`'s is whatever second THIS run rebuilds the allocation in
* -- two independent real-world timestamps that will essentially never
* agree for any replay of a real recording, even when the decision itself
* (job/host/resource/value/duration) is byte-for-byte the one that was
* recorded. Uses two `from` values far enough apart (a Unix-epoch-plausible
* value vs. 1) that they could never coincidentally collide, proving this
* is deliberate exclusion, not a fluke of nearby clock reads.
* test_dispatch_decisions_differing_from_timestamp_mismatch -
* libxce-alloc-preason-equality Requirement 5.1 (inverts the former
* bug-1838-followup "from still matches" lock): `from` used to be excluded
* because make_alloc() stamped it with time(NULL), a real-OS-clock value
* that differed between a recording and its replay. Every allocation a
* session decides now carries that session's start (Requirement 4.1,
* scheduler.c's sched_make_primary_alloc_()), and a replayed session starts
* at its recorded beginRound, so the two sides agree for a faithful replay
* and `from` is compared (by libxce's job_alloc_eq()). A `from` difference
* is therefore a real divergence: it must be reported, and the description
* must name `from` with both values.
*/
static void test_dispatch_decisions_differing_from_timestamp_still_match(void)
static void test_dispatch_decisions_differing_from_timestamp_mismatch(void)
{
xce_alloc_t* res_a[1]
= { make_test_alloc("gpu", "resv1", 1, 2.0, 1790000000ULL, 1800) };
@ -665,10 +704,13 @@ static void test_dispatch_decisions_differing_from_timestamp_still_match(void)
replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
xce_decision_t* actual = make_dispatch_decision(actual_alloc);
char* why = NULL;
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
assert_why_names(why, "allocs[0](host=hostA).allocs[0](resname=gpu)."
"from: 1790000000 != 1");
printf("test_dispatch_decisions_differing_from_timestamp_still_match: "
printf("test_dispatch_decisions_differing_from_timestamp_mismatch: "
"PASS\n");
}
@ -677,8 +719,10 @@ static void test_dispatch_decisions_differing_from_timestamp_still_match(void)
* regression: `instid` (xce_alloc_t's per-process monotonic allocation-
* instance counter, stamped from `sched->core.nextid++` at allocation-build
* time -- scheduler.c's sched_get_id()) is an execution-order bookkeeping
* value, not part of a decision's semantic identity, exactly like `from`
* above (bug-1838-followup). The ORIGINAL recording's counter may have been
* value, not part of a decision's semantic identity. It is the one field
* libxce's alloc_eq() excludes (`from`, unlike instid, is now compared: see
* test_dispatch_decisions_differing_from_timestamp_mismatch above). The
* ORIGINAL recording's counter may have been
* running, and building allocations, for far longer before this capture
* window started; a fresh replay's own counter starts whatever its loaded
* snapshot's nextid happens to be -- the two have no reason to reach the
@ -703,7 +747,7 @@ static void test_dispatch_decisions_differing_instid_still_match(void)
replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
xce_decision_t* actual = make_dispatch_decision(actual_alloc);
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL));
printf("test_dispatch_decisions_differing_instid_still_match: PASS\n");
}
@ -713,11 +757,14 @@ static void test_pend_decisions_with_identical_content_match(void)
replay_decision_t expected = make_replay_pend_decision(9001ULL, 3);
xce_pendreason_t* reason = make_pendreason(9001ULL);
xce_decision_t* actual;
char* why;
reason->reasons->main_reason = 3;
actual = make_pend_decision(reason);
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
why = (char*)"stale";
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, &why));
TEST_ASSERT_NULL(why);
printf("test_pend_decisions_with_identical_content_match: PASS\n");
}
@ -727,7 +774,16 @@ static void test_pend_decisions_differing_jobid_mismatch(void)
replay_decision_t expected = make_replay_pend_decision(9001ULL, 0);
xce_decision_t* actual = make_pend_decision(make_pendreason(9002ULL));
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
char ja[64];
char jb[64];
char needle[160];
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
fmt_jobid(ja, sizeof(ja), 9001ULL);
fmt_jobid(jb, sizeof(jb), 9002ULL);
snprintf(needle, sizeof(needle), "jobid: %s != %s", ja, jb);
assert_why_names(why, needle);
printf("test_pend_decisions_differing_jobid_mismatch: PASS\n");
}
@ -739,7 +795,10 @@ static void test_pend_decisions_differing_main_reason_mismatch(void)
/* actual's main_reason stays 0 (make_pendreason()'s own fresh-calloc
* default) -- differs from expected's 3. */
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
assert_why_names(why, "main_reason: 3 != 0");
printf("test_pend_decisions_differing_main_reason_mismatch: PASS\n");
}
@ -789,7 +848,7 @@ static void test_pend_decisions_differing_reason_detail_still_match(void)
actual = make_pend_decision(reason);
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL));
printf("test_pend_decisions_differing_reason_detail_still_match: PASS\n");
}
@ -801,8 +860,12 @@ static void test_decision_kind_mismatch_reported(void)
xce_decision_t* actual = make_dispatch_decision(alloc);
/* Recorded PEND vs. live DISPATCH for the SAME jobid: still a
* mismatch -- op/body_which disagreement is reported, not ignored. */
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
* mismatch -- op/body_which disagreement is reported, not ignored.
* The description names the kinds, recorded side first. */
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
assert_why_names(why, "kind: PEND != DISPATCH");
printf("test_decision_kind_mismatch_reported: PASS\n");
}
@ -826,7 +889,10 @@ static void test_preempt_decision_always_reports_mismatch(void)
memset(&expected, 0, sizeof(expected));
expected.body_which = ReplayDecision_body_preempt;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, &actual));
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, &actual, &why));
assert_why_names(why, "PREEMPT comparison not supported");
printf("test_preempt_decision_always_reports_mismatch: PASS\n");
}
@ -836,9 +902,18 @@ static void test_decision_eq_defends_against_null_inputs(void)
xce_decision_t* actual = make_pend_decision(make_pendreason(1ULL));
replay_decision_t expected = make_replay_pend_decision(1ULL, 0);
TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual));
TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL));
TEST_ASSERT_FALSE(sim_decision_eq(NULL, NULL));
char* why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual, NULL));
TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL, NULL));
TEST_ASSERT_FALSE(sim_decision_eq(NULL, NULL, NULL));
/* A missing side is named: which of the two has no decision. */
TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual, &why));
assert_why_names(why, "no recorded decision at this index");
why = NULL;
TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL, &why));
assert_why_names(why, "no live decision");
printf("test_decision_eq_defends_against_null_inputs: PASS\n");
}
@ -892,7 +967,15 @@ static void log_one_mismatch_for_capture(void)
replay_decision_t expected = make_replay_pend_decision(9001ULL, 3);
xce_decision_t* actual = make_pend_decision(make_pendreason(9001ULL));
sim_log_mismatch(12345ULL, 2, &expected, actual);
sim_log_mismatch(12345ULL, 2, &expected, actual, "main_reason: 3 != 0");
}
static void log_one_mismatch_without_why_for_capture(void)
{
replay_decision_t expected = make_replay_pend_decision(9001ULL, 3);
xce_decision_t* actual = make_pend_decision(make_pendreason(9001ULL));
sim_log_mismatch(12345ULL, 2, &expected, actual, NULL);
}
static void test_log_mismatch_reports_logical_time_and_decision_detail(void)
@ -925,6 +1008,59 @@ static void test_log_mismatch_reports_logical_time_and_decision_detail(void)
"PASS\n");
}
/*
* Requirement 5.3: the mismatch line keeps its logical-time and decision
* context and appends the difference description it is given; with no
* description (only when allocating it failed) the line is logged without
* one, unchanged otherwise.
*/
static void test_log_mismatch_appends_provided_description(void)
{
char* buf = capture_stdout_around(log_one_mismatch_for_capture);
char* nodiff;
TEST_ASSERT_NOT_NULL(strstr(buf, "begin_round=12345"));
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(buf, " -- diff: main_reason: 3 != 0"),
"the mismatch line must append the provided description");
free(buf);
nodiff = capture_stdout_around(log_one_mismatch_without_why_for_capture);
TEST_ASSERT_NOT_NULL(strstr(nodiff, "MISMATCH"));
TEST_ASSERT_NOT_NULL(strstr(nodiff, "decision_index=2"));
TEST_ASSERT_NULL_MESSAGE(strstr(nodiff, "-- diff:"),
"with no description the line carries no "
"diff clause");
free(nodiff);
printf("test_log_mismatch_appends_provided_description: PASS\n");
}
/* Every line of `captured` that reports a MISMATCH also carries a
* ` -- diff: ` clause (Requirement 5.3: each mismatch names its
* difference). Returns the number of MISMATCH lines seen. */
static int assert_every_mismatch_line_has_diff(const char* captured)
{
const char* line = captured;
int n = 0;
while ((line != NULL) && (*line != '\0')) {
const char* eol = strchr(line, '\n');
size_t len = (eol != NULL) ? (size_t)(eol - line) : strlen(line);
char* one = strndup(line, len);
TEST_ASSERT_NOT_NULL(one);
if (strstr(one, "MISMATCH") != NULL) {
n++;
TEST_ASSERT_NOT_NULL_MESSAGE(strstr(one, " -- diff: "), one);
}
free(one);
line = (eol != NULL) ? (eol + 1) : NULL;
}
return n;
}
/* ---------------------------------------------------------------------- */
/* Integration-level fixture helpers: a real .replay stream driving the */
/* REAL builtin_fcfs picker to a genuine live PEND decision. */
@ -1568,12 +1704,55 @@ static void test_verify_against_altered_copy_reports_exact_mismatch(void)
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, jobid_marker),
"the mismatch report must name the specific job");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, " -- diff: main_reason: 99 != 0"),
"the mismatch report must name the differing field and both values");
TEST_ASSERT_EQUAL_INT(1, assert_every_mismatch_line_has_diff(captured));
free(captured);
printf("test_verify_against_altered_copy_reports_exact_mismatch: "
"PASS\n");
}
/*
* A recorded decision left over after the live stream ends (recorded
* PEND(777), PEND(778); only job 777 was ever pushed, so the live picker
* emits only PEND(777)) is reported as a mismatch whose diff says which
* side is missing: "no live decision" (Requirement 5.3).
*/
static void test_verify_leftover_recorded_decision_names_missing_side(void)
{
char path[PATH_MAX];
FILE* f;
int rc;
char* captured;
f = write_header_and_snapshot(path, sizeof(path), "_leftover");
TEST_ASSERT_NOT_NULL(f);
append_queue_update_event(f, 1000ULL, "normal");
append_jobs_push_event(f, 1500ULL, 777ULL, "normal");
append_replay_session_with_two_pends(f, 2000ULL, 777ULL, 0, 778ULL, 0);
append_end_of_stream_marker(f);
fclose(f);
captured = run_sim_and_capture_stdout(path, true, &rc);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, "decision_index=1"),
"the left-over recorded decision must be reported at its index");
TEST_ASSERT_NOT_NULL_MESSAGE(
strstr(captured, " -- diff: no live decision"),
"a left-over recorded decision's mismatch must name the missing "
"live side");
TEST_ASSERT_EQUAL_INT(1, assert_every_mismatch_line_has_diff(captured));
free(captured);
printf("test_verify_leftover_recorded_decision_names_missing_side: "
"PASS\n");
}
/*
* sim->verify == false must skip verification entirely, even against the
* SAME deliberately-altered fixture the previous test proves DOES mismatch
@ -1873,6 +2052,12 @@ test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_mismatch(void)
"PEND-only live stub cannot reproduce it either, so --sim-verify "
"must correctly report a real mismatch here, not silently agree "
"with a decision nothing in this environment actually made");
/* Round 1: recorded DISPATCH vs live PEND; round 2: a live PEND with
* no recorded entry. Both lines name their difference. */
TEST_ASSERT_NOT_NULL(strstr(captured, " -- diff: kind: DISPATCH != PEND"));
TEST_ASSERT_NOT_NULL(
strstr(captured, " -- diff: no recorded decision at this index"));
TEST_ASSERT_TRUE(assert_every_mismatch_line_has_diff(captured) >= 2);
free(captured);
printf("test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_"
@ -1887,7 +2072,7 @@ int main(void)
RUN_TEST(test_dispatch_decisions_differing_jobid_mismatch);
RUN_TEST(test_dispatch_decisions_differing_host_mismatch);
RUN_TEST(test_dispatch_decisions_differing_resource_value_mismatch);
RUN_TEST(test_dispatch_decisions_differing_from_timestamp_still_match);
RUN_TEST(test_dispatch_decisions_differing_from_timestamp_mismatch);
RUN_TEST(test_dispatch_decisions_differing_instid_still_match);
RUN_TEST(test_pend_decisions_with_identical_content_match);
RUN_TEST(test_pend_decisions_differing_jobid_mismatch);
@ -1897,8 +2082,10 @@ int main(void)
RUN_TEST(test_preempt_decision_always_reports_mismatch);
RUN_TEST(test_decision_eq_defends_against_null_inputs);
RUN_TEST(test_log_mismatch_reports_logical_time_and_decision_detail);
RUN_TEST(test_log_mismatch_appends_provided_description);
RUN_TEST(test_verify_against_self_reports_zero_mismatches);
RUN_TEST(test_verify_against_altered_copy_reports_exact_mismatch);
RUN_TEST(test_verify_leftover_recorded_decision_names_missing_side);
RUN_TEST(test_verify_disabled_reports_no_mismatches_even_when_altered);
RUN_TEST(test_verify_with_already_resolved_job_reports_no_mismatch);
RUN_TEST(