feat(libxce-alloc-preason-equality): sim verify uses public job_alloc_eq and logs the first difference (task 4.3)

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

.kiro/specs/libxce-alloc-preason-equality/tasks.md

@@ -44,7 +44,7 @@ Test-first throughout (red → green → refactor). Every sub-task leaves the bu
   - _Requirements: 4.1, 4.4, 4.5, 6.5, 6.6_
   - _Boundary: scheduler allocation chain_
 
-- [ ] 4. Integration: move the consumers to the public API
+- [x] 4. Integration: move the consumers to the public API
 - [x] 4.1 Move the three white-box equivalence tests to the public comparators
   - Switch the three test files that call the simulator's private comparators to the public allocation and full-match pending-reason comparisons, printing the description in their failure messages and freeing it.
   - Keep their white-box inclusion of the simulator's verification source, which they still need for the decision-level functions.
@@ -60,7 +60,7 @@ Test-first throughout (red → green → refactor). Every sub-task leaves the bu
   - Done when every listed fixture carries the session-start value and the scheduler suite still passes.
   - _Requirements: 6.2, 6.7_
   - _Depends: 3.2_
-- [ ] 4.3 Migrate simulator verification to the public API and report the difference in mismatch logs
+- [x] 4.3 Migrate simulator verification to the public API and report the difference in mismatch logs
   - Write the tests first: add the new description argument to the verification unit test's direct decision-comparison calls; test that the description names the differing field and both values for a dispatch difference, a pending job-id difference, a pending main-reason difference, a kind difference, a missing side and an unsupported preempt; test that the mismatch log line appends a provided description; invert the "reservation start time difference still matches" test so a difference is now reported and named.
   - Delete the simulator's private comparators; compare dispatch decisions with the public allocation comparison; keep a simulator-local main-reason-only pending check with the bug-1845 rationale (design.md, Components: Simulator verification).
   - In the simulator's replay loop (the only code caller of the decision comparison and the mismatch log), pass the description it already computes to the mismatch log and free it, and pass a fixed "no live decision" text for recorded decisions left over; the signature changes and these callers land in this same sub-task.
@@ -90,3 +90,4 @@ Test-first throughout (red → green → refactor). Every sub-task leaves the bu
 - `xce-build.sh lib` runs only the `libxce test` binary; run `meson test -C src/xce.build.alloceq "libxce alloc OOM test"` (or plain `meson test`) for the allocation OOM binary, which also wraps `vasprintf`/`__vasprintf_chk` since 2.1.
 - Simulation mode (`sim_loop.c`) still takes its own session-begin clock reading before `xce_core_begin_session()`; by design that is left to spec scheduler-sim-discrete-clock (3.1 fixed only the normal-mode recorder).
 - 4.2 left the `from`-excluded comments at `test_sim_verify.c` ~639-681 for 4.3, which inverts the test they document.
+- `sim_verify.c` builds descriptions with `vsnprintf` + `malloc`, not `vasprintf`: about 25 scheduler tests `#include "../src/sim_verify.c"` after `<stdio.h>` without `_GNU_SOURCE`. Same ownership contract as libxce.

src/scheduler/src/sim_loop.c

@@ -138,10 +138,10 @@ extern int msg_root_has_shape(struct capn* c, capn_ptr* out, size_t want_words,
  * comment for the full rationale.
  */
 extern int  sim_decision_eq(const replay_decision_t* expected,
-                            const xce_decision_t*    actual);
+                            const xce_decision_t* actual, char** why);
 extern void sim_log_mismatch(uint64_t begin_round, int decision_index,
                              const replay_decision_t* expected,
-                             const xce_decision_t*    actual);
+                             const xce_decision_t* actual, const char* why);
 
 /*
  * Shared with sim_record.c (task 11.5, Req 5 AC 19) -- same "extern
@@ -1611,9 +1611,14 @@ int sched_sim_driver_run(xce_sched_sim_driver_t* sim,
                             ? decoded_session->decisions[di]
                             : NULL;
 
-                    if (!sim_decision_eq(expected, d)) {
+                    char* why = NULL;
-                        sim_log_mismatch(begin_round, di, expected, d);
+
+                    /* Requirement 5.3: the mismatch line names the first
+                     * difference; this loop owns and frees it. */
+                    if (!sim_decision_eq(expected, d, &why)) {
+                        sim_log_mismatch(begin_round, di, expected, d, why);
                     }
+                    free(why);
                     di++;
                 }
                 /* Req 5 AC 19 (task 11.5): "append a ReplayDecision to the
@@ -1635,7 +1640,8 @@ int sched_sim_driver_run(xce_sched_sim_driver_t* sim,
 
                 for (j = di; j < decoded_session->n_decisions; j++) {
                     sim_log_mismatch(begin_round, j,
-                                     decoded_session->decisions[j], NULL);
+                                     decoded_session->decisions[j], NULL,
+                                     "no live decision");
                 }
             }
 

src/scheduler/src/sim_verify.c

@@ -1,10 +1,13 @@
+#include <stdarg.h>
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 
 #include "controller.capnp.h"
 #include "decision.h"
 #include "utils.h"
 #include "xce/types.h"
+#include "xcealloc.h"
 
 /*
  * sim_verify.c - deterministic replay verification (scheduler-core spec,
@@ -62,164 +65,71 @@
  * make_pend_decision() calls (decision.c), the other from
  * decode_ReplaySession_ptr()'s deep-copy-from-capnp-arena walk
  * (controller.capnp.c). Pointer or memcmp equality would therefore always
- * report a false mismatch; every comparator below instead walks both trees
+ * report a false mismatch.
- * field-by-field, matching the exact xce_job_alloc_t/xce_job_preason_t
- * shapes recording_session.c's own encode side (the inverse direction)
- * already documents.
- */
-
-/*
- * sim_verify_str_eq - NULL-safe string equality. Both trees under
- * comparison route every string through STRING_DUP()/makeString()-style
- * heap copies on their own independent path (decode_* on one side,
- * make_pend_decision()/the picker's own construction on the other), so a
- * NULL/non-NULL string field is itself real signal, not noise -- (a==b==
- * NULL) counts as equal, exactly one of the two being NULL never does.
- */
-static int sim_verify_str_eq(const char* a, const char* b)
-{
-    if (a == b) {
-        return 1;
-    }
-    if ((a == NULL) || (b == NULL)) {
-        return 0;
-    }
-    return strcmp(a, b) == 0;
-}
-
-/*
- * sim_verify_alloc_eq - xce_alloc_t (one resource allocation record, e.g.
- * one GPU/memory grant) field-by-field equality. `value` is compared with
- * plain `==`: both sides are exact copies (decode/construction, no
- * arithmetic in between) of the same double, so bit-exact comparison is
- * correct here, not the usual floating-point-tolerance concern.
  *
- * `from` is DELIBERATELY excluded (bug-1838-followup): make_alloc()
+ * libxce-alloc-preason-equality (Requirements 5.1-5.3, 6.1): DISPATCH
- * (xcealloc.c) unconditionally stamps it with time(NULL), the real OS wall
+ * payloads are compared with libxce's public job_alloc_eq() (xcealloc.h),
- * clock, not core.c's virtualized sim clock. `expected->from` is whatever
+ * the single shared allocation comparison. It compares every allocation
- * real second the ORIGINAL recording happened to run in; `actual->from` is
+ * field except `instid` (a per-process counter, bug-1844) -- including
- * whatever real second THIS replay happens to run in -- two independent
+ * `from`, which every allocation now carries as its deciding session's
- * real-world timestamps with no reason to agree, even though the decision
+ * start (Requirement 4.1), so a faithful replay reproduces it exactly. PEND
- * they're attached to is semantically identical (same job, same host, same
+ * payloads keep the simulator-local main-reason-only check below
- * resource, same amount). Comparing them made every genuinely-correct live
+ * (sim_verify_pend_main_reason_eq(), bug-1845), not libxce's full-match
- * DISPATCH decision report a false --sim-verify MISMATCH whenever a replay
+ * job_preason_eq(). Each comparison can describe its first difference; the
- * runs so much as one wall-clock second away from its own recording --
+ * caller hands that description to sim_log_mismatch().
- * i.e. essentially always, for any replay of a real, non-synthetic
+ */
- * recording. `from` is reservation bookkeeping (when this grant started, for
+
- * expiry/accounting purposes), not part of the decision's own identity --
+/*
- * unlike `duration`, which IS a deterministic, request-derived value and
+ * sim_verify_why - *why = newly allocated formatted description, or NULL if
- * stays part of this comparison.
+ * formatting or the allocation fails. No-op when why is NULL. Same contract
+ * as libxce's equality functions. Built with vsnprintf() + malloc() rather
+ * than vasprintf(): this file is also white-box #include'd into test TUs
+ * whose <stdio.h> is already included without _GNU_SOURCE.
+ */
+static void sim_verify_why(char** why, const char* fmt, ...)
+    __attribute__((format(printf, 2, 3)));
+
+static void sim_verify_why(char** why, const char* fmt, ...)
+{
+    va_list ap;
+    char*   s = NULL;
+    int     n;
+
+    if (why == NULL) {
+        return;
+    }
+    *why = NULL;
+
+    va_start(ap, fmt);
+    n = vsnprintf(NULL, 0, fmt, ap);
+    va_end(ap);
+    if (n < 0) {
+        return;
+    }
+
+    s = (char*)malloc((size_t)n + 1);
+    if (s == NULL) {
+        return;
+    }
+
+    va_start(ap, fmt);
+    if (vsnprintf(s, (size_t)n + 1, fmt, ap) < 0) {
+        free(s);
+        s = NULL;
+    }
+    va_end(ap);
+
+    *why = s;
+}
+
+/*
+ * sim_verify_pend_main_reason_eq - the simulator's PEND comparison:
+ * main_reason ONLY (Requirement 5.2). Deliberately NOT libxce's full-match
+ * job_preason_eq(), which answers a different question ("are these two
+ * pending-reason objects identical") than this one ("is this the same
+ * replay outcome").
  *
- * `instid` is ALSO deliberately excluded (bug-1844, same class of issue as
+ * bug-1845: core.h documents, as a deliberate "simulation isolation"
- * `from` above): stamped from `sched->core.nextid++` (scheduler.c's
- * sched_get_id()) at allocation-build time -- a bare per-process monotonic
- * counter, not a semantic property of what got allocated. It counts how
- * many allocation instances THIS core has built over its own entire
- * execution history, so the ORIGINAL recording's counter (which may have
- * been running, and building allocations, for far longer before this
- * particular capture window even started) and a fresh replay's own counter
- * (starting from whatever its loaded snapshot's nextid happens to be) have
- * no reason to reach the same value for "the same" dispatch, even when
- * every other field -- value, duration, resname, reserve, host -- matches
- * exactly. Live-verified: a real captured .replay file showed every single
- * DISPATCH's `instid` differing (e.g. recorded=826 vs replay=1, both
- * otherwise byte-identical) while zero other fields ever differed.
- */
-static int sim_verify_alloc_eq(const xce_alloc_t* a, const xce_alloc_t* b)
-{
-    if (a == b) {
-        return 1;
-    }
-    if ((a == NULL) || (b == NULL)) {
-        return 0;
-    }
-
-    return (a->value == b->value) && (a->duration == b->duration)
-        && sim_verify_str_eq(a->resname, b->resname)
-        && sim_verify_str_eq(a->reserve, b->reserve);
-}
-
-/*
- * sim_verify_host_alloc_eq - xce_host_alloc_t equality: same host name,
- * same count and (in-order) content of per-resource xce_alloc_t entries.
- * Order-sensitive comparison is correct here: both the recorded and the
- * live tree are built by the SAME deterministic scheduling round replaying
- * against the SAME core state (Req 5's whole simulation-mode premise), so
- * the host/resource allocation arrays are expected to come out in the same
- * order, not just the same multiset.
- */
-static int sim_verify_host_alloc_eq(const xce_host_alloc_t* a,
-                                    const xce_host_alloc_t* b)
-{
-    int i;
-
-    if (a == b) {
-        return 1;
-    }
-    if ((a == NULL) || (b == NULL)) {
-        return 0;
-    }
-    if (a->nallocs != b->nallocs) {
-        return 0;
-    }
-    if (!sim_verify_str_eq(a->host, b->host)) {
-        return 0;
-    }
-
-    for (i = 0; i < a->nallocs; i++) {
-        if (!sim_verify_alloc_eq(a->allocs[i], b->allocs[i])) {
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
-/*
- * sim_verify_job_alloc_eq - xce_job_alloc_t (a DISPATCH decision's full
- * payload) equality: jobid, per-host allocation list (xce_host_alloc_t,
- * via sim_verify_host_alloc_eq above), and the shared-resource allocation
- * list (xce_alloc_t, via sim_verify_alloc_eq).
- */
-static int sim_verify_job_alloc_eq(const xce_job_alloc_t* a,
-                                   const xce_job_alloc_t* b)
-{
-    int i;
-
-    if (a == b) {
-        return 1;
-    }
-    if ((a == NULL) || (b == NULL)) {
-        return 0;
-    }
-    if ((a->jobid != b->jobid) || (a->nhosts != b->nhosts)
-        || (a->nres_shared != b->nres_shared)) {
-        return 0;
-    }
-
-    for (i = 0; i < a->nhosts; i++) {
-        if (!sim_verify_host_alloc_eq(a->allocs[i], b->allocs[i])) {
-            return 0;
-        }
-    }
-
-    for (i = 0; i < a->nres_shared; i++) {
-        if (!sim_verify_alloc_eq(a->allocs_shared[i], b->allocs_shared[i])) {
-            return 0;
-        }
-    }
-
-    return 1;
-}
-
-/*
- * sim_verify_job_preason_eq - xce_job_preason_t (a PEND decision's reason
- * payload) equality: main_reason ONLY.
- *
- * bug-1845 fix: this used to also require the full host-based/resource-
- * based reason detail lists to match (added as an optional "cheap to add"
- * deepening of the check, never proven necessary by a dedicated test).
- * That is wrong: core.h documents, as a deliberate "simulation isolation"
  * design decision, that "a simulation-mode session never touches the real
  * cache implementation at all" -- core->unsatisfiable_lookup_cb/
  * unsatisfiable_populate_cb are registered only by main.c's normal-mode
@@ -240,17 +150,56 @@ static int sim_verify_job_alloc_eq(const xce_job_alloc_t* a,
  * computation) showed {main_reason=0, nhost_reasons=0, nres_reasons=2} --
  * main_reason matched, detail did not, and never architecturally can.
  */
-static int sim_verify_job_preason_eq(const xce_job_preason_t* a,
+static int sim_verify_pend_main_reason_eq(const xce_job_preason_t* a,
-                                     const xce_job_preason_t* b)
+                                          const xce_job_preason_t* b,
+                                          char**                   why)
 {
     if (a == b) {
         return 1;
     }
     if ((a == NULL) || (b == NULL)) {
+        sim_verify_why(why, "reason: %s != %s",
+                       (a != NULL) ? "non-NULL" : "NULL",
+                       (b != NULL) ? "non-NULL" : "NULL");
+        return 0;
+    }
+    if (a->main_reason != b->main_reason) {
+        sim_verify_why(why, "main_reason: %d != %d", a->main_reason,
+                       b->main_reason);
         return 0;
     }
 
-    return a->main_reason == b->main_reason;
+    return 1;
+}
+
+/* sim_verify_kind_name - the decision kind a recorded body_which names. */
+static const char* sim_verify_kind_name(int body_which)
+{
+    switch (body_which) {
+    case ReplayDecision_body_dispatch:
+        return "DISPATCH";
+    case ReplayDecision_body_pend:
+        return "PEND";
+    case ReplayDecision_body_preempt:
+        return "PREEMPT";
+    default:
+        return "UNKNOWN";
+    }
+}
+
+/* sim_verify_op_name - the decision kind a live op names. */
+static const char* sim_verify_op_name(int op)
+{
+    switch (op) {
+    case XCE_DECISION_DISPATCH:
+        return "DISPATCH";
+    case XCE_DECISION_PEND:
+        return "PEND";
+    case XCE_DECISION_PREEMPT:
+        return "PREEMPT";
+    default:
+        return "UNKNOWN";
+    }
 }
 
 /*
@@ -263,33 +212,75 @@ static int sim_verify_job_preason_eq(const xce_job_preason_t* a,
  * body_which to name the SAME decision kind before comparing payloads --
  * an op/body_which mismatch (e.g. recorded PEND, live DISPATCH) is itself
  * reported as a mismatch, not silently ignored.
+ *
+ * `why` (optional, libxce-alloc-preason-equality Requirement 5.3): same
+ * ownership contract as libxce's job_alloc_eq(). When non-NULL, *why is
+ * set to NULL on a match, or to a newly allocated description of the first
+ * difference on a mismatch (NULL if allocating it fails), recorded side
+ * first: job_alloc_eq()'s path for DISPATCH, `jobid: A != B` or
+ * `main_reason: A != B` for PEND, `kind: A != B` for a kind mismatch,
+ * `no recorded decision at this index`/`no live decision` for a missing
+ * side, `PREEMPT comparison not supported` for PREEMPT. The caller frees.
  */
 int sim_decision_eq(const replay_decision_t* expected,
-                    const xce_decision_t*    actual)
+                    const xce_decision_t* actual, char** why)
 {
-    if ((expected == NULL) || (actual == NULL)) {
+    if (why != NULL) {
+        *why = NULL;
+    }
+
+    if (expected == NULL) {
+        sim_verify_why(why, "no recorded decision at this index");
         return 0;
     }
+    if (actual == NULL) {
+        sim_verify_why(why, "no live decision");
+        return 0;
+    }
+
+    if ((actual->op == XCE_DECISION_DISPATCH)
+        || (actual->op == XCE_DECISION_PEND)) {
+        int want = (actual->op == XCE_DECISION_DISPATCH)
+            ? ReplayDecision_body_dispatch
+            : ReplayDecision_body_pend;
+
+        if ((int)expected->body_which != want) {
+            sim_verify_why(why, "kind: %s != %s",
+                           sim_verify_kind_name(expected->body_which),
+                           sim_verify_op_name(actual->op));
+            return 0;
+        }
+    }
 
     switch (actual->op) {
     case XCE_DECISION_DISPATCH:
-        if (expected->body_which != ReplayDecision_body_dispatch) {
+        if (expected->body.dispatch == NULL) {
+            sim_verify_why(why, "dispatch: NULL != non-NULL");
             return 0;
         }
-        return (expected->body.dispatch != NULL)
+        return job_alloc_eq(expected->body.dispatch->alloc, actual->u.alloc,
-            && sim_verify_job_alloc_eq(expected->body.dispatch->alloc,
+                            why);
-                                       actual->u.alloc);
 
     case XCE_DECISION_PEND:
-        if (expected->body_which != ReplayDecision_body_pend) {
-            return 0;
-        }
         if ((expected->body.pend == NULL) || (actual->u.reason == NULL)) {
+            sim_verify_why(why, "pend: %s != %s",
+                           (expected->body.pend != NULL) ? "non-NULL" : "NULL",
+                           (actual->u.reason != NULL) ? "non-NULL" : "NULL");
             return 0;
         }
-        return (expected->body.pend->jobid == actual->u.reason->jobid)
+        if (expected->body.pend->jobid != actual->u.reason->jobid) {
-            && sim_verify_job_preason_eq(expected->body.pend->reason,
+            if (why != NULL) {
-                                         actual->u.reason->reasons);
+                char ja[64];
+                char jb[64];
+
+                fmt_jobid(ja, sizeof(ja), expected->body.pend->jobid);
+                fmt_jobid(jb, sizeof(jb), actual->u.reason->jobid);
+                sim_verify_why(why, "jobid: %s != %s", ja, jb);
+            }
+            return 0;
+        }
+        return sim_verify_pend_main_reason_eq(expected->body.pend->reason,
+                                              actual->u.reason->reasons, why);
 
     case XCE_DECISION_PREEMPT:
         /* recording_session.c's own documented reason (see its
@@ -313,9 +304,13 @@ int sim_decision_eq(const replay_decision_t* expected,
                   "as a mismatch",
                   __func__, jobidStr);
         }
+        sim_verify_why(why, "PREEMPT comparison not supported");
         return 0;
 
     default:
+        sim_verify_why(why, "kind: %s != %s",
+                       sim_verify_kind_name(expected->body_which),
+                       sim_verify_op_name(actual->op));
         return 0;
     }
 }
@@ -453,10 +448,17 @@ static void sim_verify_describe_decision(const xce_decision_t* d, char* buf,
  * or `actual` may be NULL (a length mismatch between the recorded and live
  * decision counts for this session -- sim_loop.c's own drain loop reports
  * that case through this same function rather than a separate one).
+ *
+ * `why` (libxce-alloc-preason-equality Requirement 5.3, bug-1881): the
+ * first-difference description, appended as ` -- diff: <why>` when
+ * non-NULL; NULL (only when allocating the description failed) leaves the
+ * line otherwise unchanged. This function does not compare again and does
+ * not free `why`: sim_loop.c, which calls sim_decision_eq() right before
+ * it, owns and frees the string.
  */
 void sim_log_mismatch(uint64_t begin_round, int decision_index,
                       const replay_decision_t* expected,
-                      const xce_decision_t*    actual)
+                      const xce_decision_t* actual, const char* why)
 {
     char     expected_desc[SIM_VERIFY_DESC_BUF_SZ];
     char     actual_desc[SIM_VERIFY_DESC_BUF_SZ];
@@ -467,7 +469,9 @@ void sim_log_mismatch(uint64_t begin_round, int decision_index,
     sim_verify_describe_decision(actual, actual_desc, sizeof(actual_desc));
 
     ERROR("%s decision verification MISMATCH at begin_round=%llu "
-          "decision_index=%d expected_ts=%llu -- expected: %s -- actual: %s",
+          "decision_index=%d expected_ts=%llu -- expected: %s -- actual: "
+          "%s%s%s",
           __func__, (unsigned long long)begin_round, decision_index,
-          (unsigned long long)expected_ts, expected_desc, actual_desc);
+          (unsigned long long)expected_ts, expected_desc, actual_desc,
+          (why != NULL) ? " -- diff: " : "", (why != NULL) ? why : "");
 }

src/scheduler/testsuite/test_sim_verify.c

@@ -14,9 +14,11 @@
  *   Unit-level (white-box): calls sim_decision_eq()/sim_log_mismatch()
  *   (sim_verify.c, task 11.4's own sibling file to sim_loop.c) directly
  *   against hand-built replay_decision_t/xce_decision_t pairs -- DISPATCH
- *   match/mismatch (jobid, host, resource value), PEND match/mismatch
+ *   match/mismatch (jobid, host, resource value, from), PEND match/mismatch
  *   (jobid, main_reason), a body_which/op type mismatch, and PREEMPT's
- *   documented always-false path. This is the only way to exercise DISPATCH
+ *   documented always-false path -- each mismatch also asserting the
+ *   first-difference description (libxce-alloc-preason-equality
+ *   Requirement 5.3). This is the only way to exercise DISPATCH
  *   comparison at all: sim_loop.c's real picker thread
  *   (sim_picker_thread_fn) only ever emits PEND decisions (its own doc
  *   comment: "conservatively PENDs every candidate ... rather than
@@ -558,8 +560,24 @@ static replay_decision_t make_replay_pend_decision(uint64_t jobid,
 /* Unit-level tests: sim_decision_eq() directly.                          */
 /* ---------------------------------------------------------------------- */
 
+/* Asserts that sim_decision_eq()'s description (libxce-alloc-preason-
+ * equality Requirement 5.3) was produced and contains `needle` -- the name
+ * of the differing field and both of its values -- then frees it. */
+static void assert_why_names(char* why, const char* needle)
+{
+    char msg[512];
+
+    TEST_ASSERT_NOT_NULL_MESSAGE(why, "a mismatch must produce a "
+                                      "description of the difference");
+    snprintf(msg, sizeof(msg), "description \"%s\" must contain \"%s\"",
+             why, needle);
+    TEST_ASSERT_NOT_NULL_MESSAGE(strstr(why, needle), msg);
+    free(why);
+}
+
 static void test_dispatch_decisions_with_identical_content_match(void)
 {
+    char*        why;
     xce_alloc_t* res_a[1]
         = { make_test_alloc("gpu", "resv1", 1, 2.0, 100, 1800) };
     xce_host_alloc_t* host_a[1] = { make_test_host_alloc("hostA", res_a, 1) };
@@ -577,7 +595,11 @@ static void test_dispatch_decisions_with_identical_content_match(void)
      * trees: proves the comparison is real structural equality, not a
      * pointer/memcmp shortcut (this file's own header comment). */
     TEST_ASSERT_NOT_EQUAL(expected_alloc, actual_alloc);
-    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
+    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL));
+    why = (char*)"stale";
+    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, &why));
+    TEST_ASSERT_NULL_MESSAGE(why, "an equal comparison must leave no "
+                                  "description");
 
     printf("test_dispatch_decisions_with_identical_content_match: PASS\n");
 }
@@ -595,7 +617,17 @@ static void test_dispatch_decisions_differing_jobid_mismatch(void)
     replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
     xce_decision_t*   actual   = make_dispatch_decision(actual_alloc);
 
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
+    char              ja[64];
+    char              jb[64];
+    char              needle[160];
+    char*             why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, NULL));
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    fmt_jobid(ja, sizeof(ja), 42ULL);
+    fmt_jobid(jb, sizeof(jb), 43ULL);
+    snprintf(needle, sizeof(needle), "jobid: %s != %s", ja, jb);
+    assert_why_names(why, needle);
 
     printf("test_dispatch_decisions_differing_jobid_mismatch: PASS\n");
 }
@@ -613,7 +645,11 @@ static void test_dispatch_decisions_differing_host_mismatch(void)
     replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
     xce_decision_t*   actual   = make_dispatch_decision(actual_alloc);
 
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
+    char*             why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    assert_why_names(why, "allocs[0](host=hostA).host: \"hostA\" != "
+                          "\"hostB\"");
 
     printf("test_dispatch_decisions_differing_host_mismatch: PASS\n");
 }
@@ -631,27 +667,30 @@ static void test_dispatch_decisions_differing_resource_value_mismatch(void)
     replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
     xce_decision_t*   actual   = make_dispatch_decision(actual_alloc);
 
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
+    char*             why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    assert_why_names(why, "allocs[0](host=hostA).allocs[0](resname=gpu)."
+                          "value: 1 != 2");
 
     printf("test_dispatch_decisions_differing_resource_value_mismatch: "
            "PASS\n");
 }
 
 /*
- * test_dispatch_decisions_differing_from_timestamp_still_match -
+ * test_dispatch_decisions_differing_from_timestamp_mismatch -
- * bug-1838-followup regression: `from` (xce_alloc_t's wall-clock reservation-
+ * libxce-alloc-preason-equality Requirement 5.1 (inverts the former
- * start stamp, set unconditionally by make_alloc() via time(NULL) --
+ * bug-1838-followup "from still matches" lock): `from` used to be excluded
- * xcealloc.c) is real-OS-clock-derived, not part of a decision's semantic
+ * because make_alloc() stamped it with time(NULL), a real-OS-clock value
- * identity. `expected`'s `from` is whatever second the ORIGINAL recording
+ * that differed between a recording and its replay. Every allocation a
- * ran in; `actual`'s is whatever second THIS run rebuilds the allocation in
+ * session decides now carries that session's start (Requirement 4.1,
- * -- two independent real-world timestamps that will essentially never
+ * scheduler.c's sched_make_primary_alloc_()), and a replayed session starts
- * agree for any replay of a real recording, even when the decision itself
+ * at its recorded beginRound, so the two sides agree for a faithful replay
- * (job/host/resource/value/duration) is byte-for-byte the one that was
+ * and `from` is compared (by libxce's job_alloc_eq()). A `from` difference
- * recorded. Uses two `from` values far enough apart (a Unix-epoch-plausible
+ * is therefore a real divergence: it must be reported, and the description
- * value vs. 1) that they could never coincidentally collide, proving this
+ * must name `from` with both values.
- * is deliberate exclusion, not a fluke of nearby clock reads.
  */
-static void test_dispatch_decisions_differing_from_timestamp_still_match(void)
+static void test_dispatch_decisions_differing_from_timestamp_mismatch(void)
 {
     xce_alloc_t* res_a[1]
         = { make_test_alloc("gpu", "resv1", 1, 2.0, 1790000000ULL, 1800) };
@@ -665,10 +704,13 @@ static void test_dispatch_decisions_differing_from_timestamp_still_match(void)
 
     replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
     xce_decision_t*   actual   = make_dispatch_decision(actual_alloc);
+    char*             why      = NULL;
 
-    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    assert_why_names(why, "allocs[0](host=hostA).allocs[0](resname=gpu)."
+                          "from: 1790000000 != 1");
 
-    printf("test_dispatch_decisions_differing_from_timestamp_still_match: "
+    printf("test_dispatch_decisions_differing_from_timestamp_mismatch: "
            "PASS\n");
 }
 
@@ -677,8 +719,10 @@ static void test_dispatch_decisions_differing_from_timestamp_still_match(void)
  * regression: `instid` (xce_alloc_t's per-process monotonic allocation-
  * instance counter, stamped from `sched->core.nextid++` at allocation-build
  * time -- scheduler.c's sched_get_id()) is an execution-order bookkeeping
- * value, not part of a decision's semantic identity, exactly like `from`
+ * value, not part of a decision's semantic identity. It is the one field
- * above (bug-1838-followup). The ORIGINAL recording's counter may have been
+ * libxce's alloc_eq() excludes (`from`, unlike instid, is now compared: see
+ * test_dispatch_decisions_differing_from_timestamp_mismatch above). The
+ * ORIGINAL recording's counter may have been
  * running, and building allocations, for far longer before this capture
  * window started; a fresh replay's own counter starts whatever its loaded
  * snapshot's nextid happens to be -- the two have no reason to reach the
@@ -703,7 +747,7 @@ static void test_dispatch_decisions_differing_instid_still_match(void)
     replay_decision_t expected = make_replay_dispatch_decision(expected_alloc);
     xce_decision_t*   actual   = make_dispatch_decision(actual_alloc);
 
-    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
+    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL));
 
     printf("test_dispatch_decisions_differing_instid_still_match: PASS\n");
 }
@@ -713,11 +757,14 @@ static void test_pend_decisions_with_identical_content_match(void)
     replay_decision_t expected = make_replay_pend_decision(9001ULL, 3);
     xce_pendreason_t* reason   = make_pendreason(9001ULL);
     xce_decision_t*   actual;
+    char*             why;
 
     reason->reasons->main_reason = 3;
     actual                       = make_pend_decision(reason);
 
-    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
+    why = (char*)"stale";
+    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, &why));
+    TEST_ASSERT_NULL(why);
 
     printf("test_pend_decisions_with_identical_content_match: PASS\n");
 }
@@ -727,7 +774,16 @@ static void test_pend_decisions_differing_jobid_mismatch(void)
     replay_decision_t expected = make_replay_pend_decision(9001ULL, 0);
     xce_decision_t*   actual   = make_pend_decision(make_pendreason(9002ULL));
 
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
+    char              ja[64];
+    char              jb[64];
+    char              needle[160];
+    char*             why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    fmt_jobid(ja, sizeof(ja), 9001ULL);
+    fmt_jobid(jb, sizeof(jb), 9002ULL);
+    snprintf(needle, sizeof(needle), "jobid: %s != %s", ja, jb);
+    assert_why_names(why, needle);
 
     printf("test_pend_decisions_differing_jobid_mismatch: PASS\n");
 }
@@ -739,7 +795,10 @@ static void test_pend_decisions_differing_main_reason_mismatch(void)
 
     /* actual's main_reason stays 0 (make_pendreason()'s own fresh-calloc
      * default) -- differs from expected's 3. */
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
+    char* why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    assert_why_names(why, "main_reason: 3 != 0");
 
     printf("test_pend_decisions_differing_main_reason_mismatch: PASS\n");
 }
@@ -789,7 +848,7 @@ static void test_pend_decisions_differing_reason_detail_still_match(void)
 
     actual = make_pend_decision(reason);
 
-    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual));
+    TEST_ASSERT_TRUE(sim_decision_eq(&expected, actual, NULL));
 
     printf("test_pend_decisions_differing_reason_detail_still_match: PASS\n");
 }
@@ -801,8 +860,12 @@ static void test_decision_kind_mismatch_reported(void)
     xce_decision_t*   actual   = make_dispatch_decision(alloc);
 
     /* Recorded PEND vs. live DISPATCH for the SAME jobid: still a
-     * mismatch -- op/body_which disagreement is reported, not ignored. */
+     * mismatch -- op/body_which disagreement is reported, not ignored.
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual));
+     * The description names the kinds, recorded side first. */
+    char* why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, actual, &why));
+    assert_why_names(why, "kind: PEND != DISPATCH");
 
     printf("test_decision_kind_mismatch_reported: PASS\n");
 }
@@ -826,7 +889,10 @@ static void test_preempt_decision_always_reports_mismatch(void)
     memset(&expected, 0, sizeof(expected));
     expected.body_which = ReplayDecision_body_preempt;
 
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, &actual));
+    char* why = NULL;
+
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, &actual, &why));
+    assert_why_names(why, "PREEMPT comparison not supported");
 
     printf("test_preempt_decision_always_reports_mismatch: PASS\n");
 }
@@ -836,9 +902,18 @@ static void test_decision_eq_defends_against_null_inputs(void)
     xce_decision_t*   actual   = make_pend_decision(make_pendreason(1ULL));
     replay_decision_t expected = make_replay_pend_decision(1ULL, 0);
 
-    TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual));
+    char*             why      = NULL;
-    TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL));
+
-    TEST_ASSERT_FALSE(sim_decision_eq(NULL, NULL));
+    TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual, NULL));
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL, NULL));
+    TEST_ASSERT_FALSE(sim_decision_eq(NULL, NULL, NULL));
+
+    /* A missing side is named: which of the two has no decision. */
+    TEST_ASSERT_FALSE(sim_decision_eq(NULL, actual, &why));
+    assert_why_names(why, "no recorded decision at this index");
+    why = NULL;
+    TEST_ASSERT_FALSE(sim_decision_eq(&expected, NULL, &why));
+    assert_why_names(why, "no live decision");
 
     printf("test_decision_eq_defends_against_null_inputs: PASS\n");
 }
@@ -892,7 +967,15 @@ static void log_one_mismatch_for_capture(void)
     replay_decision_t expected = make_replay_pend_decision(9001ULL, 3);
     xce_decision_t*   actual   = make_pend_decision(make_pendreason(9001ULL));
 
-    sim_log_mismatch(12345ULL, 2, &expected, actual);
+    sim_log_mismatch(12345ULL, 2, &expected, actual, "main_reason: 3 != 0");
+}
+
+static void log_one_mismatch_without_why_for_capture(void)
+{
+    replay_decision_t expected = make_replay_pend_decision(9001ULL, 3);
+    xce_decision_t*   actual   = make_pend_decision(make_pendreason(9001ULL));
+
+    sim_log_mismatch(12345ULL, 2, &expected, actual, NULL);
 }
 
 static void test_log_mismatch_reports_logical_time_and_decision_detail(void)
@@ -925,6 +1008,59 @@ static void test_log_mismatch_reports_logical_time_and_decision_detail(void)
            "PASS\n");
 }
 
+/*
+ * Requirement 5.3: the mismatch line keeps its logical-time and decision
+ * context and appends the difference description it is given; with no
+ * description (only when allocating it failed) the line is logged without
+ * one, unchanged otherwise.
+ */
+static void test_log_mismatch_appends_provided_description(void)
+{
+    char* buf = capture_stdout_around(log_one_mismatch_for_capture);
+    char* nodiff;
+
+    TEST_ASSERT_NOT_NULL(strstr(buf, "begin_round=12345"));
+    TEST_ASSERT_NOT_NULL_MESSAGE(
+        strstr(buf, " -- diff: main_reason: 3 != 0"),
+        "the mismatch line must append the provided description");
+    free(buf);
+
+    nodiff = capture_stdout_around(log_one_mismatch_without_why_for_capture);
+    TEST_ASSERT_NOT_NULL(strstr(nodiff, "MISMATCH"));
+    TEST_ASSERT_NOT_NULL(strstr(nodiff, "decision_index=2"));
+    TEST_ASSERT_NULL_MESSAGE(strstr(nodiff, "-- diff:"),
+                             "with no description the line carries no "
+                             "diff clause");
+    free(nodiff);
+
+    printf("test_log_mismatch_appends_provided_description: PASS\n");
+}
+
+/* Every line of `captured` that reports a MISMATCH also carries a
+ * ` -- diff: ` clause (Requirement 5.3: each mismatch names its
+ * difference). Returns the number of MISMATCH lines seen. */
+static int assert_every_mismatch_line_has_diff(const char* captured)
+{
+    const char* line = captured;
+    int         n    = 0;
+
+    while ((line != NULL) && (*line != '\0')) {
+        const char* eol = strchr(line, '\n');
+        size_t      len = (eol != NULL) ? (size_t)(eol - line) : strlen(line);
+        char*       one = strndup(line, len);
+
+        TEST_ASSERT_NOT_NULL(one);
+        if (strstr(one, "MISMATCH") != NULL) {
+            n++;
+            TEST_ASSERT_NOT_NULL_MESSAGE(strstr(one, " -- diff: "), one);
+        }
+        free(one);
+        line = (eol != NULL) ? (eol + 1) : NULL;
+    }
+
+    return n;
+}
+
 /* ---------------------------------------------------------------------- */
 /* Integration-level fixture helpers: a real .replay stream driving the   */
 /* REAL builtin_fcfs picker to a genuine live PEND decision.              */
@@ -1568,12 +1704,55 @@ static void test_verify_against_altered_copy_reports_exact_mismatch(void)
     TEST_ASSERT_NOT_NULL_MESSAGE(
         strstr(captured, jobid_marker),
         "the mismatch report must name the specific job");
+    TEST_ASSERT_NOT_NULL_MESSAGE(
+        strstr(captured, " -- diff: main_reason: 99 != 0"),
+        "the mismatch report must name the differing field and both values");
+    TEST_ASSERT_EQUAL_INT(1, assert_every_mismatch_line_has_diff(captured));
 
     free(captured);
     printf("test_verify_against_altered_copy_reports_exact_mismatch: "
            "PASS\n");
 }
 
+/*
+ * A recorded decision left over after the live stream ends (recorded
+ * PEND(777), PEND(778); only job 777 was ever pushed, so the live picker
+ * emits only PEND(777)) is reported as a mismatch whose diff says which
+ * side is missing: "no live decision" (Requirement 5.3).
+ */
+static void test_verify_leftover_recorded_decision_names_missing_side(void)
+{
+    char  path[PATH_MAX];
+    FILE* f;
+    int   rc;
+    char* captured;
+
+    f = write_header_and_snapshot(path, sizeof(path), "_leftover");
+    TEST_ASSERT_NOT_NULL(f);
+
+    append_queue_update_event(f, 1000ULL, "normal");
+    append_jobs_push_event(f, 1500ULL, 777ULL, "normal");
+    append_replay_session_with_two_pends(f, 2000ULL, 777ULL, 0, 778ULL, 0);
+    append_end_of_stream_marker(f);
+    fclose(f);
+
+    captured = run_sim_and_capture_stdout(path, true, &rc);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_NOT_NULL_MESSAGE(
+        strstr(captured, "decision_index=1"),
+        "the left-over recorded decision must be reported at its index");
+    TEST_ASSERT_NOT_NULL_MESSAGE(
+        strstr(captured, " -- diff: no live decision"),
+        "a left-over recorded decision's mismatch must name the missing "
+        "live side");
+    TEST_ASSERT_EQUAL_INT(1, assert_every_mismatch_line_has_diff(captured));
+
+    free(captured);
+    printf("test_verify_leftover_recorded_decision_names_missing_side: "
+           "PASS\n");
+}
+
 /*
  * sim->verify == false must skip verification entirely, even against the
  * SAME deliberately-altered fixture the previous test proves DOES mismatch
@@ -1873,6 +2052,12 @@ test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_mismatch(void)
         "PEND-only live stub cannot reproduce it either, so --sim-verify "
         "must correctly report a real mismatch here, not silently agree "
         "with a decision nothing in this environment actually made");
+    /* Round 1: recorded DISPATCH vs live PEND; round 2: a live PEND with
+     * no recorded entry. Both lines name their difference. */
+    TEST_ASSERT_NOT_NULL(strstr(captured, " -- diff: kind: DISPATCH != PEND"));
+    TEST_ASSERT_NOT_NULL(
+        strstr(captured, " -- diff: no recorded decision at this index"));
+    TEST_ASSERT_TRUE(assert_every_mismatch_line_has_diff(captured) >= 2);
 
     free(captured);
     printf("test_reconciliation_no_longer_masks_a_pend_jobs_dispatch_"
@@ -1887,7 +2072,7 @@ int main(void)
     RUN_TEST(test_dispatch_decisions_differing_jobid_mismatch);
     RUN_TEST(test_dispatch_decisions_differing_host_mismatch);
     RUN_TEST(test_dispatch_decisions_differing_resource_value_mismatch);
-    RUN_TEST(test_dispatch_decisions_differing_from_timestamp_still_match);
+    RUN_TEST(test_dispatch_decisions_differing_from_timestamp_mismatch);
     RUN_TEST(test_dispatch_decisions_differing_instid_still_match);
     RUN_TEST(test_pend_decisions_with_identical_content_match);
     RUN_TEST(test_pend_decisions_differing_jobid_mismatch);
@@ -1897,8 +2082,10 @@ int main(void)
     RUN_TEST(test_preempt_decision_always_reports_mismatch);
     RUN_TEST(test_decision_eq_defends_against_null_inputs);
     RUN_TEST(test_log_mismatch_reports_logical_time_and_decision_detail);
+    RUN_TEST(test_log_mismatch_appends_provided_description);
     RUN_TEST(test_verify_against_self_reports_zero_mismatches);
     RUN_TEST(test_verify_against_altered_copy_reports_exact_mismatch);
+    RUN_TEST(test_verify_leftover_recorded_decision_names_missing_side);
     RUN_TEST(test_verify_disabled_reports_no_mismatches_even_when_altered);
     RUN_TEST(test_verify_with_already_resolved_job_reports_no_mismatch);
     RUN_TEST(