merge: sync Rongsong's scheduler-framework (122 commits, 070c7869..f01ae353)

Introduces WAL replay field-completeness fixes (bug-1687..1697, 1731), scheduler snapshot job-allocation restore, heap UAF fixes (vtask_enq, xcealloc, builtin_fcfs, limit), HA fixes (bug-1694/1695/1696), xceadmin retry, worker signal-exit severity, and their test suites.

Conflicts resolved keeping both sides (drop-audit passed):
- worker.c: mark_worker_jobs_exit (bug-1509 B1) + fetchfile_signal_worker_disconnect (bug-1693)
- handler.go: DefaultBroker.Publish (EventJobFinished) + bug-1731 logging
- config_repl.c / test_config_repl.c: our ns-resolution comment / unique-tmpdir
- test_dispatch_commit.c: both B3-UNKWN and bug-1697 start_time assertions

Excluded Rongsong's bookkeeping per policy: .wolf/, .kiro/, CHANGES.md, reports/;
.claude/ + CLAUDE.md kept ours (his settings.json has machine-absolute paths).

src/controller/testsuite/clean_failover_preservation.sh

@@ -132,7 +132,7 @@ cleanup() {
 
 # Run a batch of xce_test commands against the live cluster; returns its exit.
 xtest() {
-    timeout 60 "$XCE_TEST" 2>&1
+    timeout 60 "$XCE_TEST" --norc 2>&1
 }
 
 # ── Preconditions ────────────────────────────────────────────────────────────

src/controller/testsuite/controller_test.c

@@ -70,6 +70,7 @@ extern void test_walcompare(void);
 extern void test_snapverify(void);
 extern void test_snapdiff(void);
 extern void test_configcompare(void);
+extern void test_rcfile(void);
 extern void test_drive_async_config(void);
 extern void test_slowlane(void);
 extern void test_commit_admission_pool(void);
@@ -84,11 +85,17 @@ extern void test_apply_seam(void);
 extern void test_apply_committed_entry(void);
 extern void test_replay_apply_seam(void);
 extern void test_wal_job_dispatch_fact(void);
+extern void test_wal_job_start_fact(void);
+extern void test_wal_job_finish_fact(void);
+extern void test_wal_job_rusage_fact(void);
+extern void test_wal_job_ckpt_fact(void);
+extern void test_wal_job_preason_fact(void);
 extern void test_job_dispatch_fail_count(void);
 extern void test_apply_reapply_idempotency(void);
 extern void test_apply_correctness_verification(void);
 extern void test_apply_pump(void);
 extern void test_apply_pump_integration(void);
+extern void test_persist_checkpoint_offload(void);
 extern void test_job_array_apply(void);
 extern void test_job_array_fence(void);
 extern void test_job_array_replay_fence(void);
@@ -96,6 +103,7 @@ extern void test_upgrade_snapshot_restore(void);
 extern void test_upgrade_snapshot_apply(void);
 extern void test_bug857_regression(void);
 extern void test_sched_admin_dispatch(void);
+extern void test_fetchfile_disconnect(void);
 extern void test_job_dispatch(void);
 extern void test_dispatch_commit(void);
 extern void test_dispatch_fail_terminal(void);
@@ -240,6 +248,7 @@ int main(int argc, char* argv[])
     TEST(test_snapverify);
     TEST(test_snapdiff);
     TEST(test_configcompare);
+    TEST(test_rcfile);
     TEST(test_drive_async_config);
     TEST(test_slowlane);
     TEST(test_commit_admission_pool);
@@ -254,11 +263,17 @@ int main(int argc, char* argv[])
     TEST(test_apply_committed_entry);
     TEST(test_replay_apply_seam);
     TEST(test_wal_job_dispatch_fact);
+    TEST(test_wal_job_start_fact);
+    TEST(test_wal_job_finish_fact);
+    TEST(test_wal_job_rusage_fact);
+    TEST(test_wal_job_ckpt_fact);
+    TEST(test_wal_job_preason_fact);
     TEST(test_job_dispatch_fail_count);
     TEST(test_apply_reapply_idempotency);
     TEST(test_apply_correctness_verification);
     TEST(test_apply_pump);
     TEST(test_apply_pump_integration);
+    TEST(test_persist_checkpoint_offload);
     TEST(test_job_array_apply);
     TEST(test_job_array_fence);
     TEST(test_job_array_replay_fence);
@@ -266,6 +281,7 @@ int main(int argc, char* argv[])
     TEST(test_upgrade_snapshot_apply);
     TEST(test_bug857_regression);
     TEST(test_sched_admin_dispatch);
+    TEST(test_fetchfile_disconnect);
     TEST(test_job_dispatch);
     TEST(test_dispatch_commit);
     TEST(test_dispatch_fail_terminal);

src/controller/testsuite/rcfile.c

@@ -0,0 +1,80 @@
+#define _GNU_SOURCE
+#include "rcfile.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Suffixes appended to the resolved base directory to form the rc-file
+ * path -- kept as two separate constants (rather than one shared tail)
+ * since the XDG_CONFIG_HOME branch is already rooted at a "config home"
+ * directory while the HOME fallback still needs the "/.config" segment. */
+#define RCFILE_XDG_SUFFIX  "/xce_test/rc"
+#define RCFILE_HOME_SUFFIX "/.config/xce_test/rc"
+
+char* rcfile_resolve_path(void)
+{
+    const char* xdg_config_home = getenv("XDG_CONFIG_HOME");
+    if (xdg_config_home && *xdg_config_home) {
+        size_t len = strlen(xdg_config_home) + strlen(RCFILE_XDG_SUFFIX) + 1;
+        char*  path = (char*)malloc(len);
+        if (!path)
+            return NULL;
+        snprintf(path, len, "%s%s", xdg_config_home, RCFILE_XDG_SUFFIX);
+        return path;
+    }
+
+    const char* home = getenv("HOME");
+    if (home && *home) {
+        size_t len = strlen(home) + strlen(RCFILE_HOME_SUFFIX) + 1;
+        char*  path = (char*)malloc(len);
+        if (!path)
+            return NULL;
+        snprintf(path, len, "%s%s", home, RCFILE_HOME_SUFFIX);
+        return path;
+    }
+
+    return NULL;
+}
+
+int rcfile_load(const char* path, int (*runner)(char* line))
+{
+    if (!path)
+        return 0;
+
+    FILE* f = fopen(path, "r");
+    if (!f) {
+        if (errno == ENOENT)
+            return 0; /* missing rc file: silent no-op success */
+        fprintf(stderr, "xce_test: warning: could not open rc file %s: %s\n",
+                path, strerror(errno));
+        return 0; /* present but unreadable: non-fatal warning */
+    }
+
+    int     rc   = 0;
+    char*   line = NULL;
+    size_t  cap  = 0;
+    ssize_t n;
+    while ((n = getline(&line, &cap, f)) != -1) {
+        if (n > 0 && line[n - 1] == '\n')
+            line[n - 1] = '\0';
+        rc = runner(line);
+        if (rc != 0)
+            break;
+    }
+
+    /* fopen() succeeds on a directory on this platform -- the failure only
+     * surfaces on the first read (getline() returns -1 with errno==EISDIR
+     * and ferror(f) set, feof(f) NOT set). Treat that the same as an open
+     * failure: one non-fatal stderr warning, no runner calls attributed to
+     * it (rc is still 0 here since no line was ever successfully read). */
+    if (rc == 0 && ferror(f)) {
+        fprintf(stderr, "xce_test: warning: could not read rc file %s: %s\n",
+                path, strerror(errno));
+    }
+
+    free(line);
+    fclose(f);
+    return rc;
+}

src/controller/testsuite/rcfile.h

@@ -0,0 +1,39 @@
+#pragma once
+/*
+ * rcfile.h -- xce_test startup rc-file path resolution and line-loading
+ * (xce-test-rc-file spec, tasks 1.1/1.2).
+ *
+ * A standalone, dependency-light module following the same pattern as
+ * configcompare.c/h -- no dependency on test.c internals, linked into both
+ * the xce_test executable and controller_unit_test.
+ *
+ * Task 1.1 implemented path resolution (rcfile_resolve_path()). Task 1.2
+ * adds line-loading (rcfile_load()) with a pluggable runner callback.
+ * Wiring rcfile_load() into xce_test's own startup/main() is a later task
+ * in the same spec (2.2) and is out of scope here -- this module has no
+ * knowledge of xce_test commands, contexts, or client state.
+ */
+
+/* Resolves the rc-file path per the XDG Base Directory Specification:
+ * "$XDG_CONFIG_HOME/xce_test/rc" when XDG_CONFIG_HOME is set and non-empty,
+ * else "$HOME/.config/xce_test/rc" when HOME is set and non-empty. Returns
+ * a heap string the caller must free(), or NULL if neither variable yields
+ * a usable base directory. */
+char* rcfile_resolve_path(void);
+
+/* Opens `path` and calls `runner(line)` once per line (trailing newline
+ * stripped), stopping early if `runner` returns nonzero. Lines are read via
+ * getline() with no length cap -- a line is never truncated or split
+ * regardless of length.
+ *
+ * Returns:
+ *   0                    - path is NULL, the file does not exist (ENOENT),
+ *                           the file could not be opened for another reason
+ *                           (a warning is printed to stderr; non-fatal), or
+ *                           every line ran and every runner call returned 0.
+ *   nonzero (== runner's) - the runner requested an abort; no further lines
+ *                           are read.
+ *
+ * Preconditions: `runner` is non-NULL when `path` is non-NULL.
+ * Postconditions: the file handle is always closed before returning. */
+int rcfile_load(const char* path, int (*runner)(char* line));

src/controller/testsuite/test.c

@@ -30,6 +30,7 @@
 #include "snapdiff.h"    /* byte-level checkpoint-file comparator (cmd_snapdiff) */
 #include "snapverify.h"  /* offline checkpoint-file structural validator (cmd_snapverify) */
 #include "configcompare.h"  /* cross-member active-config comparator (cmd_configcompare) */
+#include "rcfile.h"      /* startup rc-file path resolution + loading (xce-test-rc-file task 2.2) */
 #include "resval_typeresolve.h" /* resval_resolve_type (resource-value-verification task 2.1) */
 #include "xce/cluster.h" /* xce_parse_peers, xce_cluster_config_t */
 #include "xce/types.h"
@@ -667,12 +668,18 @@ static char args_doc[] = "";
 const char* argp_program_version = NULL;
 const char* argp_program_bug_address = BUG_EMAIL;
 
+/* Long-option key (above ASCII to avoid collisions with short opts) --
+ * mirrors src/controller/main.c's OPT_PEERS/OPT_CONF_KEEP_N idiom. */
+#define OPT_NORC 256
+
 struct arguments {
     int verbose;
+    int norc;
 };
 
 static struct argp_option options[] = {
     { "verbose", 'v', 0, 0, "Produce verbose logs" },
+    { "norc", OPT_NORC, 0, 0, "Skip loading the rc file for this invocation" },
     { 0 }
 };
 
@@ -684,6 +691,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
     case 'v':
         arguments->verbose++;
         break;
+    case OPT_NORC:
+        arguments->norc = 1;
+        break;
     default:
         return ARGP_ERR_UNKNOWN;
     }
@@ -4274,9 +4284,12 @@ static void check_run_jobs()
             int          npids  = 1;
             int          npgids = 0;
             int          nusage = 2;
+            /* bug-1735: match the real worker's collect_job_rusages()
+             * resnames (xcecd_cgroups.c), not the old never-matching
+             * "cputime"/"mem" strings. */
             xce_rusage_t data[] = {
-                { "cputime", 100.0 },
-                { "mem",     200.0 }
+                { "cpu_time_sec", 100.0 },
+                { "memory_rss",   200.0 }
             };
             xce_rusage_t* usages[] = { &(data[0]), &(data[1]) };
 
@@ -4338,9 +4351,12 @@ static void check_run_tasks()
             int          npids  = 1;
             int          npgids = 0;
             int          nusage = 2;
+            /* bug-1735: match the real worker's collect_job_rusages()
+             * resnames (xcecd_cgroups.c), not the old never-matching
+             * "cputime"/"mem" strings. */
             xce_rusage_t data[] = {
-                { "cputime", 100.0 },
-                { "mem",     200.0 }
+                { "cpu_time_sec", 100.0 },
+                { "memory_rss",   200.0 }
             };
             xce_rusage_t* usages[] = { &(data[0]), &(data[1]) };
 
@@ -4497,6 +4513,28 @@ int main(int argc, char* argv[])
     g_interactive       = isatty(STDIN_FILENO);
     g_continue_on_error = g_interactive ? 1 : 0;
 
+    /* xce-test-rc-file task 2.2: auto-load $XDG_CONFIG_HOME/xce_test/rc (or
+     * $HOME/.config/xce_test/rc) before the read loop starts, for both
+     * interactive and scripted sessions alike -- unless --norc was given.
+     * A missing/unresolvable rc file is silent (Req 1.3); a failing rc-file
+     * line is handled entirely by preprocess_and_run's existing abort/continue
+     * accounting (Req 3) -- an abort short-circuits straight to the same
+     * exit-code convention used at the bottom of this function, skipping the
+     * read loop entirely (Req 1.4/3.3). */
+    if (!arguments.norc) {
+        char* rcpath = rcfile_resolve_path();
+
+        if (rcpath != NULL) {
+            int rcrc = rcfile_load(rcpath, preprocess_and_run);
+
+            free(rcpath);
+
+            if (rcrc) {
+                return g_session_failed ? 1 : 0;
+            }
+        }
+    }
+
     while (!done) {
         char* s;
         int   rc;

src/controller/testsuite/test_apply_committed_entry.c

@@ -92,6 +92,29 @@ static int tace_array_spy_hook(int type, int situation, int detail,
     return 0;
 }
 
+/* ── Fail-detail-aware spy hook: like tace_spy_hook (counts every
+ * AFTER_EVENT firing into the same g_tace_hook_fires counter), additionally
+ * recording the job_failinfo_t.jobid seen on the most recent
+ * HOOK_DETAIL_JOB_FAIL firing (bug-1731 follow-up: HOOK_DETAIL_JOB_FAIL on
+ * WAL_JOB_FAIL apply). va_arg order mirrors job.c's own HOOK_BEFORE_EVENT
+ * fire sites for this detail -- (cluster_t*, job_failinfo_t*, job_t*). ──── */
+static int g_tace_last_fail_jobid;
+
+static int tace_faildetail_spy_hook(int type, int situation, int detail,
+                                    void* data, va_list ap)
+{
+    (void)type; (void)data;
+    if (situation != HOOK_AFTER_EVENT)
+        return 0;
+    g_tace_hook_fires++;
+    if (detail == HOOK_DETAIL_JOB_FAIL) {
+        (void)va_arg(ap, cluster_t*);
+        job_failinfo_t* info = va_arg(ap, job_failinfo_t*);
+        g_tace_last_fail_jobid = info ? (int)info->jobid : -1;
+    }
+    return 0;
+}
+
 static void tace_setup(void)
 {
     kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
@@ -131,6 +154,7 @@ static void tace_setup(void)
     g_tace_p.cluster = &g_tace_c;
 
     g_tace_hook_fires = 0;
+    g_tace_last_fail_jobid = -1;
 }
 
 static void tace_teardown(void)
@@ -403,7 +427,15 @@ test_ace_job_state_transition_rejects_backward_suppresses_notify(void)
           "notify: PASS\n");
 }
 
-static void test_ace_job_start_sets_start_time(void)
+/* bug-1697: renamed from test_ace_job_start_sets_start_time -- start_time is
+ * no longer WAL_JOB_START's to set at all (the old "fabricate time(NULL) at
+ * apply time" fallback for a bare/old-format payload is removed entirely).
+ * job.start_time is now exclusively DISPATCH-owned (WAL_JOB_DISPATCH, see
+ * wal_types.h's WAL_JOB_START doc comment for the full rationale), so this
+ * test now proves the opposite of what it used to: a WAL_JOB_START apply
+ * must leave start_time untouched, even for a genuine PEND->RUN forward
+ * transition. */
+static void test_ace_job_start_leaves_start_time_untouched(void)
 {
     tace_setup();
 
@@ -414,19 +446,18 @@ static void test_ace_job_start_sets_start_time(void)
 
     wal_jobstate_t js = { .jobid = 603, .state = JOB_STAT_RUN };
 
-    time_t before = time(NULL);
     int rc = apply_committed_entry(&g_tace_p, WAL_JOB_START, 22, &js,
                                    sizeof(js));
-    time_t after = time(NULL);
 
     TEST_ASSERT_EQUAL_INT(0, rc);
     TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
-    TEST_ASSERT_TRUE_MESSAGE(
-        (j->job.start_time >= before) && (j->job.start_time <= after),
-        "WAL_JOB_START must stamp start_time to the apply-time clock");
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, (uint64_t)j->job.start_time,
+        "bug-1697: WAL_JOB_START must leave start_time untouched (no more "
+        "apply-time time(NULL) fabrication) -- dispatch is the only owner "
+        "of this field now");
 
     tace_teardown();
-    printf("test_ace_job_start_sets_start_time: PASS\n");
+    printf("test_ace_job_start_leaves_start_time_untouched: PASS\n");
 }
 
 static void test_ace_job_finish_sets_finish_time_and_run_time(void)
@@ -458,6 +489,121 @@ static void test_ace_job_finish_sets_finish_time_and_run_time(void)
     printf("test_ace_job_finish_sets_finish_time_and_run_time: PASS\n");
 }
 
+/* ── WAL_JOB_FAIL (bug-1731 follow-up: HOOK_DETAIL_JOB_FAIL) ─────────────
+ * The identical dead-hook gap bug-1731 closed for WAL_JOB_START/FINISH
+ * existed for WAL_JOB_FAIL's deferred-fail-fact (WAL_JOB_FAIL_STAGE_FLAG)
+ * branch too -- sched_job_fail_fun (sched.c) and apiserver_job_fail_fun
+ * (apiserver.c) were permanently dead because HOOK_DETAIL_JOB_FAIL was never
+ * fired at HOOK_AFTER_EVENT. These three tests prove: (1) the STAGE_FLAG
+ * branch fires BOTH HOOK_DETAIL_JOB_FAIL and HOOK_DETAIL_JOB_UPDATE with the
+ * correct jobid, (2) the legacy _pad==0 absolute-overwrite branch fires ONLY
+ * HOOK_DETAIL_JOB_UPDATE -- it has no real job_failinfo_t to report and must
+ * not fabricate one, and (3) a guard-rejected transition still fires
+ * NEITHER hook (idempotency of this shared WAL-apply seam is unchanged). */
+
+static void test_ace_job_fail_stage_flag_fires_faildetail_and_update(void)
+{
+    tace_setup();
+
+    /* apply_job_fail_transition() (job.c) has no internal guard -- it
+     * unconditionally mutates and returns JOB_TRANSITION_APPLIED for any
+     * stage, so any seeded starting state genuinely applies. Seed a job
+     * already RUNNING with EXIT set, so the POST_EXEC branch merges
+     * JOB_STAT_PERR without the defensive "exit status not set" WARN path. */
+    job_t* j = tace_seed_job(605, JOB_STAT_RUN | JOB_STAT_EXIT);
+    TEST_ASSERT_NOT_NULL(j);
+
+    wal_jobstate_t js = { .jobid = 605,
+                          .state = 0,
+                          ._pad  = WAL_JOB_FAIL_STAGE_FLAG |
+                                   JOB_EXEC_STAGE_POST_EXEC };
+
+    TEST_ASSERT_EQUAL_INT(0,
+        xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                     tace_faildetail_spy_hook, NULL));
+
+    int rc = apply_committed_entry(&g_tace_p, WAL_JOB_FAIL, 24, &js,
+                                   sizeof(js));
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_TRUE_MESSAGE((j->job.state & JOB_STAT_PERR) != 0,
+        "a genuine STAGE_FLAG fail transition must merge JOB_STAT_PERR");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_tace_hook_fires,
+        "a genuine STAGE_FLAG fail must fire BOTH HOOK_DETAIL_JOB_FAIL and "
+        "HOOK_DETAIL_JOB_UPDATE");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(605, g_tace_last_fail_jobid,
+        "HOOK_DETAIL_JOB_FAIL must carry a job_failinfo_t with the correct "
+        "jobid");
+
+    tace_teardown();
+    printf("test_ace_job_fail_stage_flag_fires_faildetail_and_update: "
+          "PASS\n");
+}
+
+static void test_ace_job_fail_legacy_pad_zero_fires_update_only(void)
+{
+    tace_setup();
+
+    /* Legacy live path (job_ctrl's mark_job_fail): _pad == 0, `state` is
+     * already the correct absolute value -- an absolute overwrite via
+     * apply_job_update_transition(), with no real job_failinfo_t to report.
+     * Regression guard: must NOT fabricate one for HOOK_DETAIL_JOB_FAIL. */
+    job_t* j = tace_seed_job(606, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+
+    wal_jobstate_t js = { .jobid = 606, .state = JOB_STAT_EXIT, ._pad = 0 };
+
+    xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                tace_faildetail_spy_hook, NULL);
+
+    int rc = apply_committed_entry(&g_tace_p, WAL_JOB_FAIL, 25, &js,
+                                   sizeof(js));
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_tace_hook_fires,
+        "the legacy _pad==0 branch must fire exactly ONE notification "
+        "(HOOK_DETAIL_JOB_UPDATE only)");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(-1, g_tace_last_fail_jobid,
+        "the legacy _pad==0 branch has no real job_failinfo_t and must NOT "
+        "fabricate one for HOOK_DETAIL_JOB_FAIL");
+
+    tace_teardown();
+    printf("test_ace_job_fail_legacy_pad_zero_fires_update_only: PASS\n");
+}
+
+static void test_ace_job_fail_guard_rejected_fires_no_hook(void)
+{
+    tace_setup();
+
+    /* apply_job_fail_transition() (the STAGE_FLAG branch) has no guard and
+     * always applies -- a rejectable transition can only come through the
+     * legacy _pad==0 branch's apply_job_update_transition() guard. Job
+     * already terminal (DONE); a stale backward fail-derived UPDATE must be
+     * rejected, exactly mirroring
+     * test_ace_job_state_transition_rejects_backward_suppresses_notify. */
+    job_t* j = tace_seed_job(607, JOB_STAT_DONE);
+    TEST_ASSERT_NOT_NULL(j);
+
+    wal_jobstate_t js = { .jobid = 607, .state = JOB_STAT_RUN, ._pad = 0 };
+
+    xce_add_hook(&g_tace_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                tace_faildetail_spy_hook, NULL);
+
+    int rc = apply_committed_entry(&g_tace_p, WAL_JOB_FAIL, 26, &js,
+                                   sizeof(js));
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
+        "a guard-rejected backward transition is a success no-op, not an "
+        "apply failure");
+    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j->job.state,
+        "a rejected backward transition must preserve the terminal state");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_tace_hook_fires,
+        "a guard-rejected WAL_JOB_FAIL transition must fire NEITHER hook");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(-1, g_tace_last_fail_jobid,
+        "no HOOK_DETAIL_JOB_FAIL fire means no job_failinfo_t captured");
+
+    tace_teardown();
+    printf("test_ace_job_fail_guard_rejected_fires_no_hook: PASS\n");
+}
+
 /* ── WAL_QUEUE_CTRL ───────────────────────────────────────────────────── */
 
 static void test_ace_queue_ctrl_updates_and_notifies(void)
@@ -1097,8 +1243,11 @@ void test_apply_committed_entry(void)
     RUN_TEST(test_ace_job_submit_duplicate_is_noop_no_extra_notify);
     RUN_TEST(test_ace_job_state_transition_applies_forward_and_notifies);
     RUN_TEST(test_ace_job_state_transition_rejects_backward_suppresses_notify);
-    RUN_TEST(test_ace_job_start_sets_start_time);
+    RUN_TEST(test_ace_job_start_leaves_start_time_untouched);
     RUN_TEST(test_ace_job_finish_sets_finish_time_and_run_time);
+    RUN_TEST(test_ace_job_fail_stage_flag_fires_faildetail_and_update);
+    RUN_TEST(test_ace_job_fail_legacy_pad_zero_fires_update_only);
+    RUN_TEST(test_ace_job_fail_guard_rejected_fires_no_hook);
     RUN_TEST(test_ace_queue_ctrl_updates_and_notifies);
     RUN_TEST(test_ace_worker_ctrl_join_restores_and_notifies);
     RUN_TEST(test_ace_worker_ctrl_update_applies_to_existing_worker);

src/controller/testsuite/test_apply_pump.c

@@ -240,6 +240,11 @@ static void tap_setup(void)
 static void tap_teardown(void)
 {
     repl_broadcast_send_hook = NULL;
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). */
+    persist_stop(&g_tap_c.persist);
     commit_engine_fini(&g_tap_hce);
     persist_stop(&g_tap_c.persist);
     persist_fini(&g_tap_c.persist);
@@ -1223,6 +1228,13 @@ static void tpr_setup(int n_vcpus)
 static void tpr_teardown(void)
 {
     repl_broadcast_send_hook = NULL;
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). vtaskmgr_fini() stays AFTER commit_engine_fini() per the
+     * pre-existing invariant (nothing can dispatch a new vtask onto the
+     * pool mid-teardown) -- only persist_stop()'s position moves. */
+    persist_stop(&g_tpr_c.persist);
     commit_engine_fini(&g_tpr_hce);
     vtaskmgr_fini(&g_tpr_pool);
     persist_stop(&g_tpr_c.persist);

src/controller/testsuite/test_apply_pump_integration.c

@@ -418,6 +418,13 @@ static void tpi_teardown(void)
 
     tpi_scan_replies(&g_tpi_req_ep, &drain); /* free any leftover packets */
     repl_broadcast_send_hook = NULL;
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). vtaskmgr_fini() stays AFTER commit_engine_fini() per the
+     * pre-existing invariant noted below -- only persist_stop()'s position
+     * moves earlier. */
+    persist_stop(&g_tpi_c.persist);
     commit_engine_fini(&g_tpi_hce);
     if (g_tpi_pool_active) {
         /* Mirrors test_apply_pump.c's tpr_teardown ordering exactly:

src/controller/testsuite/test_apply_reapply_idempotency.c

@@ -338,9 +338,11 @@ test_tari_finish_then_stale_start_preserves_terminal_state_via_dispatch(void)
                                                    sizeof(finish)));
     TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j->job.state,
         "FINISH must drive the job to its terminal state");
-    TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_tari_hook_fires,
-        "FINISH must fire exactly one notification (SUBMIT's + FINISH's = "
-        "2 total so far)");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(3, g_tari_hook_fires,
+        "bug-1731: a genuine forward WAL_JOB_FINISH transition now fires "
+        "TWO notifications (the pre-existing HOOK_DETAIL_JOB_UPDATE plus "
+        "the new HOOK_DETAIL_JOB_EXIT), not one -- SUBMIT's 1 + FINISH's "
+        "2 = 3 total so far; tari_spy_hook counts every AFTER_EVENT detail");
 
     /* Step 3: apply a STALE, previously-unapplied START for the SAME job —
      * simulating a duplicate/out-of-order fact arriving after the job
@@ -361,9 +363,11 @@ test_tari_finish_then_stale_start_preserves_terminal_state_via_dispatch(void)
     TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j->job.state,
         "the job must remain in its FINISH-terminal state after the stale "
         "START is applied");
-    TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_tari_hook_fires,
-        "the stale, guard-rejected START must fire NO notification — the "
-        "count must stay exactly where FINISH left it");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(3, g_tari_hook_fires,
+        "the stale, guard-rejected START must fire NO notification -- no "
+        "trailing pidinfo bytes means pidinfo_just_decoded stays false too "
+        "(bug-1731's HOOK_DETAIL_JOB_START gate), so the count must stay "
+        "exactly where FINISH left it (3)");
 
     tari_teardown();
     printf("test_tari_finish_then_stale_start_preserves_terminal_state_via_"

src/controller/testsuite/test_commit_barrier_reply.c

@@ -197,6 +197,7 @@ int update_pids(job_pidinfo_t **i, int n, int *p)
 { (void)i; (void)n; (void)p; return 0; }
 int update_pgids(job_pidinfo_t **i, int n, int *p)
 { (void)i; (void)n; (void)p; return 0; }
+void aggregate_job_runusage(xce_job_runusage_t *r) { (void)r; }
 int on_scheduler_join(cluster_t *c) { (void)c; return 0; }
 int add_apiserver(cluster_t *c, char *n, endpoint_t *e)
 { (void)c; (void)n; (void)e; return 0; }
@@ -475,6 +476,22 @@ static void cbrp_teardown(void)
      * range in job.c's process-lifetime in-flight registry. Clear it here
      * so it never outlives this fixture. */
     inflight_submits_reset_for_test();
+    /* bug-1687: persist_stop() must run BEFORE commit_engine_fini() --
+     * exactly the ordering bug-1452 already fixed in test_persist_truncate.c
+     * (that fix's own note flagged this file as a same-pattern, not-yet-
+     * observed-to-crash follow-up). commit_engine_fini() tears down
+     * g_hce.lock/g_hce.cond and, as its final step, nulls g_hce.p while the
+     * fsync_worker background thread (started by persist_start(), stopped
+     * only by persist_stop()'s pthread_join) can still be mid-flight inside
+     * its unlocked p->on_durable_advance(p->durable_cb_ctx) callback
+     * (persist.c's fsync_worker) -- a genuine TOCTOU data race with no
+     * synchronization between the two threads. Calling persist_stop() here,
+     * first, guarantees the flusher thread cannot be in-flight at all by the
+     * time commit_engine_fini() runs. The later persist_stop() call below
+     * becomes a safe, idempotent no-op on this second invocation (mirrors
+     * bug-1452's fix exactly -- fsync_thread_running is already 0 and wal_fd
+     * is already -1, so both of persist_stop()'s internal branches skip). */
+    persist_stop(&g_hc.persist);
     commit_engine_fini(&g_hce);
     persist_stop(&g_hc.persist);
     persist_fini(&g_hc.persist);
@@ -2410,7 +2427,14 @@ static int cbrp_count_jobstate_entries_ex(persist_t *p, uint32_t wal_type,
             (int)pread(fd, &hdr, sizeof(hdr), (off_t)off),
             "WAL segment scan: short header read");
         if (hdr.type == wal_type
-            && hdr.payload_len == sizeof(wal_jobstate_t)) {
+            && hdr.payload_len >= sizeof(wal_jobstate_t)) {
+            /* bug-1687: >= not == -- WAL_JOB_START (like bug-1678's
+             * WAL_JOB_DISPATCH before it) can now carry trailing bytes
+             * (start_time + Cap'n Proto JobPIDInfo) beyond the fixed
+             * wal_jobstate_t header. jobid/state always live at the same
+             * fixed leading offset regardless of trailing bytes, so reading
+             * just sizeof(wal_jobstate_t) here remains correct for any
+             * payload_len >= that size. */
             wal_jobstate_t js;
 
             TEST_ASSERT_EQUAL_INT_MESSAGE(

src/controller/testsuite/test_commit_barrier_rollout.c

@@ -483,6 +483,13 @@ static void test_fact_hook_append_is_retained_and_nonblocking(void)
 
     /* Teardown */
     repl_broadcast_send_hook = NULL;
+    /* bug-1429/bug-1452: persist_stop() must run BEFORE commit_engine_fini()
+     * -- otherwise the live fsync_worker background thread can still be
+     * mid-flight inside p->on_durable_advance(ce) (persist.c's fsync_worker)
+     * while commit_engine_fini() destroys ce->lock/ce->cond, a TOCTOU race
+     * that hangs pthread_cond_destroy() (live-reproduced, gdb-confirmed).
+     * The persist_stop() call below becomes a safe, idempotent no-op. */
+    persist_stop(g_fp);
     commit_engine_fini(&g_fce);
     persist_stop(g_fp);
     persist_fini(g_fp);
@@ -1285,6 +1292,10 @@ static void cb5_teardown(void)
     cb5_drain_ep(&g_cb5_sb_ep[0]);
     cb5_drain_ep(&g_cb5_sb_ep[1]);
     cb5_drain_ep(&g_cb5_req_ep);
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * jap_teardown's comment below for the full TOCTOU-race rationale
+     * (live-reproduced, gdb-confirmed hang in this exact class of fixture). */
+    persist_stop(&g_cb5_c.persist);
     commit_engine_fini(&g_cb5_eng);
     /* persist_fini calls repl_fini internally (started-persist fixture). */
     persist_stop(&g_cb5_c.persist);
@@ -1874,6 +1885,23 @@ static void jap_setup(uint32_t sync_timeout_ms)
 static void jap_teardown(void)
 {
     repl_broadcast_send_hook = NULL;
+    /* bug-1429 (root cause of this session's live, gdb-confirmed hang --
+     * thread 1 stuck forever in commit_engine_fini -> pthread_cond_destroy
+     * at this exact call site, while two fsync_worker threads sat parked in
+     * pthread_cond_timedwait on this same g_jap): persist_stop() must run
+     * BEFORE commit_engine_fini() -- exactly the ordering bug-1452 already
+     * fixed for test_persist_truncate.c and bug-1687 for cbrp_teardown()
+     * (test_commit_barrier_reply.c). commit_engine_fini() destroys
+     * g_jace.lock/g_jace.cond; the live fsync_worker background thread
+     * (started by persist_start(), stopped only by persist_stop()'s
+     * pthread_join) can still be mid-flight inside p->on_durable_advance(ce)
+     * (persist.c's fsync_worker) -- a genuine TOCTOU data race with no
+     * synchronization between the two threads. Calling persist_stop() here
+     * first guarantees the flusher thread cannot be in-flight at all by the
+     * time commit_engine_fini() runs. The persist_stop() call below becomes
+     * a safe, idempotent no-op (fsync_thread_running and wal_fd are already
+     * cleared by the first call). */
+    persist_stop(&g_jap);
     commit_engine_fini(&g_jace);
     persist_stop(&g_jap);
     persist_fini(&g_jap);

src/controller/testsuite/test_commit_durability.c

@@ -498,8 +498,16 @@ static void test_durable_advance_drives_commit_single_member(void)
     /* committed never exceeds durable (the invariant the wake preserves). */
     TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
 
-    /* Tear the engine down BEFORE the persist (unhooks the callback so the
-     * flusher cannot fire against a destroyed engine). */
+    /* bug-1429/bug-1452: persist_stop() must run BEFORE commit_engine_fini()
+     * -- unhooking the callback (persist_set_durable_cb, inside
+     * commit_engine_fini) alone is NOT sufficient: the flusher thread can
+     * have already read the OLD, non-NULL callback pointer moments before
+     * the unhook takes effect and still be mid-flight calling back into the
+     * engine while commit_engine_fini() destroys ce->lock/ce->cond -- a
+     * TOCTOU race live-reproduced (gdb-confirmed hang) in the sibling
+     * test_commit_barrier_rollout.c fixture. teardown_durability_persist's
+     * own persist_stop() call becomes a safe, idempotent no-op. */
+    persist_stop(p);
     commit_engine_fini(&ce);
     teardown_durability_persist(p);
     printf("test_durable_advance_drives_commit_single_member: PASS\n");
@@ -778,6 +786,10 @@ static void test_commit_never_exceeds_durable_invariant(void)
     TEST_ASSERT_EQUAL_UINT64(tip, committed);
     TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_durable_advance_drives_commit_single_member above for the full
+     * TOCTOU-race rationale. */
+    persist_stop(p);
     commit_engine_fini(&ce);
     teardown_durability_persist(p);
     printf("test_commit_never_exceeds_durable_invariant: PASS\n");
@@ -967,6 +979,10 @@ static void test_durable_majority_commit_end_to_end(void)
     /* The invariant the whole spec rests on: committed never exceeds durable. */
     TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_durable_advance_drives_commit_single_member above for the full
+     * TOCTOU-race rationale. */
+    persist_stop(p);
     commit_engine_fini(&ce);
     repl_remove_standby(&p->repl, &g_dur_eps[0]);
     repl_remove_standby(&p->repl, &g_dur_eps[1]);

src/controller/testsuite/test_commit_engine.c

@@ -1497,6 +1497,12 @@ static void test_sync_append_wired_commits_on_per_entry_majority(void)
         1, atomic_load(&g_sp->committed_lsn),
         "committed_lsn must advance to the majority-acked entry");
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). sync_persist_teardown()'s own persist_stop() call becomes a
+     * safe, idempotent no-op. */
+    persist_stop(g_sp);
     commit_engine_fini(&g_ce);
     sync_persist_teardown(g_sp);
     g_sp = NULL;
@@ -1541,6 +1547,12 @@ static void test_sync_append_wired_stale_acks_time_out(void)
     TEST_ASSERT_TRUE_MESSAGE(atomic_load(&g_sp->committed_lsn) < 2,
         "committed_lsn must stay unmoved past the entry on timeout");
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). sync_persist_teardown()'s own persist_stop() call becomes a
+     * safe, idempotent no-op. */
+    persist_stop(g_sp);
     commit_engine_fini(&g_ce);
     sync_persist_teardown(g_sp);
     g_sp = NULL;
@@ -1575,6 +1587,12 @@ static void test_sync_append_wired_single_member_commits_locally(void)
         1, atomic_load(&g_sp->committed_lsn),
         "n_peers=1 sync append must advance committed_lsn to the entry");
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). sync_persist_teardown()'s own persist_stop() call becomes a
+     * safe, idempotent no-op. */
+    persist_stop(g_sp);
     commit_engine_fini(&g_ce);
     sync_persist_teardown(g_sp);
     g_sp = NULL;

src/controller/testsuite/test_convergence_verification.c

@@ -208,6 +208,12 @@ static void cv_teardown(void)
 {
     repl_broadcast_send_hook = NULL;
     ha_repl_stream_wal_hook  = NULL;
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture: a live fsync_worker thread can still call back into the
+     * commit engine while commit_engine_fini() destroys its lock/cond). */
+    persist_stop(&g_cv.persist);
     commit_engine_fini(&g_cv.commit);
     delaymgr_fini(&g_cv.delaymgr);
     persist_stop(&g_cv.persist);

src/controller/testsuite/test_demotion_convergence.c

@@ -230,6 +230,15 @@ static void dc_teardown(void)
     ha_restart_hook = NULL;
     ha_fatal_hook   = NULL;
     slowlane_fini(&g_dc.slowlane);
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- this
+     * exact site was flagged, but left unfixed, as a follow-up by bug-1452's
+     * own remediation note ("the identical commit_engine_fini-before-
+     * persist_stop ordering also exists in test_demotion_convergence.c...
+     * same latent race, not yet observed to crash there"). Fixed now as
+     * part of the bug-1429 sweep after a live, gdb-confirmed hang in the
+     * sibling test_commit_barrier_rollout.c fixture proved this class of
+     * race is a genuine, reproducible controller_unit_test hang source. */
+    persist_stop(&g_dc.persist);
     commit_engine_fini(&g_dc.commit);
     delaymgr_fini(&g_dc.delaymgr);
     persist_stop(&g_dc.persist);

src/controller/testsuite/test_dispatch_commit.c

@@ -274,6 +274,60 @@ static void test_dc_success_commits_alloc_state_and_resets_counter(void)
     printf("test_dc_success_commits_alloc_state_and_resets_counter: PASS\n");
 }
 
+#define DC_JOBID_START_TIME 91005ULL
+
+/* bug-1697's core RED/GREEN proof for the "start_time is dispatch-owned
+ * now" half of the fix: a successful dispatch must stamp job.start_time
+ * synchronously, in the SAME lock-held operation that sets job.state =
+ * JOB_STAT_RUN and commits job.allocs -- no WAL commit/apply round trip
+ * needed, since this is a plain in-memory write under the job's own write
+ * lock. RED against the pre-fix code: j->job.start_time stays 0 forever
+ * (dispatch_a_job_() never touched it, and WAL_JOB_START's own adoption is
+ * permanently dead per bug-1697's root cause, so nothing else ever sets
+ * it either). GREEN after the fix: job.start_time is nonzero immediately
+ * after dispatch_a_job_test() returns. */
+static void test_dc_success_stamps_start_time_synchronously(void)
+{
+    job_t*           j;
+    xce_job_alloc_t* alloc;
+    int              rc;
+    time_t           before, after;
+
+    dc_setup();
+
+    j = dc_seed_ready(DC_JOBID_START_TIME, 0, "dq", "wkr-dc-start-time");
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, (uint64_t)j->job.start_time,
+        "fixture must seed a job with start_time still at its zero-"
+        "initialized default");
+
+    alloc = make_job_alloc(DC_JOBID_START_TIME, 1,
+        make_host_allocs(1,
+            make_host_alloc((char*)"wkr-dc-start-time", 1,
+                make_allocs(1,
+                    make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
+    TEST_ASSERT_NOT_NULL(alloc);
+
+    before = time(NULL);
+    rc     = dispatch_a_job_test(&g_dc, NULL, alloc);
+    after  = time(NULL);
+    TEST_ASSERT_TRUE_MESSAGE(rc >= 0,
+        "a dispatch to a live, no-iomgr worker endpoint must succeed");
+
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (uint64_t)j->job.start_time,
+        "bug-1697: a successful dispatch must stamp job.start_time "
+        "synchronously -- it must never be left at 0");
+    TEST_ASSERT_TRUE_MESSAGE(
+        (j->job.start_time >= before) && (j->job.start_time <= after),
+        "job.start_time must be the dispatch-time clock, captured within "
+        "this call's own [before, after] window");
+
+    free_JobAlloc_ptr(&alloc);
+    dc_drain_ep(&g_dc_worker_ep);
+    dc_teardown();
+    printf("test_dc_success_stamps_start_time_synchronously: PASS\n");
+}
+
 /* bug-1679: spy hook counting HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires
  * specifically -- the exact hook apiserver_job_update_fun (apiserver.c)
  * subscribes to in production to push a job's current state to every
@@ -506,7 +560,9 @@ static int dc_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
 
 static void test_dc_success_appends_wal_job_dispatch_durably(void)
 {
-    char tmp[256];
+    char   tmp[256];
+    time_t captured_start_time = 0; /* bug-1697: set in Phase 1, checked
+                                     * against Phase 2's recovered value */
 
     strcpy(tmp, "/tmp/xce_dc31_replay_XXXXXX");
     TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
@@ -637,6 +693,10 @@ static void test_dc_success_appends_wal_job_dispatch_durably(void)
         rc = dispatch_a_job_test(&c, NULL, alloc);
         TEST_ASSERT_TRUE(rc >= 0);
         TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+        TEST_ASSERT_NOT_EQUAL_MESSAGE(0, (uint64_t)j->job.start_time,
+            "bug-1697: a successful dispatch must stamp job.start_time "
+            "synchronously");
+        captured_start_time = j->job.start_time;
 
         TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
 
@@ -686,6 +746,11 @@ static void test_dc_success_appends_wal_job_dispatch_durably(void)
         TEST_ASSERT_TRUE_MESSAGE((j2->job.state & JOB_STAT_UNKWN) != 0,
             "bug-1509 B3: a RUN job recovered on restart is marked UNKWN "
             "(in-memory) awaiting worker reconnect/resend reconciliation");
+        TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)captured_start_time,
+            (uint64_t)j2->job.start_time,
+            "bug-1697: restart replay must recover the exact dispatch-time "
+            "start_time stamped by the real dispatch_a_job_() call in "
+            "Phase 1, from the WAL_JOB_DISPATCH fact's extended payload");
 
         /* bug-1678: restart replay must recover not just STATE but the
          * resource allocation itself, or EXEC_HOST is permanently blank for
@@ -734,6 +799,7 @@ void test_dispatch_commit(void)
           "5.1)\n\n");
 
     RUN_TEST(test_dc_success_commits_alloc_state_and_resets_counter);
+    RUN_TEST(test_dc_success_stamps_start_time_synchronously);
     RUN_TEST(test_dc_success_notifies_job_update_hook_exactly_once);
     RUN_TEST(test_dc_worker_not_found_leaves_job_pend_and_alloc_uncommitted);
     RUN_TEST(test_dc_success_appends_wal_job_dispatch_durably);

src/controller/testsuite/test_egress_filter.c

@@ -217,6 +217,11 @@ static void ef_setup(void)
 static void ef_teardown(void)
 {
     repl_broadcast_send_hook = NULL;
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). */
+    persist_stop(&g_c.persist);
     commit_engine_fini(&g_ce);
     persist_stop(&g_c.persist);
     persist_fini(&g_c.persist);

src/controller/testsuite/test_fetchfile_disconnect.c

@@ -0,0 +1,187 @@
+/*
+ * test_fetchfile_disconnect.c — bug-1693 regression: a worker disconnect
+ * mid-wait wakes on_fetchfile()'s pending waiter immediately (Decision Log
+ * fidelity fix).
+ *
+ * Background: cluster.c's OPCode_schedAdmin dispatch case documents itself
+ * as mirroring on_fetchfile's proxy pattern (job.c) -- register a waiter,
+ * forward async, block on a condvar with a deadline. An invariant audit
+ * found BOTH sites violated the same recorded Decision Log entry: "No
+ * client reply-wait timeout as a fix for a hung request -- fail loud at the
+ * real transport/source instead of masking with a deadline backstop."
+ * Before this fix, handle_worker_leave() (worker.c) only ever nulled w->ep
+ * on a worker disconnect -- it never woke a pending on_fetchfile() waiter
+ * proxied to that worker, so a disconnect mid-wait could only ever be
+ * observed at on_fetchfile's own 10s deadline.
+ *
+ * The fix: fetchfile_signal_worker_disconnect(worker_t*) (job.c) walks
+ * every registered fetchfile_pending_t and wakes each one whose ->worker
+ * matches the disconnected worker with a distinguishable -ECONNRESET
+ * status (not the deadline's -ETIMEDOUT). handle_worker_leave() now calls
+ * it immediately, right after nulling w->ep.
+ *
+ * Test strategy for THIS file: on_fetchfile() itself needs a real,
+ * RTTI-registered job_t/worker_t pair reachable through c->jobmgr/
+ * c->workermgr (key_find + lock()/reference() -- see on_fetchfile's own
+ * body, job.c) to run end-to-end; building that fixture is squarely a
+ * job/worker-lookup concern this fix's scope explicitly does not touch
+ * (only the disconnect-to-waiter SIGNAL path is in scope). So this test
+ * exercises the real fetchfile_signal_worker_disconnect() function against
+ * a real fetchfile_pending_t wait, using job.c's XCE_TESTING-only seam
+ * fetchfile_wait_for_test() (mirrors on_fetchfile's own register/wait/
+ * unregister sequence exactly, including a real pthread_cond_timedwait
+ * deadline) instead of the full on_fetchfile()/handle_worker_leave() call
+ * chain -- a "direct unit test of the disconnect-handler's new signal call
+ * reaching the right wait context", per this task's own stated minimum bar
+ * for whichever of the two sites doesn't get the full live two-thread
+ * reproduction (that's test_sched_admin_dispatch.c's
+ * test_schedadmin_disconnect_wakes_immediately, which does drive the real
+ * handle_scheduler_leave() end-to-end against a real dispatch wait).
+ * worker_t itself is a plain, stack-owned struct here (not routed through
+ * makeWorker()/wobj.c's RTTI machinery) -- unneeded, since neither
+ * fetchfile_signal_worker_disconnect() nor fetchfile_wait_for_test() ever
+ * locks/references it; only pointer identity (matching ->worker) matters,
+ * mirroring test_sched_worker_change.c's identical "plain struct, not
+ * through the real object constructor" rationale.
+ */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <pthread.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+
+#include "cluster.h"
+#include "xce/cluster_def.h"
+
+/* Test-only seam (job.c, XCE_TESTING-only) -- see this file's header
+ * comment above for why it's used instead of a full on_fetchfile()
+ * fixture. */
+extern int fetchfile_wait_for_test(worker_t* w, long timeout_ms,
+                                   int* out_status);
+
+/* Monotonic milliseconds -- same construction as test_sched_admin_
+ * dispatch.c's sa_mono_ms() / test_commit_resolve.c's mono_ms_r(). */
+static uint64_t ff_mono_ms(void)
+{
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
+}
+
+typedef struct {
+    worker_t* w;
+} ff_disconnect_arg_t;
+
+static void* ff_deliver_disconnect(void* argp)
+{
+    ff_disconnect_arg_t* arg = (ff_disconnect_arg_t*)argp;
+
+    /* Give the main thread time to register the pending waiter and start
+     * blocking on the condvar first -- comfortably inside the backstop
+     * deadline below, same margin convention as test_sched_admin_
+     * dispatch.c's sa_deliver_scheduler_reply. */
+    usleep(200000);
+
+    fetchfile_signal_worker_disconnect(arg->w);
+    return NULL;
+}
+
+/* ── Regression: disconnect wakes the waiter immediately, not at the
+ * deadline ──────────────────────────────────────────────────────────────── */
+static void test_fetchfile_worker_disconnect_wakes_immediately(void)
+{
+    worker_t w;
+    memset(&w, 0, sizeof(w));
+
+    ff_disconnect_arg_t arg = { .w = &w };
+    pthread_t            tid;
+    TEST_ASSERT_EQUAL_INT(0,
+                          pthread_create(&tid, NULL, ff_deliver_disconnect, &arg));
+
+    /* Backstop deadline: much larger than the ~200ms disconnect delay above,
+     * so a genuine wake is trivially distinguishable from a regression that
+     * falls through to the deadline (asserted on below via elapsed time) --
+     * but far shorter than production's real 10s on_fetchfile deadline, so
+     * a regression fails this test in bounded time rather than hanging for
+     * 10s. */
+    const long timeout_ms = 5000;
+    int        status     = 12345; /* poison -- must be overwritten */
+
+    uint64_t t0   = ff_mono_ms();
+    int      woke = fetchfile_wait_for_test(&w, timeout_ms, &status);
+    uint64_t took = ff_mono_ms() - t0;
+
+    pthread_join(tid, NULL);
+
+    TEST_ASSERT_TRUE_MESSAGE(
+        woke,
+        "must be woken by the disconnect signal, not merely reach the "
+        "backstop deadline with pending.done still 0");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        -ECONNRESET, status,
+        "a worker disconnect must be relayed with a distinguishable "
+        "-ECONNRESET status, never the deadline's -ETIMEDOUT");
+    TEST_ASSERT_TRUE_MESSAGE(
+        took < 1000,
+        "a worker disconnect mid-wait must wake the waiter immediately "
+        "(well under the 5s backstop used by this test, and far under "
+        "production's real 10s on_fetchfile deadline) -- this is the "
+        "regression this test locks: pre-fix, handle_worker_leave() never "
+        "signalled fetchfile waiters at all, so this would only ever "
+        "complete at the deadline with status -ETIMEDOUT");
+
+    printf("test_fetchfile_worker_disconnect_wakes_immediately: PASS\n");
+}
+
+/* ── A disconnect on an UNRELATED worker must not wake this waiter ────────
+ *
+ * Proves fetchfile_signal_worker_disconnect() actually matches by worker
+ * identity (the ->worker field on fetchfile_pending_t) rather than waking
+ * every registered waiter indiscriminately -- multiple fetchfile requests
+ * against different workers must not cross-signal each other. */
+static void test_fetchfile_worker_disconnect_does_not_wake_other_worker(void)
+{
+    worker_t w_target, w_other;
+    memset(&w_target, 0, sizeof(w_target));
+    memset(&w_other, 0, sizeof(w_other));
+
+    ff_disconnect_arg_t arg = { .w = &w_other };
+    pthread_t            tid;
+    TEST_ASSERT_EQUAL_INT(0,
+                          pthread_create(&tid, NULL, ff_deliver_disconnect, &arg));
+
+    /* Short backstop -- this test's whole point is that NOTHING wakes it
+     * early, so the wait genuinely runs out; keep it small to keep the test
+     * fast rather than proving a large deadline elapses. */
+    const long timeout_ms = 400;
+    int        status     = 12345;
+
+    int woke = fetchfile_wait_for_test(&w_target, timeout_ms, &status);
+
+    pthread_join(tid, NULL);
+
+    TEST_ASSERT_FALSE_MESSAGE(
+        woke,
+        "a disconnect on a DIFFERENT worker must not wake this waiter -- "
+        "fetchfile_signal_worker_disconnect() must match by worker "
+        "identity, not signal indiscriminately");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, status,
+                                  "an unmatched waiter must fall through to "
+                                  "the ordinary deadline outcome");
+
+    printf(
+        "test_fetchfile_worker_disconnect_does_not_wake_other_worker: PASS\n");
+}
+
+/* ── Suite entry point ──────────────────────────────────────────────────── */
+void test_fetchfile_disconnect(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_fetchfile_worker_disconnect_wakes_immediately);
+    RUN_TEST(test_fetchfile_worker_disconnect_does_not_wake_other_worker);
+}

src/controller/testsuite/test_ha_connection_reuse.c

@@ -427,6 +427,127 @@ static void test_handle_ctrlhello_sets_primary_ctrl_fd(void)
     printf("test_handle_ctrlhello_sets_primary_ctrl_fd: PASS\n");
 }
 
+/* ── Test 8b (bug-1701): handle_ctrlhello populates primary_host/port from
+ * the sender's name matched against our own static peers_cfg candidate list.
+ * This is the local-lookup fix that unblocks bug-1700's fill_redirect_reply()
+ * mechanism (join.c:27), which is fed by ha->primary_host/primary_port and
+ * was permanently empty before this fix — no wire-protocol change, just a
+ * boot-time-known static lookup keyed by req->from->name. ──────────────── */
+
+static void test_handle_ctrlhello_populates_primary_host_from_known_peer(void)
+{
+    cluster_t c;
+    memset(&c, 0, sizeof(c));
+    ha_init(&c.ha, &g_cfg, 0);
+    c.ha.primary_ctrl_fd = -1;
+    atomic_store_explicit(&c.ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
+
+    /* Static peer list (as parsed at boot from --peers name=host:port,...)
+     * containing the candidate that will send us this ctrlhello. */
+    myconf_t conf;
+    memset(&conf, 0, sizeof(conf));
+    conf.peers_cfg.candidates_count = 1;
+    strncpy(conf.peers_cfg.candidates[0].name, "primary1",
+            sizeof(conf.peers_cfg.candidates[0].name) - 1);
+    strncpy(conf.peers_cfg.candidates[0].hostname, "10.0.1.7",
+            sizeof(conf.peers_cfg.candidates[0].hostname) - 1);
+    conf.peers_cfg.candidates[0].port = 9991;
+    c.myconf = &conf;
+
+    endpoint_t ep;
+    memset(&ep, 0, sizeof(ep));
+    ep.fd = 42;
+
+    ctrl_hello_req_t hello;
+    memset(&hello, 0, sizeof(hello));
+    hello.name        = "primary1";
+    hello.applied_lsn = 0;
+    hello.config_tag  = 0;
+
+    /* req->from is populated in production by request.c:379
+     * ("req->from = ep->peer;") for EVERY incoming request, carrying the
+     * sender's name= identity from --peers name=host:port. */
+    xce_peerinfo_t from;
+    memset(&from, 0, sizeof(from));
+    from.name = "primary1";
+    from.key  = NULL;
+
+    xce_request_t req;
+    memset(&req, 0, sizeof(req));
+    req.from              = &from;
+    req.body_which        = Request_body_ctrlhellomsg;
+    req.body.ctrlhellomsg = &hello;
+
+    handle_ctrlhello(&c, &ep, &req);
+
+    TEST_ASSERT_EQUAL_STRING_MESSAGE("10.0.1.7", c.ha.primary_host,
+        "bug-1701: handle_ctrlhello must copy the matched candidate's "
+        "hostname into ha->primary_host so fill_redirect_reply() has a "
+        "real target");
+    TEST_ASSERT_EQUAL_UINT32_MESSAGE(9991, c.ha.primary_port,
+        "bug-1701: handle_ctrlhello must copy the matched candidate's port "
+        "into ha->primary_port");
+
+    ha_fini(&c.ha);
+    printf("test_handle_ctrlhello_populates_primary_host_from_known_peer: PASS\n");
+}
+
+/* ── Test 8c (bug-1701 edge case): unknown sender name must not crash and
+ * must leave primary_host/primary_port unchanged (not cleared to empty) — a
+ * real config mismatch is surfaced via WARN, not a crash or data wipe. ───── */
+
+static void test_handle_ctrlhello_unknown_peer_leaves_primary_host_unchanged(void)
+{
+    cluster_t c;
+    memset(&c, 0, sizeof(c));
+    ha_init(&c.ha, &g_cfg, 0);
+    c.ha.primary_ctrl_fd = -1;
+    atomic_store_explicit(&c.ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
+
+    myconf_t conf;
+    memset(&conf, 0, sizeof(conf));
+    conf.peers_cfg.candidates_count = 1;
+    strncpy(conf.peers_cfg.candidates[0].name, "primary1",
+            sizeof(conf.peers_cfg.candidates[0].name) - 1);
+    strncpy(conf.peers_cfg.candidates[0].hostname, "10.0.1.7",
+            sizeof(conf.peers_cfg.candidates[0].hostname) - 1);
+    conf.peers_cfg.candidates[0].port = 9991;
+    c.myconf = &conf;
+
+    endpoint_t ep;
+    memset(&ep, 0, sizeof(ep));
+    ep.fd = 42;
+
+    ctrl_hello_req_t hello;
+    memset(&hello, 0, sizeof(hello));
+    hello.name = "stranger";
+
+    xce_peerinfo_t from;
+    memset(&from, 0, sizeof(from));
+    from.name = "stranger"; /* does not match any configured candidate */
+
+    xce_request_t req;
+    memset(&req, 0, sizeof(req));
+    req.from              = &from;
+    req.body_which        = Request_body_ctrlhellomsg;
+    req.body.ctrlhellomsg = &hello;
+
+    handle_ctrlhello(&c, &ep, &req); /* must not crash */
+
+    TEST_ASSERT_EQUAL_STRING_MESSAGE("", c.ha.primary_host,
+        "bug-1701: an unmatched sender name must leave primary_host at its "
+        "prior (fresh-node empty) value, never crash");
+    TEST_ASSERT_EQUAL_UINT32_MESSAGE(0, c.ha.primary_port,
+        "bug-1701: an unmatched sender name must leave primary_port at its "
+        "prior (fresh-node zero) value");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(42, c.ha.primary_ctrl_fd,
+        "bug-1701: primary_ctrl_fd bookkeeping must proceed normally even "
+        "when the name lookup misses");
+
+    ha_fini(&c.ha);
+    printf("test_handle_ctrlhello_unknown_peer_leaves_primary_host_unchanged: PASS\n");
+}
+
 /* ── Test 9 (Task 7.2): follower path is passive — no outbound connect ──────
  *
  * Req 2.1, 2.4: after raft_cb_become_follower the role is CTRL_STANDBY and
@@ -511,6 +632,11 @@ void test_ha_connection_reuse(void)
     /* Req 3.1 — handle_ctrlhello designates primary control channel */
     RUN_TEST(test_handle_ctrlhello_sets_primary_ctrl_fd);
 
+    /* bug-1701 — handle_ctrlhello populates primary_host/primary_port via a
+     * local peers_cfg lookup keyed by req->from->name */
+    RUN_TEST(test_handle_ctrlhello_populates_primary_host_from_known_peer);
+    RUN_TEST(test_handle_ctrlhello_unknown_peer_leaves_primary_host_unchanged);
+
     /* Task 7.2 — Req 2.1, 2.4: follower path is fully passive */
     RUN_TEST(test_follower_passive_no_outbound_connect);
 

src/controller/testsuite/test_ha_durability.c

@@ -34,6 +34,7 @@
 
 #include "ha.h"
 #include "config.h"    /* XCE_MAXNAME_LEN */
+#include "xce/cluster_def.h"  /* cluster_t, for bug-1684's raft_on_vote_request test */
 
 /* ── On-disk format (mirrors ha.c internals, duplicated here for white-box tests) ─
  *
@@ -381,6 +382,133 @@ static void test_persist_atomicity_tmp_cleaned_up(void)
     printf("test_persist_atomicity_tmp_cleaned_up: PASS\n");
 }
 
+/* ── Test 6: ha_persist_term() returns a real error on a genuine write
+ * failure (bug-1684) ─────────────────────────────────────────────────────
+ *
+ * Before this fix, ha_persist_term() had a void return -- every open/write/
+ * fsync/rename failure was silently swallowed ("best-effort; caller must
+ * not rely on this for correctness"). Its sole vote-granting caller,
+ * raft_cb_persist(), always returned 0 to the raft library regardless, so a
+ * vote was granted even when the term/voted_for record never reached disk.
+ *
+ * Points workdir at a path whose PARENT directory does not exist, so
+ * open(O_CREAT) on the .tmp file deterministically fails ENOENT -- no
+ * permission-bit tricks needed (those misbehave when running as root,
+ * common in sandboxes), and no real filesystem quota/full-disk simulation
+ * required either.
+ */
+static void test_persist_term_returns_error_on_write_failure(void)
+{
+    ha_state_t ha;
+    ha_init(&ha, &g_durability_cfg, 0);
+
+    atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
+    ha.voted_for_term = 3;
+    strncpy(ha.voted_for_name, "ctrl-node-9", sizeof(ha.voted_for_name) - 1);
+
+    const char *bad_workdir =
+        "/tmp/xce_ha_durability_test_nonexistent_parent_xyz/subdir";
+    int rc = ha_persist_term(&ha, bad_workdir);
+    TEST_ASSERT_TRUE_MESSAGE(
+        rc < 0,
+        "ha_persist_term must return a negative errno when the write "
+        "genuinely fails (bug-1684)");
+
+    ha_fini(&ha);
+    printf("test_persist_term_returns_error_on_write_failure: PASS\n");
+}
+
+/* ── Test 7: NULL workdir stays a no-op SUCCESS, not folded into the new
+ * error path (bug-1684 regression guard) ──────────────────────────────── */
+static void test_persist_term_null_workdir_returns_success(void)
+{
+    ha_state_t ha;
+    ha_init(&ha, &g_durability_cfg, 0);
+
+    int rc = ha_persist_term(&ha, NULL);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        0, rc,
+        "ha_persist_term(..., NULL) must return 0 -- test-mode skip is not "
+        "a failure");
+
+    ha_fini(&ha);
+    printf("test_persist_term_null_workdir_returns_success: PASS\n");
+}
+
+/* ── Test 8: a vote denied by a real persist failure leaves ha's in-memory
+ * voted_for state UNCHANGED (bug-1684) ─────────────────────────────────────
+ *
+ * Drives the REAL production path: ha_init_raft() wires raft_cb_persist as
+ * the raft library's on_persist callback (s_real_cbs, ha.c); the function
+ * under test, raft_on_vote_request() (raft.c, the pure library), is the
+ * exact function cluster.c's production dispatch calls. ha_init_raft(ha,
+ * NULL, n_peers) avoids needing a real persist_t (only used for
+ * initial_log_len/initial_commit_index when c is non-NULL) -- ha->raft_c is
+ * then set directly afterward, the same test-shim pattern ha.h's own
+ * raft_cb_become_leader_for_test/raft_cb_become_follower_for_test comments
+ * document ("this shim sets ha->raft_c = c first").
+ *
+ * The fixture's myconf populates ONE real candidate name (index 1) so
+ * raft_cb_persist's voted_for_name lookup resolves to a genuine non-empty
+ * name -- not the self/empty fallback -- making the rollback assertion
+ * meaningful: before this fix, ha.voted_for_name/voted_for_term would show
+ * that candidate's vote despite the vote being denied and never durably
+ * recorded.
+ */
+static void test_vote_denied_and_state_rolled_back_on_persist_failure(void)
+{
+    ha_state_t ha;
+    ha_init(&ha, &g_durability_cfg, 0);
+    ha_init_raft(&ha, NULL, 2);
+
+    myconf_t test_myconf;
+    memset(&test_myconf, 0, sizeof(test_myconf));
+    test_myconf.workdir =
+        (char *)"/tmp/xce_ha_durability_test_nonexistent_parent_xyz/subdir";
+    test_myconf.name = (char *)"self-node";
+    test_myconf.peers_cfg.candidates_count = 2;
+    strncpy(test_myconf.peers_cfg.candidates[1].name, "candidate-1",
+            sizeof(test_myconf.peers_cfg.candidates[1].name) - 1);
+
+    cluster_t test_c;
+    memset(&test_c, 0, sizeof(test_c));
+    test_c.myconf = &test_myconf;
+
+    /* Test shim: same pattern ha.h documents for the other raft_cb_*_for_test
+     * helpers -- set ha->raft_c directly rather than driving a full
+     * persist_t through ha_init_raft(). */
+    ha.raft_c = &test_c;
+
+    /* Baseline: no prior vote. */
+    TEST_ASSERT_EQUAL_UINT64(0ULL, ha.voted_for_term);
+    TEST_ASSERT_EQUAL_STRING("", ha.voted_for_name);
+
+    uint64_t reply_term = 0;
+    int result = raft_on_vote_request(&ha.raft, /*from_idx=*/1, /*term=*/5,
+                                      /*cand_last_log_term=*/0,
+                                      /*cand_last_log_index=*/0, &reply_term);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        RAFT_DENY, result,
+        "vote must be DENIED when the term/vote persist genuinely fails "
+        "(bug-1684)");
+
+    /* voted_for state must be UNCHANGED -- the vote was never durably
+     * recorded, so this node must not believe it already voted. */
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(
+        0ULL, ha.voted_for_term,
+        "voted_for_term must roll back to its pre-call value on persist "
+        "failure (bug-1684)");
+    TEST_ASSERT_EQUAL_STRING_MESSAGE(
+        "", ha.voted_for_name,
+        "voted_for_name must roll back to its pre-call value on persist "
+        "failure (bug-1684)");
+
+    ha_fini(&ha);
+    printf("test_vote_denied_and_state_rolled_back_on_persist_failure: "
+          "PASS\n");
+}
+
 /* ── Test suite entry point ───────────────────────────────────────────────── */
 
 void test_ha_durability(void)
@@ -400,4 +528,11 @@ void test_ha_durability(void)
 
     /* Req 9.1: write+rename atomicity — no .tmp artifact left */
     RUN_TEST(test_persist_atomicity_tmp_cleaned_up);
+
+    /* bug-1684: ha_persist_term() must return a real error on write failure,
+     * NULL-workdir must stay a no-op success, and raft_cb_persist() must
+     * deny the vote AND roll back in-memory voted_for state on failure. */
+    RUN_TEST(test_persist_term_returns_error_on_write_failure);
+    RUN_TEST(test_persist_term_null_workdir_returns_success);
+    RUN_TEST(test_vote_denied_and_state_rolled_back_on_persist_failure);
 }

src/controller/testsuite/test_ha_role_lifecycle.c

@@ -691,6 +691,11 @@ static void lifecycle_cluster_teardown(void)
 #ifdef XCE_TESTING
     repl_broadcast_send_hook = NULL;
 #endif
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). */
+    persist_stop(&g_lc.persist);
     commit_engine_fini(&g_lc.commit);
     delaymgr_fini(&g_lc.delaymgr);
     persist_stop(&g_lc.persist);

src/controller/testsuite/test_ha_startup.c

@@ -47,6 +47,12 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
                              uint64_t my_config_tag, uint64_t my_lsn,
                              const char *my_name);
 
+/* bug-1695 — myapp_run's post-stagger "leader already known" bootstrap check,
+ * extracted from myapp.c for unit testability. See myapp.c's doc comment on
+ * myapp_leader_already_known for the full contract. */
+int myapp_leader_already_known(ha_state_t *ha, const char *my_name,
+                               uint64_t term_at_boot);
+
 /* ── Dispatch gate (task 3.3) — does NOT exist yet; linker failure expected ──
  *
  * ha_dispatch_gate: role-based operation filter for the vtaskmgr dispatch loop.
@@ -715,6 +721,192 @@ static void test_nonblocking_election_returns_in_under_10ms(void)
            elapsed_ms_val);
 }
 
+/* ── bug-1695: myapp_run's "leader already known" bootstrap check ──────────
+ *
+ * myapp_leader_already_known(ha, my_name, term_at_boot) is myapp.c's
+ * extracted, unit-testable decision function (see myapp.c's doc comment on
+ * myapp_leader_already_known for the full contract).  These tests reproduce
+ * the exact live-soak-confirmed regression: a RESTORED (from ha-term.bin),
+ * never-since-advanced raft term must NOT be treated as "a leader is known"
+ * -- only a term that genuinely advances during the startup-stagger nap
+ * (real raft traffic processed by iomgr) may.
+ */
+
+static void test_bug1695_stale_restored_term_does_not_skip_election(void)
+{
+    /* Core regression: a node reboots after the cluster's first-ever
+     * election. ha_restore_term() + ha_init_raft() would have seeded
+     * ha->raft.term = 5 (the persisted, historical term) -- and the caller
+     * snapshots term_at_boot = 5 at that same moment, BEFORE the nap. No new
+     * raft traffic is processed during the nap (no vote grant, no AE/
+     * heartbeat observed) -- raft.term is still 5 when this check runs. No
+     * live primary_ctrl_fd connection exists either (ha_init defaults it to
+     * -1, left untouched here). This must NOT report "leader known": the
+     * cluster has no live, reachable primary this node has actually
+     * observed, so it must proceed to ha_start_election_async instead of
+     * permanently stalling (bug-1695's exact live-soak symptom: "raft_
+     * term=1, primary_fd=-1) -- skipping own election", repeating forever
+     * on every restart). */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    ha.raft.term = 5;
+    memset(ha.voted_for_name, 0, sizeof(ha.voted_for_name));
+    ha.voted_for_term = 0;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   5 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, leader_known,
+        "bug-1695: a merely-restored, never-since-advanced raft term must "
+        "NOT be treated as evidence a leader is known -- the node must "
+        "proceed to its own election, not permanently skip_election");
+
+    ha_fini(&ha);
+    printf("test_bug1695_stale_restored_term_does_not_skip_election: PASS\n");
+}
+
+static void test_bug1695_term_advanced_during_nap_skips_election(void)
+{
+    /* Correct fast path #1 (must still work): the term genuinely ADVANCED
+     * during this boot's nap (e.g. iomgr processed a real incoming
+     * RequestVote/AppendEntries at a higher term while this thread napped).
+     * This IS real evidence a leader/candidate is active -- skip own
+     * election. */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    ha.raft.term = 6; /* advanced past term_at_boot=5 during the nap */
+    memset(ha.voted_for_name, 0, sizeof(ha.voted_for_name));
+    ha.voted_for_term = 0;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   5 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, leader_known,
+        "a raft term that genuinely advanced past term_at_boot during the "
+        "nap must still be treated as evidence a leader/candidate is known");
+
+    ha_fini(&ha);
+    printf("test_bug1695_term_advanced_during_nap_skips_election: PASS\n");
+}
+
+static void test_bug1695_live_primary_fd_skips_election_without_term_advance(void)
+{
+    /* Correct fast path #2 (must still work): primary_ctrl_fd is already
+     * connected (winner's ctrlhello arrived) even though raft.term has not
+     * advanced past term_at_boot. Must still skip. */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    ha.raft.term = 5; /* no advance */
+    ha.primary_ctrl_fd = 42; /* simulated live connection; never opened/
+                                closed by this test or ha_fini */
+    memset(ha.voted_for_name, 0, sizeof(ha.voted_for_name));
+    ha.voted_for_term = 0;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   5 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, leader_known,
+        "an already-connected primary_ctrl_fd must still count as leader "
+        "known, independent of the raft.term fallback");
+
+    ha.primary_ctrl_fd = -1;
+    ha_fini(&ha);
+    printf("test_bug1695_live_primary_fd_skips_election_without_term_advance: PASS\n");
+}
+
+static void test_bug1695_first_boot_zero_term_no_advance_does_not_skip(void)
+{
+    /* First-ever-boot case: no ha-term.bin exists yet, so term_at_boot == 0
+     * and raft.term == 0 (unadvanced). Must proceed to election exactly as
+     * before this fix (the old `raft.term > 0` check was already false here
+     * too) -- confirms the fix does not regress first-boot behavior. */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    ha.raft.term = 0;
+    memset(ha.voted_for_name, 0, sizeof(ha.voted_for_name));
+    ha.voted_for_term = 0;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   0 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, leader_known,
+        "first-ever-boot (term_at_boot=0, raft.term=0) must proceed to its "
+        "own election, same as before this fix");
+
+    ha_fini(&ha);
+    printf("test_bug1695_first_boot_zero_term_no_advance_does_not_skip: PASS\n");
+}
+
+static void test_bug1695_first_boot_term_advances_from_zero_skips(void)
+{
+    /* First-ever-boot, but real vote-request traffic advances the term from
+     * 0 to 1 during the nap -- must skip, same as before this fix. */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    ha.raft.term = 1;
+    memset(ha.voted_for_name, 0, sizeof(ha.voted_for_name));
+    ha.voted_for_term = 0;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   0 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, leader_known,
+        "first-ever-boot with a genuine term advance (0 -> 1) during the "
+        "nap must still skip its own election");
+
+    ha_fini(&ha);
+    printf("test_bug1695_first_boot_term_advances_from_zero_skips: PASS\n");
+}
+
+static void test_bug1695_voted_for_other_in_current_term_skips_unaffected(void)
+{
+    /* Sanity/no-regression: the voted_for_name/vft==cur fast path is
+     * untouched by this fix and must keep working -- this node granted a
+     * vote to a DIFFERENT candidate in the CURRENT term, so it is a
+     * follower and must skip its own election, even with no raft.term
+     * advance and no live primary_ctrl_fd. */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    atomic_store(&ha.term, 3ULL);
+    ha.raft.term = 3;
+    strncpy(ha.voted_for_name, "ctrl-1", sizeof(ha.voted_for_name) - 1);
+    ha.voted_for_term = 3;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   3 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, leader_known,
+        "a vote granted to a different candidate in the current term must "
+        "still skip own election (untouched by this fix)");
+
+    ha_fini(&ha);
+    printf("test_bug1695_voted_for_other_in_current_term_skips_unaffected: PASS\n");
+}
+
+static void test_bug1695_self_vote_does_not_skip_election(void)
+{
+    /* Sanity/no-regression: a self-vote (voted_for_name == own name) must
+     * NOT count as "leader known" -- it means THIS node was the candidate,
+     * not that an external leader was observed. With no term advance and no
+     * live primary_ctrl_fd, this must proceed to its own election. */
+    ha_state_t ha;
+    ha_init(&ha, &g_startup_cfg, 0);
+    atomic_store(&ha.term, 3ULL);
+    ha.raft.term = 3;
+    strncpy(ha.voted_for_name, "ctrl-2", sizeof(ha.voted_for_name) - 1);
+    ha.voted_for_term = 3;
+
+    int leader_known = myapp_leader_already_known(&ha, "ctrl-2",
+                                                   3 /* term_at_boot */);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, leader_known,
+        "a self-vote must not be treated as evidence an external leader is "
+        "known");
+
+    ha_fini(&ha);
+    printf("test_bug1695_self_vote_does_not_skip_election: PASS\n");
+}
+
 /* ── Req 2.4: heartbeat timing validation at startup ─────────────────────── */
 
 /*
@@ -829,6 +1021,17 @@ void test_ha_startup(void)
     RUN_TEST(test_nonblocking_single_node_election_enters_primary_directly);
     RUN_TEST(test_nonblocking_election_returns_in_under_10ms);
 
+    /* bug-1695 — myapp_run's "leader already known" bootstrap check must
+     * require a genuine term ADVANCE during the startup nap, not merely a
+     * nonzero restored term */
+    RUN_TEST(test_bug1695_stale_restored_term_does_not_skip_election);
+    RUN_TEST(test_bug1695_term_advanced_during_nap_skips_election);
+    RUN_TEST(test_bug1695_live_primary_fd_skips_election_without_term_advance);
+    RUN_TEST(test_bug1695_first_boot_zero_term_no_advance_does_not_skip);
+    RUN_TEST(test_bug1695_first_boot_term_advances_from_zero_skips);
+    RUN_TEST(test_bug1695_voted_for_other_in_current_term_skips_unaffected);
+    RUN_TEST(test_bug1695_self_vote_does_not_skip_election);
+
     /* Task 1.1 (ha-heartbeat-liveness) — ha_init zero-initializes last_activity */
     RUN_TEST(test_ha_init_last_activity_is_zero);
 

src/controller/testsuite/test_host_runusage.c

@@ -0,0 +1,340 @@
+/* test_host_runusage.c -- bug-1735/bug-1736 regression suite
+ *
+ * bug-1735: job memory usage always reads 0 in the scheduler's simulation
+ * record file, despite the worker daemon genuinely measuring it. Two
+ * independent, confirmed root causes in misc.c/job.c:
+ *
+ *   1. update_host_runusage() (misc.c) only ever matched resnames
+ *      "cputime"/"mem" -- but the REAL worker's rusage collector
+ *      (collect_job_rusages(), xcecd_cgroups.c) sends "cpu_time_sec"/
+ *      "memory_rss" (see that file's res_names[] array). The two never
+ *      matched, so xce_host_runusage_t.cpu_time/mem_usage stayed at their
+ *      calloc()'d zero forever, for every real (non-simulated) job.
+ *
+ *   2. update_job_rusages() (job.c) had a standing FIXME ("update other
+ *      fields of job rusage later") -- xce_job_runusage_t's own job-level
+ *      max_mem/avg_mem/used_mem/cpu_time were never derived from the
+ *      per-host host_runusage[] entries update_host_runusage() populates,
+ *      so a multi-host job's aggregate usage stayed 0 even once (1) above
+ *      is fixed.
+ *
+ * bug-1736 (split from bug-1735, lower priority, display/reporting only):
+ * two more gaps in the same area.
+ *
+ *   3. xce_host_runusage_t had NO field at all to receive the worker's
+ *      "memory_swap"/"tasks_current" resnames -- a complete dead end, not
+ *      a naming mismatch. Added mem_swap/tasks_current fields plus parsing.
+ *
+ *   4. xce_job_runusage_t.peak_cpu_time was never computed. Added as the
+ *      running MAX of aggregate_job_runusage()'s per-call cpu_time sum
+ *      across every call for a given job (not just the latest report).
+ *
+ * peak_cpu_duration/mem_efficiency/avg_cpu_efficiency/peak_cpu_efficiency
+ * are DELIBERATELY still not covered -- see aggregate_job_runusage()'s own
+ * doc comment (misc.c) for why: no unambiguous formula/definition exists
+ * for these without a product decision, unlike peak_cpu_time.
+ *
+ * This file unit-tests all four fixes directly against the REAL misc.c
+ * (linked standalone, mirroring resval_typeresolve_unit_test's/
+ * sched_worker_change_unit_test's isolation rationale -- misc.c's
+ * update_host_runusage()/aggregate_job_runusage() are calloc/realloc/
+ * strcmp-only, no RTTI locking, no networking, so they link and run with
+ * zero fake/stub scaffolding). get_ep_worker()/get_ep_role() (same file)
+ * are untouched by this fix and unused by these tests, but still link
+ * fine via controller_unit_test_deps' existing librtti_dep.
+ */
+#include <math.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unity/unity.h>
+
+#include "cluster.h"
+
+/* ------------------------------------------------------------------ */
+/* Helpers                                                             */
+/* ------------------------------------------------------------------ */
+
+/* This project's shared unity_dep is built with double-precision support
+ * excluded (TEST_ASSERT_EQUAL_DOUBLE compiles to an unconditional "Unity
+ * Double Precision Disabled" failure) -- no other test file in this
+ * codebase uses it either. Plain epsilon comparison in ordinary C avoids
+ * Unity's own double formatting/compare path entirely. */
+#define ASSERT_DOUBLE_EQ_MSG(expected, actual, msg)                          \
+    TEST_ASSERT_TRUE_MESSAGE(fabs((expected) - (actual)) < 1e-6, msg)
+#define ASSERT_DOUBLE_EQ(expected, actual)                                   \
+    ASSERT_DOUBLE_EQ_MSG(expected, actual, #actual " != " #expected)
+
+static xce_rusage_t** make_usages(const char* name1, float val1,
+                                  const char* name2, float val2)
+{
+    xce_rusage_t** usages = (xce_rusage_t**)calloc(2, sizeof(xce_rusage_t*));
+
+    usages[0]          = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t));
+    usages[0]->resname = strdup(name1);
+    usages[0]->value   = val1;
+
+    usages[1]          = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t));
+    usages[1]->resname = strdup(name2);
+    usages[1]->value   = val2;
+
+    return usages;
+}
+
+static void free_usages(xce_rusage_t** usages)
+{
+    free(usages[0]->resname);
+    free(usages[0]);
+    free(usages[1]->resname);
+    free(usages[1]);
+    free(usages);
+}
+
+static xce_rusage_t** make_usages4(const char* n1, float v1, const char* n2,
+                                   float v2, const char* n3, float v3,
+                                   const char* n4, float v4)
+{
+    const char*    names[4]  = { n1, n2, n3, n4 };
+    float          values[4] = { v1, v2, v3, v4 };
+    xce_rusage_t** usages    = (xce_rusage_t**)calloc(4, sizeof(xce_rusage_t*));
+    int            i;
+
+    for (i = 0; i < 4; i++) {
+        usages[i]          = (xce_rusage_t*)calloc(1, sizeof(xce_rusage_t));
+        usages[i]->resname = strdup(names[i]);
+        usages[i]->value   = values[i];
+    }
+
+    return usages;
+}
+
+static void free_usages4(xce_rusage_t** usages)
+{
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        free(usages[i]->resname);
+        free(usages[i]);
+    }
+    free(usages);
+}
+
+static void free_runusage(xce_job_runusage_t* r)
+{
+    int i;
+
+    if (r == NULL) {
+        return;
+    }
+
+    for (i = 0; i < r->num_hosts; i++) {
+        free(r->host_runusage[i]->host_name);
+        free(r->host_runusage[i]);
+    }
+    free(r->host_runusage);
+    free(r);
+}
+
+/* ------------------------------------------------------------------ */
+/* Tests -- update_host_runusage() resname alignment (root cause 1)   */
+/* ------------------------------------------------------------------ */
+
+static void
+test_update_host_runusage_matches_real_worker_resnames(void)
+{
+    xce_job_runusage_t* runusage = NULL;
+    xce_rusage_t**       usages
+        = make_usages("cpu_time_sec", 42.0f, "memory_rss", 128.5f);
+
+    int rc = update_host_runusage(&runusage, strdup("host-a"), 2, usages);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_NOT_NULL(runusage);
+    TEST_ASSERT_EQUAL_INT(1, runusage->num_hosts);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        42, runusage->host_runusage[0]->cpu_time,
+        "cpu_time must be set from the worker's real 'cpu_time_sec' resname "
+        "(bug-1735 root cause 1) -- NOT the old, never-matching 'cputime'");
+    ASSERT_DOUBLE_EQ_MSG(
+        128.5, runusage->host_runusage[0]->mem_usage,
+        "mem_usage must be set from the worker's real 'memory_rss' resname "
+        "(bug-1735 root cause 1) -- NOT the old, never-matching 'mem'");
+
+    free_usages(usages);
+    free_runusage(runusage);
+}
+
+static void test_update_host_runusage_updates_existing_host_in_place(void)
+{
+    xce_job_runusage_t* runusage = NULL;
+    xce_rusage_t** first  = make_usages("cpu_time_sec", 10.0f, "memory_rss", 50.0f);
+    xce_rusage_t** second = make_usages("cpu_time_sec", 20.0f, "memory_rss", 75.0f);
+
+    update_host_runusage(&runusage, strdup("host-a"), 2, first);
+    update_host_runusage(&runusage, strdup("host-a"), 2, second);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        1, runusage->num_hosts,
+        "a second report for the SAME host must update in place, not "
+        "append a duplicate host entry");
+    TEST_ASSERT_EQUAL_INT(20, runusage->host_runusage[0]->cpu_time);
+    ASSERT_DOUBLE_EQ(75.0, runusage->host_runusage[0]->mem_usage);
+
+    free_usages(first);
+    free_usages(second);
+    free_runusage(runusage);
+}
+
+/* ------------------------------------------------------------------ */
+/* Tests -- bug-1736: memory_swap/tasks_current (root cause 3)         */
+/* ------------------------------------------------------------------ */
+
+static void test_update_host_runusage_parses_swap_and_task_count(void)
+{
+    xce_job_runusage_t* runusage = NULL;
+    xce_rusage_t**       usages  = make_usages4(
+        "cpu_time_sec", 5.0f, "memory_rss", 64.0f, "memory_swap", 12.5f,
+        "tasks_current", 3.0f);
+
+    int rc = update_host_runusage(&runusage, strdup("host-a"), 4, usages);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    ASSERT_DOUBLE_EQ_MSG(
+        12.5, runusage->host_runusage[0]->mem_swap,
+        "mem_swap must be set from the worker's 'memory_swap' resname "
+        "(bug-1736) -- previously had no field to land in at all");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        3, runusage->host_runusage[0]->tasks_current,
+        "tasks_current must be set from the worker's 'tasks_current' "
+        "resname (bug-1736) -- previously had no field to land in at all");
+
+    free_usages4(usages);
+    free_runusage(runusage);
+}
+
+/* ------------------------------------------------------------------ */
+/* Tests -- aggregate_job_runusage() (root cause 2)                    */
+/* ------------------------------------------------------------------ */
+
+static xce_job_runusage_t* make_runusage_with_hosts(int n, const int* cpu,
+                                                     const double* mem)
+{
+    xce_job_runusage_t* r = (xce_job_runusage_t*)calloc(1, sizeof(*r));
+    int                  i;
+
+    r->num_hosts     = n;
+    r->host_runusage
+        = (xce_host_runusage_t**)calloc(n, sizeof(xce_host_runusage_t*));
+
+    for (i = 0; i < n; i++) {
+        r->host_runusage[i]
+            = (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
+        r->host_runusage[i]->host_name = strdup("h");
+        r->host_runusage[i]->cpu_time  = cpu[i];
+        r->host_runusage[i]->mem_usage = mem[i];
+    }
+
+    return r;
+}
+
+static void test_aggregate_job_runusage_single_host(void)
+{
+    int    cpu[] = { 30 };
+    double mem[] = { 256.0 };
+    xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu, mem);
+
+    aggregate_job_runusage(r);
+
+    ASSERT_DOUBLE_EQ(256.0, r->used_mem);
+    ASSERT_DOUBLE_EQ(256.0, r->max_mem);
+    ASSERT_DOUBLE_EQ(256.0, r->avg_mem);
+    TEST_ASSERT_EQUAL_INT(30, r->cpu_time);
+
+    free_runusage(r);
+}
+
+static void test_aggregate_job_runusage_multi_host_sum_max_avg(void)
+{
+    int    cpu[] = { 10, 20, 30 };
+    double mem[] = { 100.0, 300.0, 200.0 };
+    xce_job_runusage_t* r = make_runusage_with_hosts(3, cpu, mem);
+
+    aggregate_job_runusage(r);
+
+    ASSERT_DOUBLE_EQ_MSG(
+        600.0, r->used_mem,
+        "used_mem is the SUM of every host's mem_usage (total footprint of "
+        "a multi-host job), not any single host's value");
+    ASSERT_DOUBLE_EQ_MSG(300.0, r->max_mem, "max_mem is the MAX across hosts");
+    ASSERT_DOUBLE_EQ_MSG(200.0, r->avg_mem, "avg_mem is the AVERAGE across hosts");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        60, r->cpu_time, "job-level cpu_time is the SUM across hosts");
+
+    free_runusage(r);
+}
+
+static void test_aggregate_job_runusage_peak_cpu_time_is_running_max(void)
+{
+    int    cpu1[] = { 50 };
+    double mem1[] = { 100.0 };
+    xce_job_runusage_t* r = make_runusage_with_hosts(1, cpu1, mem1);
+
+    aggregate_job_runusage(r);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(50, r->peak_cpu_time,
+                                  "peak_cpu_time tracks the first report");
+
+    /* Second report: a LOWER cpu_time (e.g. a merge/replay delivered an
+     * older report after a newer one) -- peak must NOT regress. */
+    r->host_runusage[0]->cpu_time = 20;
+    aggregate_job_runusage(r);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        50, r->peak_cpu_time,
+        "peak_cpu_time (bug-1736) is the running MAX across every report "
+        "this job has ever received, not just the latest one -- a lower "
+        "subsequent report must not regress it");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        20, r->cpu_time, "cpu_time itself IS the latest report, unlike peak");
+
+    /* Third report: a NEW high -- peak must advance. */
+    r->host_runusage[0]->cpu_time = 90;
+    aggregate_job_runusage(r);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(90, r->peak_cpu_time,
+                                  "peak_cpu_time advances on a new high");
+
+    free_runusage(r);
+}
+
+static void test_aggregate_job_runusage_null_and_empty_are_safe(void)
+{
+    xce_job_runusage_t empty = { 0 };
+
+    aggregate_job_runusage(NULL); /* must not crash */
+    aggregate_job_runusage(&empty); /* num_hosts == 0, must not crash */
+
+    ASSERT_DOUBLE_EQ(0.0, empty.used_mem);
+}
+
+/* ------------------------------------------------------------------ */
+/* Standalone Unity main (this binary is its own test executable -- see */
+/* meson.build's host_runusage_unit_test)                               */
+/* ------------------------------------------------------------------ */
+
+void setUp(void) { return; }
+void tearDown(void) { return; }
+
+int main(void)
+{
+    printf("\n>> bug-1735: host/job rusage resname + aggregation fix\n\n");
+
+    UNITY_BEGIN();
+
+    RUN_TEST(test_update_host_runusage_matches_real_worker_resnames);
+    RUN_TEST(test_update_host_runusage_updates_existing_host_in_place);
+    RUN_TEST(test_update_host_runusage_parses_swap_and_task_count);
+    RUN_TEST(test_aggregate_job_runusage_single_host);
+    RUN_TEST(test_aggregate_job_runusage_multi_host_sum_max_avg);
+    RUN_TEST(test_aggregate_job_runusage_peak_cpu_time_is_running_max);
+    RUN_TEST(test_aggregate_job_runusage_null_and_empty_are_safe);
+
+    return UNITY_END();
+}

src/controller/testsuite/test_job_array_fence.c

@@ -222,6 +222,11 @@ static void tf_teardown(void)
     ha_fini(&g_tf_c.ha);
 
     repl_broadcast_send_hook = NULL;
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). */
+    persist_stop(&g_tf_c.persist);
     commit_engine_fini(&g_tf_hce);
     persist_stop(&g_tf_c.persist);
     persist_fini(&g_tf_c.persist);

src/controller/testsuite/test_job_array_replay_fence.c

@@ -66,6 +66,7 @@
 
 #include <capnp_c.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <inttypes.h>
 #include <stdatomic.h>
 #include <stdbool.h>
@@ -657,6 +658,441 @@ test_first_boot_promotion_evaluates_own_first_replay_fence(void)
           "PASS\n");
 }
 
+/* ── bug-1696 (positive-path sanity check, NOT the RED/GREEN discriminator
+ * -- see test_first_boot_promotion_refuses_when_own_first_recovery_finds_
+ * corruption below for that): a node's very FIRST-EVER promotion evaluation
+ * must still succeed and preserve its own genuinely durable WAL when that
+ * WAL is intact ─────────────────────────────────────────────────────────
+ *
+ * IMPORTANT CORRECTION (found while proving this test's RED phase against
+ * the unfixed ha.c -- do not re-introduce the original hypothesis this
+ * comment used to state): the original bug-1696 write-up hypothesized that
+ * the pre-recovery promotion-truncation-floor computation
+ * (persist_durable_floor()==0 before recovery) would make
+ * persist_truncate_wal_above() PHYSICALLY DESTROY real durable WAL data left
+ * over from a prior life of this process. Source-tracing
+ * truncate_wal_above_locked() (persist.c) disproves that specific mechanism:
+ * its own no-op guard is `if (p->current_lsn == 0 || cutoff_lsn + 1 >=
+ * p->current_lsn) return 0;`, and persist_init() unconditionally resets
+ * current_lsn=1 on every fresh boot regardless of what is physically on
+ * disk -- so a truncate call issued BEFORE persist_recover() has ever
+ * re-derived current_lsn from the real WAL scan is ALWAYS a no-op (nothing
+ * looks "written this boot" yet to truncate). This test proves that
+ * empirically: it reproduces the exact fixture the original hypothesis
+ * described (N real, fsynced entries from a prior life, then a truly fresh
+ * persist_t on the same workdir, is_first_boot=true) and confirms the data
+ * survives with or WITHOUT the bug-1696 reorder fix applied -- i.e. this
+ * specific scenario was never actually destructive. It is kept as a
+ * regression/sanity guard (the reorder must not newly break the healthy
+ * first-boot-promotion path) but does NOT discriminate the bug.
+ *
+ * The REAL, mechanically-provable bug-1696 hazard is different: the
+ * promotion-ELIGIBILITY GATE (persist_recovery_truncated()/persist_
+ * durable_floor()/persist_apply_verdict()) is ALSO evaluated before
+ * cluster_init_phase2()/persist_recover() has run, so it can wrongly judge
+ * a genuinely CORRUPT or DIVERGENT node ELIGIBLE (reading the fresh-
+ * persist_init "clean" defaults) and let it become CTRL_PRIMARY -- only
+ * discovering the real corruption a few lines later, too late to matter.
+ * See test_first_boot_promotion_refuses_when_own_first_recovery_finds_
+ * corruption below for the actual RED/GREEN proof of that hazard, and
+ * buglog.json's bug-1696 entry for the full corrected root-cause writeup.
+ */
+static void test_first_boot_promotion_never_truncates_own_durable_wal(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_trf_nodataloss_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    const uint64_t n_committed_entries = 5;
+
+    /* ── "boot 1": physically append N real, fsynced WAL_NOOP entries, then
+     * tear down -- no cluster needed (persist_wal_async is a physical-write
+     * -only path when p->cluster is NULL, mirrors the sibling test's phase
+     * 1 and test_replicated_apply_chain_survives_cold_restart's writer). ── */
+    {
+        persist_config_t cfg1 = persist_config_defaults();
+        cfg1.wal_sync_ms = 5000;
+        persist_t *p1 = calloc(1, sizeof(persist_t));
+        TEST_ASSERT_NOT_NULL(p1);
+        persist_set_workdir(p1, tmp, "ctrl-test");
+        TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL));
+        TEST_ASSERT_EQUAL_INT(0, persist_start(p1));
+        for (uint64_t i = 0; i < n_committed_entries; i++) {
+            TEST_ASSERT_EQUAL_INT(0,
+                persist_wal_async(p1, WAL_NOOP, NULL, 0));
+        }
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(p1));
+        persist_stop(p1);
+        persist_fini(p1);
+        free(p1);
+    }
+
+    /* ── "boot 2" / restart: fresh cluster_t + its EMBEDDED persist_t point
+     * at the SAME on-disk workdir. Fresh persist_t -> durable_lsn=0 until
+     * persist_recover() runs -- exactly a real restart. ─────────────────── */
+    trf_build_cluster();
+
+    persist_config_t cfg2 = persist_config_defaults();
+    cfg2.wal_sync_ms = 5000;
+    persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test");
+    TEST_ASSERT_EQUAL_INT(0,
+        persist_init(&g_trf_c.persist, &cfg2, &g_trf_c));
+    TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist));
+    g_trf_c.persist.recovery_verdict = RECOVER_CLEAN;
+
+    /* Precondition: never recovered yet -- durable_floor reads 0 regardless
+     * of the N real entries physically on disk. This is the exact blind
+     * spot bug-1696 exploits. */
+    uint64_t durable_floor_precondition =
+        persist_durable_floor(&g_trf_c.persist);
+
+    ha_config_t hcfg = {
+        .heartbeat_interval_ms    = 3000,
+        .heartbeat_timeout_ms     = 10000,
+        .election_timeout_ms      = 2000,
+        .probe_connect_timeout_ms = 500,
+        .max_promotion_retries    = 5,
+    };
+    TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0));
+    g_trf_c.ha.raft_c = &g_trf_c;
+    g_trf_c.ha.repl   = NULL; /* no healthy peer reachable at this level */
+
+    TEST_ASSERT_EQUAL_INT(0,
+        delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task));
+    TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr));
+
+    /* raft_cb_become_leader's tail (on a successful promotion) reaches
+     * ha_arm_own_term_gate -> commit_submit/commit_set_writable_lsn, which
+     * needs a wired commit engine -- unlike the sibling REFUSAL tests above,
+     * THIS test drives promotion all the way through to CTRL_PRIMARY, so it
+     * must be initialized (mirrors test_convergence_verification.c's
+     * cv_setup, the only other fixture in this suite that reaches a
+     * successful raft_cb_become_leader_with_cluster_for_test promotion). */
+    TEST_ASSERT_EQUAL_INT(0,
+        commit_engine_init(&g_trf_c.commit, &g_trf_c.persist,
+                           /*n_peers=*/0, /*n_vcpus=*/4));
+
+    /* Single-node-shaped elect_ctx: n_peers=0, self_lsn=0 -- required to even
+     * exercise the truncation code path (gated on ha->elect_ctx != NULL) and
+     * reproduces the bug narrative's "self also advertises ~0 before its own
+     * recovery" condition. */
+    ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx));
+    TEST_ASSERT_NOT_NULL(ctx);
+    ctx->n_peers  = 0;
+    ctx->self_lsn = 0;
+    ctx->self_tag = 0;
+    pthread_mutex_init(&ctx->lock, NULL);
+    sem_init(&ctx->done, 0, 0);
+    atomic_store_explicit(&g_trf_c.ha.elect_ctx, (void *)ctx,
+                          memory_order_release);
+
+    /* The crux of the reproduction: ha->phase2_done is 0 (fresh ha_init
+     * above), so raft_cb_become_leader's is_first_boot block fires and calls
+     * this hook -- which calls persist_recover() for real, from INSIDE this
+     * same election evaluation, racing the truncation/eligibility code
+     * exactly like production. */
+    ha_phase2_hook = trf_phase2_hook_calls_persist_recover;
+
+    raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1);
+
+    ha_phase2_hook = NULL; /* restore -- shared process-global test seam */
+
+    int32_t role_after_promotion =
+        atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire);
+    uint64_t durable_floor_after_promotion =
+        persist_durable_floor(&g_trf_c.persist);
+    bool truncated_after_promotion =
+        persist_recovery_truncated(&g_trf_c.persist);
+
+    atomic_store_explicit(&g_trf_c.ha.elect_ctx, NULL, memory_order_release);
+    pthread_mutex_destroy(&ctx->lock);
+    sem_destroy(&ctx->done);
+    free(ctx);
+
+    delaymgr_stop(&g_trf_c.delaymgr);
+    delaymgr_fini(&g_trf_c.delaymgr);
+    ha_fini(&g_trf_c.ha);
+
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). */
+    persist_stop(&g_trf_c.persist);
+    commit_engine_fini(&g_trf_c.commit);
+    persist_stop(&g_trf_c.persist);
+    persist_fini(&g_trf_c.persist);
+    trf_teardown_cluster();
+
+    /* ── Independent THIRD-boot verification: a brand fresh persist_t (no
+     * cluster involved at all, so this reflects ONLY what is physically on
+     * disk after everything above) recovering the SAME workdir -- the
+     * `waldump`-equivalent proof the real bug was confirmed with. ────────── */
+    uint64_t third_boot_recovered_current_lsn = 0;
+    {
+        persist_t *p3 = calloc(1, sizeof(persist_t));
+        TEST_ASSERT_NOT_NULL(p3);
+        persist_set_workdir(p3, tmp, "ctrl-test");
+        persist_config_t cfg3 = persist_config_defaults();
+        cfg3.wal_sync_ms = 5000;
+        TEST_ASSERT_EQUAL_INT(0, persist_init(p3, &cfg3, NULL));
+        TEST_ASSERT_EQUAL_INT(0, persist_start(p3));
+        TEST_ASSERT_EQUAL_INT(0, persist_recover(p3));
+        third_boot_recovered_current_lsn = p3->current_lsn;
+        persist_stop(p3);
+        persist_fini(p3);
+        free(p3);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    /* ── Assertions ───────────────────────────────────────────────────── */
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, durable_floor_precondition,
+        "fixture precondition: a never-recovered persist_t must read "
+        "durable_floor=0 regardless of the N real entries on disk -- this "
+        "is the exact blind spot bug-1696 exploits");
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_PRIMARY, role_after_promotion,
+        "this node is structurally healthy (clean recovery, no faults) and "
+        "must actually become CTRL_PRIMARY on this election -- the bug is "
+        "that it used to get there by first destroying its own WAL, not "
+        "that it fails to promote");
+    TEST_ASSERT_FALSE_MESSAGE(truncated_after_promotion,
+        "a clean recovery of genuinely-intact WAL data must never be "
+        "reported as recovery-truncated");
+
+    TEST_ASSERT_GREATER_OR_EQUAL_UINT64_MESSAGE(
+        n_committed_entries, durable_floor_after_promotion,
+        "bug-1696: promotion must NEVER truncate this node's own durable "
+        "WAL to a bogus pre-recovery floor -- durable_floor after promotion "
+        "must still reflect the N entries committed before this restart, "
+        "not collapse to ~0/1 because the truncation floor was computed "
+        "before cluster_init_phase2()/persist_recover() ever ran");
+
+    TEST_ASSERT_GREATER_OR_EQUAL_UINT64_MESSAGE(
+        n_committed_entries + 1, third_boot_recovered_current_lsn,
+        "bug-1696 (independent physical proof): a THIRD boot recovering the "
+        "SAME on-disk workdir after everything above must still find all N "
+        "originally-committed entries physically present -- current_lsn == "
+        "N+1 (one past the highest surviving LSN) proves the real on-disk "
+        "WAL was never truncated away by the second boot's promotion");
+
+    printf("test_first_boot_promotion_never_truncates_own_durable_wal: "
+          "PASS\n");
+}
+
+/* ── bug-1696 (ACTUAL RED/GREEN proof): the promotion-eligibility gate must
+ * NEVER judge a node ELIGIBLE using pre-recovery-blind defaults when its own
+ * first-ever replay (racing this SAME election evaluation) would have
+ * discovered the log is genuinely CORRUPT ───────────────────────────────────
+ *
+ * raft_cb_become_leader()'s main eligibility gate reads persist_recovery_
+ * truncated()/persist_durable_floor()/persist_apply_verdict() to decide
+ * ELIGIBLE vs REFUSE. Before the bug-1696 fix, that gate ran BEFORE cluster_
+ * init_phase2()/persist_recover() -- the ONLY call that can discover a real
+ * on-disk WAL is structurally corrupt for THIS process's first-ever boot.
+ * A freshly persist_init()'d persist_t defaults recovery_verdict=
+ * RECOVER_CLEAN (persist.c ~1168) and durable_lsn=0 (persist.c ~1163) --
+ * "clean and empty", not "unknown" -- so the gate necessarily judged a node
+ * whose WAL is ACTUALLY corrupt as ELIGIBLE, let it fall through to
+ * CTRL_PRIMARY, and only discovered the real corruption a few lines later
+ * (too late: the bug-1445 re-check immediately after phase2 only re-checks
+ * persist_apply_resource_fenced(), never persist_recovery_truncated()).
+ * This is the general form of the hazard bug-1445 already patched narrowly
+ * for ONE flag; bug-1696 is the same hazard for the far more consequential
+ * durable_floor/recovery_truncated gate inputs.
+ *
+ * Fixture: "boot 1" writes two real, fsynced WAL entries, then the FIRST
+ * entry's on-disk CRC is corrupted (identical technique to test_commit_
+ * durability.c's test_recover_verdict_crc_break_first_entry_truncated) --
+ * persist_recover(), once it actually runs, will classify this as
+ * RECOVER_TRUNCATED. "boot 2" points a fresh cluster_t + persist_t at the
+ * SAME corrupted workdir with ha->phase2_done fresh (0, is_first_boot=true)
+ * and ha_phase2_hook wired to call persist_recover() for real, from INSIDE
+ * this same election evaluation -- exactly production's ordering. A single-
+ * node-shaped ha_elect_ctx_t (n_peers=0, self_lsn=0, no reachable peer)
+ * isolates the recovery_truncated conjunct: promotion_safe_lsn=0 means the
+ * HWM comparison alone could never explain a refusal, so if this node is
+ * refused, recovery_truncated is what did it.
+ *
+ * RED before the fix: role_after_election == CTRL_PRIMARY (the gate judged
+ * this node ELIGIBLE while blind, and nothing after phase2 re-checked
+ * recovery_truncated before falling through to CTRL_PRIMARY) despite
+ * persist_recover() -- which DID run moments later, inside the very same
+ * call -- correctly detecting RECOVER_TRUNCATED.
+ * GREEN after the fix: phase2/persist_recover() runs BEFORE the gate, so the
+ * gate sees the real RECOVER_TRUNCATED verdict and REFUSES promotion (stays
+ * alive, fenced, in_catch_up=1, never self-exits).
+ */
+static void
+test_first_boot_promotion_refuses_when_own_first_recovery_finds_corruption(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_trf_corruptboot_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    /* ── "boot 1": two real, fsynced WAL_NOOP entries, then die. ─────────── */
+    {
+        persist_config_t cfg1 = persist_config_defaults();
+        cfg1.wal_sync_ms = 5000;
+        persist_t *p1 = calloc(1, sizeof(persist_t));
+        TEST_ASSERT_NOT_NULL(p1);
+        persist_set_workdir(p1, tmp, "ctrl-test");
+        TEST_ASSERT_EQUAL_INT(0, persist_init(p1, &cfg1, NULL));
+        TEST_ASSERT_EQUAL_INT(0, persist_start(p1));
+        TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_NOOP, NULL, 0));
+        TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p1, WAL_NOOP, NULL, 0));
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(p1));
+        persist_stop(p1);
+        persist_fini(p1);
+        free(p1);
+    }
+
+    /* Corrupt the FIRST entry's on-disk CRC -- breaks the chain at LSN 1
+     * (identical technique to test_commit_durability.c's
+     * test_recover_verdict_crc_break_first_entry_truncated). A future
+     * persist_recover() over this workdir will classify RECOVER_TRUNCATED. */
+    {
+        char seg_path[512];
+        snprintf(seg_path, sizeof(seg_path),
+                 "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal", tmp,
+                 (uint64_t)1);
+        int fd = open(seg_path, O_RDWR);
+        TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
+        wal_entry_hdr_t hdr;
+        TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
+                              (int)pread(fd, &hdr, sizeof(hdr), 0));
+        hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */
+        TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
+                              (int)pwrite(fd, &hdr, sizeof(hdr), 0));
+        fsync(fd);
+        close(fd);
+    }
+
+    /* ── "boot 2" / restart: fresh cluster_t + persist_t on the SAME
+     * corrupted workdir. Never recovered yet -- the pre-recovery defaults
+     * (recovery_verdict=RECOVER_CLEAN, durable_lsn=0) read "clean and
+     * empty" regardless of the real on-disk corruption. This is the exact
+     * blind spot bug-1696 exploits. ──────────────────────────────────────── */
+    trf_build_cluster();
+
+    persist_config_t cfg2 = persist_config_defaults();
+    cfg2.wal_sync_ms = 5000;
+    persist_set_workdir(&g_trf_c.persist, tmp, "ctrl-test");
+    TEST_ASSERT_EQUAL_INT(0,
+        persist_init(&g_trf_c.persist, &cfg2, &g_trf_c));
+    TEST_ASSERT_EQUAL_INT(0, persist_start(&g_trf_c.persist));
+
+    bool recovery_truncated_precondition =
+        persist_recovery_truncated(&g_trf_c.persist);
+    recover_verdict_t recovery_verdict_precondition =
+        persist_recovery_verdict(&g_trf_c.persist);
+
+    ha_config_t hcfg = {
+        .heartbeat_interval_ms    = 3000,
+        .heartbeat_timeout_ms     = 10000,
+        .election_timeout_ms      = 2000,
+        .probe_connect_timeout_ms = 500,
+        .max_promotion_retries    = 5,
+    };
+    TEST_ASSERT_EQUAL_INT(0, ha_init(&g_trf_c.ha, &hcfg, 0));
+    g_trf_c.ha.raft_c = &g_trf_c;
+    g_trf_c.ha.repl   = NULL; /* no healthy peer reachable at this level */
+
+    TEST_ASSERT_EQUAL_INT(0,
+        delaymgr_init(&g_trf_c.delaymgr, 5, trf_run_delay_task));
+    TEST_ASSERT_EQUAL_INT(0, delaymgr_start(&g_trf_c.delaymgr));
+    ha_set_promotion_fence_window_ms(60);
+
+    /* Single-node-shaped elect_ctx: n_peers=0, self_lsn=0, no reachable peer
+     * -- isolates the recovery_truncated conjunct (promotion_safe_lsn=0
+     * means the HWM comparison alone can never explain a refusal here). */
+    ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx));
+    TEST_ASSERT_NOT_NULL(ctx);
+    ctx->n_peers  = 0;
+    ctx->self_lsn = 0;
+    ctx->self_tag = 0;
+    pthread_mutex_init(&ctx->lock, NULL);
+    sem_init(&ctx->done, 0, 0);
+    atomic_store_explicit(&g_trf_c.ha.elect_ctx, (void *)ctx,
+                          memory_order_release);
+
+    /* The crux of the reproduction: ha->phase2_done is 0 (fresh ha_init
+     * above), so raft_cb_become_leader's is_first_boot block fires and calls
+     * this hook -- which calls persist_recover() for real, from INSIDE this
+     * same election evaluation, racing the eligibility gate exactly like
+     * production. */
+    ha_phase2_hook = trf_phase2_hook_calls_persist_recover;
+
+    raft_cb_become_leader_with_cluster_for_test(&g_trf_c, /*term=*/1);
+
+    ha_phase2_hook = NULL; /* restore -- shared process-global test seam */
+
+    int32_t role_after_election =
+        atomic_load_explicit(&g_trf_c.ha.role, memory_order_acquire);
+    int in_catch_up_after_election =
+        atomic_load_explicit(&g_trf_c.ha.in_catch_up, memory_order_acquire);
+    bool recovery_truncated_after_election =
+        persist_recovery_truncated(&g_trf_c.persist);
+    recover_verdict_t recovery_verdict_after_election =
+        persist_recovery_verdict(&g_trf_c.persist);
+
+    atomic_store_explicit(&g_trf_c.ha.elect_ctx, NULL, memory_order_release);
+    pthread_mutex_destroy(&ctx->lock);
+    sem_destroy(&ctx->done);
+    free(ctx);
+
+    ha_set_promotion_fence_window_ms(0); /* restore default */
+    ha_cancel_promotion_fence_timer(&g_trf_c.ha);
+    delaymgr_stop(&g_trf_c.delaymgr);
+    delaymgr_fini(&g_trf_c.delaymgr);
+    ha_fini(&g_trf_c.ha);
+
+    persist_stop(&g_trf_c.persist);
+    persist_fini(&g_trf_c.persist);
+    trf_teardown_cluster();
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    /* ── Assertions ───────────────────────────────────────────────────── */
+    TEST_ASSERT_FALSE_MESSAGE(recovery_truncated_precondition,
+        "fixture precondition: a never-recovered persist_t must read "
+        "recovery_truncated=false (the fresh-persist_init default) "
+        "regardless of the real on-disk corruption -- this is the exact "
+        "blind spot bug-1696 exploits");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(RECOVER_CLEAN, recovery_verdict_precondition,
+        "fixture precondition: the pre-recovery default verdict reads CLEAN, "
+        "not the real corruption");
+
+    TEST_ASSERT_TRUE_MESSAGE(recovery_truncated_after_election,
+        "this node's own first-ever replay (triggered from inside this SAME "
+        "election evaluation) must have discovered and recorded the real "
+        "on-disk corruption by the time this call returns");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(RECOVER_TRUNCATED,
+        recovery_verdict_after_election,
+        "the real recovery verdict must be RECOVER_TRUNCATED once this "
+        "node's own replay has actually run");
+
+    TEST_ASSERT_NOT_EQUAL_MESSAGE((int32_t)CTRL_PRIMARY, role_after_election,
+        "bug-1696: a node whose OWN first-ever replay (racing this SAME "
+        "election evaluation) discovers a genuinely CORRUPT/TRUNCATED WAL "
+        "must be refused promotion -- not silently become CTRL_PRIMARY on "
+        "an eligibility decision made blind, before that first replay ever "
+        "ran");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_catch_up_after_election,
+        "a node refused for this reason must be fenced via the SAME "
+        "existing in_catch_up mechanism every other refusal reason uses, "
+        "and stay ALIVE (never self-exit)");
+
+    printf("test_first_boot_promotion_refuses_when_own_first_recovery_finds_"
+          "corruption: PASS\n");
+}
+
 #endif /* XCE_TESTING */
 
 void test_job_array_replay_fence(void)
@@ -665,6 +1101,8 @@ void test_job_array_replay_fence(void)
 #ifdef XCE_TESTING
     RUN_TEST(test_replay_wal_engages_fence_on_resource_exhaustion_failure);
     RUN_TEST(test_first_boot_promotion_evaluates_own_first_replay_fence);
+    RUN_TEST(test_first_boot_promotion_never_truncates_own_durable_wal);
+    RUN_TEST(test_first_boot_promotion_refuses_when_own_first_recovery_finds_corruption);
 #else
     printf("test_job_array_replay_fence: skipped (no XCE_TESTING)\n");
 #endif

src/controller/testsuite/test_job_array_sched_push.c

@@ -64,6 +64,15 @@ extern int on_scheduler_join(cluster_t*);
  * rather than linking all of opget.c, mirroring opget.c's own trivial body. */
 int check_worker_(xce_worker_t* w) { return (w != NULL) ? 1 : 0; }
 
+/* bug-1693: sched.c's handle_scheduler_leave() now calls
+ * schedadmin_signal_disconnect() (defined in cluster.c, which this
+ * standalone binary does not link -- see this file's own comment above for
+ * why). This test never triggers a scheduler-leave/disconnect path, so the
+ * real behavior is never exercised here; the symbol only needs to resolve
+ * at link time, mirroring check_worker_()'s identical stub rationale just
+ * above. */
+int schedadmin_signal_disconnect(void) { return 0; }
+
 /* ------------------------------------------------------------------ */
 /* Fixture                                                              */
 /* ------------------------------------------------------------------ */

src/controller/testsuite/test_p2_integration.c

@@ -277,6 +277,12 @@ static void test_single_node_committed_lsn_advances(void)
     /* WAL LSN did advance (local write succeeded) */
     TEST_ASSERT_TRUE(p->current_lsn > 0);
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_commit_barrier_rollout.c's jap_teardown for the full TOCTOU-race
+     * rationale (live-reproduced, gdb-confirmed hang in this exact class of
+     * fixture). teardown_p2_persist()'s own persist_stop() call becomes a
+     * safe, idempotent no-op. */
+    persist_stop(p);
     commit_engine_fini(&ce);
     teardown_p2_persist(p);
     printf("test_single_node_committed_lsn_advances: PASS\n");
@@ -344,6 +350,10 @@ static void test_committed_lsn_does_not_advance_on_timeout(void)
     TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, committed_after,
         "committed_lsn must stay unmoved when the entry never reaches majority");
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_single_node_committed_lsn_advances above for the full TOCTOU-race
+     * rationale. */
+    persist_stop(p);
     commit_engine_fini(&ce);
     teardown_p2_persist(p);
     printf("test_committed_lsn_does_not_advance_on_timeout: PASS\n");
@@ -448,6 +458,10 @@ static void test_committed_lsn_advances_after_majority_ack(void)
         committed_after > committed_before || committed_after == written_lsn,
         "committed_lsn must advance after majority ACK (Req 4.1)");
 
+    /* bug-1429/bug-1452: persist_stop() before commit_engine_fini() -- see
+     * test_single_node_committed_lsn_advances above for the full TOCTOU-race
+     * rationale. */
+    persist_stop(g_p9);
     commit_engine_fini(&ce);
     teardown_p2_persist(g_p9);
     g_p9 = NULL;

src/controller/testsuite/test_persist_checkpoint_offload.c

@@ -0,0 +1,320 @@
+/* test_persist_checkpoint_offload.c — xce-audit-invariants controller pass,
+ * dimension 1 async-I/O thread-role fix, bug-1694.
+ *
+ * Proves checkpoint_task_fn (persist.c) no longer blocks the delaymgr thread
+ * with real (unbounded) checkpoint I/O: the trigger now only releases the
+ * "scheduled" slot and submits the real work to the cluster's slow lane
+ * (slowlane_submit), returning immediately regardless of how long the real
+ * checkpoint body takes. Mirrors test_ha_liveness.c's reconnect/configpush
+ * slow-lane offload tests (rc_run_offload_tick / test_configpush_offload_
+ * runs_on_lane) exactly: a gated XCE_TESTING hook (persist_checkpoint_
+ * test_hook) substitutes for the real (disk-backed) persist_checkpoint()
+ * call so the test can park the checkpoint body in flight and observe which
+ * thread it runs on, without needing a full WAL/snapshot fixture.
+ *
+ * checkpoint_task_fn's only production call site is delay_task (1s
+ * coalescing delay) — persist_checkpoint_task_fn_for_test drives it directly
+ * so the test doesn't have to wait out a real delaymgr timer.
+ *
+ * Fixture is deliberately minimal (bare zeroed persist_t + cluster_t, no
+ * persist_init/persist_start): every field checkpoint_task_fn/checkpoint_
+ * lane_fn/run_checkpoint_body touch — checkpoint_in_progress,
+ * wal_record_count (plain atomics), cluster->slowlane — is satisfied without
+ * a real WAL. The XCE_TESTING hook path never reaches the real
+ * persist_checkpoint() (which WOULD need a real disk-backed persist_t), so
+ * this is safe. Mirrors test_ha_liveness.c's rc_setup_primary /
+ * test_configpush_offload_runs_on_lane, which likewise use a bare
+ * memset-zeroed cluster_t with only ha_init + slowlane_init.
+ */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+#include <unity/unity.h>
+
+#include "persist.h"
+#include "slowlane.h"
+#include "xce/cluster_def.h"
+
+/* Test seams (persist.c, compiled under XCE_TESTING — controller_unit_test
+ * always builds with it). */
+extern int  persist_checkpoint_task_fn_for_test(persist_t *p);
+extern void (*persist_checkpoint_test_hook)(persist_t *);
+
+/* ── park/release gate + thread-recording probe ──────────────────────────
+ * Mirrors test_ha_liveness.c's cp_probe/cp_wait_parked/cp_wait_runs/
+ * cp_release_gate/cp_reset_probe (configpush slow-lane offload, raft-drive-
+ * async task 3.3) exactly, renamed for checkpoint. */
+static _Atomic int       g_ckpt_runs = 0;
+static _Atomic pthread_t g_ckpt_last_thread;
+static _Atomic int       g_ckpt_gate_on = 0;
+static pthread_mutex_t   g_ckpt_mu         = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t    g_ckpt_release_cv = PTHREAD_COND_INITIALIZER;
+static int               g_ckpt_parked  = 0;
+static int               g_ckpt_release = 0;
+
+static void ckpt_probe(persist_t *p)
+{
+    (void)p;
+    atomic_store_explicit(&g_ckpt_last_thread, pthread_self(),
+                          memory_order_relaxed);
+    if (atomic_load(&g_ckpt_gate_on)) {
+        pthread_mutex_lock(&g_ckpt_mu);
+        g_ckpt_parked = 1;
+        while (!g_ckpt_release)
+            pthread_cond_wait(&g_ckpt_release_cv, &g_ckpt_mu);
+        pthread_mutex_unlock(&g_ckpt_mu);
+    }
+    atomic_fetch_add(&g_ckpt_runs, 1);
+}
+
+/* Bounded poll until the gated body parks; 0 on park, -1 on timeout. */
+static int ckpt_wait_parked(int timeout_ms)
+{
+    pthread_mutex_lock(&g_ckpt_mu);
+    int parked = 0;
+    for (int i = 0; i < timeout_ms; i++) {
+        if (g_ckpt_parked) { parked = 1; break; }
+        pthread_mutex_unlock(&g_ckpt_mu);
+        struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; /* 1 ms */
+        nanosleep(&ts, NULL);
+        pthread_mutex_lock(&g_ckpt_mu);
+    }
+    pthread_mutex_unlock(&g_ckpt_mu);
+    return parked ? 0 : -1;
+}
+
+static int ckpt_wait_runs(int want, int timeout_ms)
+{
+    for (int i = 0; i < timeout_ms; i++) {
+        if (atomic_load(&g_ckpt_runs) >= want) break;
+        struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; /* 1 ms */
+        nanosleep(&ts, NULL);
+    }
+    return atomic_load(&g_ckpt_runs);
+}
+
+static void ckpt_release_gate(void)
+{
+    pthread_mutex_lock(&g_ckpt_mu);
+    g_ckpt_release = 1;
+    pthread_cond_broadcast(&g_ckpt_release_cv);
+    pthread_mutex_unlock(&g_ckpt_mu);
+}
+
+static void ckpt_reset_probe(void)
+{
+    atomic_store(&g_ckpt_runs, 0);
+    atomic_store(&g_ckpt_gate_on, 0);
+    pthread_mutex_lock(&g_ckpt_mu);
+    g_ckpt_parked  = 0;
+    g_ckpt_release = 0;
+    pthread_mutex_unlock(&g_ckpt_mu);
+}
+
+/* Elapsed milliseconds between two CLOCK_MONOTONIC timespecs. */
+static double ms_since(const struct timespec *t0, const struct timespec *t1)
+{
+    return (double)(t1->tv_sec - t0->tv_sec) * 1000.0 +
+           (double)(t1->tv_nsec - t0->tv_nsec) / 1e6;
+}
+
+/* Minimal fixture: bare zeroed persist_t + cluster_t, real slow lane. See
+ * file header for why this is sufficient (the hook path never reaches real
+ * disk I/O). */
+static void ckpt_setup(persist_t *p, cluster_t *c)
+{
+    memset(p, 0, sizeof(*p));
+    memset(c, 0, sizeof(*c));
+    TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c->slowlane, 2, 4));
+    p->cluster = c;
+    atomic_store(&p->checkpoint_in_progress, 1); /* mirrors schedule_checkpoint's
+                                                   * CAS having claimed the slot
+                                                   * before delay_task fires the
+                                                   * trigger */
+    atomic_store(&p->wal_record_count, 999);
+}
+
+static void ckpt_teardown(cluster_t *c)
+{
+    slowlane_fini(&c->slowlane);
+}
+
+/*
+ * GREEN (Req: delaymgr thread must never block on checkpoint I/O): with the
+ * offload flag ON (the production default), triggering checkpoint_task_fn
+ * returns promptly — well under the gate's hold duration — while the real
+ * checkpoint body is still parked in flight on a SLOW-LANE worker thread,
+ * not the caller's own thread. This is the direct proof that delaymgr's own
+ * worker thread (which, in production, is the one calling checkpoint_task_fn)
+ * would never observe the checkpoint body's blocking duration.
+ */
+static void test_checkpoint_offload_trigger_returns_promptly(void)
+{
+    persist_t p;
+    cluster_t c;
+    ckpt_setup(&p, &c);
+
+    ckpt_reset_probe();
+    atomic_store(&g_ckpt_gate_on, 1); /* lane body parks at the gate */
+    persist_checkpoint_test_hook = ckpt_probe;
+    persist_set_checkpoint_offload_for_test(1);
+
+    pthread_t caller = pthread_self();
+
+    struct timespec t0, t1;
+    clock_gettime(CLOCK_MONOTONIC, &t0);
+    int rc = persist_checkpoint_task_fn_for_test(&p);
+    clock_gettime(CLOCK_MONOTONIC, &t1);
+    TEST_ASSERT_EQUAL_INT(0, rc);
+
+    double trigger_ms = ms_since(&t0, &t1);
+    TEST_ASSERT_TRUE_MESSAGE(trigger_ms < 200.0,
+        "checkpoint_task_fn (the delaymgr-thread trigger) must return "
+        "promptly — it must never block on the real checkpoint body");
+
+    /* The "scheduled" slot must already be released so a future
+     * schedule_checkpoint() call is not permanently locked out. */
+    TEST_ASSERT_EQUAL_INT(0, atomic_load(&p.checkpoint_in_progress));
+
+    /* The real checkpoint body must actually be running (parked) — proving
+     * the trigger's prompt return is NOT because the work was dropped. */
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ckpt_wait_parked(2000),
+        "the offloaded checkpoint body must reach the gate, running "
+        "somewhere off the trigger's own call stack");
+    TEST_ASSERT_FALSE_MESSAGE(
+        pthread_equal(atomic_load_explicit(&g_ckpt_last_thread,
+                                            memory_order_relaxed), caller),
+        "the checkpoint body must run on a SLOW-LANE worker thread, NOT the "
+        "trigger's own (delaymgr, in production) thread");
+
+    /* wal_record_count must NOT have been reset yet — the body hasn't
+     * finished (still parked at the gate). */
+    TEST_ASSERT_EQUAL_UINT(999, atomic_load(&p.wal_record_count));
+
+    ckpt_release_gate();
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, ckpt_wait_runs(1, 2000),
+        "the checkpoint body must complete exactly once after release");
+    /* Only after the body actually completes does the counter reset. */
+    TEST_ASSERT_EQUAL_UINT(0, atomic_load(&p.wal_record_count));
+
+    persist_checkpoint_test_hook = NULL;
+    persist_set_checkpoint_offload_for_test(1); /* restore production default */
+    ckpt_teardown(&c);
+    printf("test_checkpoint_offload_trigger_returns_promptly: PASS\n");
+}
+
+/*
+ * RED (pre-fix behavior, demonstrated via the flag's OFF/kill-switch path):
+ * with the offload flag OFF, checkpoint_task_fn falls back to running the
+ * checkpoint body INLINE, on the calling thread — exactly the delaymgr-
+ * blocking violation this fix closes. The trigger call itself now BLOCKS
+ * until the gate is released, and the body runs on the CALLER's own thread.
+ * This is what checkpoint_task_fn did unconditionally before this fix; with
+ * the flag flipped OFF the same legacy code path is still reachable, so this
+ * test demonstrates the exact violation the GREEN test above proves fixed.
+ */
+typedef struct { persist_t *p; _Atomic int done; int rc; } ckpt_trigger_arg_t;
+
+static void *ckpt_trigger_thread(void *arg)
+{
+    ckpt_trigger_arg_t *a = (ckpt_trigger_arg_t *)arg;
+    a->rc = persist_checkpoint_task_fn_for_test(a->p);
+    atomic_store(&a->done, 1);
+    return NULL;
+}
+
+static void test_checkpoint_inline_blocks_delaymgr_thread_RED(void)
+{
+    persist_t p;
+    cluster_t c;
+    ckpt_setup(&p, &c);
+
+    ckpt_reset_probe();
+    atomic_store(&g_ckpt_gate_on, 1);
+    persist_checkpoint_test_hook = ckpt_probe;
+    persist_set_checkpoint_offload_for_test(0); /* RED: legacy inline path */
+
+    ckpt_trigger_arg_t arg = { .p = &p };
+    atomic_store(&arg.done, 0);
+
+    pthread_t th;
+    pthread_create(&th, NULL, ckpt_trigger_thread, &arg);
+
+    /* The body must reach the gate — proving it ran (inline, synchronously,
+     * as part of the trigger call itself), unlike the offloaded case. */
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, ckpt_wait_parked(2000),
+        "RED: the inline checkpoint body must still run (on the delaymgr/"
+        "trigger thread itself) with the offload flag OFF");
+
+    /* While parked, the trigger call has NOT returned — this is the
+     * blocking-delaymgr-thread violation. */
+    struct timespec ts = { .tv_sec = 0, .tv_nsec = 100 * 1000000L }; /* 100ms */
+    nanosleep(&ts, NULL);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&arg.done),
+        "RED: with the offload flag OFF, checkpoint_task_fn must still be "
+        "blocked inside the checkpoint body — reproducing the pre-fix "
+        "delaymgr-thread-blocking violation");
+
+    /* The body ran on the SAME thread as the trigger — no offload. */
+    TEST_ASSERT_TRUE_MESSAGE(
+        pthread_equal(atomic_load_explicit(&g_ckpt_last_thread,
+                                            memory_order_relaxed), th),
+        "RED: with the offload flag OFF the checkpoint body must run on the "
+        "trigger's own (delaymgr) thread, not a lane worker");
+
+    ckpt_release_gate();
+    pthread_join(th, NULL);
+    TEST_ASSERT_EQUAL_INT(0, arg.rc);
+    TEST_ASSERT_EQUAL_INT(1, atomic_load(&arg.done));
+
+    persist_checkpoint_test_hook = NULL;
+    persist_set_checkpoint_offload_for_test(1); /* restore production default */
+    ckpt_teardown(&c);
+    printf("test_checkpoint_inline_blocks_delaymgr_thread_RED: PASS\n");
+}
+
+/*
+ * No-cluster fallback (unit-test / bare persist_t, e.g. most of this file's
+ * siblings): checkpoint_task_fn's own production call site is only ever
+ * reached via schedule_checkpoint's cluster branch, but this proves the
+ * defensive persist->cluster == NULL check inside checkpoint_task_fn itself
+ * also takes the inline path (no crash dereferencing a NULL cluster's
+ * slowlane).
+ */
+static void test_checkpoint_no_cluster_runs_inline(void)
+{
+    persist_t p;
+    memset(&p, 0, sizeof(p));
+    p.cluster = NULL;
+    atomic_store(&p.checkpoint_in_progress, 1);
+    atomic_store(&p.wal_record_count, 42);
+
+    ckpt_reset_probe();
+    persist_checkpoint_test_hook = ckpt_probe;
+    persist_set_checkpoint_offload_for_test(1); /* even with flag ON */
+
+    int rc = persist_checkpoint_task_fn_for_test(&p);
+    TEST_ASSERT_EQUAL_INT(0, rc);
+
+    /* No lane to submit to (cluster == NULL) — must run synchronously,
+     * inline, before the trigger call returns. */
+    TEST_ASSERT_EQUAL_INT(1, atomic_load(&g_ckpt_runs));
+    TEST_ASSERT_EQUAL_UINT(0, atomic_load(&p.wal_record_count));
+    TEST_ASSERT_EQUAL_INT(0, atomic_load(&p.checkpoint_in_progress));
+
+    persist_checkpoint_test_hook = NULL;
+    printf("test_checkpoint_no_cluster_runs_inline: PASS\n");
+}
+
+void test_persist_checkpoint_offload(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_checkpoint_offload_trigger_returns_promptly);
+    RUN_TEST(test_checkpoint_inline_blocks_delaymgr_thread_RED);
+    RUN_TEST(test_checkpoint_no_cluster_runs_inline);
+}

src/controller/testsuite/test_rcfile.c

@@ -0,0 +1,419 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+
+#include "rcfile.h"
+
+/* Saved prior state of one environment variable, captured before a test
+ * case mutates it. Distinguishes "was unset" (getenv() returned NULL) from
+ * "was set to X" (including X == "") so restoration is exact -- required
+ * because controller_unit_test links 80+ test_*.c files into ONE process
+ * running sequentially, and a leaked HOME/XDG_CONFIG_HOME value would
+ * silently affect every test that runs after this file. */
+typedef struct {
+    int   was_set; /* 1 if getenv() returned non-NULL before mutation */
+    char* value;   /* heap copy of the prior value; valid only if was_set */
+} rcfile_envsave_t;
+
+static rcfile_envsave_t env_save(const char* name)
+{
+    rcfile_envsave_t s;
+    const char* cur = getenv(name);
+    if (cur) {
+        s.was_set = 1;
+        s.value   = strdup(cur);
+        TEST_ASSERT_NOT_NULL(s.value);
+    } else {
+        s.was_set = 0;
+        s.value   = NULL;
+    }
+    return s;
+}
+
+/* Restores `name` to exactly the state `s` captured, then frees `s`'s heap
+ * copy. Safe to call even if the test already unset/reset the variable
+ * itself -- always drives it back to the captured prior state. */
+static void env_restore(const char* name, rcfile_envsave_t* s)
+{
+    if (s->was_set) {
+        TEST_ASSERT_EQUAL_INT(0, setenv(name, s->value, 1));
+        free(s->value);
+    } else {
+        TEST_ASSERT_EQUAL_INT(0, unsetenv(name));
+    }
+    s->value = NULL;
+}
+
+static void test_resolve_uses_xdg_config_home_when_set_and_nonempty(void)
+{
+    rcfile_envsave_t save_xdg  = env_save("XDG_CONFIG_HOME");
+    rcfile_envsave_t save_home = env_save("HOME");
+
+    TEST_ASSERT_EQUAL_INT(
+        0, setenv("XDG_CONFIG_HOME", "/tmp/xce_rcfile_test_xdg", 1));
+    /* HOME is deliberately set to something recognizably different, so a
+     * pass here proves HOME was NOT consulted, not just that the result
+     * happens to match by coincidence. */
+    TEST_ASSERT_EQUAL_INT(
+        0, setenv("HOME", "/tmp/xce_rcfile_test_home_unused", 1));
+
+    char* path = rcfile_resolve_path();
+    TEST_ASSERT_NOT_NULL(path);
+    TEST_ASSERT_EQUAL_STRING("/tmp/xce_rcfile_test_xdg/xce_test/rc", path);
+    free(path);
+
+    env_restore("HOME", &save_home);
+    env_restore("XDG_CONFIG_HOME", &save_xdg);
+}
+
+static void test_resolve_falls_back_to_home_when_xdg_unset(void)
+{
+    rcfile_envsave_t save_xdg  = env_save("XDG_CONFIG_HOME");
+    rcfile_envsave_t save_home = env_save("HOME");
+
+    TEST_ASSERT_EQUAL_INT(0, unsetenv("XDG_CONFIG_HOME"));
+    TEST_ASSERT_EQUAL_INT(0, setenv("HOME", "/tmp/xce_rcfile_test_home", 1));
+
+    char* path = rcfile_resolve_path();
+    TEST_ASSERT_NOT_NULL(path);
+    TEST_ASSERT_EQUAL_STRING("/tmp/xce_rcfile_test_home/.config/xce_test/rc",
+                              path);
+    free(path);
+
+    env_restore("HOME", &save_home);
+    env_restore("XDG_CONFIG_HOME", &save_xdg);
+}
+
+static void test_resolve_falls_back_to_home_when_xdg_empty(void)
+{
+    rcfile_envsave_t save_xdg  = env_save("XDG_CONFIG_HOME");
+    rcfile_envsave_t save_home = env_save("HOME");
+
+    TEST_ASSERT_EQUAL_INT(0, setenv("XDG_CONFIG_HOME", "", 1));
+    TEST_ASSERT_EQUAL_INT(
+        0, setenv("HOME", "/tmp/xce_rcfile_test_home2", 1));
+
+    char* path = rcfile_resolve_path();
+    TEST_ASSERT_NOT_NULL(path);
+    TEST_ASSERT_EQUAL_STRING("/tmp/xce_rcfile_test_home2/.config/xce_test/rc",
+                              path);
+    free(path);
+
+    env_restore("HOME", &save_home);
+    env_restore("XDG_CONFIG_HOME", &save_xdg);
+}
+
+static void test_resolve_returns_null_when_neither_var_usable_both_unset(void)
+{
+    rcfile_envsave_t save_xdg  = env_save("XDG_CONFIG_HOME");
+    rcfile_envsave_t save_home = env_save("HOME");
+
+    TEST_ASSERT_EQUAL_INT(0, unsetenv("XDG_CONFIG_HOME"));
+    TEST_ASSERT_EQUAL_INT(0, unsetenv("HOME"));
+
+    char* path = rcfile_resolve_path();
+    TEST_ASSERT_NULL(path);
+
+    env_restore("HOME", &save_home);
+    env_restore("XDG_CONFIG_HOME", &save_xdg);
+}
+
+static void test_resolve_returns_null_when_neither_var_usable_both_empty(void)
+{
+    rcfile_envsave_t save_xdg  = env_save("XDG_CONFIG_HOME");
+    rcfile_envsave_t save_home = env_save("HOME");
+
+    TEST_ASSERT_EQUAL_INT(0, setenv("XDG_CONFIG_HOME", "", 1));
+    TEST_ASSERT_EQUAL_INT(0, setenv("HOME", "", 1));
+
+    char* path = rcfile_resolve_path();
+    TEST_ASSERT_NULL(path);
+
+    env_restore("HOME", &save_home);
+    env_restore("XDG_CONFIG_HOME", &save_xdg);
+}
+
+/* Proves the save/restore mechanism itself round-trips exactly, for all
+ * three prior-state shapes (unset / empty / a real value), regardless of
+ * whatever ambient state this test happened to start with -- this is what
+ * guarantees a test run AFTER this file never observes state leaked by it,
+ * since there is no later test in this same file to leak into. */
+static void test_env_save_restore_roundtrips_all_three_prior_states(void)
+{
+    const char* name = "XDG_CONFIG_HOME";
+    rcfile_envsave_t real_prior = env_save(name);
+
+    /* Case 1: prior state was unset. */
+    TEST_ASSERT_EQUAL_INT(0, unsetenv(name));
+    rcfile_envsave_t s1 = env_save(name);
+    TEST_ASSERT_EQUAL_INT(0, s1.was_set);
+    TEST_ASSERT_EQUAL_INT(0, setenv(name, "/tmp/mutated_during_case1", 1));
+    env_restore(name, &s1);
+    TEST_ASSERT_NULL(getenv(name));
+
+    /* Case 2: prior state was set to the empty string. */
+    TEST_ASSERT_EQUAL_INT(0, setenv(name, "", 1));
+    rcfile_envsave_t s2 = env_save(name);
+    TEST_ASSERT_EQUAL_INT(1, s2.was_set);
+    TEST_ASSERT_EQUAL_STRING("", s2.value);
+    TEST_ASSERT_EQUAL_INT(0, setenv(name, "/tmp/mutated_during_case2", 1));
+    env_restore(name, &s2);
+    TEST_ASSERT_NOT_NULL(getenv(name));
+    TEST_ASSERT_EQUAL_STRING("", getenv(name));
+
+    /* Case 3: prior state was set to a real value. */
+    TEST_ASSERT_EQUAL_INT(0, setenv(name, "/tmp/xce_rcfile_prior_value", 1));
+    rcfile_envsave_t s3 = env_save(name);
+    TEST_ASSERT_EQUAL_INT(1, s3.was_set);
+    TEST_ASSERT_EQUAL_STRING("/tmp/xce_rcfile_prior_value", s3.value);
+    TEST_ASSERT_EQUAL_INT(0, unsetenv(name));
+    env_restore(name, &s3);
+    TEST_ASSERT_NOT_NULL(getenv(name));
+    TEST_ASSERT_EQUAL_STRING("/tmp/xce_rcfile_prior_value", getenv(name));
+
+    env_restore(name, &real_prior);
+}
+
+/* ---- rcfile_load() tests (task 1.2) ---------------------------------- */
+
+/* Test-local temp dir helper, mirroring test_configcompare.c's
+ * fresh_dir()/cleanup_dir() pattern -- keeps rc-file fixtures out of the
+ * shared /tmp leak this sandbox already tracks (bug-924) by cleaning up
+ * after itself instead of leaving an mkdtemp() dir behind forever. */
+static char g_dir[256];
+
+static void fresh_dir(void)
+{
+    strcpy(g_dir, "/tmp/xce_rcfile_test_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(g_dir));
+}
+
+static void cleanup_dir(void)
+{
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", g_dir);
+    system(cmd);
+}
+
+static void write_file(const char* path, const char* content, size_t len)
+{
+    FILE* f = fopen(path, "wb");
+    TEST_ASSERT_NOT_NULL(f);
+    if (len > 0)
+        TEST_ASSERT_EQUAL_INT((int)len, (int)fwrite(content, 1, len, f));
+    fclose(f);
+}
+
+/* Fake runner: records every line it is called with (heap copies, in call
+ * order) and, once scripted via fail_at_call, returns fail_rc on that call
+ * (1-based) -- lets a test assert both the exact propagated return value
+ * and the exact point iteration stopped. */
+#define RCFILE_TEST_MAX_LINES 8
+
+typedef struct {
+    char* lines[RCFILE_TEST_MAX_LINES];
+    int   count;
+    int   fail_at_call; /* 1-based; 0 = never fail */
+    int   fail_rc;
+} rcfile_fake_runner_state_t;
+
+static rcfile_fake_runner_state_t g_fake;
+
+static void fake_runner_reset(void)
+{
+    for (int i = 0; i < g_fake.count; i++) {
+        free(g_fake.lines[i]);
+        g_fake.lines[i] = NULL;
+    }
+    g_fake.count        = 0;
+    g_fake.fail_at_call = 0;
+    g_fake.fail_rc      = 0;
+}
+
+static int fake_runner(char* line)
+{
+    TEST_ASSERT_LESS_THAN_INT(RCFILE_TEST_MAX_LINES, g_fake.count);
+    g_fake.lines[g_fake.count] = strdup(line);
+    TEST_ASSERT_NOT_NULL(g_fake.lines[g_fake.count]);
+    g_fake.count++;
+    if (g_fake.fail_at_call != 0 && g_fake.count == g_fake.fail_at_call)
+        return g_fake.fail_rc;
+    return 0;
+}
+
+static void test_rcfile_load_returns_0_and_never_calls_runner_when_path_missing(void)
+{
+    fresh_dir();
+    fake_runner_reset();
+
+    char missing_path[300];
+    snprintf(missing_path, sizeof(missing_path), "%s/does_not_exist", g_dir);
+
+    int rc = rcfile_load(missing_path, fake_runner);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_INT(0, g_fake.count);
+    cleanup_dir();
+}
+
+static void test_rcfile_load_calls_runner_once_per_line_in_order_newline_stripped(void)
+{
+    fresh_dir();
+    fake_runner_reset();
+
+    char path[300];
+    snprintf(path, sizeof(path), "%s/rc", g_dir);
+    /* Last line deliberately has no trailing newline -- getline() must
+     * still hand it to the runner intact, not require a final \n. */
+    const char* content =
+        "set masters host1:1,host2:2\nworker\njoin\nquit";
+    write_file(path, content, strlen(content));
+
+    int rc = rcfile_load(path, fake_runner);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_INT(4, g_fake.count);
+    TEST_ASSERT_EQUAL_STRING("set masters host1:1,host2:2", g_fake.lines[0]);
+    TEST_ASSERT_EQUAL_STRING("worker", g_fake.lines[1]);
+    TEST_ASSERT_EQUAL_STRING("join", g_fake.lines[2]);
+    TEST_ASSERT_EQUAL_STRING("quit", g_fake.lines[3]);
+    cleanup_dir();
+}
+
+static void test_rcfile_load_stops_after_nonzero_runner_and_propagates_value(void)
+{
+    fresh_dir();
+    fake_runner_reset();
+    g_fake.fail_at_call = 2;
+    g_fake.fail_rc       = 42;
+
+    char path[300];
+    snprintf(path, sizeof(path), "%s/rc", g_dir);
+    const char* content = "line1\nline2\nline3\nline4\n";
+    write_file(path, content, strlen(content));
+
+    int rc = rcfile_load(path, fake_runner);
+
+    TEST_ASSERT_EQUAL_INT(42, rc);
+    TEST_ASSERT_EQUAL_INT(2, g_fake.count);
+    TEST_ASSERT_EQUAL_STRING("line1", g_fake.lines[0]);
+    TEST_ASSERT_EQUAL_STRING("line2", g_fake.lines[1]);
+    cleanup_dir();
+}
+
+/* A present-but-unreadable-as-a-regular-file path (a directory) must warn
+ * to stderr (non-fatal) and return 0 without ever calling the runner --
+ * proven here by actually capturing stderr via dup/dup2 (not just
+ * asserting the return value), then restoring the original fd exactly so
+ * later test files in this same combined binary see stderr unaffected. */
+static void test_rcfile_load_directory_path_warns_and_returns_0(void)
+{
+    fresh_dir();
+    fake_runner_reset();
+
+    fflush(stderr);
+    int saved_stderr_fd = dup(fileno(stderr));
+    TEST_ASSERT_TRUE(saved_stderr_fd >= 0);
+
+    char capture_path[300];
+    snprintf(capture_path, sizeof(capture_path), "%s/stderr_capture", g_dir);
+    FILE* redirected = freopen(capture_path, "w", stderr);
+    TEST_ASSERT_NOT_NULL(redirected);
+
+    int rc = rcfile_load(g_dir, fake_runner); /* g_dir is a directory */
+
+    fflush(stderr);
+    TEST_ASSERT_TRUE(dup2(saved_stderr_fd, fileno(stderr)) >= 0);
+    close(saved_stderr_fd);
+    clearerr(stderr);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_INT(0, g_fake.count);
+
+    FILE* f = fopen(capture_path, "r");
+    TEST_ASSERT_NOT_NULL(f);
+    char   buf[512];
+    size_t n = fread(buf, 1, sizeof(buf) - 1, f);
+    buf[n]   = '\0';
+    fclose(f);
+    TEST_ASSERT_TRUE(n > 0);
+    TEST_ASSERT_NOT_NULL(strstr(buf, g_dir));
+
+    cleanup_dir();
+}
+
+/* Proves the getline()-not-fgets() design decision actually holds: a
+ * single line well beyond any typical small fixed-size buffer (e.g. the
+ * historic 256/1024-byte stack buffers this design note calls out) must
+ * reach the runner completely intact and unsplit -- a real content/length
+ * equality check, not just "didn't crash". */
+static void test_rcfile_load_long_line_reaches_runner_intact_and_unsplit(void)
+{
+    fresh_dir();
+    fake_runner_reset();
+
+    char path[300];
+    snprintf(path, sizeof(path), "%s/rc", g_dir);
+
+    /* Build a >4KB single line shaped like a real long `set masters` rc
+     * command: many comma-separated host:port pairs on one line. */
+    size_t cap = 8192;
+    char*  long_line = (char*)malloc(cap);
+    TEST_ASSERT_NOT_NULL(long_line);
+    size_t off = (size_t)snprintf(long_line, cap, "set masters ");
+    int    pair_no = 0;
+    while (off < 4096 + 200) {
+        int written = snprintf(long_line + off, cap - off,
+                                "%shost%d:1000%d", pair_no ? "," : "",
+                                pair_no, pair_no % 10);
+        TEST_ASSERT_TRUE(written > 0);
+        off += (size_t)written;
+        pair_no++;
+    }
+    TEST_ASSERT_TRUE(off > 4096);
+
+    /* File content: the long line, then a short trailer line, so a
+     * truncation/splitting bug would also show up as an extra/garbled
+     * runner call rather than only a length mismatch. */
+    size_t file_len = off + 1 /* \n */ + strlen("quit") + 1 /* \n */;
+    char*  file_content = (char*)malloc(file_len + 1);
+    TEST_ASSERT_NOT_NULL(file_content);
+    int fc_len = snprintf(file_content, file_len + 1, "%s\nquit\n", long_line);
+    TEST_ASSERT_EQUAL_INT((int)file_len, fc_len);
+
+    write_file(path, file_content, file_len);
+
+    int rc = rcfile_load(path, fake_runner);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_INT(2, g_fake.count);
+    TEST_ASSERT_TRUE(strlen(g_fake.lines[0]) > 4096);
+    TEST_ASSERT_EQUAL_INT((int)off, (int)strlen(g_fake.lines[0]));
+    TEST_ASSERT_EQUAL_STRING(long_line, g_fake.lines[0]);
+    TEST_ASSERT_EQUAL_STRING("quit", g_fake.lines[1]);
+
+    free(long_line);
+    free(file_content);
+    cleanup_dir();
+}
+
+void test_rcfile(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_resolve_uses_xdg_config_home_when_set_and_nonempty);
+    RUN_TEST(test_resolve_falls_back_to_home_when_xdg_unset);
+    RUN_TEST(test_resolve_falls_back_to_home_when_xdg_empty);
+    RUN_TEST(test_resolve_returns_null_when_neither_var_usable_both_unset);
+    RUN_TEST(test_resolve_returns_null_when_neither_var_usable_both_empty);
+    RUN_TEST(test_env_save_restore_roundtrips_all_three_prior_states);
+    RUN_TEST(test_rcfile_load_returns_0_and_never_calls_runner_when_path_missing);
+    RUN_TEST(test_rcfile_load_calls_runner_once_per_line_in_order_newline_stripped);
+    RUN_TEST(test_rcfile_load_stops_after_nonzero_runner_and_propagates_value);
+    RUN_TEST(test_rcfile_load_directory_path_warns_and_returns_0);
+    RUN_TEST(test_rcfile_load_long_line_reaches_runner_intact_and_unsplit);
+}

src/controller/testsuite/test_rcfile_integration.sh

@@ -0,0 +1,377 @@
+#!/usr/bin/env bash
+#
+# test_rcfile_integration.sh — xce-test-rc-file integration test, shell-driven
+# against the REAL built `xce_test` binary. Mirrors clean_failover_preservation.sh's
+# harness conventions in this directory: mktemp -d scratch fixtures, a
+# log()/pass()/failcase() reporting style, explicit per-scenario PASS/FAIL, and
+# a real process exit code (0 == every scenario passed).
+#
+# Design: .kiro/specs/xce-test-rc-file/design.md, "Testing Strategy" ->
+# "Integration Tests". Requirements: .kiro/specs/xce-test-rc-file/requirements.md
+# Requirement 1 (rc-file discovery/loading) and Requirement 2 (--norc opt-out).
+#
+# Task 3.1 scope (implemented here):
+#   test_rcfile_takes_effect_piped()       Req 1.1, 1.2, 1.4, 1.5, 2.3
+#   test_rcfile_takes_effect_interactive() Req 1.1, 1.2, 1.4, 1.5, 2.3
+#   test_norc_skips_rcfile()               Req 2.1, 2.3
+#
+# Task 3.2 scope (implemented here), reusing the same setup_rc_home-style
+# fixture pattern/cleanup_rc_home/run_piped/run_interactive_pty helpers and
+# the same pass()/failcase() accounting:
+#   test_rcfile_failure_aborts_piped()          Req 3.1, 3.3, 3.4
+#   test_rcfile_failure_continues_interactive() Req 3.1, 3.2, 3.4
+#
+# Exit 0 == every scenario PASSed; nonzero == at least one scenario FAILed.
+
+set -u
+
+# ── Config ───────────────────────────────────────────────────────────────────
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$HERE/../../.." && pwd)"
+XCE_TEST="${XCE_TEST_BIN:-$REPO_ROOT/src/xce.build/controller/xce_test}"
+
+# xce_test's `masters` setting defaults to "localhost:9990" (test.c's
+# settings[] table) and prints as "<not set>" until something actually
+# assigns it (KVAL() macro in cmd_set). Pick an override value that can never
+# collide with either, so a positive match is unambiguous.
+OVERRIDE_MASTERS="rchost-a:19991,rchost-b:19992"
+NOT_SET="<not set>"
+
+# A command name guaranteed absent from every xce_test command table (checked
+# against every `{ "name", cmd_fn, ... }` entry in test.c) -- used as an rc-file
+# line that deterministically fails via execute_line()'s find_command() miss
+# ("<name>: No such command for xce test." on stderr, execute_line returns -1),
+# the simplest reliable way to force a command failure (design.md Testing
+# Strategy / task 3.2).
+RCFILE_BOGUS_CMD="xce_rcfile_test_bogus_cmd_does_not_exist"
+
+RUN_TIMEOUT_S="${XCE_RCFILE_TEST_TIMEOUT_S:-20}"
+
+FAILURES=0
+RESULTS=()
+
+# The interactive-pty driver (see run_interactive_pty below) is written ONCE
+# to its own temp file rather than fed via a heredoc on the python3 command
+# line -- a heredoc there would compete with the SEPARATE pipe that must also
+# deliver the xce_test command script to the same process's stdin (`python3 -`
+# reads its whole program from stdin, consuming it, leaving nothing for the
+# script itself to read via sys.stdin). Cleaned up on exit.
+PTY_DRIVER="$(mktemp "${TMPDIR:-/tmp}/xce-rcfile-pty-driver.XXXXXX")"
+trap 'rm -f "$PTY_DRIVER"' EXIT
+
+cat >"$PTY_DRIVER" <<'PYEOF'
+import os, pty, select, signal, sys, time
+
+def read_available(fd, timeout):
+    r, _, _ = select.select([fd], [], [], timeout)
+    if fd in r:
+        try:
+            return os.read(fd, 65536)
+        except OSError:
+            return b""
+    return b""
+
+def main():
+    xce_test_bin, homedir, xdg, extra, timeout_s = sys.argv[1:6]
+    timeout_s = float(timeout_s)
+    commands = sys.stdin.read().splitlines()
+
+    env = dict(os.environ)
+    env["HOME"] = homedir
+    if xdg:
+        env["XDG_CONFIG_HOME"] = xdg
+    else:
+        env.pop("XDG_CONFIG_HOME", None)
+
+    args = [xce_test_bin] + ([extra] if extra else [])
+
+    pid, master_fd = pty.fork()
+    if pid == 0:
+        os.execvpe(args[0], args, env)
+        os._exit(127)
+
+    output = b""
+    deadline = time.time() + timeout_s
+
+    # Let xce_test start up and print its first prompt before sending input.
+    time.sleep(0.4)
+    output += read_available(master_fd, 0.4)
+    for cmd in commands:
+        os.write(master_fd, (cmd + "\n").encode())
+        time.sleep(0.3)
+        output += read_available(master_fd, 0.3)
+
+    status = None
+    while time.time() < deadline:
+        chunk = read_available(master_fd, 0.3)
+        if chunk:
+            output += chunk
+            continue
+        pid_done, st = os.waitpid(pid, os.WNOHANG)
+        if pid_done == pid:
+            status = st
+            break
+    if status is None:
+        try:
+            os.kill(pid, signal.SIGKILL)
+        except OSError:
+            pass
+        _, status = os.waitpid(pid, 0)
+
+    while True:
+        chunk = read_available(master_fd, 0.2)
+        if not chunk:
+            break
+        output += chunk
+
+    try:
+        os.close(master_fd)
+    except OSError:
+        pass
+
+    exitcode = os.waitstatus_to_exitcode(status)
+    sys.stdout.buffer.write(output)
+    sys.exit(exitcode if exitcode >= 0 else 1)
+
+main()
+PYEOF
+
+# ── Reporting helpers ────────────────────────────────────────────────────────
+log()      { printf '[rcfile-integration] %s\n' "$*" >&2; }
+pass()     { RESULTS+=("PASS: $1"); log "PASS: $1"; }
+failcase() {
+    RESULTS+=("FAIL: $1 -- $2")
+    log "FAIL: $1 -- $2"
+    FAILURES=$((FAILURES + 1))
+}
+
+# ── Fixture helpers ──────────────────────────────────────────────────────────
+
+# Creates a scratch $HOME with $HOME/config/xce_test/rc containing an
+# OVERRIDE_MASTERS `set masters` line, and echoes "<homedir>|<xdg_config_home>".
+setup_rc_home() {
+    local dir
+    dir="$(mktemp -d "${TMPDIR:-/tmp}/xce-rcfile-it.XXXXXX")" || return 1
+    mkdir -p "$dir/config/xce_test" || return 1
+    printf 'set masters %s\n' "$OVERRIDE_MASTERS" >"$dir/config/xce_test/rc"
+    printf '%s|%s\n' "$dir" "$dir/config"
+}
+
+cleanup_rc_home() {
+    local dir="$1"
+    [ -n "$dir" ] && [ -d "$dir" ] && rm -rf "$dir"
+}
+
+# Creates a scratch $HOME with $HOME/config/xce_test/rc containing a
+# deliberately-failing line (RCFILE_BOGUS_CMD, an unrecognized command name)
+# FOLLOWED by an OVERRIDE_MASTERS `set masters` line. One fixture serves both
+# task 3.2 scenarios: the piped-abort scenario only needs a failing line to be
+# present (rcfile_load stops at the first nonzero runner result, so the
+# trailing `set masters` line is never reached -- design.md's Architecture
+# Pattern & Boundary Map "Stopped -- yes --> Exit" edge); the interactive
+# scenario relies on that SAME trailing line still running afterward, to
+# prove Requirement 3.2's "continue executing the remaining rc-file commands"
+# rather than merely "didn't crash". Echoes "<homedir>|<xdg_config_home>",
+# mirroring setup_rc_home().
+setup_rc_home_failing() {
+    local dir
+    dir="$(mktemp -d "${TMPDIR:-/tmp}/xce-rcfile-it-fail.XXXXXX")" || return 1
+    mkdir -p "$dir/config/xce_test" || return 1
+    printf '%s\nset masters %s\n' "$RCFILE_BOGUS_CMD" "$OVERRIDE_MASTERS" \
+        >"$dir/config/xce_test/rc"
+    printf '%s|%s\n' "$dir" "$dir/config"
+}
+
+# Extracts the current-value column of the `masters` line from a `set`
+# command's output (cmd_set's "%-12s:%s default: %s\n" format). Robust to the
+# exact column padding and to a value containing an embedded space (the
+# unset-sentinel "<not set>") by capturing everything between the first ':'
+# and the literal " default:" tail.
+masters_value_in_output() {
+    printf '%s\n' "$1" | grep '^masters' | head -1 \
+        | sed -E 's/^masters[[:space:]]*:(.*) default:.*/\1/'
+}
+
+# Runs xce_test PIPED (stdin is not a tty -> scripted/batch mode -- the
+# session's existing abort-on-first-failure default). Args:
+#   $1 homedir  $2 xdg_config_home  $3 script (newline-joined commands)
+#   $4... extra xce_test flags (e.g. --norc)
+# Prints combined stdout+stderr; returns xce_test's own exit code.
+run_piped() {
+    local homedir="$1" xdg="$2" script="$3"
+    shift 3
+    printf '%s\n' "$script" \
+        | env HOME="$homedir" XDG_CONFIG_HOME="$xdg" \
+              timeout "$RUN_TIMEOUT_S" "$XCE_TEST" "$@" 2>&1
+}
+
+# Runs xce_test behind a REAL pty (genuine interactive/TTY session), so
+# libedit's readline() and isatty(STDIN_FILENO)-gated g_interactive/
+# g_continue_on_error behave exactly as they do for a human at a terminal --
+# a piped run cannot exercise this. Uses a small embedded Python driver
+# (pty.fork()), the same technique task 2.2's own implementer used for manual
+# interactive verification of this feature.
+# Args: $1 homedir  $2 xdg_config_home  $3 script  $4 extra flag ("" or e.g. "--norc")
+# Prints the captured pty transcript; returns xce_test's own exit code.
+run_interactive_pty() {
+    local homedir="$1" xdg="$2" script="$3" extra="${4:-}"
+    printf '%s\n' "$script" \
+        | python3 "$PTY_DRIVER" "$XCE_TEST" "$homedir" "$xdg" "$extra" "$RUN_TIMEOUT_S"
+}
+
+# ── Scenarios (task 3.1) ─────────────────────────────────────────────────────
+
+test_rcfile_takes_effect_piped() {
+    local name="rcfile_takes_effect_piped"
+    local fixture homedir xdg out rc mval
+    fixture="$(setup_rc_home)" || { failcase "$name" "setup_rc_home failed"; return; }
+    homedir="${fixture%%|*}"; xdg="${fixture##*|}"
+
+    out="$(run_piped "$homedir" "$xdg" $'set\nquit')"
+    rc=$?
+    mval="$(masters_value_in_output "$out")"
+
+    if [ "$rc" -ne 0 ]; then
+        failcase "$name" "xce_test exited $rc (expected 0); output:"$'\n'"$out"
+    elif [ "$mval" != "$OVERRIDE_MASTERS" ]; then
+        failcase "$name" "expected masters=='$OVERRIDE_MASTERS', got '$mval'; output:"$'\n'"$out"
+    else
+        pass "$name"
+    fi
+    cleanup_rc_home "$homedir"
+}
+
+test_rcfile_takes_effect_interactive() {
+    local name="rcfile_takes_effect_interactive"
+    local fixture homedir xdg out rc mval
+    fixture="$(setup_rc_home)" || { failcase "$name" "setup_rc_home failed"; return; }
+    homedir="${fixture%%|*}"; xdg="${fixture##*|}"
+
+    out="$(run_interactive_pty "$homedir" "$xdg" $'set\nquit')"
+    rc=$?
+    mval="$(masters_value_in_output "$out")"
+
+    if [ "$rc" -ne 0 ]; then
+        failcase "$name" "xce_test exited $rc (expected 0); transcript:"$'\n'"$out"
+    elif [ "$mval" != "$OVERRIDE_MASTERS" ]; then
+        failcase "$name" "expected masters=='$OVERRIDE_MASTERS', got '$mval'; transcript:"$'\n'"$out"
+    elif ! grep -qF 'xce>' <<<"$out"; then
+        failcase "$name" "no 'xce>' interactive prompt seen in transcript -- session may not have been a real tty; transcript:"$'\n'"$out"
+    else
+        pass "$name"
+    fi
+    cleanup_rc_home "$homedir"
+}
+
+test_norc_skips_rcfile() {
+    local name="norc_skips_rcfile"
+    local fixture homedir xdg out rc mval
+    fixture="$(setup_rc_home)" || { failcase "$name" "setup_rc_home failed"; return; }
+    homedir="${fixture%%|*}"; xdg="${fixture##*|}"
+
+    out="$(run_piped "$homedir" "$xdg" $'set\nquit' --norc)"
+    rc=$?
+    mval="$(masters_value_in_output "$out")"
+
+    if [ "$rc" -ne 0 ]; then
+        failcase "$name" "xce_test --norc exited $rc (expected 0); output:"$'\n'"$out"
+    elif [ "$mval" = "$OVERRIDE_MASTERS" ]; then
+        failcase "$name" "--norc did not skip rc-file load -- override value leaked into 'set' output; output:"$'\n'"$out"
+    elif [ "$mval" != "$NOT_SET" ]; then
+        failcase "$name" "expected masters=='$NOT_SET' (never assigned) with --norc, got '$mval'; output:"$'\n'"$out"
+    else
+        pass "$name"
+    fi
+    cleanup_rc_home "$homedir"
+}
+
+# ── Scenarios (task 3.2) ─────────────────────────────────────────────────────
+
+# A failing rc-file command in scripted/piped mode must abort BEFORE any of
+# the piped script's own commands run, exiting nonzero (Req 3.1, 3.3, 3.4).
+# The piped script's own `echo PIPED_SCRIPT_MARKER` is the observable that
+# proves the read loop never got to read the piped script at all -- not just
+# that the run "failed" for some other reason.
+test_rcfile_failure_aborts_piped() {
+    local name="rcfile_failure_aborts_piped"
+    local fixture homedir xdg out rc
+    fixture="$(setup_rc_home_failing)" || { failcase "$name" "setup_rc_home_failing failed"; return; }
+    homedir="${fixture%%|*}"; xdg="${fixture##*|}"
+
+    out="$(run_piped "$homedir" "$xdg" $'echo PIPED_SCRIPT_MARKER\nquit')"
+    rc=$?
+
+    if [ "$rc" -eq 0 ]; then
+        failcase "$name" "xce_test exited 0 (expected nonzero -- a failing rc-file command should fail the session, Req 3.4); output:"$'\n'"$out"
+    elif grep -qF 'PIPED_SCRIPT_MARKER' <<<"$out"; then
+        failcase "$name" "piped script's own PIPED_SCRIPT_MARKER output was present -- rc-file abort did not happen before the read loop consumed the piped script (Req 3.3); output:"$'\n'"$out"
+    elif ! grep -qF "$RCFILE_BOGUS_CMD" <<<"$out"; then
+        failcase "$name" "rc-file command failure was not reported (expected '$RCFILE_BOGUS_CMD: No such command...', Req 3.1); output:"$'\n'"$out"
+    else
+        pass "$name"
+    fi
+    cleanup_rc_home "$homedir"
+}
+
+# The same failing rc-file command in an interactive (TTY-simulated) session
+# must NOT abort: the rc file's remaining lines still run (proving "continue",
+# not just "didn't crash" -- Req 3.2) and the session still reaches the normal
+# interactive prompt afterward. Per Req 3.4 the same session-verdict/exit-code
+# accounting still applies as for any other failing command, so -- exactly as
+# for a failing command typed directly at the prompt -- the process's own
+# final exit code stays nonzero even though the session itself continued
+# normally through to `quit`; a manual mutation check (rc file with no failing
+# line) confirms this flips to exit 0, so this is asserting real behavior, not
+# vacuously true.
+test_rcfile_failure_continues_interactive() {
+    local name="rcfile_failure_continues_interactive"
+    local fixture homedir xdg out rc mval
+    fixture="$(setup_rc_home_failing)" || { failcase "$name" "setup_rc_home_failing failed"; return; }
+    homedir="${fixture%%|*}"; xdg="${fixture##*|}"
+
+    out="$(run_interactive_pty "$homedir" "$xdg" $'set\nquit')"
+    rc=$?
+    mval="$(masters_value_in_output "$out")"
+
+    if [ "$rc" -eq 0 ]; then
+        failcase "$name" "xce_test exited 0 (expected nonzero -- a failing rc-file command should still fail the session's verdict, Req 3.4); transcript:"$'\n'"$out"
+    elif ! grep -qF "$RCFILE_BOGUS_CMD" <<<"$out"; then
+        failcase "$name" "rc-file command failure was not reported (expected '$RCFILE_BOGUS_CMD: No such command...', Req 3.1); transcript:"$'\n'"$out"
+    elif [ "$mval" != "$OVERRIDE_MASTERS" ]; then
+        failcase "$name" "expected masters=='$OVERRIDE_MASTERS' -- the rc-file line AFTER the failing one should still have run (Req 3.2), got '$mval'; transcript:"$'\n'"$out"
+    elif ! grep -qF 'xce>' <<<"$out"; then
+        failcase "$name" "no 'xce>' interactive prompt seen after the rc-file failure -- session did not reach the normal prompt (Req 3.2); transcript:"$'\n'"$out"
+    else
+        pass "$name"
+    fi
+    cleanup_rc_home "$homedir"
+}
+
+# ── Main ──────────────────────────────────────────────────────────────────────
+
+main() {
+    [ -x "$XCE_TEST" ] \
+        || { log "FAIL: xce_test not found/executable: $XCE_TEST (run /xce-build controller)"; exit 1; }
+
+    test_rcfile_takes_effect_piped
+    test_rcfile_takes_effect_interactive
+    test_norc_skips_rcfile
+    test_rcfile_failure_aborts_piped
+    test_rcfile_failure_continues_interactive
+
+    log "---- summary ----"
+    local r
+    for r in "${RESULTS[@]}"; do
+        log "$r"
+    done
+
+    if [ "$FAILURES" -eq 0 ]; then
+        log "ALL SCENARIOS PASSED"
+        exit 0
+    else
+        log "$FAILURES SCENARIO(S) FAILED"
+        exit 1
+    fi
+}
+
+main "$@"

src/controller/testsuite/test_repl_apply.c

@@ -52,9 +52,9 @@ int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state)
 
 __attribute__((weak))
 int restore_job_dispatch_state(cluster_t* c, uint64_t jobid, uint32_t state,
-                               xce_job_alloc_t* alloc)
+                               uint64_t start_time, xce_job_alloc_t* alloc)
 {
-    (void)c; (void)jobid; (void)state; (void)alloc;
+    (void)c; (void)jobid; (void)state; (void)start_time; (void)alloc;
     return 0;
 }
 

src/controller/testsuite/test_sched_admin_dispatch.c

@@ -235,6 +235,106 @@ static void test_schedadmin_rejects_when_not_primary(void)
     printf("test_schedadmin_rejects_when_not_primary: PASS\n");
 }
 
+/* ── bug-1700: a standby dispatch-gate rejection must redirect to the known
+ * primary ──────────────────────────────────────────────────────────────────
+ *
+ * ha_dispatch_gate() rejects OPCode_schedAdmin on a CTRL_STANDBY node with
+ * -XCE_ERR_STANDBY (ha.c's CTRL_STANDBY case default branch, ~line 4774) --
+ * this node MAY know who the primary is (c->ha.primary_host/primary_port).
+ * Before the fix, myreq_handler's generic ha_dispatch_gate rejection block
+ * (cluster.c ~406-414) never called fill_redirect_reply() (the exact same
+ * one-line helper OPCode_join's handler already uses, join.c:277), so
+ * redirect_host/redirect_port were left at their zero-value defaults on
+ * EVERY dispatch-gate rejection -- xceadmin's client-side, already fully
+ * implemented and generic follow_redirects() had nothing to follow and
+ * surfaced the bare -XCE_ERR_STANDBY status as-is instead of transparently
+ * retrying against the primary. This is the exact scenario the user hit
+ * running `xceadmin scheduler record status` against a standby. */
+static void test_schedadmin_standby_with_known_primary_redirects(void)
+{
+    cluster_t* c = sa_make_cluster();
+    atomic_store(&c->ha.role, CTRL_STANDBY);
+    strncpy(c->ha.primary_host, "10.0.1.7", sizeof(c->ha.primary_host) - 1);
+    c->ha.primary_port = 7001;
+
+    /* Poison sentinel mirrors test_schedadmin_rejects_when_not_primary above
+     * -- a gate rejection must never reach the forwarding code. */
+    endpoint_t* poison = (endpoint_t*)(uintptr_t)0x1;
+    c->sched.ep         = poison;
+
+    endpoint_t ep;
+    memset(&ep, 0, sizeof(ep));
+    ep.owner = c;
+
+    io_packet_t* pkt = sa_build_request_pkt();
+    int          rc  = myreq_handler_for_test(&ep, pkt);
+
+    TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
+    TEST_ASSERT_TRUE_MESSAGE(
+        c->sched.ep == poison,
+        "c->sched.ep must be untouched -- a gate rejection never forwards");
+
+    xce_request_t decoded;
+    sa_decode_queued_reply(&ep, &decoded);
+    TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
+    xce_reply_t* rr = decoded.body.replymsg;
+    TEST_ASSERT_NOT_NULL(rr);
+    TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
+    TEST_ASSERT_NOT_NULL_MESSAGE(
+        rr->redirect_host,
+        "bug-1700: a standby dispatch-gate rejection with a known primary "
+        "must carry that primary's address so xceadmin's follow_redirects() "
+        "can transparently retry there");
+    TEST_ASSERT_EQUAL_STRING("10.0.1.7", rr->redirect_host);
+    TEST_ASSERT_EQUAL_UINT16(7001, rr->redirect_port);
+    xce_free_request(&decoded);
+
+    sa_teardown(c);
+    printf("test_schedadmin_standby_with_known_primary_redirects: PASS\n");
+}
+
+/* Regression companion: a standby that does NOT yet know any live primary
+ * (fresh ha_state_t straight out of calloc -- primary_host empty,
+ * primary_port 0, e.g. very early boot / mid-election / primary just died
+ * with no successor elected yet) must NOT crash and must produce a reply
+ * the client correctly treats as non-redirecting: discovery.py's
+ * `_redirect_target()` requires BOTH a non-empty host AND a non-zero port
+ * to treat a reply as a redirect, so an empty host / zero port here is
+ * harmless and falls back to today's bare-rejection behavior. Proves the
+ * fix does not regress the genuinely-no-primary-yet case. */
+static void test_schedadmin_standby_no_known_primary_no_redirect(void)
+{
+    cluster_t* c = sa_make_cluster();
+    atomic_store(&c->ha.role, CTRL_STANDBY);
+    /* c->ha.primary_host/primary_port left at their calloc'd zero value. */
+
+    endpoint_t ep;
+    memset(&ep, 0, sizeof(ep));
+    ep.owner = c;
+
+    io_packet_t* pkt = sa_build_request_pkt();
+    int          rc  = myreq_handler_for_test(&ep, pkt);
+
+    TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rc);
+
+    xce_request_t decoded;
+    sa_decode_queued_reply(&ep, &decoded);
+    TEST_ASSERT_EQUAL_INT(Request_body_replymsg, decoded.body_which);
+    xce_reply_t* rr = decoded.body.replymsg;
+    TEST_ASSERT_NOT_NULL(rr);
+    TEST_ASSERT_EQUAL_INT(-XCE_ERR_STANDBY, rr->status);
+    TEST_ASSERT_TRUE_MESSAGE(
+        rr->redirect_host == NULL || rr->redirect_host[0] == '\0',
+        "no known primary: redirect_host must decode as empty/absent, "
+        "never garbage -- the client's _redirect_target() treats an empty "
+        "host as 'no redirect, use this reply as-is'");
+    TEST_ASSERT_EQUAL_UINT16(0, rr->redirect_port);
+    xce_free_request(&decoded);
+
+    sa_teardown(c);
+    printf("test_schedadmin_standby_no_known_primary_no_redirect: PASS\n");
+}
+
 /* ── Observable 2: primary with no scheduler connected rejects immediately ──
  */
 static void test_schedadmin_rejects_when_not_connected(void)
@@ -667,15 +767,121 @@ static void test_schedadmin_round_trip_business_error(void)
     printf("test_schedadmin_round_trip_business_error: PASS\n");
 }
 
+/* ── bug-1693 regression: scheduler disconnect wakes the waiter immediately
+ * (Decision Log fidelity fix) ───────────────────────────────────────────────
+ *
+ * Before this fix, handle_scheduler_leave() (sched.c) only ever nulled
+ * c->sched.ep on a scheduler disconnect -- it never signalled the pending
+ * OPCode_schedAdmin waiter, so a disconnect mid-wait could only ever be
+ * observed at the FULL SCHEDADMIN_TIMEOUT_MS deadline (5s in production).
+ * That is exactly the Decision Log violation this task fixes: "No client
+ * reply-wait timeout as a fix for a hung request -- fail loud at the real
+ * transport/source instead of masking with a deadline backstop."
+ *
+ * c->sched.ep points at a real (if unconnected) endpoint_t, exactly like
+ * test_schedadmin_round_trip_success above, so the dispatch case's
+ * forward+wait path runs for real. A second thread plays the disconnect by
+ * calling schedadmin_signal_disconnect() (cluster.c) directly -- the exact,
+ * real, unmodified production function handle_scheduler_leave() (sched.c)
+ * calls to wake the waiter, and the entire fix under test here.
+ *
+ * NOT calling handle_scheduler_leave() itself: this file's binary
+ * (controller_unit_test) does not link the real sched.c at all (see
+ * meson.build's comment on sched_push_unit_test/sched_worker_change_unit_
+ * test for why -- sched.c's on_scheduler_join()/collect_confvars() collide
+ * at link time with fake definitions several OTHER controller_unit_test_src
+ * files already provide). test_config_repl.c (also in this binary) already
+ * supplies its OWN no-op handle_scheduler_leave() stub for that reason,
+ * predating this fix -- calling handle_scheduler_leave() here would
+ * silently exercise THAT stub, not sched.c's real implementation, and prove
+ * nothing. handle_scheduler_leave()'s own body is a thin, 2-statement
+ * wrapper (call this function, then null c->sched.ep under c->sched.lock)
+ * -- calling schedadmin_signal_disconnect() directly exercises the entire
+ * real fix mechanism (the new function, and the real dispatch-side wait it
+ * wakes) with zero loss of fidelity versus going through that wrapper. */
+static void* sa_deliver_scheduler_disconnect(void* argp)
+{
+    (void)argp;
+
+    /* Same margin as sa_deliver_scheduler_reply above -- comfortably inside
+     * the 5s deadline, giving the main thread time to register the pending
+     * waiter and start blocking on the condvar first. */
+    usleep(200000);
+
+    schedadmin_signal_disconnect();
+    return NULL;
+}
+
+static void test_schedadmin_disconnect_wakes_immediately(void)
+{
+    cluster_t* c = sa_make_cluster();
+    atomic_store(&c->ha.role, CTRL_PRIMARY);
+
+    endpoint_t sched_conn;
+    memset(&sched_conn, 0, sizeof(sched_conn));
+    c->sched.ep = &sched_conn;
+
+    pthread_t tid;
+    TEST_ASSERT_EQUAL_INT(
+        0, pthread_create(&tid, NULL, sa_deliver_scheduler_disconnect, c));
+
+    endpoint_t caller_ep;
+    memset(&caller_ep, 0, sizeof(caller_ep));
+    caller_ep.owner = c;
+
+    io_packet_t* pkt = sa_build_request_pkt();
+
+    uint64_t t0   = sa_mono_ms();
+    int      rc   = myreq_handler_for_test(&caller_ep, pkt);
+    uint64_t took = sa_mono_ms() - t0;
+
+    pthread_join(tid, NULL);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        0, rc, "the dispatch case always replies, even on a disconnect");
+
+    TEST_ASSERT_TRUE_MESSAGE(
+        took < 1000,
+        "a scheduler disconnect mid-wait must wake the waiter immediately "
+        "(well under the real, unshrunk 5s SCHEDADMIN_TIMEOUT_MS deadline) "
+        "-- this is the regression this test locks: pre-fix, nothing ever "
+        "signalled the waiter on disconnect, so this only ever completed at "
+        "the full 5s deadline");
+
+    xce_request_t decoded;
+    sa_decode_queued_reply(&caller_ep, &decoded);
+    xce_reply_t* rr = decoded.body.replymsg;
+    TEST_ASSERT_NOT_NULL(rr);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        0, rr->status,
+        "a disconnect is relayed as an ok-RPC SchedAdminReply carrying a "
+        "distinguishable error, not a negative RPC status");
+    TEST_ASSERT_EQUAL_INT(RequestReply_data_schedadminreplymsg, rr->data_which);
+    sched_admin_reply_t* got = rr->data.schedadminreplymsg;
+    TEST_ASSERT_NOT_NULL(got);
+    TEST_ASSERT_EQUAL_INT(0, got->ok);
+    TEST_ASSERT_EQUAL_STRING_MESSAGE(
+        "scheduler disconnected", got->error,
+        "must be distinguishable from \"request timeout\" -- proves this "
+        "woke via the disconnect signal, not the deadline");
+    xce_free_request(&decoded);
+
+    sa_teardown(c);
+    printf("test_schedadmin_disconnect_wakes_immediately: PASS\n");
+}
+
 /* ── Suite entry point ──────────────────────────────────────────────────── */
 void test_sched_admin_dispatch(void)
 {
     UnitySetTestFile(__FILE__);
     RUN_TEST(test_schedadmin_rejects_when_not_primary);
+    RUN_TEST(test_schedadmin_standby_with_known_primary_redirects);
+    RUN_TEST(test_schedadmin_standby_no_known_primary_no_redirect);
     RUN_TEST(test_schedadmin_rejects_when_not_connected);
     RUN_TEST(test_schedadmin_fast_rejects_when_admission_exhausted);
     RUN_TEST(test_schedadmin_round_trip_success);
     RUN_TEST(test_schedadmin_forward_timeout);
     RUN_TEST(test_schedadmin_round_trip_rpc_status_only);
     RUN_TEST(test_schedadmin_round_trip_business_error);
+    RUN_TEST(test_schedadmin_disconnect_wakes_immediately);
 }

src/controller/testsuite/test_sched_worker_change.c

@@ -90,6 +90,43 @@ extern int on_scheduler_join(cluster_t*);
  * push.c's own stub. */
 int check_worker_(xce_worker_t* w) { return (w != NULL) ? 1 : 0; }
 
+/* bug-1693: sched.c's handle_scheduler_leave() now calls
+ * schedadmin_signal_disconnect() (defined for real in cluster.c, which this
+ * standalone binary does not link -- see this file's own header comment
+ * for why real sched.c/dataex.c/egress.c are linked here in isolation).
+ * check_worker_() just above is a pure link-satisfying no-op stub since
+ * this file's own tests never reach it at runtime; this one is instead a
+ * counting SPY -- sched.c IS linked for real here, so handle_scheduler_
+ * leave() itself (unlike the OPCode_schedAdmin dispatch/wait machinery,
+ * which lives in cluster.c and is NOT linked here) is exercisable for
+ * real. test_scheduler_leave_signals_schedadmin_before_clearing_ep below
+ * uses this spy to prove handle_scheduler_leave() actually calls this
+ * function (not just that it compiles/links) -- the one piece of the
+ * bug-1693 fix that test_sched_admin_dispatch.c's own regression test
+ * (controller_unit_test, where cluster.c IS real but sched.c is stubbed by
+ * test_config_repl.c) cannot reach, since the two real halves of this fix
+ * (cluster.c's schedadmin_signal_disconnect() and sched.c's
+ * handle_scheduler_leave()) are never both linked-for-real in the same
+ * binary anywhere in this project's current test topology. */
+static int         g_schedadmin_signal_disconnect_calls;
+static cluster_t*  g_schedadmin_spy_c; /* set by the test below; NULL elsewhere */
+static endpoint_t* g_schedadmin_spy_ep_at_signal_time
+    = (endpoint_t*)(uintptr_t)0x1; /* poison -- must be overwritten */
+
+int schedadmin_signal_disconnect(void)
+{
+    g_schedadmin_signal_disconnect_calls++;
+    /* Captures c->sched.ep's value AT THE MOMENT this is called -- proves
+     * handle_scheduler_leave() calls this BEFORE nulling c->sched.ep, not
+     * after (see this fix's lock-ordering rationale in sched.c's doc
+     * comment: calling it after would deadlock against the OPCode_
+     * schedAdmin dispatch case in production, which holds c->sched.lock for
+     * its whole wait). */
+    if (g_schedadmin_spy_c != NULL)
+        g_schedadmin_spy_ep_at_signal_time = g_schedadmin_spy_c->sched.ep;
+    return 1;
+}
+
 /* ------------------------------------------------------------------ */
 /* Fixture                                                              */
 /* ------------------------------------------------------------------ */
@@ -331,6 +368,66 @@ static void test_worker_dead_then_encode_worker_is_asan_clean(void)
         "test_worker_dead_then_encode_worker_is_asan_clean: PASS\n");
 }
 
+/* ------------------------------------------------------------------ */
+/* bug-1693: handle_scheduler_leave() signals schedadmin BEFORE clearing */
+/* c->sched.ep (Decision Log fidelity fix)                               */
+/* ------------------------------------------------------------------ */
+
+/* Complements test_sched_admin_dispatch.c's own bug-1693 regression test
+ * (controller_unit_test): that file proves schedadmin_signal_disconnect()
+ * (cluster.c, real there) genuinely wakes a real pending OPCode_schedAdmin
+ * wait immediately, but cannot exercise the real handle_scheduler_leave()
+ * (sched.c is stubbed by test_config_repl.c in that binary). This binary
+ * has the opposite shape: sched.c is real, cluster.c is not linked at all,
+ * so schedadmin_signal_disconnect() above is a counting spy rather than the
+ * real implementation. Together the two prove both halves of the fix for
+ * real: the signal function's real effect on a real wait (there), and that
+ * handle_scheduler_leave() really calls it, unconditionally and before
+ * clearing c->sched.ep (here). */
+static void test_scheduler_leave_signals_schedadmin_before_clearing_ep(void)
+{
+    cluster_t* c = calloc(1, sizeof(cluster_t));
+    TEST_ASSERT_NOT_NULL(c);
+    mylock_init(&(c->sched.lock));
+
+    endpoint_t sentinel;
+    memset(&sentinel, 0, sizeof(sentinel));
+    c->sched.ep = &sentinel;
+
+    g_schedadmin_signal_disconnect_calls  = 0;
+    g_schedadmin_spy_c                    = c;
+    g_schedadmin_spy_ep_at_signal_time    = (endpoint_t*)(uintptr_t)0x1;
+
+    int rc = handle_scheduler_leave(c);
+    TEST_ASSERT_EQUAL_INT(0, rc);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        1, g_schedadmin_signal_disconnect_calls,
+        "bug-1693 regression: handle_scheduler_leave() must call "
+        "schedadmin_signal_disconnect() exactly once on every disconnect -- "
+        "pre-fix, it never called it at all, so a pending OPCode_schedAdmin "
+        "waiter could only ever be woken by the 5s deadline");
+    TEST_ASSERT_EQUAL_PTR_MESSAGE(
+        &sentinel, g_schedadmin_spy_ep_at_signal_time,
+        "handle_scheduler_leave() must call schedadmin_signal_disconnect() "
+        "BEFORE nulling c->sched.ep -- calling it after would deadlock "
+        "against the OPCode_schedAdmin dispatch case in production, which "
+        "holds c->sched.lock for its entire wait duration (see the "
+        "lock-ordering rationale in sched.c's handle_scheduler_leave() doc "
+        "comment)");
+    TEST_ASSERT_NULL_MESSAGE(c->sched.ep,
+                             "handle_scheduler_leave() must still clear "
+                             "c->sched.ep as before -- this fix only adds "
+                             "the signal call, it does not change the "
+                             "pre-existing ep-clearing behavior");
+
+    g_schedadmin_spy_c = NULL;
+    mylock_fini(&(c->sched.lock));
+    free(c);
+    printf(
+        "test_scheduler_leave_signals_schedadmin_before_clearing_ep: PASS\n");
+}
+
 /* ------------------------------------------------------------------ */
 /* Standalone Unity main (this binary is its own test executable -- see */
 /* meson.build's sched_worker_change_unit_test)                         */
@@ -355,6 +452,7 @@ int main(void)
 
     RUN_TEST(test_worker_dead_leaves_resvals_to_sched_null_and_zero);
     RUN_TEST(test_worker_dead_then_encode_worker_is_asan_clean);
+    RUN_TEST(test_scheduler_leave_signals_schedadmin_before_clearing_ep);
 
     return UNITY_END();
 }

src/controller/testsuite/test_wal_job_ckpt_fact.c

@@ -0,0 +1,682 @@
+/* test_wal_job_ckpt_fact.c — bug-1691 regression suite
+ *
+ * Proves WAL_JOB_CKPT's payload -- previously bare wal_jobstate_t{jobid,
+ * state}, exactly the same silent-field-drop bug class as bug-1678
+ * (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+
+ * start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), and
+ * bug-1690 (WAL_JOB_RUSAGE/job.run_usage) -- now carries the worker/
+ * scheduler-reported checkpoint info string (job.ckptinfo, a plain char*)
+ * through BOTH of persist.c's apply seams:
+ *
+ *   - apply_committed_entry() (live-commit apply / boot replay dispatch)
+ *   - apply_wal_entry()       (standby catch-up / replicated-apply dispatch)
+ *
+ * job.ckptinfo (xce/types.h) is a plain NUL-terminated char* -- NOT the
+ * richer xce_ckptinfo_t/CheckpointInfo capnp struct used elsewhere for a
+ * different purpose (that struct is on_job_ckpt()'s OWN parameter type,
+ * used only to carry the incoming report; the FIELD this fix restores is
+ * job.ckptinfo itself). Being a plain string, NO capnp struct is needed at
+ * all -- the fix appends job.ckptinfo as a NUL-terminated string directly
+ * after the wal_jobstate_t header, mirroring WAL_QUEUE_CTRL's own
+ * [header][name NUL-terminated] wire shape exactly (persist_queue_hook_fun /
+ * apply_wal_entry's WAL_QUEUE_CTRL case), rather than DISPATCH/START/RUSAGE's
+ * capnp-wrapper approach or FINISH's fixed-size native trailer.
+ *
+ * IMPORTANT DIVERGENCE FROM WAL_JOB_START/FINISH/DISPATCH's regression
+ * suites, SAME AS bug-1690's WAL_JOB_RUSAGE: on_job_ckpt() (job.c) never
+ * changes job.state -- it only ever sets job.ckptinfo. persist_job_hook_fun's
+ * new HOOK_DETAIL_JOB_CKPT case captures js.state verbatim from the job's
+ * CURRENT (unchanged) state, so apply_job_update_transition() sees wire
+ * state == current state and reports JOB_TRANSITION_NOOP_DUPLICATE on every
+ * single real CKPT replay. Gating ckptinfo adoption on
+ * result==JOB_TRANSITION_APPLIED (the way START/FINISH/DISPATCH do) would
+ * therefore make it dead code for the realistic, only-ever-occurring case.
+ * The fix decodes/adopts ckptinfo UNCONDITIONALLY, before that gate.
+ * test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change below is this
+ * suite's most important test -- it is the scenario a naive copy of the
+ * START/FINISH/DISPATCH pattern would have silently failed to cover.
+ *
+ * ALSO DIVERGES FROM bug-1690's WAL_JOB_RUSAGE fix in one respect: adoption
+ * here is a plain free-and-replace (dereference() the old value, adopt a
+ * fresh makeString()'d copy), NOT a merge. job.ckptinfo IS xstring-backed
+ * (makeString(), xstring.c) exactly like RUSAGE's host_name field was, and
+ * CAN be the job's own live value when the primary's own live apply pump
+ * re-applies a fact moments after on_job_ckpt() already set it -- but
+ * dereference() is the correct RTTI release verb for an xstring regardless
+ * of whether the pointer is "old" or coincidentally "the current live one"
+ * (it decrements a refcount rather than raw free()ing a non-owned pointer),
+ * so no merge-vs-replace hazard exists here the way it did for RUSAGE's
+ * capnp-generated free_JobRunusage_ptr() (a plain-malloc-oriented free
+ * function that would have corrupted the heap on an xstring pointer).
+ *
+ * Also proves backward compatibility: a WAL_JOB_CKPT entry with the OLD
+ * payload shape (exactly sizeof(wal_jobstate_t), no trailing bytes --
+ * simulating an entry written before this fix, or one fired while
+ * job.ckptinfo was still NULL at write time) still decodes cleanly, with
+ * ckptinfo left untouched exactly as before this fix -- no crash, no
+ * misread.
+ *
+ * Fixtures mirror test_wal_job_finish_fact.c's own conventions exactly:
+ *   - wjc_setup/wjc_teardown: bare cluster_t (jobmgr + hooksys) + in-memory
+ *     persist_t.
+ *   - wjc_make_fresh_persist/wjc_teardown_persist: real disk-backed
+ *     persist_t + persist_recover(), for restart-replay tests.
+ *
+ * test_wjc_real_on_job_ckpt_survives_restart drives the REAL, unmodified
+ * on_job_ckpt() (the actual worker/scheduler-CKPT receipt handler --
+ * cluster.c's NotifyData_data_ckptinfo dispatch's only callee) through a
+ * real WAL-write + persist_recover() restart cycle -- unlike bug-1690's
+ * WAL_JOB_RUSAGE suite, on_job_ckpt() has no misc.c-style stubbed dependency
+ * in this test binary, so it can be driven directly (mirrors
+ * test_wjf_real_on_job_finish_survives_restart's simpler on_job_finish()
+ * pattern, not RUSAGE's persist_job_hook_fun trampoline workaround).
+ */
+#include <rtti.h>
+
+#include <capnp_c.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unity/unity.h>
+
+#include "cluster.h"
+#include "controller.capnp.h"
+#include "hook.h"
+#include "jobj.h"
+#include "persist.h"
+#include "wal_types.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+#include "xcealloc.h"
+#include "xstring.h"
+
+/* jobj.c's jobsub table and the process-global xstring table are shared
+ * across every test file in the controller_unit_test binary — guard
+ * idempotently, mirroring every sibling fixture in this suite. */
+static int g_wjc_jobobj_ready;
+
+static cluster_t g_wjc_c;
+static persist_t g_wjc_p;
+
+/* ── Spy hook: counts AFTER_EVENT firings, ignores all payload args ─────── */
+static int g_wjc_hook_fires;
+
+static int wjc_spy_hook(int type, int situation, int detail, void* data,
+                        va_list ap)
+{
+    (void)type; (void)detail; (void)data; (void)ap;
+    if (situation == HOOK_AFTER_EVENT)
+        g_wjc_hook_fires++;
+    return 0;
+}
+
+static void wjc_setup(void)
+{
+    kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                  .nslots   = 32,
+                                  .delta    = 16 };
+
+    if (makeString((char*)"wjc-probe") == NULL)
+        xstring_table_init(64);
+
+    if (!g_wjc_jobobj_ready) {
+        TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
+        g_wjc_jobobj_ready = 1;
+    }
+
+    memset(&g_wjc_c, 0, sizeof(g_wjc_c));
+    TEST_ASSERT_EQUAL_INT(0,
+        xce_hook_init(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
+
+    g_wjc_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+    TEST_ASSERT_NOT_NULL(g_wjc_c.jobmgr);
+
+    memset(&g_wjc_p, 0, sizeof(g_wjc_p));
+    g_wjc_p.cluster = &g_wjc_c;
+
+    g_wjc_hook_fires = 0;
+}
+
+static void wjc_teardown(void)
+{
+    xce_hook_fini(&g_wjc_c.hooksys);
+    if (g_wjc_c.jobmgr)
+        dereference(g_wjc_c.jobmgr);
+}
+
+/* Look up a job_t wrapper by jobid (NULL if absent). */
+static job_t* wjc_find_job(uint64_t jobid)
+{
+    job_t* j = NULL;
+    lock(g_wjc_c.jobmgr, RTTI_LOCK_READ);
+    int found = key_find(g_wjc_c.jobmgr, (void*)jobid, (void**)&j);
+    unlock(g_wjc_c.jobmgr);
+    return (found > 0) ? j : NULL;
+}
+
+/* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay
+ * entirely — for seeding fact-transition tests, mirroring
+ * test_wal_job_finish_fact.c's wjf_seed_job / test_wal_job_rusage_fact.c's
+ * wru_seed_job. */
+static job_submit_data_t g_wjc_seed_submit;
+
+static job_t* wjc_seed_job(uint64_t jobid, uint32_t state)
+{
+    xce_job_t jdef;
+
+    memset(&g_wjc_seed_submit, 0, sizeof(g_wjc_seed_submit));
+    g_wjc_seed_submit.queue = (char*)"q";
+    g_wjc_seed_submit.user  = (char*)"tester";
+
+    memset(&jdef, 0, sizeof(jdef));
+    jdef.jobid  = jobid;
+    jdef.state  = state;
+    jdef.submit = &g_wjc_seed_submit;
+
+    return makeJob(g_wjc_c.jobmgr, &jdef);
+}
+
+/* Build a wal_jobstate_t header + trailing NUL-terminated ckptinfo string --
+ * persist_job_hook_fun's WAL_JOB_CKPT payload shape (bug-1691), mirroring
+ * WAL_QUEUE_CTRL's own [header][name NUL-terminated] wire shape. Caller owns
+ * *out. */
+static void wjc_build_ckpt_payload(uint64_t jobid, uint32_t state,
+                                   const char* ckptinfo,
+                                   uint8_t** out, size_t* out_len)
+{
+    wal_jobstate_t js       = { .jobid = jobid, .state = state, ._pad = 0 };
+    size_t         info_len = strlen(ckptinfo) + 1;
+
+    *out_len = sizeof(js) + info_len;
+    *out     = malloc(*out_len);
+    TEST_ASSERT_NOT_NULL(*out);
+    memcpy(*out, &js, sizeof(js));
+    memcpy(*out + sizeof(js), ckptinfo, info_len);
+}
+
+static void wjc_assert_ckptinfo_matches(job_t* j, const char* expected)
+{
+    TEST_ASSERT_NOT_NULL_MESSAGE(j->job.ckptinfo,
+        "a WAL_JOB_CKPT payload with a trailing ckptinfo string must set "
+        "job.ckptinfo (bug-1691), not just state");
+    TEST_ASSERT_EQUAL_STRING(expected, j->job.ckptinfo);
+}
+
+/* ── apply_committed_entry(): live-commit apply / boot replay dispatch ──── */
+
+static void test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change(void)
+{
+    wjc_setup();
+
+    job_t* j = wjc_seed_job(74001, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjc_build_ckpt_payload(74001, JOB_STAT_RUN, "ckpt-blob-1", &payload,
+                          &plen);
+
+    xce_add_hook(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjc_spy_hook,
+                NULL);
+
+    int rc = apply_committed_entry(&g_wjc_p, WAL_JOB_CKPT, 300, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    wjc_assert_ckptinfo_matches(j, "ckpt-blob-1");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjc_hook_fires,
+        "a same-state (duplicate) CKPT replay -- the realistic, only-ever-"
+        "occurring case, since on_job_ckpt() never changes job.state -- "
+        "must NOT fire a notification hook, unchanged from before this fix, "
+        "even though ckptinfo WAS adopted");
+
+    wjc_teardown();
+    printf("test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change: "
+          "PASS\n");
+}
+
+/* Secondary scenario: a WAL_JOB_CKPT fact that DOES carry a forward state
+ * transition (mechanism-level test, mirroring how bug-1690's WAL_JOB_RUSAGE
+ * suite covers this same divergence). ckptinfo must be adopted here too, and
+ * the existing forward-transition hook-fire behavior must be unchanged. */
+static void
+test_wjc_apply_committed_entry_sets_ckptinfo_forward_transition(void)
+{
+    wjc_setup();
+
+    job_t* j = wjc_seed_job(74002, JOB_STAT_PEND);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjc_build_ckpt_payload(74002, JOB_STAT_RUN, "ckpt-blob-2", &payload,
+                          &plen);
+
+    xce_add_hook(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjc_spy_hook,
+                NULL);
+
+    int rc = apply_committed_entry(&g_wjc_p, WAL_JOB_CKPT, 301, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    wjc_assert_ckptinfo_matches(j, "ckpt-blob-2");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wjc_hook_fires,
+        "a forward CKPT-carried transition must still fire exactly one "
+        "notification, unchanged from before this fix");
+
+    wjc_teardown();
+    printf(
+        "test_wjc_apply_committed_entry_sets_ckptinfo_forward_transition: "
+        "PASS\n");
+}
+
+static void
+test_wjc_apply_committed_entry_backward_compat_old_payload(void)
+{
+    wjc_setup();
+
+    job_t* j = wjc_seed_job(74003, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+    TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo,
+        "fixture must seed a job with ckptinfo still unset");
+
+    wal_jobstate_t js = { .jobid = 74003, .state = JOB_STAT_RUN, ._pad = 0 };
+
+    int rc = apply_committed_entry(&g_wjc_p, WAL_JOB_CKPT, 302, &js,
+                                   sizeof(js));
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo,
+        "an old-format (exactly sizeof(wal_jobstate_t)) WAL_JOB_CKPT "
+        "payload must leave ckptinfo NULL -- no crash, no misread of "
+        "absent trailing bytes");
+
+    wjc_teardown();
+    printf("test_wjc_apply_committed_entry_backward_compat_old_payload: "
+          "PASS\n");
+}
+
+/* ── apply_wal_entry(): standby catch-up / replicated-apply dispatch ────── */
+
+static void test_wjc_apply_wal_entry_sets_ckptinfo_no_hook(void)
+{
+    wjc_setup();
+
+    job_t* j = wjc_seed_job(74004, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjc_build_ckpt_payload(74004, JOB_STAT_RUN, "ckpt-blob-4", &payload,
+                          &plen);
+
+    xce_add_hook(&g_wjc_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjc_spy_hook,
+                NULL);
+
+    int rc = apply_wal_entry(&g_wjc_p, WAL_JOB_CKPT, 303, payload,
+                             (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    wjc_assert_ckptinfo_matches(j, "ckpt-blob-4");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjc_hook_fires,
+        "apply_wal_entry's catch-up dispatch must never fire a "
+        "notification hook");
+
+    wjc_teardown();
+    printf("test_wjc_apply_wal_entry_sets_ckptinfo_no_hook: PASS\n");
+}
+
+static void test_wjc_apply_wal_entry_backward_compat_old_payload(void)
+{
+    wjc_setup();
+
+    job_t* j = wjc_seed_job(74005, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+    TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo,
+        "fixture must seed a job with ckptinfo still unset");
+
+    wal_jobstate_t js = { .jobid = 74005, .state = JOB_STAT_RUN, ._pad = 0 };
+
+    int rc = apply_wal_entry(&g_wjc_p, WAL_JOB_CKPT, 304, &js, sizeof(js));
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_NULL_MESSAGE(j->job.ckptinfo,
+        "an old-format WAL_JOB_CKPT payload must leave ckptinfo NULL on "
+        "this seam too");
+
+    wjc_teardown();
+    printf("test_wjc_apply_wal_entry_backward_compat_old_payload: PASS\n");
+}
+
+/* ── Restart-replay Observables ──────────────────────────────────────── */
+
+static persist_t* wjc_make_fresh_persist(const char* tmpdir)
+{
+    persist_t* p = calloc(1, sizeof(persist_t));
+    if (!p) return NULL;
+
+    persist_config_t cfg = persist_config_defaults();
+    cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
+
+    persist_set_workdir(p, tmpdir, "ctrl-test");
+
+    if (persist_init(p, &cfg, NULL) != 0) {
+        free(p);
+        return NULL;
+    }
+    if (persist_start(p) != 0) {
+        persist_fini(p);
+        free(p);
+        return NULL;
+    }
+    return p;
+}
+
+static void wjc_teardown_persist(persist_t* p)
+{
+    persist_stop(p);
+    persist_fini(p);
+    free(p);
+}
+
+/* Encode helper mirroring test_wal_job_finish_fact.c's wjf_encode_job
+ * (capnp_encode_job is file-static in persist.c). */
+static int wjc_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
+{
+    struct capn          c;
+    struct capn_segment* cs;
+    struct Job           cj;
+    Job_ptr              jp;
+
+    capn_init_malloc(&c);
+    cs = capn_root(&c).seg;
+    encode_Job(cs, &cj, (xce_job_t*)j);
+    jp = new_Job(cs);
+    write_Job(&cj, jp);
+    capn_setp(capn_root(&c), 0, jp.p);
+
+    *out_len = capn_size(&c);
+    *out     = malloc(*out_len);
+    if (!*out) { capn_free(&c); return -ENOMEM; }
+    capn_write_mem(&c, *out, *out_len, 0);
+    capn_free(&c);
+    return 0;
+}
+
+/* Direct-payload restart replay: appends WAL_JOB_SUBMIT + a new-format
+ * WAL_JOB_CKPT (real ckptinfo string) straight onto disk via
+ * persist_wal_async, then replays via a fresh persist_t's persist_recover()
+ * (a real restart). Isolates the WAL fact shape from the live caller,
+ * mirroring test_wal_job_finish_fact.c's own restart-replay test. */
+static void test_wjc_restart_replay_recovers_ckptinfo(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_wjc_replay_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    const char* wire_ckptinfo = "ckpt-blob-replay";
+
+    /* Phase 1: write the log to real disk. */
+    {
+        persist_t* p = wjc_make_fresh_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p);
+
+        job_submit_data_t submit;
+        memset(&submit, 0, sizeof(submit));
+        submit.queue = (char*)"q";
+        submit.user  = (char*)"tester";
+
+        xce_job_t jdef;
+        memset(&jdef, 0, sizeof(jdef));
+        jdef.jobid  = 74006;
+        jdef.state  = JOB_STAT_RUN;
+        jdef.submit = &submit;
+
+        uint8_t* job_payload = NULL;
+        size_t   job_plen    = 0;
+        TEST_ASSERT_EQUAL_INT(0,
+            wjc_encode_job(&jdef, &job_payload, &job_plen));
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_SUBMIT, job_payload,
+                              (uint32_t)job_plen));
+        free(job_payload);
+
+        uint8_t* ckpt_payload = NULL;
+        size_t   ckpt_plen    = 0;
+        wjc_build_ckpt_payload(74006, JOB_STAT_RUN, wire_ckptinfo,
+                              &ckpt_payload, &ckpt_plen);
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_CKPT, ckpt_payload,
+                              (uint32_t)ckpt_plen));
+        free(ckpt_payload);
+
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
+        wjc_teardown_persist(p);
+    }
+
+    /* Phase 2: fresh persist_t + fresh cluster_t, recover from disk —
+     * simulates a controller restart. */
+    {
+        persist_t* p2 = wjc_make_fresh_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p2);
+
+        wjc_setup();
+        p2->cluster = &g_wjc_c;
+
+        int rc = persist_recover(p2);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        job_t* j = wjc_find_job(74006);
+        TEST_ASSERT_NOT_NULL_MESSAGE(j,
+            "restart replay must construct the job via WAL_JOB_SUBMIT");
+        TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+        wjc_assert_ckptinfo_matches(j, wire_ckptinfo);
+
+        wjc_teardown();
+        wjc_teardown_persist(p2);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    printf("test_wjc_restart_replay_recovers_ckptinfo: PASS\n");
+}
+
+/* bug-1691's primary regression test: drives the REAL production
+ * on_job_ckpt() path (the actual worker/scheduler-CKPT receipt handler --
+ * cluster.c's NotifyData_data_ckptinfo dispatch's only callee) with a real
+ * xce_ckptinfo_t, through a real WAL-write + persist_recover() restart cycle
+ * -- the exact technique test_wjf_real_on_job_finish_survives_restart
+ * (bug-1689) already established. Asserts the job's ckptinfo survives the
+ * restart intact, not just state -- the audit's exact confirmed root
+ * cause. */
+static void test_wjc_real_on_job_ckpt_survives_restart(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_wjc_onckpt_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    const uint64_t jobid              = 74007;
+    const char*    real_ckptinfo_wire = "real-ckpt-info-from-on-job-ckpt";
+
+    /* Phase 1: real cluster + real disk persist. Seed WAL_JOB_SUBMIT
+     * directly (so replay can reconstruct the job) AND a live in-memory job
+     * object (so the real on_job_ckpt() call site has something to
+     * find/lock) -- mirrors test_wjf_real_on_job_finish_survives_restart's
+     * dual-seed pattern exactly. Then drive an actual on_job_ckpt() call
+     * and let IT append WAL_JOB_CKPT via persist_job_hook_fun -- this is
+     * the production code path under test, not a hand-built WAL entry. */
+    {
+        persist_config_t cfg;
+
+        if (makeString((char*)"wjc-onckpt-probe") == NULL)
+            xstring_table_init(64);
+        if (!g_wjc_jobobj_ready) {
+            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
+            g_wjc_jobobj_ready = 1;
+        }
+
+        cluster_t c;
+        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                      .nslots = 32, .delta = 16 };
+        memset(&c, 0, sizeof(c));
+        TEST_ASSERT_EQUAL_INT(0,
+            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
+        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+        TEST_ASSERT_NOT_NULL(c.jobmgr);
+
+        cfg             = persist_config_defaults();
+        cfg.wal_sync_ms = 5000;
+        persist_set_workdir(&c.persist, tmp, "ctrl-test");
+        /* Unlike test_wjf_real_on_job_finish_survives_restart's identical-
+         * looking fixture (bug-1689, a DIRECT persist_submit_job_finish_
+         * fact() call site -- no hook registration needed, so that sibling
+         * test can pass NULL here), on_job_ckpt() (job.c) is hook-driven:
+         * it only ever calls xce_run_hooks(..., HOOK_DETAIL_JOB_CKPT, ...)
+         * and relies ENTIRELY on persist_job_hook_fun (this fix's actual
+         * changed code) being registered as a c.hooksys subscriber to reach
+         * the WAL at all. persist_start() only registers that hook `if
+         * (p->cluster != NULL)` -- passing &c (not NULL) here is required
+         * for the real production path to append anything. */
+        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, &c));
+        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
+
+        /* Durable WAL_JOB_SUBMIT seed. */
+        {
+            job_submit_data_t wal_submit;
+            xce_job_t         wal_jdef;
+            uint8_t*          buf = NULL;
+            size_t            sz  = 0;
+
+            memset(&wal_submit, 0, sizeof(wal_submit));
+            wal_submit.queue = (char*)"oq";
+            wal_submit.user  = (char*)"tester";
+
+            memset(&wal_jdef, 0, sizeof(wal_jdef));
+            wal_jdef.jobid  = jobid;
+            wal_jdef.state  = JOB_STAT_RUN;
+            wal_jdef.submit = &wal_submit;
+
+            TEST_ASSERT_EQUAL_INT(0,
+                wjc_encode_job(&wal_jdef, &buf, &sz));
+            TEST_ASSERT_EQUAL_INT(0,
+                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
+                                  (uint32_t)sz));
+            free(buf);
+        }
+
+        /* Live in-memory job object, already RUN, for on_job_ckpt() to
+         * find/lock. */
+        {
+            job_submit_data_t submit;
+            xce_job_t         jdef;
+
+            memset(&submit, 0, sizeof(submit));
+            submit.queue = (char*)"oq";
+            submit.user  = (char*)"tester";
+
+            memset(&jdef, 0, sizeof(jdef));
+            jdef.jobid  = jobid;
+            jdef.state  = JOB_STAT_RUN;
+            jdef.submit = &submit;
+
+            TEST_ASSERT_NOT_NULL(makeJob(c.jobmgr, &jdef));
+        }
+
+        /* The real worker/scheduler-reported checkpoint report. `ep` is
+         * unused by on_job_ckpt()'s body -- NULL is safe, mirroring how
+         * production call sites pass whatever endpoint_t* the transport
+         * layer happened to receive it on, which this fact-recording path
+         * never dereferences. */
+        xce_ckptinfo_t ckptinfo = { .jobid    = jobid,
+                                    .ckptinfo = (char*)real_ckptinfo_wire };
+
+        int rc = on_job_ckpt(&c, NULL, &ckptinfo);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
+
+        persist_stop(&c.persist);
+        persist_fini(&c.persist);
+        dereference(c.jobmgr);
+        xce_hook_fini(&c.hooksys);
+    }
+
+    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
+     * recover from disk — simulates a controller restart. */
+    {
+        persist_t* p2 = wjc_make_fresh_persist(tmp);
+        cluster_t  c2;
+        job_t*     j2 = NULL;
+        int        rc;
+        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                      .nslots = 32, .delta = 16 };
+
+        TEST_ASSERT_NOT_NULL(p2);
+
+        memset(&c2, 0, sizeof(c2));
+        TEST_ASSERT_EQUAL_INT(0,
+            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
+        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+        TEST_ASSERT_NOT_NULL(c2.jobmgr);
+
+        p2->cluster = &c2;
+        rc          = persist_recover(p2);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        lock(c2.jobmgr, RTTI_LOCK_READ);
+        key_find(c2.jobmgr, (void*)jobid, (void**)&j2);
+        unlock(c2.jobmgr);
+        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
+            "restart replay must reconstruct the job via WAL_JOB_SUBMIT");
+        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state,
+            "restart replay must recover the job's state, unchanged by the "
+            "real on_job_ckpt() call");
+
+        /* bug-1691: restart replay must recover the worker/scheduler-
+         * reported ckptinfo string -- the audit's exact confirmed root
+         * cause (mirrors bug-1678/1687/1689/1690's own acceptance
+         * criterion). */
+        wjc_assert_ckptinfo_matches(j2, real_ckptinfo_wire);
+
+        xce_hook_fini(&c2.hooksys);
+        if (c2.jobmgr) dereference(c2.jobmgr);
+        wjc_teardown_persist(p2);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    printf("test_wjc_real_on_job_ckpt_survives_restart: PASS\n");
+}
+
+/* ── Test group entry point ───────────────────────────────────────────── */
+
+void test_wal_job_ckpt_fact(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_wjc_apply_committed_entry_sets_ckptinfo_no_state_change);
+    RUN_TEST(test_wjc_apply_committed_entry_sets_ckptinfo_forward_transition);
+    RUN_TEST(test_wjc_apply_committed_entry_backward_compat_old_payload);
+    RUN_TEST(test_wjc_apply_wal_entry_sets_ckptinfo_no_hook);
+    RUN_TEST(test_wjc_apply_wal_entry_backward_compat_old_payload);
+    RUN_TEST(test_wjc_restart_replay_recovers_ckptinfo);
+    RUN_TEST(test_wjc_real_on_job_ckpt_survives_restart);
+}

src/controller/testsuite/test_wal_job_dispatch_fact.c

@@ -162,13 +162,21 @@ static void test_wjd_apply_committed_entry_transitions_pend_to_run(void)
     printf("test_wjd_apply_committed_entry_transitions_pend_to_run: PASS\n");
 }
 
-/* bug-1678: build a wal_jobstate_t header + trailing Cap'n Proto JobAlloc
- * bytes -- persist_submit_job_dispatch_fact()'s payload shape -- mirroring
+/* bug-1678 (extended bug-1697): build a wal_jobstate_t header + optional
+ * fixed uint64_t start_time + trailing Cap'n Proto JobAlloc bytes --
+ * persist_submit_job_dispatch_fact()'s payload shape -- mirroring
  * capnp_encode_joballoc (persist.c, file-static) since job.c/persist.c's own
- * static helpers aren't linkable from a test TU. Caller owns *out. */
-static int wjd_build_dispatch_payload(uint64_t jobid, uint32_t state,
-                                      xce_job_alloc_t* alloc,
-                                      uint8_t** out, size_t* out_len)
+ * static helpers aren't linkable from a test TU. Caller owns *out.
+ * `has_start_time` sets WAL_JOB_DISPATCH_HAS_START_TIME_FLAG in _pad and
+ * inserts `start_time` between the header and the JobAlloc bytes (the
+ * post-bug-1697 shape); when false, `start_time` is ignored and the payload
+ * is exactly the pre-bug-1697 shape (header immediately followed by
+ * JobAlloc bytes, _pad=0). */
+static int wjd_build_dispatch_payload_ex(uint64_t jobid, uint32_t state,
+                                         bool has_start_time,
+                                         uint64_t start_time,
+                                         xce_job_alloc_t* alloc,
+                                         uint8_t** out, size_t* out_len)
 {
     struct capn          c;
     struct capn_segment* cs;
@@ -176,7 +184,12 @@ static int wjd_build_dispatch_payload(uint64_t jobid, uint32_t state,
     JobAlloc_ptr         jap;
     uint8_t*             allocbuf;
     size_t               alloclen;
-    wal_jobstate_t        js = { .jobid = jobid, .state = state, ._pad = 0 };
+    size_t               start_field = has_start_time ? sizeof(start_time)
+                                                       : 0;
+    wal_jobstate_t       js = { .jobid = jobid, .state = state,
+                               ._pad = has_start_time
+                                   ? WAL_JOB_DISPATCH_HAS_START_TIME_FLAG
+                                   : 0 };
 
     capn_init_malloc(&c);
     cs = capn_root(&c).seg;
@@ -191,15 +204,27 @@ static int wjd_build_dispatch_payload(uint64_t jobid, uint32_t state,
     capn_write_mem(&c, allocbuf, alloclen, 0);
     capn_free(&c);
 
-    *out_len = sizeof(js) + alloclen;
+    *out_len = sizeof(js) + start_field + alloclen;
     *out     = malloc(*out_len);
     if (!*out) { free(allocbuf); return -ENOMEM; }
     memcpy(*out, &js, sizeof(js));
-    memcpy(*out + sizeof(js), allocbuf, alloclen);
+    if (has_start_time)
+        memcpy(*out + sizeof(js), &start_time, sizeof(start_time));
+    memcpy(*out + sizeof(js) + start_field, allocbuf, alloclen);
     free(allocbuf);
     return 0;
 }
 
+/* Pre-bug-1697 shape wrapper -- every call site that predates the start_time
+ * field keeps working unchanged. */
+static int wjd_build_dispatch_payload(uint64_t jobid, uint32_t state,
+                                      xce_job_alloc_t* alloc,
+                                      uint8_t** out, size_t* out_len)
+{
+    return wjd_build_dispatch_payload_ex(jobid, state, false, 0, alloc, out,
+                                         out_len);
+}
+
 /* One host ("wjd-worker"), one resource ("slots", value 1.0) -- the same
  * minimal shape test_dispatch_commit.c's own dispatch tests use. Caller owns
  * the result (release via free_JobAlloc_ptr). */
@@ -242,12 +267,60 @@ static void test_wjd_apply_committed_entry_transitions_and_sets_allocs(void)
     TEST_ASSERT_EQUAL_STRING("wjd-worker", j->job.allocs->allocs[0]->host);
     TEST_ASSERT_EQUAL_STRING("slots",
         j->job.allocs->allocs[0]->allocs[0]->resname);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, (uint64_t)j->job.start_time,
+        "bug-1697 backward compat: a pre-bug-1697 WAL_JOB_DISPATCH payload "
+        "(no WAL_JOB_DISPATCH_HAS_START_TIME_FLAG, header+allocs only) must "
+        "leave job.start_time untouched -- no crash, no misread of absent "
+        "start_time bytes");
 
     wjd_teardown();
     printf("test_wjd_apply_committed_entry_transitions_and_sets_allocs: "
           "PASS\n");
 }
 
+/* bug-1697: WAL_JOB_DISPATCH_HAS_START_TIME_FLAG set -- the post-fix shape
+ * carrying a fixed uint64_t start_time between the header and the trailing
+ * JobAlloc bytes. Proves apply_committed_entry() decodes/adopts it. */
+static void test_wjd_apply_committed_entry_sets_start_time_and_allocs(void)
+{
+    wjd_setup();
+
+    job_t* j = wjd_seed_job(71007, JOB_STAT_PEND);
+    TEST_ASSERT_NOT_NULL(j);
+
+    xce_job_alloc_t* alloc = wjd_make_test_alloc(71007);
+    TEST_ASSERT_NOT_NULL(alloc);
+
+    uint64_t wire_start_time = 1700000123ULL;
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    TEST_ASSERT_EQUAL_INT(0,
+        wjd_build_dispatch_payload_ex(71007, JOB_STAT_RUN, true,
+                                      wire_start_time, alloc, &payload,
+                                      &plen));
+    free_JobAlloc_ptr(&alloc);
+
+    int rc = apply_committed_entry(&g_wjd_p, WAL_JOB_DISPATCH, 104, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(wire_start_time,
+        (uint64_t)j->job.start_time,
+        "bug-1697: a WAL_JOB_DISPATCH payload with "
+        "WAL_JOB_DISPATCH_HAS_START_TIME_FLAG set must adopt the "
+        "wire-carried start_time, not leave it at 0");
+    TEST_ASSERT_NOT_NULL_MESSAGE(j->job.allocs,
+        "job.allocs must still be recovered alongside start_time");
+    TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
+    TEST_ASSERT_EQUAL_STRING("wjd-worker", j->job.allocs->allocs[0]->host);
+
+    wjd_teardown();
+    printf("test_wjd_apply_committed_entry_sets_start_time_and_allocs: "
+          "PASS\n");
+}
+
 /* ── apply_wal_entry(): standby catch-up / replicated-apply dispatch ────── */
 
 static void
@@ -309,11 +382,54 @@ static void test_wjd_apply_wal_entry_transitions_and_sets_allocs(void)
         "primary's own boot replay");
     TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
     TEST_ASSERT_EQUAL_STRING("wjd-worker", j->job.allocs->allocs[0]->host);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(0, (uint64_t)j->job.start_time,
+        "bug-1697 backward compat: a pre-bug-1697 WAL_JOB_DISPATCH payload "
+        "must leave job.start_time untouched on this seam too");
 
     wjd_teardown();
     printf("test_wjd_apply_wal_entry_transitions_and_sets_allocs: PASS\n");
 }
 
+/* bug-1697: sibling of test_wjd_apply_committed_entry_sets_start_time_and_
+ * allocs, for apply_wal_entry's standby catch-up / replicated-apply seam. */
+static void test_wjd_apply_wal_entry_sets_start_time_and_allocs(void)
+{
+    wjd_setup();
+
+    job_t* j = wjd_seed_job(71008, JOB_STAT_PEND);
+    TEST_ASSERT_NOT_NULL(j);
+
+    xce_job_alloc_t* alloc = wjd_make_test_alloc(71008);
+    TEST_ASSERT_NOT_NULL(alloc);
+
+    uint64_t wire_start_time = 1700000456ULL;
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    TEST_ASSERT_EQUAL_INT(0,
+        wjd_build_dispatch_payload_ex(71008, JOB_STAT_RUN, true,
+                                      wire_start_time, alloc, &payload,
+                                      &plen));
+    free_JobAlloc_ptr(&alloc);
+
+    int rc = apply_wal_entry(&g_wjd_p, WAL_JOB_DISPATCH, 105, payload,
+                             (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(wire_start_time,
+        (uint64_t)j->job.start_time,
+        "bug-1697: apply_wal_entry's catch-up dispatch must also recover "
+        "the wire-carried start_time when "
+        "WAL_JOB_DISPATCH_HAS_START_TIME_FLAG is set, required for a "
+        "promoted standby");
+    TEST_ASSERT_NOT_NULL(j->job.allocs);
+    TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
+
+    wjd_teardown();
+    printf("test_wjd_apply_wal_entry_sets_start_time_and_allocs: PASS\n");
+}
+
 /* ── Restart-replay Observable (this task's completion criterion) ───────── */
 
 static persist_t* wjd_make_fresh_persist(const char* tmpdir)
@@ -502,11 +618,15 @@ test_wjd_restart_replay_recovers_running_and_allocs_with_dispatch_fact(void)
         xce_job_alloc_t* alloc = wjd_make_test_alloc(71006);
         TEST_ASSERT_NOT_NULL(alloc);
 
+        /* bug-1697: new-format payload (WAL_JOB_DISPATCH_HAS_START_TIME_
+         * FLAG set), so this restart-replay test also proves start_time
+         * -- not just allocs -- survives a real restart. */
         uint8_t* dispatch_payload = NULL;
         size_t   dispatch_plen    = 0;
         TEST_ASSERT_EQUAL_INT(0,
-            wjd_build_dispatch_payload(71006, JOB_STAT_RUN, alloc,
-                                       &dispatch_payload, &dispatch_plen));
+            wjd_build_dispatch_payload_ex(71006, JOB_STAT_RUN, true,
+                                          71006000ULL, alloc,
+                                          &dispatch_payload, &dispatch_plen));
         free_JobAlloc_ptr(&alloc);
 
         TEST_ASSERT_EQUAL_INT(0,
@@ -546,6 +666,11 @@ test_wjd_restart_replay_recovers_running_and_allocs_with_dispatch_fact(void)
         TEST_ASSERT_EQUAL_INT(1, j->job.allocs->nhosts);
         TEST_ASSERT_EQUAL_STRING("wjd-worker",
             j->job.allocs->allocs[0]->host);
+        TEST_ASSERT_EQUAL_UINT64_MESSAGE(71006000ULL,
+            (uint64_t)j->job.start_time,
+            "bug-1697: restart replay must also recover the dispatch-time "
+            "start_time from the WAL_JOB_DISPATCH fact's wire-carried "
+            "field, not leave it at 0");
 
         wjd_teardown();
         wjd_teardown_persist(p2);
@@ -567,8 +692,10 @@ void test_wal_job_dispatch_fact(void)
     UnitySetTestFile(__FILE__);
     RUN_TEST(test_wjd_apply_committed_entry_transitions_pend_to_run);
     RUN_TEST(test_wjd_apply_committed_entry_transitions_and_sets_allocs);
+    RUN_TEST(test_wjd_apply_committed_entry_sets_start_time_and_allocs);
     RUN_TEST(test_wjd_apply_wal_entry_transitions_pend_to_run_no_hook);
     RUN_TEST(test_wjd_apply_wal_entry_transitions_and_sets_allocs);
+    RUN_TEST(test_wjd_apply_wal_entry_sets_start_time_and_allocs);
     RUN_TEST(test_wjd_restart_replay_recovers_running_without_start_fact);
     RUN_TEST(
         test_wjd_restart_replay_recovers_running_and_allocs_with_dispatch_fact);

src/controller/testsuite/test_wal_job_finish_fact.c

@@ -0,0 +1,865 @@
+/* test_wal_job_finish_fact.c — bug-1689 regression suite
+ *
+ * Proves WAL_JOB_FINISH's payload -- previously bare wal_jobstate_t{jobid,
+ * state}, exactly the same silent-field-drop bug class as bug-1678's
+ * WAL_JOB_DISPATCH and bug-1687's WAL_JOB_START -- now carries a
+ * receipt-time finish_time AND the worker-reported exit_code (job.exit_info)
+ * through BOTH of persist.c's apply seams:
+ *
+ *   - apply_committed_entry() (live-commit apply / boot replay dispatch)
+ *   - apply_wal_entry()       (standby catch-up / replicated-apply dispatch)
+ *
+ * and that a real, production on_job_finish() call (the actual worker-FINISH
+ * receipt handler) survives a real controller restart (WAL write +
+ * persist_recover() over a real on-disk WAL) with job.exit_info and
+ * job.finish_time (and the derived job.run_time) intact, not just
+ * job.state (Req: mirrors bug-1678/bug-1687's own acceptance criterion,
+ * "after recover the job should get the same as it's state before
+ * shutdown").
+ *
+ * Also proves backward compatibility: a WAL_JOB_FINISH entry with the OLD
+ * payload shape (exactly sizeof(wal_jobstate_t), no trailing bytes --
+ * simulating an entry written before this fix) still decodes cleanly, with
+ * exit_info left NULL and finish_time fabricated at apply time exactly as
+ * before this fix (apply_committed_entry) / left untouched exactly as
+ * before this fix (apply_wal_entry) -- no crash, no misread.
+ *
+ * job_exitinfo_t is a single int (exitcode) with NO existing Cap'n Proto
+ * mapping (confirmed -- no JobExitInfo struct in controller.capnp), so the
+ * trailing payload is a small NATIVE (non-capnp) fixed-size trailer,
+ * mirroring wal_jobstate_t's own native-struct style -- unlike bug-1687's
+ * WAL_JOB_START, which needed capnp for the variable-length JobPIDInfo.
+ *
+ * Fixtures mirror test_wal_job_start_fact.c's own conventions exactly:
+ *   - wjf_setup/wjf_teardown: bare cluster_t (jobmgr + hooksys) + in-memory
+ *     persist_t.
+ *   - wjf_make_fresh_persist/wjf_teardown_persist: real disk-backed
+ *     persist_t + persist_recover(), for restart-replay tests.
+ */
+#include <rtti.h>
+
+#include <capnp_c.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unity/unity.h>
+
+#include "cluster.h"
+#include "controller.capnp.h"
+#include "hook.h"
+#include "jobj.h"
+#include "persist.h"
+#include "wal_types.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+#include "xcealloc.h"
+#include "xstring.h"
+
+/* jobj.c's jobsub table and the process-global xstring table are shared
+ * across every test file in the controller_unit_test binary — guard
+ * idempotently, mirroring every sibling fixture in this suite. */
+static int g_wjf_jobobj_ready;
+
+static cluster_t g_wjf_c;
+static persist_t g_wjf_p;
+
+/* ── Spy hook: counts AFTER_EVENT firings, ignores all payload args ─────── */
+static int g_wjf_hook_fires;
+
+static int wjf_spy_hook(int type, int situation, int detail, void* data,
+                        va_list ap)
+{
+    (void)type; (void)detail; (void)data; (void)ap;
+    if (situation == HOOK_AFTER_EVENT)
+        g_wjf_hook_fires++;
+    return 0;
+}
+
+/* ── bug-1731: detail-aware spy hook, distinguishing HOOK_DETAIL_JOB_EXIT
+ * from HOOK_DETAIL_JOB_UPDATE and capturing the JOB_EXIT payload -- unlike
+ * wjf_spy_hook above (which just counts every AFTER_EVENT fire regardless
+ * of detail), this proves the NEW fire happens (not just "a" fire) and that
+ * the pre-existing JOB_UPDATE fire is not replaced or interfered with. Arg
+ * shapes mirror sched_job_exit_fun/apiserver_job_exit_fun's own va_arg
+ * reads (cluster_t*, xce_jobfinish_t*) and persist_job_hook_fun's
+ * HOOK_DETAIL_JOB_UPDATE case (cluster_t*, job_t*). */
+static int             g_wjf_exit_hook_fires;
+static int             g_wjf_update_hook_fires;
+static xce_jobfinish_t g_wjf_captured_finish;
+static int             g_wjf_captured_finish_valid;
+
+static int wjf_detail_spy_hook(int type, int situation, int detail,
+                               void* data, va_list ap)
+{
+    (void)type; (void)data;
+    if (situation != HOOK_AFTER_EVENT)
+        return 0;
+
+    switch (detail) {
+    case HOOK_DETAIL_JOB_EXIT: {
+        (void)va_arg(ap, cluster_t*);
+        xce_jobfinish_t* info = va_arg(ap, xce_jobfinish_t*);
+        g_wjf_exit_hook_fires++;
+        if (info != NULL) {
+            g_wjf_captured_finish       = *info;
+            g_wjf_captured_finish_valid = 1;
+        }
+        break;
+    }
+    case HOOK_DETAIL_JOB_UPDATE:
+        g_wjf_update_hook_fires++;
+        break;
+    default:
+        break;
+    }
+    return 0;
+}
+
+static void wjf_setup(void)
+{
+    kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                  .nslots   = 32,
+                                  .delta    = 16 };
+
+    if (makeString((char*)"wjf-probe") == NULL)
+        xstring_table_init(64);
+
+    if (!g_wjf_jobobj_ready) {
+        TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
+        g_wjf_jobobj_ready = 1;
+    }
+
+    memset(&g_wjf_c, 0, sizeof(g_wjf_c));
+    TEST_ASSERT_EQUAL_INT(0,
+        xce_hook_init(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
+
+    g_wjf_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+    TEST_ASSERT_NOT_NULL(g_wjf_c.jobmgr);
+
+    memset(&g_wjf_p, 0, sizeof(g_wjf_p));
+    g_wjf_p.cluster = &g_wjf_c;
+
+    g_wjf_hook_fires = 0;
+}
+
+static void wjf_teardown(void)
+{
+    xce_hook_fini(&g_wjf_c.hooksys);
+    if (g_wjf_c.jobmgr)
+        dereference(g_wjf_c.jobmgr);
+}
+
+/* Look up a job_t wrapper by jobid (NULL if absent). */
+static job_t* wjf_find_job(uint64_t jobid)
+{
+    job_t* j = NULL;
+    lock(g_wjf_c.jobmgr, RTTI_LOCK_READ);
+    int found = key_find(g_wjf_c.jobmgr, (void*)jobid, (void**)&j);
+    unlock(g_wjf_c.jobmgr);
+    return (found > 0) ? j : NULL;
+}
+
+/* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay
+ * entirely — for seeding fact-transition tests, mirroring
+ * test_wal_job_start_fact.c's wjs_seed_job. seed_start_time lets FINISH
+ * tests seed a known job.start_time so run_time is derivable/assertable. */
+static job_submit_data_t g_wjf_seed_submit;
+
+static job_t* wjf_seed_job(uint64_t jobid, uint32_t state,
+                           time_t seed_start_time)
+{
+    xce_job_t jdef;
+
+    memset(&g_wjf_seed_submit, 0, sizeof(g_wjf_seed_submit));
+    g_wjf_seed_submit.queue = (char*)"q";
+    g_wjf_seed_submit.user  = (char*)"tester";
+
+    memset(&jdef, 0, sizeof(jdef));
+    jdef.jobid      = jobid;
+    jdef.state      = state;
+    jdef.start_time = seed_start_time;
+    jdef.submit     = &g_wjf_seed_submit;
+
+    return makeJob(g_wjf_c.jobmgr, &jdef);
+}
+
+/* Build a wal_jobstate_t header + fixed uint64_t finish_time + fixed
+ * int32_t exit_code -- persist_submit_job_finish_fact()'s payload shape.
+ * No capnp needed (job_exitinfo_t is a single int with no existing capnp
+ * mapping), unlike wjs_build_start_payload()'s JobPIDInfo trailer. Caller
+ * owns *out. */
+static void wjf_build_finish_payload(uint64_t jobid, uint32_t state,
+                                     uint64_t finish_time, int32_t exit_code,
+                                     uint8_t** out, size_t* out_len)
+{
+    wal_jobstate_t js = { .jobid = jobid, .state = state, ._pad = 0 };
+
+    *out_len = sizeof(js) + sizeof(finish_time) + sizeof(exit_code);
+    *out     = malloc(*out_len);
+    TEST_ASSERT_NOT_NULL(*out);
+    memcpy(*out, &js, sizeof(js));
+    memcpy(*out + sizeof(js), &finish_time, sizeof(finish_time));
+    memcpy(*out + sizeof(js) + sizeof(finish_time), &exit_code,
+          sizeof(exit_code));
+}
+
+static void wjf_assert_exit_info_matches(job_exitinfo_t* got,
+                                         int32_t expected_exitcode)
+{
+    TEST_ASSERT_NOT_NULL_MESSAGE(got,
+        "a WAL_JOB_FINISH payload with a trailing exit_code must set "
+        "job.exit_info (bug-1689), not just state");
+    TEST_ASSERT_EQUAL_INT(expected_exitcode, got->exitcode);
+}
+
+/* ── apply_committed_entry(): live-commit apply / boot replay dispatch ──── */
+
+static void test_wjf_apply_committed_entry_sets_finish_time_and_exit_info(void)
+{
+    wjf_setup();
+
+    time_t seed_start = 1699999000;
+    job_t* j = wjf_seed_job(73001, JOB_STAT_RUN, seed_start);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint64_t wire_finish_time = 1700000050ULL;
+    int32_t  wire_exit_code   = 17;
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjf_build_finish_payload(73001, JOB_STAT_EXIT, wire_finish_time,
+                             wire_exit_code, &payload, &plen);
+
+    xce_add_hook(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjf_spy_hook,
+                NULL);
+
+    int rc = apply_committed_entry(&g_wjf_p, WAL_JOB_FINISH, 300, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(wire_finish_time,
+        (uint64_t)j->job.finish_time,
+        "WAL_JOB_FINISH must adopt the wire-carried finish_time (bug-1689), "
+        "not fabricate time(NULL) when one is present");
+    wjf_assert_exit_info_matches(j->job.exit_info, wire_exit_code);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        (int)(wire_finish_time - (uint64_t)seed_start), j->job.run_time,
+        "run_time must be re-derived from the wire-carried finish_time minus "
+        "start_time");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(2, g_wjf_hook_fires,
+        "bug-1731: a forward WAL_JOB_FINISH transition now fires TWO "
+        "notifications, not one -- the pre-existing HOOK_DETAIL_JOB_UPDATE "
+        "plus the new HOOK_DETAIL_JOB_EXIT; wjf_spy_hook counts every "
+        "AFTER_EVENT detail, so both land here. See "
+        "test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once "
+        "below for a detail-aware assertion isolating the new fire "
+        "specifically");
+
+    wjf_teardown();
+    printf("test_wjf_apply_committed_entry_sets_finish_time_and_exit_info: "
+          "PASS\n");
+}
+
+static void
+test_wjf_apply_committed_entry_backward_compat_old_payload(void)
+{
+    wjf_setup();
+
+    time_t seed_start = 1699999000;
+    job_t* j = wjf_seed_job(73002, JOB_STAT_RUN, seed_start);
+    TEST_ASSERT_NOT_NULL(j);
+    TEST_ASSERT_NULL_MESSAGE(j->job.exit_info,
+        "fixture must seed a job with exit_info still unset");
+
+    wal_jobstate_t js = { .jobid = 73002, .state = JOB_STAT_DONE, ._pad = 0 };
+
+    time_t before = time(NULL);
+    int rc = apply_committed_entry(&g_wjf_p, WAL_JOB_FINISH, 301, &js,
+                                   sizeof(js));
+    time_t after = time(NULL);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, j->job.state);
+    TEST_ASSERT_TRUE_MESSAGE(
+        (j->job.finish_time >= before) && (j->job.finish_time <= after),
+        "an old-format (exactly sizeof(wal_jobstate_t)) WAL_JOB_FINISH "
+        "payload must still fabricate finish_time at apply time, unchanged "
+        "from before this fix");
+    TEST_ASSERT_NULL_MESSAGE(j->job.exit_info,
+        "an old-format WAL_JOB_FINISH payload must leave exit_info NULL -- "
+        "no crash, no misread of absent trailing bytes");
+
+    wjf_teardown();
+    printf("test_wjf_apply_committed_entry_backward_compat_old_payload: "
+          "PASS\n");
+}
+
+/* bug-1731's primary regression test: apply_committed_entry()'s
+ * WAL_JOB_START/FINISH/... shared case fires exactly ONE generic
+ * HOOK_DETAIL_JOB_UPDATE at HOOK_AFTER_EVENT for every job-state fact,
+ * including WAL_JOB_FINISH -- but never the type-specific
+ * HOOK_DETAIL_JOB_EXIT, so the apiserver's fully-wired `case
+ * NotifyData_data_jobFinishInfo:` (apiserver_job_exit_fun -> OnJobFinished)
+ * was permanently starved of data. This is now fixed: a genuine forward
+ * WAL_JOB_FINISH transition (result == JOB_TRANSITION_APPLIED) also fires
+ * HOOK_DETAIL_JOB_EXIT with a real xce_jobfinish_t{jobid, exit_code} -- RED
+ * before the fix (g_wjf_exit_hook_fires stays 0), GREEN after. Also
+ * confirms the pre-existing HOOK_DETAIL_JOB_UPDATE fire is untouched
+ * (regression guard -- the new fire must not replace or interfere with
+ * it). */
+static void
+test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once(void)
+{
+    wjf_setup();
+    g_wjf_exit_hook_fires       = 0;
+    g_wjf_update_hook_fires     = 0;
+    g_wjf_captured_finish_valid = 0;
+    memset(&g_wjf_captured_finish, 0, sizeof(g_wjf_captured_finish));
+
+    time_t seed_start = 1700100000;
+    job_t* j = wjf_seed_job(73010, JOB_STAT_RUN, seed_start);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint64_t wire_finish_time = 1700100050ULL;
+    int32_t  wire_exit_code   = 7;
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjf_build_finish_payload(73010, JOB_STAT_EXIT, wire_finish_time,
+                             wire_exit_code, &payload, &plen);
+
+    xce_add_hook(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                wjf_detail_spy_hook, NULL);
+
+    int rc = apply_committed_entry(&g_wjf_p, WAL_JOB_FINISH, 400, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wjf_exit_hook_fires,
+        "bug-1731: a genuine forward WAL_JOB_FINISH apply must fire "
+        "HOOK_DETAIL_JOB_EXIT at HOOK_AFTER_EVENT exactly once -- this is "
+        "the ONLY mechanism that drives the apiserver's OnJobFinished, "
+        "which was permanently starved of data before this fix (the dead "
+        "handler pair sched_job_exit_fun/apiserver_job_exit_fun only ever "
+        "subscribes to HOOK_AFTER_EVENT, but job.c only ever fires this "
+        "detail at HOOK_BEFORE_EVENT)");
+    TEST_ASSERT_TRUE_MESSAGE(g_wjf_captured_finish_valid,
+        "HOOK_DETAIL_JOB_EXIT must fire with a non-NULL xce_jobfinish_t*");
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(73010, g_wjf_captured_finish.jobid,
+        "the fired xce_jobfinish_t's jobid must match the applied job");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(wire_exit_code,
+        g_wjf_captured_finish.exit_code,
+        "the fired xce_jobfinish_t's exit_code must match the wire-carried "
+        "exit_code this WAL_JOB_FINISH payload encoded");
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wjf_update_hook_fires,
+        "bug-1731 regression guard: the pre-existing HOOK_DETAIL_JOB_UPDATE "
+        "fire (sched_job_update_fun/apiserver_job_update_fun's live path) "
+        "must still happen too -- the new JOB_EXIT fire must not replace or "
+        "interfere with it");
+
+    xce_disable_hook(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                     wjf_detail_spy_hook);
+    wjf_teardown();
+    printf(
+        "test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once: "
+        "PASS\n");
+}
+
+/* bug-1731 idempotency guard (Constraints: "re-applying an already-applied
+ * WAL_JOB_FINISH should hit the existing result != JOB_TRANSITION_APPLIED
+ * early-return and fire NOTHING, exactly as it does today for
+ * HOOK_DETAIL_JOB_UPDATE"): a job already in a terminal state re-receiving
+ * the identical WAL_JOB_FINISH fact (simulating a restart-replay/standby
+ * catch-up re-application of an already-applied entry) must fire NEITHER
+ * the new HOOK_DETAIL_JOB_EXIT NOR the pre-existing HOOK_DETAIL_JOB_UPDATE
+ * -- the new fire sits inside the SAME `result == JOB_TRANSITION_APPLIED`
+ * guard as JOB_UPDATE, so it inherits this idempotency for free. */
+static void
+test_wjf_apply_committed_entry_reapply_finish_fires_no_hook(void)
+{
+    wjf_setup();
+    g_wjf_exit_hook_fires   = 0;
+    g_wjf_update_hook_fires = 0;
+
+    time_t seed_start = 1700200000;
+    /* Already terminal -- as if a prior WAL_JOB_FINISH already applied. */
+    job_t* j = wjf_seed_job(73011, JOB_STAT_EXIT, seed_start);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint64_t wire_finish_time = 1700200050ULL;
+    int32_t  wire_exit_code   = 3;
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjf_build_finish_payload(73011, JOB_STAT_EXIT, wire_finish_time,
+                             wire_exit_code, &payload, &plen);
+
+    xce_add_hook(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                wjf_detail_spy_hook, NULL);
+
+    int rc = apply_committed_entry(&g_wjf_p, WAL_JOB_FINISH, 401, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjf_exit_hook_fires,
+        "bug-1731: re-applying an already-terminal WAL_JOB_FINISH "
+        "(rejected-backward/no-op-duplicate) must fire NO "
+        "HOOK_DETAIL_JOB_EXIT -- confirms the new fire inherits the "
+        "existing apply-once idempotency guard for free");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjf_update_hook_fires,
+        "the pre-existing HOOK_DETAIL_JOB_UPDATE fire must likewise stay "
+        "silent on a duplicate re-apply, unchanged from before this fix");
+
+    xce_disable_hook(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_JOB,
+                     wjf_detail_spy_hook);
+    wjf_teardown();
+    printf(
+        "test_wjf_apply_committed_entry_reapply_finish_fires_no_hook: "
+        "PASS\n");
+}
+
+/* ── apply_wal_entry(): standby catch-up / replicated-apply dispatch ────── */
+
+static void
+test_wjf_apply_wal_entry_sets_finish_time_and_exit_info_no_hook(void)
+{
+    wjf_setup();
+
+    time_t seed_start = 1699999100;
+    job_t* j = wjf_seed_job(73003, JOB_STAT_RUN, seed_start);
+    TEST_ASSERT_NOT_NULL(j);
+
+    uint64_t wire_finish_time = 1700000150ULL;
+    int32_t  wire_exit_code   = 0;
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    wjf_build_finish_payload(73003, JOB_STAT_DONE, wire_finish_time,
+                             wire_exit_code, &payload, &plen);
+
+    xce_add_hook(&g_wjf_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wjf_spy_hook,
+                NULL);
+
+    int rc = apply_wal_entry(&g_wjf_p, WAL_JOB_FINISH, 302, payload,
+                             (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, j->job.state);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(wire_finish_time,
+        (uint64_t)j->job.finish_time,
+        "apply_wal_entry's catch-up dispatch must also recover the "
+        "wire-carried finish_time (bug-1689), required for a promoted "
+        "standby");
+    wjf_assert_exit_info_matches(j->job.exit_info, wire_exit_code);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        (int)(wire_finish_time - (uint64_t)seed_start), j->job.run_time,
+        "apply_wal_entry's catch-up dispatch must also re-derive run_time "
+        "from the wire-carried finish_time");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wjf_hook_fires,
+        "apply_wal_entry's catch-up dispatch must never fire a notification "
+        "hook");
+
+    wjf_teardown();
+    printf("test_wjf_apply_wal_entry_sets_finish_time_and_exit_info_no_hook: "
+          "PASS\n");
+}
+
+static void test_wjf_apply_wal_entry_backward_compat_old_payload(void)
+{
+    wjf_setup();
+
+    time_t seed_start = 1699999100;
+    job_t* j = wjf_seed_job(73004, JOB_STAT_RUN, seed_start);
+    TEST_ASSERT_NOT_NULL(j);
+    j->job.finish_time = 54321; /* sentinel -- must survive untouched */
+    j->job.run_time    = -1;    /* sentinel -- must survive untouched */
+
+    wal_jobstate_t js = { .jobid = 73004, .state = JOB_STAT_DONE, ._pad = 0 };
+
+    int rc = apply_wal_entry(&g_wjf_p, WAL_JOB_FINISH, 303, &js, sizeof(js));
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_DONE, j->job.state);
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(54321, (uint64_t)j->job.finish_time,
+        "an old-format WAL_JOB_FINISH payload must leave finish_time "
+        "untouched on this seam, unchanged from before this fix");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(-1, j->job.run_time,
+        "an old-format WAL_JOB_FINISH payload must leave run_time untouched "
+        "on this seam, unchanged from before this fix");
+    TEST_ASSERT_NULL_MESSAGE(j->job.exit_info,
+        "an old-format WAL_JOB_FINISH payload must leave exit_info NULL");
+
+    wjf_teardown();
+    printf("test_wjf_apply_wal_entry_backward_compat_old_payload: PASS\n");
+}
+
+/* ── Restart-replay Observables ──────────────────────────────────────── */
+
+static persist_t* wjf_make_fresh_persist(const char* tmpdir)
+{
+    persist_t* p = calloc(1, sizeof(persist_t));
+    if (!p) return NULL;
+
+    persist_config_t cfg = persist_config_defaults();
+    cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
+
+    persist_set_workdir(p, tmpdir, "ctrl-test");
+
+    if (persist_init(p, &cfg, NULL) != 0) {
+        free(p);
+        return NULL;
+    }
+    if (persist_start(p) != 0) {
+        persist_fini(p);
+        free(p);
+        return NULL;
+    }
+    return p;
+}
+
+static void wjf_teardown_persist(persist_t* p)
+{
+    persist_stop(p);
+    persist_fini(p);
+    free(p);
+}
+
+/* Encode helper mirroring test_wal_job_start_fact.c's wjs_encode_job
+ * (capnp_encode_job is file-static in persist.c). */
+static int wjf_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
+{
+    struct capn          c;
+    struct capn_segment* cs;
+    struct Job           cj;
+    Job_ptr              jp;
+
+    capn_init_malloc(&c);
+    cs = capn_root(&c).seg;
+    encode_Job(cs, &cj, (xce_job_t*)j);
+    jp = new_Job(cs);
+    write_Job(&cj, jp);
+    capn_setp(capn_root(&c), 0, jp.p);
+
+    *out_len = capn_size(&c);
+    *out     = malloc(*out_len);
+    if (!*out) { capn_free(&c); return -ENOMEM; }
+    capn_write_mem(&c, *out, *out_len, 0);
+    capn_free(&c);
+    return 0;
+}
+
+/* Direct-payload restart replay: appends WAL_JOB_SUBMIT + an old-format
+ * WAL_JOB_START (just to give the job a start_time so run_time is
+ * derivable -- WAL_JOB_START's own richness is bug-1687's concern, not this
+ * file's) + a new-format WAL_JOB_FINISH (real exit_code + a fixed
+ * finish_time) straight onto disk via persist_wal_async, then replays via a
+ * fresh persist_t's persist_recover() (a real restart). Isolates the WAL
+ * fact shape from the live caller, mirroring test_wal_job_start_fact.c's own
+ * restart-replay test. */
+static void test_wjf_restart_replay_recovers_finish_time_and_exit_info(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_wjf_replay_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    uint64_t wire_finish_time = 1700019999ULL;
+    int32_t  wire_exit_code   = 42;
+
+    /* Phase 1: write the log to real disk. */
+    {
+        persist_t* p = wjf_make_fresh_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p);
+
+        job_submit_data_t submit;
+        memset(&submit, 0, sizeof(submit));
+        submit.queue = (char*)"q";
+        submit.user  = (char*)"tester";
+
+        xce_job_t jdef;
+        memset(&jdef, 0, sizeof(jdef));
+        jdef.jobid  = 73005;
+        jdef.state  = JOB_STAT_PEND;
+        jdef.submit = &submit;
+
+        uint8_t* job_payload = NULL;
+        size_t   job_plen    = 0;
+        TEST_ASSERT_EQUAL_INT(0,
+            wjf_encode_job(&jdef, &job_payload, &job_plen));
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_SUBMIT, job_payload,
+                              (uint32_t)job_plen));
+        free(job_payload);
+
+        /* Old-format WAL_JOB_START — just enough to move the job to RUN and
+         * give it SOME start_time (fabricated at apply time); run_time's
+         * correctness is checked below as a derivation from whatever
+         * start_time replay actually produces, not a hardcoded literal. */
+        wal_jobstate_t start_js = { .jobid = 73005, .state = JOB_STAT_RUN,
+                                    ._pad = 0 };
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_START, &start_js, sizeof(start_js)));
+
+        uint8_t* finish_payload = NULL;
+        size_t   finish_plen    = 0;
+        wjf_build_finish_payload(73005, JOB_STAT_EXIT, wire_finish_time,
+                                 wire_exit_code, &finish_payload,
+                                 &finish_plen);
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_FINISH, finish_payload,
+                              (uint32_t)finish_plen));
+        free(finish_payload);
+
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
+        wjf_teardown_persist(p);
+    }
+
+    /* Phase 2: fresh persist_t + fresh cluster_t, recover from disk —
+     * simulates a controller restart. */
+    {
+        persist_t* p2 = wjf_make_fresh_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p2);
+
+        wjf_setup();
+        p2->cluster = &g_wjf_c;
+
+        int rc = persist_recover(p2);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        job_t* j = wjf_find_job(73005);
+        TEST_ASSERT_NOT_NULL_MESSAGE(j,
+            "restart replay must construct the job via WAL_JOB_SUBMIT");
+        TEST_ASSERT_EQUAL_UINT32(JOB_STAT_EXIT, j->job.state);
+        TEST_ASSERT_EQUAL_UINT64_MESSAGE(wire_finish_time,
+            (uint64_t)j->job.finish_time,
+            "restart replay must recover the exact wire-carried finish_time "
+            "(bug-1689), not a freshly fabricated time(NULL)");
+        wjf_assert_exit_info_matches(j->job.exit_info, wire_exit_code);
+        TEST_ASSERT_EQUAL_INT_MESSAGE(
+            (int)(wire_finish_time - (uint64_t)j->job.start_time),
+            j->job.run_time,
+            "restart replay must re-derive run_time from the recovered "
+            "finish_time and start_time");
+
+        wjf_teardown();
+        wjf_teardown_persist(p2);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    printf("test_wjf_restart_replay_recovers_finish_time_and_exit_info: "
+          "PASS\n");
+}
+
+/* bug-1689's primary regression test: drives the REAL production
+ * on_job_finish() path (the actual worker-FINISH receipt handler --
+ * record_job_finish_fact()'s only caller) with a real xce_jobfinish_t,
+ * through a real WAL-write + persist_recover() restart cycle -- the exact
+ * technique test_wjs_real_on_job_start_survives_restart (bug-1687) already
+ * established. Asserts the job's exit_info AND finish_time survive the
+ * restart intact, not just state -- the audit's exact confirmed root
+ * cause. */
+static void test_wjf_real_on_job_finish_survives_restart(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_wjf_onfinish_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    const uint64_t jobid          = 73006;
+    const time_t   seed_start     = 1699999500;
+    const int      real_exit_code = 137;
+    time_t before_call, after_call;
+
+    /* Phase 1: real cluster + real disk persist. Seed WAL_JOB_SUBMIT
+     * directly (so replay can reconstruct the job) AND a live in-memory job
+     * object already RUN with a start_time (so the real on_job_finish() call
+     * site has something to find/lock and a meaningful run_time can be
+     * derived) -- mirrors test_wjs_real_on_job_start_survives_restart's
+     * dual-seed pattern exactly. Then drive an actual on_job_finish() call
+     * and let IT append WAL_JOB_FINISH -- this is the production code path
+     * under test, not a hand-built WAL entry. */
+    {
+        persist_config_t cfg;
+
+        if (makeString((char*)"wjf-onfinish-probe") == NULL)
+            xstring_table_init(64);
+        if (!g_wjf_jobobj_ready) {
+            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
+            g_wjf_jobobj_ready = 1;
+        }
+
+        cluster_t c;
+        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                      .nslots = 32, .delta = 16 };
+        memset(&c, 0, sizeof(c));
+        TEST_ASSERT_EQUAL_INT(0,
+            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
+        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+        TEST_ASSERT_NOT_NULL(c.jobmgr);
+
+        cfg             = persist_config_defaults();
+        cfg.wal_sync_ms = 5000;
+        persist_set_workdir(&c.persist, tmp, "ctrl-test");
+        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
+        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
+
+        /* Durable WAL_JOB_SUBMIT seed. */
+        {
+            job_submit_data_t wal_submit;
+            xce_job_t         wal_jdef;
+            uint8_t*          buf = NULL;
+            size_t            sz  = 0;
+
+            memset(&wal_submit, 0, sizeof(wal_submit));
+            wal_submit.queue = (char*)"oq";
+            wal_submit.user  = (char*)"tester";
+
+            memset(&wal_jdef, 0, sizeof(wal_jdef));
+            wal_jdef.jobid  = jobid;
+            wal_jdef.state  = JOB_STAT_PEND;
+            wal_jdef.submit = &wal_submit;
+
+            TEST_ASSERT_EQUAL_INT(0,
+                wjf_encode_job(&wal_jdef, &buf, &sz));
+            TEST_ASSERT_EQUAL_INT(0,
+                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
+                                  (uint32_t)sz));
+            free(buf);
+        }
+
+        /* Durable WAL_JOB_START seed (old-format is fine — bug-1687's own
+         * concern) so replay's own start_time is well-defined. */
+        {
+            wal_jobstate_t start_js = { .jobid = jobid, .state = JOB_STAT_RUN,
+                                        ._pad = 0 };
+            TEST_ASSERT_EQUAL_INT(0,
+                persist_wal_async(&c.persist, WAL_JOB_START, &start_js,
+                                  sizeof(start_js)));
+        }
+
+        /* Live in-memory job object, already RUN with a known start_time,
+         * for on_job_finish() to find/lock. */
+        {
+            job_submit_data_t submit;
+            xce_job_t         jdef;
+
+            memset(&submit, 0, sizeof(submit));
+            submit.queue = (char*)"oq";
+            submit.user  = (char*)"tester";
+
+            memset(&jdef, 0, sizeof(jdef));
+            jdef.jobid      = jobid;
+            jdef.state      = JOB_STAT_RUN;
+            jdef.start_time = seed_start;
+            jdef.submit     = &submit;
+
+            TEST_ASSERT_NOT_NULL(makeJob(c.jobmgr, &jdef));
+        }
+
+        /* The real worker-reported finish report. */
+        xce_jobfinish_t finish = { .jobid = jobid,
+                                   .exit_code = real_exit_code };
+
+        before_call = time(NULL);
+        int rc = on_job_finish(&c, &finish);
+        after_call = time(NULL);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
+
+        persist_stop(&c.persist);
+        persist_fini(&c.persist);
+        dereference(c.jobmgr);
+        xce_hook_fini(&c.hooksys);
+    }
+
+    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
+     * recover from disk — simulates a controller restart. */
+    {
+        persist_t* p2 = wjf_make_fresh_persist(tmp);
+        cluster_t  c2;
+        job_t*     j2 = NULL;
+        int        rc;
+        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                      .nslots = 32, .delta = 16 };
+
+        TEST_ASSERT_NOT_NULL(p2);
+
+        memset(&c2, 0, sizeof(c2));
+        TEST_ASSERT_EQUAL_INT(0,
+            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
+        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+        TEST_ASSERT_NOT_NULL(c2.jobmgr);
+
+        p2->cluster = &c2;
+        rc          = persist_recover(p2);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        lock(c2.jobmgr, RTTI_LOCK_READ);
+        key_find(c2.jobmgr, (void*)jobid, (void**)&j2);
+        unlock(c2.jobmgr);
+        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
+            "restart replay must reconstruct the job via WAL_JOB_SUBMIT");
+        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j2->job.state,
+            "restart replay must recover the job as EXIT (nonzero exit code) "
+            "from the WAL_JOB_FINISH fact appended by the real "
+            "on_job_finish() call");
+
+        /* bug-1689: restart replay must recover not just STATE but the
+         * worker-reported exit_info AND the true finish_time -- the audit's
+         * exact confirmed root cause (mirrors bug-1678/bug-1687's own
+         * acceptance criterion). */
+        wjf_assert_exit_info_matches(j2->job.exit_info, real_exit_code);
+        TEST_ASSERT_TRUE_MESSAGE(
+            (j2->job.finish_time >= before_call)
+                && (j2->job.finish_time <= after_call),
+            "restart replay must recover the true, receipt-time finish_time "
+            "captured by the real on_job_finish() call, not a fresh "
+            "apply-time time(NULL) fabrication");
+        TEST_ASSERT_EQUAL_INT_MESSAGE(
+            (int)(j2->job.finish_time - j2->job.start_time), j2->job.run_time,
+            "restart replay must re-derive run_time from the recovered "
+            "finish_time and start_time");
+
+        xce_hook_fini(&c2.hooksys);
+        if (c2.jobmgr) dereference(c2.jobmgr);
+        wjf_teardown_persist(p2);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    printf("test_wjf_real_on_job_finish_survives_restart: PASS\n");
+}
+
+/* ── Test group entry point ───────────────────────────────────────────── */
+
+void test_wal_job_finish_fact(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_wjf_apply_committed_entry_sets_finish_time_and_exit_info);
+    RUN_TEST(test_wjf_apply_committed_entry_backward_compat_old_payload);
+    RUN_TEST(test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once);
+    RUN_TEST(test_wjf_apply_committed_entry_reapply_finish_fires_no_hook);
+    RUN_TEST(test_wjf_apply_wal_entry_sets_finish_time_and_exit_info_no_hook);
+    RUN_TEST(test_wjf_apply_wal_entry_backward_compat_old_payload);
+    RUN_TEST(test_wjf_restart_replay_recovers_finish_time_and_exit_info);
+    RUN_TEST(test_wjf_real_on_job_finish_survives_restart);
+}

src/controller/testsuite/test_wal_job_rusage_fact.c

@@ -0,0 +1,912 @@
+/* test_wal_job_rusage_fact.c — bug-1690 regression suite
+ *
+ * Proves WAL_JOB_RUSAGE's payload -- previously bare wal_jobstate_t{jobid,
+ * state}, exactly the same silent-field-drop bug class as bug-1678
+ * (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+
+ * start_time) and bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time) --
+ * now carries the client-reported, accumulated per-host CPU/mem usage
+ * (job.run_usage, an xce_job_runusage_t*) through BOTH of persist.c's apply
+ * seams:
+ *
+ *   - apply_committed_entry() (live-commit apply / boot replay dispatch)
+ *   - apply_wal_entry()       (standby catch-up / replicated-apply dispatch)
+ *
+ * SECOND, EQUALLY IMPORTANT DIVERGENCE: adoption is a MERGE
+ * (merge_job_runusage(), job.c), never a free-and-replace. job.run_usage
+ * can be the job's LIVE structure -- update_job_rusages()/update_host_
+ * runusage() (job.c/misc.c) mutate it synchronously and in place, with a
+ * host_name sourced from makeString() (an RTTI-managed, header-before-data
+ * pointer, NOT a plain malloc()) -- and the primary's own live apply pump
+ * (persist_apply_pump_run()) re-applies that SAME fact, on that SAME live
+ * job, moments after the live mutation already happened. Freeing it via
+ * the capnp-generated free_JobRunusage_ptr() (written for plain malloc'd,
+ * capnp-decoded Text fields) would free a non-malloc'd pointer -- heap
+ * corruption under ASAN. test_wru_apply_committed_entry_merges_without_
+ * freeing_live_host below proves this directly with a real makeString()'d
+ * host_name.
+ *
+ * IMPORTANT DIVERGENCE FROM WAL_JOB_START/FINISH/DISPATCH's regression
+ * suites: a rusage report almost never changes job.state (the job stays RUN
+ * across every repeat report) -- unlike a START/FINISH/DISPATCH, which are
+ * genuine one-time PEND->RUN / RUN->terminal transitions. apply_committed_
+ * entry()'s shared job-fact group only stamps type-specific fields when
+ * apply_job_update_transition() reports JOB_TRANSITION_APPLIED (a forward
+ * state change) -- gating run_usage adoption on that same condition would
+ * make it dead code for the realistic, common case (every report after the
+ * first hits JOB_TRANSITION_NOOP_DUPLICATE). The fix decodes/adopts
+ * run_usage UNCONDITIONALLY, before that gate. test_wru_apply_committed_
+ * entry_sets_runusage_no_state_change below is this suite's most important
+ * test: it proves run_usage still lands even when state does NOT change --
+ * the scenario a naive copy of the START/FINISH/DISPATCH pattern would have
+ * silently failed to cover.
+ *
+ * xce_job_runusage_t (xce/types.h) had NO existing Cap'n Proto mapping at
+ * all -- unlike JobAlloc/JobPIDInfo, which bug-1678/1687 could reuse
+ * standalone from the existing Job struct's own sub-fields. This fix adds a
+ * genuine new schema pair, HostRunusage + JobRunusage (controller.capnp),
+ * distinct from the similar-looking but unrelated xce_job_rusages_t/
+ * JobRusages/SchedJobRusages (the client's WIRE UPDATE REQUEST shape).
+ *
+ * Also proves backward compatibility: a WAL_JOB_RUSAGE entry with the OLD
+ * payload shape (exactly sizeof(wal_jobstate_t), no trailing bytes --
+ * simulating an entry written before this fix, or one fired while
+ * job.run_usage was still NULL at write time) still decodes cleanly, with
+ * run_usage left untouched exactly as before this fix -- no crash, no
+ * misread.
+ *
+ * Fixtures mirror test_wal_job_start_fact.c's own conventions exactly:
+ *   - wru_setup/wru_teardown: bare cluster_t (jobmgr + hooksys) + in-memory
+ *     persist_t.
+ *   - wru_make_fresh_persist/wru_teardown_persist: real disk-backed
+ *     persist_t + persist_recover(), for restart-replay tests.
+ *
+ * test_wru_real_hook_appends_and_survives_restart drives the REAL,
+ * unmodified persist_job_hook_fun() (the actual production hook subscriber
+ * this fix changed) via the "variadic trampoline" pattern established by
+ * test_commit_barrier_rollout.c/test_persist_hooks.c, rather than going
+ * through update_job_rusages() (job.c) itself: this same controller_unit_
+ * test binary already links test_commit_barrier_reply.c, which defines its
+ * OWN non-static stub of update_host_runusage() (a no-op, since misc.c is
+ * not part of controller_unit_test_src) -- calling update_job_rusages()
+ * from this file would silently hit that no-op stub and never populate
+ * job.run_usage at all, testing nothing new. Driving persist_job_hook_fun()
+ * directly with a job_t whose run_usage is set on the fixture (bypassing
+ * update_host_runusage entirely, which this fix does not touch) exercises
+ * the actual changed function on the actual production code path a real
+ * update_job_rusages() call reaches it from.
+ */
+#include <rtti.h>
+
+#include <capnp_c.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <unity/unity.h>
+
+#include "cluster.h"
+#include "controller.capnp.h"
+#include "hook.h"
+#include "jobj.h"
+#include "persist.h"
+#include "wal_types.h"
+#include "xce/cluster_def.h"
+#include "xce/types.h"
+#include "xcealloc.h"
+#include "xstring.h"
+
+/* jobj.c's jobsub table and the process-global xstring table are shared
+ * across every test file in the controller_unit_test binary — guard
+ * idempotently, mirroring every sibling fixture in this suite. */
+static int g_wru_jobobj_ready;
+
+static cluster_t g_wru_c;
+static persist_t g_wru_p;
+
+/* ── Spy hook: counts AFTER_EVENT firings, ignores all payload args ─────── */
+static int g_wru_hook_fires;
+
+static int wru_spy_hook(int type, int situation, int detail, void* data,
+                        va_list ap)
+{
+    (void)type; (void)detail; (void)data; (void)ap;
+    if (situation == HOOK_AFTER_EVENT)
+        g_wru_hook_fires++;
+    return 0;
+}
+
+static void wru_setup(void)
+{
+    kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                  .nslots   = 32,
+                                  .delta    = 16 };
+
+    if (makeString((char*)"wru-probe") == NULL)
+        xstring_table_init(64);
+
+    if (!g_wru_jobobj_ready) {
+        TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
+        g_wru_jobobj_ready = 1;
+    }
+
+    memset(&g_wru_c, 0, sizeof(g_wru_c));
+    TEST_ASSERT_EQUAL_INT(0,
+        xce_hook_init(&g_wru_c.hooksys, XCE_CONTROLLER_HOOK_MAX));
+
+    g_wru_c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+    TEST_ASSERT_NOT_NULL(g_wru_c.jobmgr);
+
+    memset(&g_wru_p, 0, sizeof(g_wru_p));
+    g_wru_p.cluster = &g_wru_c;
+
+    g_wru_hook_fires = 0;
+}
+
+static void wru_teardown(void)
+{
+    xce_hook_fini(&g_wru_c.hooksys);
+    if (g_wru_c.jobmgr)
+        dereference(g_wru_c.jobmgr);
+}
+
+/* Look up a job_t wrapper by jobid (NULL if absent). */
+static job_t* wru_find_job(uint64_t jobid)
+{
+    job_t* j = NULL;
+    lock(g_wru_c.jobmgr, RTTI_LOCK_READ);
+    int found = key_find(g_wru_c.jobmgr, (void*)jobid, (void**)&j);
+    unlock(g_wru_c.jobmgr);
+    return (found > 0) ? j : NULL;
+}
+
+/* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay
+ * entirely — for seeding fact-transition tests, mirroring
+ * test_wal_job_dispatch_fact.c's wjd_seed_job / test_wal_job_start_fact.c's
+ * wjs_seed_job. */
+static job_submit_data_t g_wru_seed_submit;
+
+static job_t* wru_seed_job(uint64_t jobid, uint32_t state)
+{
+    xce_job_t jdef;
+
+    memset(&g_wru_seed_submit, 0, sizeof(g_wru_seed_submit));
+    g_wru_seed_submit.queue = (char*)"q";
+    g_wru_seed_submit.user  = (char*)"tester";
+
+    memset(&jdef, 0, sizeof(jdef));
+    jdef.jobid  = jobid;
+    jdef.state  = state;
+    jdef.submit = &g_wru_seed_submit;
+
+    return makeJob(g_wru_c.jobmgr, &jdef);
+}
+
+/* ── Real fixture data: 2 hosts' worth of accumulated CPU/mem usage ──────── */
+
+static xce_host_runusage_t g_wru_host_a = {
+    .host_name = (char*)"wru-host-a", .cpu_time = 1200, .mem_usage = 512.5
+};
+static xce_host_runusage_t g_wru_host_b = {
+    .host_name = (char*)"wru-host-b", .cpu_time = 900, .mem_usage = 256.25
+};
+static xce_host_runusage_t* g_wru_host_ptrs[2];
+
+static xce_job_runusage_t wru_make_test_runusage(void)
+{
+    xce_job_runusage_t r;
+
+    memset(&r, 0, sizeof(r));
+    r.max_mem            = 1024.5;
+    r.avg_mem             = 768.25;
+    r.used_mem            = 900.0;
+    r.cpu_time            = 2100;
+    r.peak_cpu_time       = 1500;
+    r.peak_cpu_duration   = 30;
+    r.mem_efficiency      = 0.75f;
+    r.avg_cpu_efficiency  = 0.6f;
+    r.peak_cpu_efficiency = 0.9f;
+    r.num_hosts           = 2;
+
+    g_wru_host_ptrs[0] = &g_wru_host_a;
+    g_wru_host_ptrs[1] = &g_wru_host_b;
+    r.host_runusage     = g_wru_host_ptrs;
+
+    return r;
+}
+
+static void wru_assert_runusage_matches(xce_job_runusage_t* got)
+{
+    TEST_ASSERT_NOT_NULL_MESSAGE(got,
+        "a WAL_JOB_RUSAGE payload with trailing JobRunusage bytes must set "
+        "job.run_usage (bug-1690), not just state");
+    TEST_ASSERT_EQUAL_FLOAT(1024.5, got->max_mem);
+    TEST_ASSERT_EQUAL_FLOAT(768.25, got->avg_mem);
+    TEST_ASSERT_EQUAL_FLOAT(900.0, got->used_mem);
+    TEST_ASSERT_EQUAL_INT(2100, got->cpu_time);
+    TEST_ASSERT_EQUAL_INT(1500, got->peak_cpu_time);
+    TEST_ASSERT_EQUAL_INT(30, got->peak_cpu_duration);
+    TEST_ASSERT_EQUAL_FLOAT(0.75f, got->mem_efficiency);
+    TEST_ASSERT_EQUAL_FLOAT(0.6f, got->avg_cpu_efficiency);
+    TEST_ASSERT_EQUAL_FLOAT(0.9f, got->peak_cpu_efficiency);
+    TEST_ASSERT_EQUAL_INT(2, got->num_hosts);
+    TEST_ASSERT_NOT_NULL(got->host_runusage);
+    TEST_ASSERT_EQUAL_STRING("wru-host-a", got->host_runusage[0]->host_name);
+    TEST_ASSERT_EQUAL_INT(1200, got->host_runusage[0]->cpu_time);
+    TEST_ASSERT_EQUAL_FLOAT(512.5, got->host_runusage[0]->mem_usage);
+    TEST_ASSERT_EQUAL_STRING("wru-host-b", got->host_runusage[1]->host_name);
+    TEST_ASSERT_EQUAL_INT(900, got->host_runusage[1]->cpu_time);
+    TEST_ASSERT_EQUAL_FLOAT(256.25, got->host_runusage[1]->mem_usage);
+}
+
+/* Build a wal_jobstate_t header + trailing Cap'n Proto JobRunusage bytes --
+ * persist_job_hook_fun's WAL_JOB_RUSAGE payload shape -- mirroring
+ * wjs_build_start_payload() (test_wal_job_start_fact.c) since persist.c's
+ * own static capnp_encode_jobrunusage() isn't linkable from a test TU.
+ * Caller owns *out. */
+static int wru_build_rusage_payload(uint64_t jobid, uint32_t state,
+                                    xce_job_runusage_t* r,
+                                    uint8_t** out, size_t* out_len)
+{
+    struct capn          c;
+    struct capn_segment* cs;
+    struct JobRunusage   cjr;
+    JobRunusage_ptr      jrp;
+    uint8_t*             rbuf;
+    size_t               rlen;
+    wal_jobstate_t       js = { .jobid = jobid, .state = state, ._pad = 0 };
+
+    capn_init_malloc(&c);
+    cs = capn_root(&c).seg;
+    encode_JobRunusage(cs, &cjr, r);
+    jrp = new_JobRunusage(cs);
+    write_JobRunusage(&cjr, jrp);
+    capn_setp(capn_root(&c), 0, jrp.p);
+
+    rlen = capn_size(&c);
+    rbuf = malloc(rlen);
+    if (!rbuf) { capn_free(&c); return -ENOMEM; }
+    capn_write_mem(&c, rbuf, rlen, 0);
+    capn_free(&c);
+
+    *out_len = sizeof(js) + rlen;
+    *out     = malloc(*out_len);
+    if (!*out) { free(rbuf); return -ENOMEM; }
+    memcpy(*out, &js, sizeof(js));
+    memcpy(*out + sizeof(js), rbuf, rlen);
+    free(rbuf);
+    return 0;
+}
+
+/* ── apply_committed_entry(): live-commit apply / boot replay dispatch ──── */
+
+/* THE key regression test: seeds the job already RUNNING and replays a
+ * WAL_JOB_RUSAGE fact whose own state is ALSO RUN -- the realistic,
+ * overwhelmingly common shape of a real rusage report (job.state never
+ * actually changes across repeat reports). apply_job_update_transition()
+ * reports JOB_TRANSITION_NOOP_DUPLICATE for this (state == cur), so a fix
+ * that (incorrectly) gated run_usage adoption on JOB_TRANSITION_APPLIED --
+ * mirroring WAL_JOB_START/FINISH/DISPATCH's own gated fields -- would leave
+ * job.run_usage NULL here. This test fails against that naive shape and
+ * passes against the actual fix (unconditional decode/adopt). */
+static void test_wru_apply_committed_entry_sets_runusage_no_state_change(void)
+{
+    wru_setup();
+
+    job_t* j = wru_seed_job(73001, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+
+    xce_job_runusage_t r = wru_make_test_runusage();
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    TEST_ASSERT_EQUAL_INT(0,
+        wru_build_rusage_payload(73001, JOB_STAT_RUN, &r, &payload, &plen));
+
+    xce_add_hook(&g_wru_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wru_spy_hook,
+                NULL);
+
+    int rc = apply_committed_entry(&g_wru_p, WAL_JOB_RUSAGE, 300, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    wru_assert_runusage_matches(j->job.run_usage);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wru_hook_fires,
+        "a same-state (duplicate) RUSAGE replay must NOT fire a "
+        "notification hook -- unchanged from before this fix -- even though "
+        "run_usage WAS adopted");
+
+    wru_teardown();
+    printf("test_wru_apply_committed_entry_sets_runusage_no_state_change: "
+          "PASS\n");
+}
+
+/* Secondary scenario: a WAL_JOB_RUSAGE fact that DOES carry a forward state
+ * transition (mechanism-level test, mirroring how the START/DISPATCH suites
+ * use PEND->RUN generically). run_usage must be adopted here too, and the
+ * existing forward-transition hook-fire behavior must be unchanged. */
+static void
+test_wru_apply_committed_entry_sets_runusage_forward_transition(void)
+{
+    wru_setup();
+
+    job_t* j = wru_seed_job(73002, JOB_STAT_PEND);
+    TEST_ASSERT_NOT_NULL(j);
+
+    xce_job_runusage_t r = wru_make_test_runusage();
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    TEST_ASSERT_EQUAL_INT(0,
+        wru_build_rusage_payload(73002, JOB_STAT_RUN, &r, &payload, &plen));
+
+    xce_add_hook(&g_wru_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wru_spy_hook,
+                NULL);
+
+    int rc = apply_committed_entry(&g_wru_p, WAL_JOB_RUSAGE, 301, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    wru_assert_runusage_matches(j->job.run_usage);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_wru_hook_fires,
+        "a forward RUSAGE-carried transition must still fire exactly one "
+        "notification, unchanged from before this fix");
+
+    wru_teardown();
+    printf(
+        "test_wru_apply_committed_entry_sets_runusage_forward_transition: "
+        "PASS\n");
+}
+
+static void
+test_wru_apply_committed_entry_backward_compat_old_payload(void)
+{
+    wru_setup();
+
+    job_t* j = wru_seed_job(73003, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+    TEST_ASSERT_NULL_MESSAGE(j->job.run_usage,
+        "fixture must seed a job with run_usage still unset");
+
+    wal_jobstate_t js = { .jobid = 73003, .state = JOB_STAT_RUN, ._pad = 0 };
+
+    int rc = apply_committed_entry(&g_wru_p, WAL_JOB_RUSAGE, 302, &js,
+                                   sizeof(js));
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_NULL_MESSAGE(j->job.run_usage,
+        "an old-format (exactly sizeof(wal_jobstate_t)) WAL_JOB_RUSAGE "
+        "payload must leave run_usage NULL -- no crash, no misread of "
+        "absent trailing bytes");
+
+    wru_teardown();
+    printf("test_wru_apply_committed_entry_backward_compat_old_payload: "
+          "PASS\n");
+}
+
+/* ── Merge safety: never free a LIVE (possibly xstring-backed) host_name ──
+ *
+ * bug-1690's most important safety finding: update_job_rusages()/
+ * update_host_runusage() (job.c/misc.c) build the LIVE job.run_usage with
+ * host_name sourced from makeString() (src/lib/xstring.c) -- an
+ * RTTI-managed, header-before-data pointer (RTTI_DATA_TO_OBJ, librtti), NOT
+ * a plain malloc() allocation. apply_committed_entry()'s WAL_JOB_RUSAGE
+ * branch can run against this EXACT live structure -- the primary's own
+ * live apply pump (persist_apply_pump_run()) re-applies a fact moments
+ * after the fact's own live mutation already happened. A naive
+ * free-and-replace (the capnp-generated free_JobRunusage_ptr(), written for
+ * plain malloc'd Text fields) would free() a non-malloc'd pointer there:
+ * guaranteed heap corruption under ASAN. This test builds a run_usage with
+ * a REAL makeString()'d host_name (mirroring the live-mutation flavor
+ * exactly), then applies a WAL_JOB_RUSAGE fact reporting an update for that
+ * SAME host plus a brand-new second host, and asserts: (a) no crash
+ * (implicit -- ASAN aborts mid-test on a corrupting free), (b) the existing
+ * host's host_name POINTER IDENTITY survives (proving it was merged in
+ * place, never freed+reallocated), (c) its cpu_time/mem_usage reflect the
+ * new report, (d) the new host is correctly appended. */
+static void
+test_wru_apply_committed_entry_merges_without_freeing_live_host(void)
+{
+    wru_setup();
+
+    job_t* j = wru_seed_job(73008, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+
+    /* Build a LIVE-flavor run_usage by hand, mirroring update_host_
+     * runusage()'s own construction exactly: calloc'd struct/array/entries,
+     * but a REAL makeString()'d host_name -- the exact allocator mismatch
+     * this test exists to catch. */
+    char* live_host_name = makeString((char*)"wru-live-host");
+    TEST_ASSERT_NOT_NULL(live_host_name);
+
+    xce_job_runusage_t* live = (xce_job_runusage_t*)calloc(1, sizeof(*live));
+    TEST_ASSERT_NOT_NULL(live);
+    live->max_mem   = 100.0;
+    live->cpu_time  = 50;
+    live->num_hosts = 1;
+    live->host_runusage =
+        (xce_host_runusage_t**)calloc(1, sizeof(xce_host_runusage_t*));
+    TEST_ASSERT_NOT_NULL(live->host_runusage);
+    live->host_runusage[0] =
+        (xce_host_runusage_t*)calloc(1, sizeof(xce_host_runusage_t));
+    TEST_ASSERT_NOT_NULL(live->host_runusage[0]);
+    live->host_runusage[0]->host_name = live_host_name;
+    live->host_runusage[0]->cpu_time  = 111;
+    live->host_runusage[0]->mem_usage = 222.0;
+
+    j->job.run_usage = live;
+
+    /* Fact payload: an update for "wru-live-host" plus a brand-new
+     * "wru-new-host" -- exactly what a real subsequent rusage report for
+     * this job would carry. */
+    xce_host_runusage_t update_host = {
+        .host_name = (char*)"wru-live-host", .cpu_time = 999,
+        .mem_usage = 888.0
+    };
+    xce_host_runusage_t new_host = {
+        .host_name = (char*)"wru-new-host", .cpu_time = 42, .mem_usage = 84.0
+    };
+    xce_host_runusage_t* host_ptrs[2] = { &update_host, &new_host };
+
+    xce_job_runusage_t r;
+    memset(&r, 0, sizeof(r));
+    r.max_mem       = 500.0;
+    r.cpu_time      = 250;
+    r.num_hosts     = 2;
+    r.host_runusage = host_ptrs;
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    TEST_ASSERT_EQUAL_INT(0,
+        wru_build_rusage_payload(73008, JOB_STAT_RUN, &r, &payload, &plen));
+
+    int rc = apply_committed_entry(&g_wru_p, WAL_JOB_RUSAGE, 305, payload,
+                                   (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+
+    xce_job_runusage_t* got = j->job.run_usage;
+    TEST_ASSERT_NOT_NULL(got);
+    TEST_ASSERT_EQUAL_FLOAT(500.0, got->max_mem);
+    TEST_ASSERT_EQUAL_INT(250, got->cpu_time);
+    TEST_ASSERT_EQUAL_INT(2, got->num_hosts);
+
+    int live_idx = -1, found_new = 0;
+    for (int i = 0; i < got->num_hosts; i++) {
+        xce_host_runusage_t* h = got->host_runusage[i];
+        TEST_ASSERT_NOT_NULL(h);
+        if (h->host_name != NULL &&
+            strcmp(h->host_name, "wru-live-host") == 0) {
+            live_idx = i;
+            TEST_ASSERT_TRUE_MESSAGE(h->host_name == live_host_name,
+                "an existing host's host_name pointer must be preserved by "
+                "identity -- merge must never free+replace it (bug-1690 "
+                "heap-corruption hazard: it may be an xstring, not a "
+                "malloc'd pointer)");
+            TEST_ASSERT_EQUAL_INT(999, h->cpu_time);
+            TEST_ASSERT_EQUAL_FLOAT(888.0, h->mem_usage);
+        } else if (h->host_name != NULL &&
+                  strcmp(h->host_name, "wru-new-host") == 0) {
+            found_new = 1;
+            TEST_ASSERT_EQUAL_INT(42, h->cpu_time);
+            TEST_ASSERT_EQUAL_FLOAT(84.0, h->mem_usage);
+        }
+    }
+    TEST_ASSERT_TRUE_MESSAGE(live_idx >= 0,
+        "existing host must survive the merge");
+    TEST_ASSERT_TRUE_MESSAGE(found_new,
+        "new host must be appended by the merge");
+
+    /* Cleanup: job_destroy() (jobj.c) releases run_usage via a manual
+     * FREEUP-based walk that (independent of this fix -- a separate,
+     * pre-existing, out-of-scope issue) assumes every host_name is plain
+     * malloc'd. Detach the still-live, still-xstring-backed host_name
+     * before teardown so THAT unrelated issue can't fire inside this
+     * test's own cleanup, and release it correctly ourselves. */
+    if (live_idx >= 0)
+        got->host_runusage[live_idx]->host_name = NULL;
+    dereference(live_host_name);
+
+    wru_teardown();
+    printf(
+        "test_wru_apply_committed_entry_merges_without_freeing_live_host: "
+        "PASS\n");
+}
+
+/* ── apply_wal_entry(): standby catch-up / replicated-apply dispatch ────── */
+
+static void test_wru_apply_wal_entry_sets_runusage_no_hook(void)
+{
+    wru_setup();
+
+    job_t* j = wru_seed_job(73004, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+
+    xce_job_runusage_t r = wru_make_test_runusage();
+
+    uint8_t* payload = NULL;
+    size_t   plen    = 0;
+    TEST_ASSERT_EQUAL_INT(0,
+        wru_build_rusage_payload(73004, JOB_STAT_RUN, &r, &payload, &plen));
+
+    xce_add_hook(&g_wru_c.hooksys, XCE_CONTROLLER_HOOK_JOB, wru_spy_hook,
+                NULL);
+
+    int rc = apply_wal_entry(&g_wru_p, WAL_JOB_RUSAGE, 303, payload,
+                             (uint32_t)plen);
+    free(payload);
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    wru_assert_runusage_matches(j->job.run_usage);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_wru_hook_fires,
+        "apply_wal_entry's catch-up dispatch must never fire a "
+        "notification hook");
+
+    wru_teardown();
+    printf("test_wru_apply_wal_entry_sets_runusage_no_hook: PASS\n");
+}
+
+static void test_wru_apply_wal_entry_backward_compat_old_payload(void)
+{
+    wru_setup();
+
+    job_t* j = wru_seed_job(73005, JOB_STAT_RUN);
+    TEST_ASSERT_NOT_NULL(j);
+    TEST_ASSERT_NULL_MESSAGE(j->job.run_usage,
+        "fixture must seed a job with run_usage still unset");
+
+    wal_jobstate_t js = { .jobid = 73005, .state = JOB_STAT_RUN, ._pad = 0 };
+
+    int rc = apply_wal_entry(&g_wru_p, WAL_JOB_RUSAGE, 304, &js, sizeof(js));
+
+    TEST_ASSERT_EQUAL_INT(0, rc);
+    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+    TEST_ASSERT_NULL_MESSAGE(j->job.run_usage,
+        "an old-format WAL_JOB_RUSAGE payload must leave run_usage NULL on "
+        "this seam, unchanged from before this fix");
+
+    wru_teardown();
+    printf("test_wru_apply_wal_entry_backward_compat_old_payload: PASS\n");
+}
+
+/* ── Restart-replay Observables ──────────────────────────────────────── */
+
+static persist_t* wru_make_fresh_persist(const char* tmpdir)
+{
+    persist_t* p = calloc(1, sizeof(persist_t));
+    if (!p) return NULL;
+
+    persist_config_t cfg = persist_config_defaults();
+    cfg.wal_sync_ms = 5000; /* slow fsync — no races in tests */
+
+    persist_set_workdir(p, tmpdir, "ctrl-test");
+
+    if (persist_init(p, &cfg, NULL) != 0) {
+        free(p);
+        return NULL;
+    }
+    if (persist_start(p) != 0) {
+        persist_fini(p);
+        free(p);
+        return NULL;
+    }
+    return p;
+}
+
+static void wru_teardown_persist(persist_t* p)
+{
+    persist_stop(p);
+    persist_fini(p);
+    free(p);
+}
+
+/* Encode helper mirroring test_wal_job_start_fact.c's wjs_encode_job
+ * (capnp_encode_job is file-static in persist.c). */
+static int wru_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
+{
+    struct capn          c;
+    struct capn_segment* cs;
+    struct Job           cj;
+    Job_ptr              jp;
+
+    capn_init_malloc(&c);
+    cs = capn_root(&c).seg;
+    encode_Job(cs, &cj, (xce_job_t*)j);
+    jp = new_Job(cs);
+    write_Job(&cj, jp);
+    capn_setp(capn_root(&c), 0, jp.p);
+
+    *out_len = capn_size(&c);
+    *out     = malloc(*out_len);
+    if (!*out) { capn_free(&c); return -ENOMEM; }
+    capn_write_mem(&c, *out, *out_len, 0);
+    capn_free(&c);
+    return 0;
+}
+
+/* Direct-payload restart replay: appends WAL_JOB_SUBMIT + a new-format
+ * WAL_JOB_RUSAGE (real run_usage, same-state RUN->RUN -- the realistic
+ * shape) straight onto disk via persist_wal_async, then replays via a fresh
+ * persist_t's persist_recover() (a real restart). Isolates the WAL fact
+ * shape from the live caller, mirroring test_wal_job_start_fact.c's own
+ * restart-replay test. */
+static void test_wru_restart_replay_recovers_runusage(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_wru_replay_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    /* Phase 1: write the log to real disk. */
+    {
+        persist_t* p = wru_make_fresh_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p);
+
+        job_submit_data_t submit;
+        memset(&submit, 0, sizeof(submit));
+        submit.queue = (char*)"q";
+        submit.user  = (char*)"tester";
+
+        xce_job_t jdef;
+        memset(&jdef, 0, sizeof(jdef));
+        jdef.jobid  = 73006;
+        jdef.state  = JOB_STAT_RUN;
+        jdef.submit = &submit;
+
+        uint8_t* job_payload = NULL;
+        size_t   job_plen    = 0;
+        TEST_ASSERT_EQUAL_INT(0,
+            wru_encode_job(&jdef, &job_payload, &job_plen));
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_SUBMIT, job_payload,
+                              (uint32_t)job_plen));
+        free(job_payload);
+
+        xce_job_runusage_t r = wru_make_test_runusage();
+        uint8_t* rusage_payload = NULL;
+        size_t   rusage_plen    = 0;
+        TEST_ASSERT_EQUAL_INT(0,
+            wru_build_rusage_payload(73006, JOB_STAT_RUN, &r,
+                                     &rusage_payload, &rusage_plen));
+        TEST_ASSERT_EQUAL_INT(0,
+            persist_wal_async(p, WAL_JOB_RUSAGE, rusage_payload,
+                              (uint32_t)rusage_plen));
+        free(rusage_payload);
+
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
+        wru_teardown_persist(p);
+    }
+
+    /* Phase 2: fresh persist_t + fresh cluster_t, recover from disk —
+     * simulates a controller restart. */
+    {
+        persist_t* p2 = wru_make_fresh_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p2);
+
+        wru_setup();
+        p2->cluster = &g_wru_c;
+
+        int rc = persist_recover(p2);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        job_t* j = wru_find_job(73006);
+        TEST_ASSERT_NOT_NULL_MESSAGE(j,
+            "restart replay must construct the job via WAL_JOB_SUBMIT");
+        TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
+        wru_assert_runusage_matches(j->job.run_usage);
+
+        wru_teardown();
+        wru_teardown_persist(p2);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    printf("test_wru_restart_replay_recovers_runusage: PASS\n");
+}
+
+/* Variadic trampoline so the test passes typed args through a real va_list
+ * -- pattern from test_persist_hooks.c / test_commit_barrier_rollout.c's
+ * call_job_hook, reproduced here (file-local) since that one is static to
+ * its own TU. */
+static int wru_call_job_hook(int type, int sit, int detail, persist_t* p, ...)
+{
+    va_list ap;
+    int     rc;
+
+    va_start(ap, p);
+    rc = persist_job_hook_fun(type, sit, detail, p, ap);
+    va_end(ap);
+    return rc;
+}
+
+/* bug-1690's primary regression test: drives the REAL, unmodified
+ * persist_job_hook_fun() -- the actual production hook subscriber this fix
+ * changed -- with a job_t whose run_usage is genuinely populated (see this
+ * file's header comment for why this goes around update_job_rusages()
+ * itself rather than through it), through a real WAL-write +
+ * persist_recover() restart cycle. Asserts run_usage survives the restart
+ * intact, not just state -- the audit's exact confirmed root cause (mirrors
+ * bug-1678/1687/1689's own acceptance criterion). */
+static void test_wru_real_hook_appends_and_survives_restart(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_wru_realhook_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    const uint64_t jobid = 73007;
+
+    /* Phase 1: real cluster + real disk persist. Seed WAL_JOB_SUBMIT
+     * directly (so replay can reconstruct the job) AND a live in-memory job
+     * object with run_usage already populated (so the real
+     * persist_job_hook_fun() call site has something to encode) -- mirrors
+     * test_wjs_real_on_job_start_survives_restart's dual-seed pattern. */
+    {
+        persist_config_t cfg;
+
+        if (makeString((char*)"wru-realhook-probe") == NULL)
+            xstring_table_init(64);
+        if (!g_wru_jobobj_ready) {
+            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
+            g_wru_jobobj_ready = 1;
+        }
+
+        cluster_t c;
+        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                      .nslots = 32, .delta = 16 };
+        memset(&c, 0, sizeof(c));
+        TEST_ASSERT_EQUAL_INT(0,
+            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
+        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+        TEST_ASSERT_NOT_NULL(c.jobmgr);
+
+        cfg             = persist_config_defaults();
+        cfg.wal_sync_ms = 5000;
+        persist_set_workdir(&c.persist, tmp, "ctrl-test");
+        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
+        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));
+
+        /* Durable WAL_JOB_SUBMIT seed. */
+        {
+            job_submit_data_t wal_submit;
+            xce_job_t         wal_jdef;
+            uint8_t*          buf = NULL;
+            size_t            sz  = 0;
+
+            memset(&wal_submit, 0, sizeof(wal_submit));
+            wal_submit.queue = (char*)"oq";
+            wal_submit.user  = (char*)"tester";
+
+            memset(&wal_jdef, 0, sizeof(wal_jdef));
+            wal_jdef.jobid  = jobid;
+            wal_jdef.state  = JOB_STAT_RUN;
+            wal_jdef.submit = &wal_submit;
+
+            TEST_ASSERT_EQUAL_INT(0,
+                wru_encode_job(&wal_jdef, &buf, &sz));
+            TEST_ASSERT_EQUAL_INT(0,
+                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
+                                  (uint32_t)sz));
+            free(buf);
+        }
+
+        /* Live in-memory job object, with run_usage populated directly
+         * (standing in for update_host_runusage()'s real accumulation --
+         * see this file's header comment on why that function can't be
+         * exercised for real in this binary), for persist_job_hook_fun()
+         * to find/read. */
+        job_t* live_job;
+        {
+            job_submit_data_t submit;
+            xce_job_t         jdef;
+
+            memset(&submit, 0, sizeof(submit));
+            submit.queue = (char*)"oq";
+            submit.user  = (char*)"tester";
+
+            memset(&jdef, 0, sizeof(jdef));
+            jdef.jobid  = jobid;
+            jdef.state  = JOB_STAT_RUN;
+            jdef.submit = &submit;
+
+            live_job = makeJob(c.jobmgr, &jdef);
+            TEST_ASSERT_NOT_NULL(live_job);
+        }
+
+        xce_job_runusage_t r = wru_make_test_runusage();
+        live_job->job.run_usage = &r;
+
+        xce_job_rusages_t rusages;
+        memset(&rusages, 0, sizeof(rusages));
+        rusages.jobid = jobid;
+
+        int rc = wru_call_job_hook(XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
+                                   HOOK_DETAIL_JOB_RUSAGE_UPDATE, &c.persist,
+                                   &c, &rusages);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        /* Prevent the stack-local `r` from being double-freed by teardown
+         * below — this fixture's job_t doesn't own it. */
+        live_job->job.run_usage = NULL;
+
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));
+
+        persist_stop(&c.persist);
+        persist_fini(&c.persist);
+        dereference(c.jobmgr);
+        xce_hook_fini(&c.hooksys);
+    }
+
+    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
+     * recover from disk — simulates a controller restart. */
+    {
+        persist_t* p2 = wru_make_fresh_persist(tmp);
+        cluster_t  c2;
+        job_t*     j2 = NULL;
+        int        rc;
+        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
+                                      .nslots = 32, .delta = 16 };
+
+        TEST_ASSERT_NOT_NULL(p2);
+
+        memset(&c2, 0, sizeof(c2));
+        TEST_ASSERT_EQUAL_INT(0,
+            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
+        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
+        TEST_ASSERT_NOT_NULL(c2.jobmgr);
+
+        p2->cluster = &c2;
+        rc          = persist_recover(p2);
+        TEST_ASSERT_EQUAL_INT(0, rc);
+
+        lock(c2.jobmgr, RTTI_LOCK_READ);
+        key_find(c2.jobmgr, (void*)jobid, (void**)&j2);
+        unlock(c2.jobmgr);
+        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
+            "restart replay must reconstruct the job via WAL_JOB_SUBMIT");
+        TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j2->job.state);
+
+        /* bug-1690: restart replay must recover the client-reported
+         * run_usage, not just state -- the audit's exact confirmed root
+         * cause (mirrors bug-1678/1687/1689's own acceptance criterion for
+         * their own fields). */
+        wru_assert_runusage_matches(j2->job.run_usage);
+
+        xce_hook_fini(&c2.hooksys);
+        if (c2.jobmgr) dereference(c2.jobmgr);
+        wru_teardown_persist(p2);
+    }
+
+    char cmd[512];
+    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
+    if (system(cmd) != 0)
+        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
+
+    printf("test_wru_real_hook_appends_and_survives_restart: PASS\n");
+}
+
+/* ── Test group entry point ───────────────────────────────────────────── */
+
+void test_wal_job_rusage_fact(void)
+{
+    UnitySetTestFile(__FILE__);
+    RUN_TEST(test_wru_apply_committed_entry_sets_runusage_no_state_change);
+    RUN_TEST(test_wru_apply_committed_entry_sets_runusage_forward_transition);
+    RUN_TEST(test_wru_apply_committed_entry_backward_compat_old_payload);
+    RUN_TEST(test_wru_apply_committed_entry_merges_without_freeing_live_host);
+    RUN_TEST(test_wru_apply_wal_entry_sets_runusage_no_hook);
+    RUN_TEST(test_wru_apply_wal_entry_backward_compat_old_payload);
+    RUN_TEST(test_wru_restart_replay_recovers_runusage);
+    RUN_TEST(test_wru_real_hook_appends_and_survives_restart);
+}