/* test_worker_resync.c — worker-rejoin-resync task 2.1
 *
 * Proves record_job_reconcile_pending_fact() (job.c) — the Requirement 4.6
 * new fact-recording function that returns a worker-unrecognized job to
 * PEND, releasing any resource allocation it was carrying (bug-1631's own
 * deliberately-deferred trigger condition, closed here — see design.md's
 * Error Handling section) and durably appending a plain WAL_JOB_UPDATE fact
 * -- following dispatch_a_job_()'s eager-mutation-plus-explicit-notify
 * pattern (design.md "Controller: job.c — record_job_reconcile_pending_
 * fact, record_job_reconcile_assign_fact"), NOT record_job_finish_fact's
 * deferred one.
 *
 * This task implements ONLY record_job_reconcile_pending_fact() itself --
 * no live caller exists yet (dispatch_a_job_'s disconnected-worker split is
 * task 2.3; resync.c's report-apply glue is a later task), so every test
 * here directly exercises the exact sequence a real caller will perform:
 * lock -> call record_job_reconcile_pending_fact() -> check the returned
 * job_transition_result_t -> unlock -> (only when APPLIED) call
 * persist_notify_job_updated_already_persisted() itself -- proving the full
 * intended sequence end-to-end even though production wiring of that
 * sequence is a later task's job (design.md's Service Interface
 * Postconditions/Invariants: the caller, not this function, owns that
 * post-unlock notify call).
 *
 * Fixture mirrors test_dispatch_commit.c's dc_setup/dc_teardown exactly: a
 * bare cluster_t (jobmgr + hooksys) plus a real disk-backed persist_t
 * initialized with cluster == NULL, so wal_hook_append_classified()
 * (persist.c) takes the direct persist_wal_async() path for
 * persist_submit_job_fact()'s plain RETAIN-mode WAL_JOB_UPDATE append -- no
 * raft/commit-engine machinery needed. p->current_lsn before/after each
 * call proves (or disproves) a durable append happened, mirroring
 * test_dispatch_commit.c's test_dc_worker_not_found_leaves_job_pend_and_
 * alloc_uncommitted's own LSN-delta idiom.
 */
#include <rtti.h>

#include <capnp_c.h>
#include <errno.h>
#include <pthread.h>
#include <stdarg.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <unity/unity.h>

#include "cluster.h"
#include "controller.capnp.h"
#include "hook.h"
#include "jobj.h"
#include "kvmgr.h"
#include "netio.h" /* worker-rejoin-resync task 7.1: endpoint_get/
                    * endpoint_deq/ITPTR_TO_PACKET, for real wire capture of
                    * the round-trip test's query send / report send */
#include "persist.h"
#include "request.h" /* worker-rejoin-resync task 7.1: xce_decode,
                      * xce_free_request, xce_job_state_report_req */
#include "resobj.h" /* worker-rejoin-resync task 7.1: makeResource/
                     * xce_resource_t, for the round-trip test's real
                     * resmgr (apply_join_state, reached via rejoin_worker,
                     * needs one) */
#include "resync.h" /* worker-rejoin-resync task 3.1 */
#include "wal_types.h"
#include "wobj.h" /* worker-rejoin-resync task 3.1: makeWorker, for a real
                   * RTTI-managed worker_t the timeout test can genuinely
                   * lock() */
#include "xce/cluster_def.h"
#include "xce/types.h"
#include "xcealloc.h"
#include "xstring.h"

/* worker-rejoin-resync task 7.1: rejoin_worker/worker_send_resync_query are
 * non-static in worker.c when XCE_TESTING is defined -- forward-declared
 * directly here rather than via a header, mirroring test_worker_join.c's
 * own identical forward declarations exactly (these functions have no
 * public header of their own; test_worker_join.c already establishes this
 * as the file's own convention for driving the real join path from a
 * test). collect_confvars (needed for worker.c to link, since on_worker_
 * join -> worker_push_config -> collect_confvars) is already provided by
 * test_worker_join.c elsewhere in this SAME controller_unit_test binary --
 * not redefined here to avoid a duplicate-symbol link error; this test
 * never calls worker_push_config anyway. */
int rejoin_worker(cluster_t* c, worker_t* w, endpoint_t* ep,
                  hostattrs_t* attrs, xce_hostconfig_t* hconfig);
int worker_send_resync_query(cluster_t* c, worker_t* w, endpoint_t* ep);

/* jobj.c's job type table and the process-global xstring table are shared
 * across every test file in the controller_unit_test binary -- guard
 * idempotently, mirroring every sibling fixture in this suite
 * (test_wal_job_dispatch_fact.c's g_wjd_jobobj_ready). */
static int g_twr_jobobj_ready;

static cluster_t g_twr;
static char      g_twr_tmpdir[256];

/* worker-rejoin-resync task 3.1: no-op kvmgr datakiller for g_twr.workermgr,
 * mirroring production's own worker_obj_killer (cluster.c) -- a worker_t,
 * once created, is never explicitly torn down for the life of the process
 * (no key_del(c->workermgr, ...) call site exists anywhere in this
 * codebase), so there is nothing for this to actually free. */
static void twr_worker_obj_killer(void* arg)
{
    (void)arg;
}

static void twr_setup(void)
{
    kvmgr_params_t   int_params = { .key_type = KVMGR_KEY_INT,
                                    .nslots = 32, .delta = 16 };
    persist_config_t cfg;

    if (makeString((char*)"twr-probe") == NULL)
        xstring_table_init(64);

    if (!g_twr_jobobj_ready) {
        TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
        g_twr_jobobj_ready = 1;
    }

    memset(&g_twr, 0, sizeof(g_twr));
    TEST_ASSERT_EQUAL_INT(0,
        xce_hook_init(&g_twr.hooksys, XCE_CONTROLLER_HOOK_MAX));

    g_twr.jobmgr = makeInstance(&xkvmgr_type, &int_params);
    TEST_ASSERT_NOT_NULL(g_twr.jobmgr);

    /* worker-rejoin-resync task 3.1: a real workermgr so the round-lifecycle
     * tests below can create a genuine RTTI-managed worker_t (via
     * makeWorker) where that matters (the timeout test, whose internal
     * resync_on_timeout callback genuinely lock()s `w`) -- key type mirrors
     * production's own workermgr_params (cluster.c, KVMGR_KEY_STRING,
     * keyed by hostname). */
    {
        kvmgr_params_t workermgr_params
            = { .key_type   = KVMGR_KEY_STRING,
                .nslots     = 32,
                .delta      = 16,
                .datakiller = twr_worker_obj_killer };

        g_twr.workermgr = makeInstance(&xkvmgr_type, &workermgr_params);
        TEST_ASSERT_NOT_NULL(g_twr.workermgr);
    }

    /* Real disk-backed persist, cluster == NULL: wal_hook_append_
     * classified() (persist.c) takes the direct persist_wal_async() path --
     * no raft/commit-engine machinery needed for a plain RETAIN-mode fact
     * append (mirrors test_dispatch_commit.c's dc_setup exactly, which
     * exercises dispatch_a_job_'s own real persist_submit_job_fact()-family
     * call the same way). */
    cfg             = persist_config_defaults();
    cfg.wal_sync_ms = 5000;
    strcpy(g_twr_tmpdir, "/tmp/xce_twr21_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(g_twr_tmpdir));
    persist_set_workdir(&g_twr.persist, g_twr_tmpdir, "ctrl-test");
    TEST_ASSERT_EQUAL_INT(0, persist_init(&g_twr.persist, &cfg, NULL));
    TEST_ASSERT_EQUAL_INT(0, persist_start(&g_twr.persist));
}

static void twr_teardown(void)
{
    char cmd[512];

    persist_stop(&g_twr.persist);
    persist_fini(&g_twr.persist);

    if (g_twr.jobmgr)
        dereference(g_twr.jobmgr);
    if (g_twr.workermgr)
        dereference(g_twr.workermgr);
    xce_hook_fini(&g_twr.hooksys);

    snprintf(cmd, sizeof(cmd), "rm -rf %s", g_twr_tmpdir);
    if (system(cmd) != 0)
        fprintf(stderr, "cleanup warning: rm -rf %s failed\n", g_twr_tmpdir);
}

/* Creates a job directly in the fixture's jobmgr, bypassing dispatch/replay
 * entirely -- for seeding fact-transition tests, mirroring
 * test_wal_job_dispatch_fact.c's wjd_seed_job / test_dispatch_commit.c's
 * dc_seed_ready. */
static job_submit_data_t g_twr_seed_submit;

static job_t* twr_seed_job(uint64_t jobid, uint32_t state)
{
    xce_job_t jdef;

    memset(&g_twr_seed_submit, 0, sizeof(g_twr_seed_submit));
    g_twr_seed_submit.queue = (char*)"q";
    g_twr_seed_submit.user  = (char*)"tester";

    memset(&jdef, 0, sizeof(jdef));
    jdef.jobid  = jobid;
    jdef.state  = state;
    jdef.submit = &g_twr_seed_submit;

    return makeJob(g_twr.jobmgr, &jdef);
}

/* One host ("twr-worker"), one resource ("slots", value 1.0) -- the same
 * minimal shape test_wal_job_dispatch_fact.c's/test_dispatch_commit.c's own
 * dispatch-alloc tests use. Caller owns the result (release via
 * free_JobAlloc_ptr, or hand ownership to job->job.allocs). */
static xce_job_alloc_t* twr_make_test_alloc(uint64_t jobid)
{
    return make_job_alloc(jobid, 1,
        make_host_allocs(1,
            make_host_alloc((char*)"twr-worker", 1,
                make_allocs(1,
                    make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}

static uint64_t twr_current_lsn(void)
{
    uint64_t lsn;

    pthread_mutex_lock(&g_twr.persist.lock);
    lsn = g_twr.persist.current_lsn;
    pthread_mutex_unlock(&g_twr.persist.lock);
    return lsn;
}

/* Spy hook: counts HOOK_AFTER_EVENT/HOOK_DETAIL_JOB_UPDATE fires -- the
 * exact hook sched_job_update_fun/apiserver_job_update_fun subscribe to in
 * production, mirroring test_dispatch_commit.c's dc_job_update_spy_hook. */
static int g_twr_update_hook_fires;

static int twr_job_update_spy_hook(int type, int situation, int detail,
                                   void* data, va_list ap)
{
    (void)type; (void)data; (void)ap;
    if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_JOB_UPDATE)
        g_twr_update_hook_fires++;
    return 0;
}

#define TWR_JOBID_APPLIED  81001ULL
#define TWR_JOBID_NOOP     81002ULL
#define TWR_JOBID_TERMINAL 81003ULL

/* THE core proof (Req 4.6, 7.1-7.3; bug-1631 regression): a job that was
 * dispatched (real allocs committed, state RUN) and reconciles to
 * unrecognized ends the call -- synchronously, BEFORE any commit round-trip,
 * matching dispatch_a_job_'s own eager pattern, NOT record_job_finish_
 * fact's deferred one -- with PEND state and no allocation, a durably
 * appended fact, and (once the caller unlocks and notifies, exactly as a
 * real caller must) the scheduler's notify path fired exactly once. */
static void test_reconcile_pending_applies_releases_allocs_and_notifies(void)
{
    job_t*           j;
    xce_job_alloc_t* alloc;
    int              rc;
    uint64_t         lsn_before, lsn_after;

    twr_setup();
    g_twr_update_hook_fires = 0;
    TEST_ASSERT_EQUAL_INT(0,
        xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
                    twr_job_update_spy_hook, NULL));

    j = twr_seed_job(TWR_JOBID_APPLIED, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);

    alloc = twr_make_test_alloc(TWR_JOBID_APPLIED);
    TEST_ASSERT_NOT_NULL(alloc);
    j->job.allocs = alloc; /* simulate dispatch_a_job_'s own committed alloc,
                             * ownership transferred to job->job.allocs. */

    lsn_before = twr_current_lsn();

    lock(j, RTTI_LOCK_WRITE);
    rc = record_job_reconcile_pending_fact(&g_twr, j);

    /* Observable completion (design.md): PEND + no allocation, asserted
     * IMMEDIATELY after the call returns, still under the write lock,
     * before any commit round-trip -- the mutation is eager, not
     * deferred. */
    TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_APPLIED, rc,
        "an unrecognized RUN job must apply the PEND transition");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j->job.state,
        "job must be PEND synchronously, before any WAL commit/apply "
        "round trip");
    TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
        "bug-1631: a job reconciled from RUN back to PEND must release its "
        "resource allocation immediately, so a later redispatch never "
        "inherits a stale allocation from the prior dispatch");
    unlock(j);

    lsn_after = twr_current_lsn();
    TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
        "record_job_reconcile_pending_fact must durably append a "
        "WAL_JOB_UPDATE fact for an applied transition");

    /* Caller's responsibility, AFTER unlocking (design.md Postconditions/
     * Invariants): notify the scheduler through the existing notify path,
     * exactly once, for this actual mutation (Req 7.1-7.3). */
    persist_notify_job_updated_already_persisted(&g_twr, j);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
        "the scheduler notify path must fire exactly once for this "
        "reconciled pending transition (Req 7.2) when the caller notifies "
        "after unlocking -- omitting it would reproduce bug-1679 at this "
        "new call site");

    xce_disable_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
                     twr_job_update_spy_hook);
    twr_teardown();
    printf("test_reconcile_pending_applies_releases_allocs_and_notifies: "
          "PASS\n");
}

/* A job with no allocation at all (e.g. never dispatched, already PEND):
 * reconciling it again must be a no-op per apply_job_update_transition's
 * own duplicate-state guard (JOB_TRANSITION_NOOP_DUPLICATE) -- no mutation,
 * no allocs touch (there is none to release), no durable fact, no crash
 * (Req 4.3). */
static void test_reconcile_pending_noop_on_already_pending_job(void)
{
    job_t*   j;
    int      rc;
    uint64_t lsn_before, lsn_after;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_NOOP, JOB_STAT_PEND);
    TEST_ASSERT_NOT_NULL(j);
    TEST_ASSERT_NULL(j->job.allocs);

    lsn_before = twr_current_lsn();

    lock(j, RTTI_LOCK_WRITE);
    rc = record_job_reconcile_pending_fact(&g_twr, j);
    TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_NOOP_DUPLICATE, rc,
        "reconciling an already-PEND job must be treated as a duplicate "
        "no-op, matching apply_job_update_transition's own guard");
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
    TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
        "a job that never held an allocation must still have none after a "
        "no-op reconcile");
    unlock(j);

    lsn_after = twr_current_lsn();
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
        "a no-op reconcile must append no duplicate WAL fact (Req 4.3)");

    twr_teardown();
    printf("test_reconcile_pending_noop_on_already_pending_job: PASS\n");
}

/* A job the controller already knows is terminal (DONE) must never be
 * clobbered back to PEND by a reconciliation report (Req 4.4) -- the
 * existing job_state_is_backward guard inside apply_job_update_transition,
 * reused unmodified, rejects it. */
static void test_reconcile_pending_rejects_terminal_job(void)
{
    job_t*   j;
    int      rc;
    uint64_t lsn_before, lsn_after;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_TERMINAL, JOB_STAT_DONE);
    TEST_ASSERT_NOT_NULL(j);

    lsn_before = twr_current_lsn();

    lock(j, RTTI_LOCK_WRITE);
    rc = record_job_reconcile_pending_fact(&g_twr, j);
    TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_REJECTED_BACKWARD, rc,
        "a terminal job must reject the backward PEND transition, never "
        "clobbering more-final state (Req 4.4)");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j->job.state,
        "the job's terminal state must be preserved exactly");
    unlock(j);

    lsn_after = twr_current_lsn();
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
        "a rejected reconcile must append no WAL fact");

    twr_teardown();
    printf("test_reconcile_pending_rejects_terminal_job: PASS\n");
}

/* ── record_job_reconcile_assign_fact() tests (task 2.2) ─────────────────
 *
 * Proves record_job_reconcile_assign_fact() (job.c) -- the Requirement 4.8
 * new fact-recording function that records a job the worker reports running
 * that the controller has no existing non-terminal record of ("unqueried
 * extra", Req 3.6) as assigned to the reporting worker, through the same
 * dispatch_a_job_()-style eager-mutation-plus-explicit-notify pattern as
 * record_job_reconcile_pending_fact above, reusing bug-1678's NULL-alloc
 * persist_submit_job_dispatch_fact() path (design.md "Controller: job.c --
 * record_job_reconcile_pending_fact, record_job_reconcile_assign_fact").
 *
 * As with task 2.1, no live caller exists yet (resync.c's report-apply glue
 * is a later task), so every test here directly exercises the exact
 * sequence a real caller will perform: lock -> call
 * record_job_reconcile_assign_fact() -> check the result -> unlock -> call
 * persist_notify_job_updated_already_persisted() itself. */

#define TWR_JOBID_ASSIGN         81004ULL
#define TWR_JOBID_ASSIGN_REPLAY  81005ULL

/* THE core proof (Req 4.1, 4.3, 4.4, 4.7, 4.8, 7.1-7.3): a job the
 * controller never previously tracked, reported RUNNING by a worker, ends
 * the call -- synchronously, before any commit round-trip -- recorded as
 * running with no resource allocation (the controller genuinely does not
 * know what the scheduler originally computed, design.md's "Documented
 * scope decision"), a durably appended fact, and (once the caller unlocks
 * and notifies, exactly as a real caller must) the scheduler's notify path
 * fired exactly once. */
static void
test_reconcile_assign_applies_running_no_allocs_and_notifies(void)
{
    job_t*                 j;
    worker_t                w;
    xce_job_state_entry_t   report;
    int                     rc;
    uint64_t                lsn_before, lsn_after;

    twr_setup();
    g_twr_update_hook_fires = 0;
    TEST_ASSERT_EQUAL_INT(0,
        xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
                    twr_job_update_spy_hook, NULL));

    /* Seeded with no allocation, mirroring a job the caller (task 3.2, not
     * yet implemented) constructed fresh for a jobid the controller had
     * never seen before -- there is nothing to release here, only
     * something new to record. */
    j = twr_seed_job(TWR_JOBID_ASSIGN, JOB_STAT_PEND);
    TEST_ASSERT_NOT_NULL(j);
    TEST_ASSERT_NULL(j->job.allocs);

    memset(&w, 0, sizeof(w));
    w.host.name = (char*)"twr-worker";

    report = (xce_job_state_entry_t){
        .jobid          = TWR_JOBID_ASSIGN,
        .status         = JobStateStatus_running,
        .exit_code      = 0,
        .finish_time_ms = 0,
        .queried        = 0
    };

    lsn_before = twr_current_lsn();

    lock(j, RTTI_LOCK_WRITE);
    rc = record_job_reconcile_assign_fact(&g_twr, j, &w, &report);

    /* Observable completion (design.md / this task): RUN + no allocation,
     * asserted IMMEDIATELY after the call returns, still under the write
     * lock, before any commit round-trip -- the mutation is eager, not
     * deferred. */
    TEST_ASSERT_EQUAL_INT_MESSAGE(JOB_TRANSITION_APPLIED, rc,
        "an unqueried running job must always apply (no reject/no-op case)");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
        "job must be RUN synchronously, before any WAL commit/apply round "
        "trip");
    TEST_ASSERT_NULL_MESSAGE(j->job.allocs,
        "Req 4.8: a reconciliation-assigned job must never carry a "
        "fabricated resource allocation -- the controller genuinely does "
        "not know what the scheduler originally computed for it");
    unlock(j);

    lsn_after = twr_current_lsn();
    TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
        "record_job_reconcile_assign_fact must durably append a "
        "WAL_JOB_DISPATCH fact");

    /* Caller's responsibility, AFTER unlocking (design.md Postconditions/
     * Invariants): notify the scheduler through the existing notify path,
     * exactly once, for this actual mutation (Req 7.1-7.3). */
    persist_notify_job_updated_already_persisted(&g_twr, j);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
        "the scheduler notify path must fire exactly once for this "
        "reconciled assignment (Req 7.2) when the caller notifies after "
        "unlocking -- omitting it would reproduce bug-1679 at this new "
        "call site");

    xce_disable_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_JOB,
                     twr_job_update_spy_hook);
    twr_teardown();
    printf("test_reconcile_assign_applies_running_no_allocs_and_notifies: "
          "PASS\n");
}

/* Encode helper mirroring test_dispatch_commit.c's dc_encode_job /
 * test_wal_job_dispatch_fact.c's wjd_encode_job (capnp_encode_job is
 * file-static in persist.c, so a direct WAL_JOB_SUBMIT seed needs its own
 * encoder built from the exported encode_Job/new_Job/write_Job codec
 * chain). */
static int twr_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
{
    struct capn          c;
    struct capn_segment* cs;
    struct Job           cj;
    Job_ptr              jp;

    capn_init_malloc(&c);
    cs = capn_root(&c).seg;
    encode_Job(cs, &cj, (xce_job_t*)j);
    jp = new_Job(cs);
    write_Job(&cj, jp);
    capn_setp(capn_root(&c), 0, jp.p);

    *out_len = capn_size(&c);
    *out     = malloc(*out_len);
    if (!*out) { capn_free(&c); return -ENOMEM; }
    capn_write_mem(&c, *out, *out_len, 0);
    capn_free(&c);
    return 0;
}

/* Fresh disk-backed persist_t pointed at an existing tmpdir -- mirrors
 * test_dispatch_commit.c's dc_make_fresh_persist/dc_teardown_persist
 * exactly, for this test's own Phase 2 (simulated restart). */
static persist_t* twr_make_fresh_persist(const char* tmpdir)
{
    persist_t*       p = calloc(1, sizeof(persist_t));
    persist_config_t cfg;

    if (!p) return NULL;

    cfg             = persist_config_defaults();
    cfg.wal_sync_ms = 5000;

    persist_set_workdir(p, tmpdir, "ctrl-test");

    if (persist_init(p, &cfg, NULL) != 0) {
        free(p);
        return NULL;
    }
    if (persist_start(p) != 0) {
        persist_fini(p);
        free(p);
        return NULL;
    }
    return p;
}

static void twr_teardown_persist(persist_t* p)
{
    persist_stop(p);
    persist_fini(p);
    free(p);
}

/* Restart-replay round trip -- this task's own explicit completion
 * criterion ("durably survives a restart replay with the same result"),
 * mirroring test_dispatch_commit.c's test_dc_success_appends_wal_job_
 * dispatch_durably's Phase 1/Phase 2 shape exactly, but driving Phase 1's
 * WAL_JOB_DISPATCH fact via record_job_reconcile_assign_fact() instead of
 * dispatch_a_job_() -- proving bug-1678's NULL-alloc decode branch (already
 * tested on both apply seams by test_wal_job_dispatch_fact.c) round-trips
 * this new call site's real fact correctly too. Uses its own local
 * cluster_t/persist_t (not the shared g_twr fixture) so Phase 1's disk
 * state survives past that phase's own teardown, exactly like dc's
 * original. */
static void test_reconcile_assign_survives_restart_replay(void)
{
    char     tmp[256];
    uint64_t captured_start_time = 0;

    strcpy(tmp, "/tmp/xce_twr22_replay_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    /* Phase 1: real cluster + real disk persist. Seed a durable
     * WAL_JOB_SUBMIT fact directly (so replay can reconstruct the job --
     * this job was never dispatched through any controller, so there is no
     * other WAL_JOB_SUBMIT/DISPATCH in this log), seed the in-memory job_t
     * the same way twr_seed_job() does (bypassing dispatch/replay --
     * record_job_reconcile_assign_fact() operates on a job_t* the caller
     * has already located/constructed, not this function's concern), then
     * drive the real function under test. */
    {
        cluster_t              c;
        kvmgr_params_t         int_params = { .key_type = KVMGR_KEY_INT,
                                              .nslots = 32, .delta = 16 };
        persist_config_t       cfg;
        job_t*                 j;
        worker_t                w;
        xce_job_state_entry_t   report;
        int                     rc;

        if (makeString((char*)"twr22-probe") == NULL)
            xstring_table_init(64);
        if (!g_twr_jobobj_ready) {
            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
            g_twr_jobobj_ready = 1;
        }

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";

            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = TWR_JOBID_ASSIGN_REPLAY;
            wal_jdef.state  = JOB_STAT_PEND;
            wal_jdef.submit = &wal_submit;

            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);
        }

        {
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";

            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = TWR_JOBID_ASSIGN_REPLAY;
            jdef.state  = JOB_STAT_PEND;
            jdef.submit = &submit;
            j = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(j);
        }

        memset(&w, 0, sizeof(w));
        w.host.name = (char*)"twr-worker-replay";

        report = (xce_job_state_entry_t){
            .jobid          = TWR_JOBID_ASSIGN_REPLAY,
            .status         = JobStateStatus_running,
            .exit_code      = 0,
            .finish_time_ms = 0,
            .queried        = 0
        };

        lock(j, RTTI_LOCK_WRITE);
        rc = record_job_reconcile_assign_fact(&c, j, &w, &report);
        TEST_ASSERT_EQUAL_INT(JOB_TRANSITION_APPLIED, rc);
        TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, j->job.state);
        TEST_ASSERT_NULL(j->job.allocs);
        captured_start_time = (uint64_t)j->job.start_time;
        TEST_ASSERT_NOT_EQUAL(0, captured_start_time);
        unlock(j);

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
     * recover from disk -- simulates a controller restart. */
    {
        persist_t* p2 = twr_make_fresh_persist(tmp);
        cluster_t  c2;
        job_t*     j2 = NULL;
        int        rc;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 32, .delta = 16 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        key_find(c2.jobmgr, (void*)TWR_JOBID_ASSIGN_REPLAY, (void**)&j2);
        unlock(c2.jobmgr);
        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
            "restart replay must reconstruct the job via WAL_JOB_SUBMIT");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(TWR_JOBID_ASSIGN_REPLAY,
            j2->job.jobid, "recovered job must be the same jobid");
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state,
            "restart replay must recover the job as RUNNING from the "
            "WAL_JOB_DISPATCH fact appended by record_job_reconcile_"
            "assign_fact()'s real call (Req 4.8) -- no WAL_JOB_START "
            "anywhere in this log");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(captured_start_time,
            (uint64_t)j2->job.start_time,
            "restart replay must recover the exact dispatch-time "
            "start_time stamped by the real call in Phase 1");
        TEST_ASSERT_NULL_MESSAGE(j2->job.allocs,
            "Req 4.8 / design.md's documented scope decision: replay must "
            "NOT fabricate a resource allocation for a reconciliation-"
            "assigned job -- allocs stays NULL exactly as it was set "
            "eagerly in Phase 1, exercising bug-1678's NULL-alloc decode "
            "branch");

        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    printf("test_reconcile_assign_survives_restart_replay: PASS\n");
}

/* ── resync.c round lifecycle tests (task 3.1) ───────────────────────────
 *
 * Proves resync_arm_round()/resync_resolve_round()/resync_round_is_current()
 * (resync.c) -- the per-worker job-state reconciliation round lifecycle:
 * query-set construction (Req 1.1, 2.1, 2.2), a fresh monotonic round token
 * per arm distinguishing this round from any prior one for the same worker
 * (Req 6.1, 6.3), the XCE_WORKER_RECONCILING status bit (Req 5.1, 5.2), and
 * the Req 5.3 bounded timeout resolving the round without any blocking
 * call. Reuses the g_twr/twr_setup/twr_teardown fixture and twr_seed_job()
 * from task 2.1/2.2's tests above -- resync_arm_round only needs a
 * cluster_t with a real c->jobmgr (which g_twr already provides).
 *
 * No live caller exists yet (worker.c's rejoin_worker/apply_join_state
 * wiring is task 6.1; the report-apply glue that would call
 * resync_round_is_current for real is task 3.2) -- every test here calls
 * resync_arm_round/resync_resolve_round/resync_round_is_current directly,
 * exactly as a real caller (task 6.1) eventually will.
 *
 * resync_arm_round/resync_resolve_round/resync_round_is_current never
 * lock/unlock `w` themselves (their documented precondition is that the
 * CALLER already holds it) -- so a bare stack worker_t is sufficient and
 * safe for those tests, exactly mirroring how task 2.2's own
 * record_job_reconcile_assign_fact tests above already use a bare stack
 * `worker_t w` with no locking. The ONE exception is the timeout test:
 * resync_on_timeout (the internal delaymgr callback resync_test_fire_
 * timeout drives directly) runs with no caller-held lock -- because a real
 * delaymgr fire has none -- so it genuinely calls lock()/unlock() on `w`
 * itself, which requires a real RTTI-managed worker_t (via makeWorker
 * against g_twr.workermgr), not a bare stack struct. */

#define TWR_JOBID_RESYNC_A       81101ULL
#define TWR_JOBID_RESYNC_B       81102ULL
#define TWR_JOBID_RESYNC_OTHER   81103ULL
#define TWR_JOBID_RESYNC_DONE    81104ULL
#define TWR_JOBID_RESYNC_NOALLOC 81105ULL
#define TWR_JOBID_RESYNC_SOLO    81106ULL
#define TWR_JOBID_RESYNC_TIMEOUT 81107ULL

/* Same builder chain as twr_make_test_alloc() above, parameterized by host
 * -- twr_make_test_alloc() itself hardcodes "twr-worker", which does not
 * fit these tests' need for several DISTINCT hosts within one job table. */
static xce_job_alloc_t* twr_make_test_alloc_for_host(uint64_t    jobid,
                                                      const char* host)
{
    return make_job_alloc(jobid, 1,
        make_host_allocs(1,
            make_host_alloc((char*)host, 1,
                make_allocs(1,
                    make_alloc((char*)"slots", NULL, 0, 1.0f, 0)))));
}

/* True iff `jobid` appears anywhere in ids[0..n) -- used below to prove the
 * query set contains exactly the expected members, neither missing a real
 * match nor carrying an extra one. */
static int twr_ids_contains(const uint64_t* ids, int n, uint64_t jobid)
{
    int i;

    for (i = 0; i < n; i++) {
        if (ids[i] == jobid)
            return 1;
    }
    return 0;
}

/* A real RTTI-managed worker_t (via makeWorker against g_twr.workermgr),
 * for the one test below that needs genuine lock()/unlock() to succeed on
 * `w` -- mirrors worker.c's own init_worker() construction
 * (`tmp.name = (char*)host; tmp.attrs = attrs; w = makeWorker(c->workermgr,
 * &tmp);`). */
static worker_t* twr_make_test_worker(const char* hostname)
{
    hostattrs_t  attrs;
    xce_worker_t tmp;

    memset(&attrs, 0, sizeof(attrs));
    memset(&tmp, 0, sizeof(tmp));
    tmp.name  = (char*)hostname;
    tmp.attrs = &attrs;

    return makeWorker(g_twr.workermgr, &tmp);
}

/* Req 1.1, 2.1, 2.2: the query set matches EXACTLY the non-terminal jobs
 * assigned to the given host -- not fewer (a genuine match omitted), not
 * more (a different host's job, a terminal job on the SAME host, or a
 * never-dispatched job with no host assignment at all, leaking in). */
static void test_resync_arm_round_query_set_matches_exactly(void)
{
    job_t*   ja;
    job_t*   jb;
    job_t*   jother;
    job_t*   jdone;
    job_t*   jnoalloc;
    worker_t w;
    int      rc;

    twr_setup();

    ja = twr_seed_job(TWR_JOBID_RESYNC_A, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(ja);
    ja->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_A, "resync-host-A");

    jb = twr_seed_job(TWR_JOBID_RESYNC_B, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(jb);
    jb->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_B, "resync-host-A");

    /* Same non-terminal state, DIFFERENT host -- must NOT appear in
     * resync-host-A's query set. */
    jother = twr_seed_job(TWR_JOBID_RESYNC_OTHER, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(jother);
    jother->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_OTHER,
                                                       "resync-host-B");

    /* SAME host, but terminal (DONE) -- must NOT appear (Req 1.1's
     * "non-terminal" qualifier). */
    jdone = twr_seed_job(TWR_JOBID_RESYNC_DONE, JOB_STAT_DONE);
    TEST_ASSERT_NOT_NULL(jdone);
    jdone->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_DONE, "resync-host-A");

    /* Never dispatched (no allocation at all) -- not assigned to ANY host,
     * so must NOT appear regardless of state. */
    jnoalloc = twr_seed_job(TWR_JOBID_RESYNC_NOALLOC, JOB_STAT_PEND);
    TEST_ASSERT_NOT_NULL(jnoalloc);
    TEST_ASSERT_NULL(jnoalloc->job.allocs);

    memset(&w, 0, sizeof(w));
    w.host.name = (char*)"resync-host-A";

    rc = resync_arm_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT_MESSAGE(2, rc,
        "exactly the 2 non-terminal resync-host-A jobs must be armed into "
        "the round");
    TEST_ASSERT_NOT_NULL(w.resync);
    TEST_ASSERT_EQUAL_INT(2, w.resync->n_queried_jobids);
    TEST_ASSERT_TRUE_MESSAGE(
        twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
                         TWR_JOBID_RESYNC_A),
        "job A (non-terminal, resync-host-A) must be in the query set");
    TEST_ASSERT_TRUE_MESSAGE(
        twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
                         TWR_JOBID_RESYNC_B),
        "job B (non-terminal, resync-host-A) must be in the query set");
    TEST_ASSERT_FALSE_MESSAGE(
        twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
                         TWR_JOBID_RESYNC_OTHER),
        "a job assigned to a DIFFERENT host must not appear");
    TEST_ASSERT_FALSE_MESSAGE(
        twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
                         TWR_JOBID_RESYNC_DONE),
        "a TERMINAL job on the same host must not appear (Req 1.1)");
    TEST_ASSERT_FALSE_MESSAGE(
        twr_ids_contains(w.resync->queried_jobids, w.resync->n_queried_jobids,
                         TWR_JOBID_RESYNC_NOALLOC),
        "a never-dispatched job (no host assignment at all) must not "
        "appear");
    TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
        "XCE_WORKER_RECONCILING must be set once a round is armed");

    resync_resolve_round(&g_twr, &w);
    TEST_ASSERT_NULL(w.resync);
    TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));

    twr_teardown();
    printf("test_resync_arm_round_query_set_matches_exactly: PASS\n");
}

/* Req 1.2: a worker with zero non-terminal jobs assigned is a documented
 * no-op -- no round armed, no status bit set. */
static void test_resync_arm_round_no_jobs_is_noop(void)
{
    worker_t w;
    int      rc;

    twr_setup();

    memset(&w, 0, sizeof(w));
    w.host.name = (char*)"resync-host-empty";

    rc = resync_arm_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
        "a worker with no non-terminal jobs must arm zero jobs (Req 1.2)");
    TEST_ASSERT_NULL_MESSAGE(w.resync,
        "no resync_state_t must be attached for the Req 1.2 no-op case");
    TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
        "XCE_WORKER_RECONCILING must not be set for the Req 1.2 no-op case");

    twr_teardown();
    printf("test_resync_arm_round_no_jobs_is_noop: PASS\n");
}

/* Req 6.1, 6.3: arming a round for a worker that already carries one
 * (simulating a disconnect interrupting an outstanding round, then a
 * rejoin) discards the prior round and produces a FRESH, DIFFERENT token --
 * and the original token is provably no longer current once superseded. */
static void test_resync_arm_round_fresh_token_supersedes_prior(void)
{
    job_t*   j;
    worker_t w;
    int      rc;
    uint64_t token1;
    uint64_t token2;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
                                                  "resync-host-solo");

    memset(&w, 0, sizeof(w));
    w.host.name = (char*)"resync-host-solo";

    rc = resync_arm_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    TEST_ASSERT_NOT_NULL(w.resync);
    token1 = w.resync->round_token;
    TEST_ASSERT_TRUE_MESSAGE(resync_round_is_current(&w, token1),
        "the round just armed must report its own token as current");

    /* Simulated disconnect-then-rejoin: the round never resolved, but the
     * worker rejoins and is armed again. */
    rc = resync_arm_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    TEST_ASSERT_NOT_NULL(w.resync);
    token2 = w.resync->round_token;

    TEST_ASSERT_NOT_EQUAL_MESSAGE(token1, token2,
        "re-arming must produce a token distinct from the prior round's "
        "(Req 6.1, 6.3)");
    TEST_ASSERT_FALSE_MESSAGE(resync_round_is_current(&w, token1),
        "the ORIGINAL token must now report stale/superseded (Req 6.3) -- "
        "the mechanism a later report-apply caller uses to discard a "
        "report carrying it");
    TEST_ASSERT_TRUE_MESSAGE(resync_round_is_current(&w, token2),
        "the NEW token must report current");

    resync_resolve_round(&g_twr, &w);
    twr_teardown();
    printf("test_resync_arm_round_fresh_token_supersedes_prior: PASS\n");
}

/* Req 5.1, 5.2: XCE_WORKER_RECONCILING is set once a round is armed and
 * cleared once resync_resolve_round runs -- resync_round_is_current also
 * correctly reports the resolved round as no longer current. */
static void test_resync_reconciling_flag_set_after_arm_cleared_after_resolve(
    void)
{
    job_t*   j;
    worker_t w;
    int      rc;
    uint64_t token;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
                                                  "resync-host-flag");

    memset(&w, 0, sizeof(w));
    w.host.name = (char*)"resync-host-flag";
    TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));

    rc = resync_arm_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
        "XCE_WORKER_RECONCILING must be set once armed (Req 5.1)");
    token = w.resync->round_token;

    resync_resolve_round(&g_twr, &w);
    TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w.host.status),
        "XCE_WORKER_RECONCILING must clear once resolved (Req 5.2)");
    TEST_ASSERT_NULL_MESSAGE(w.resync,
        "resync_resolve_round must detach/free w->resync");
    TEST_ASSERT_FALSE_MESSAGE(resync_round_is_current(&w, token),
        "a resolved round's token must no longer report current");

    /* Idempotent: resolving an already-resolved (NULL) round is a safe
     * no-op, never a double-free. */
    resync_resolve_round(&g_twr, &w);
    TEST_ASSERT_NULL(w.resync);

    twr_teardown();
    printf("test_resync_reconciling_flag_set_after_arm_cleared_after_"
          "resolve: PASS\n");
}

/* bug-1841 (new): resync_resolve_round() must notify HOOK_DETAIL_WORKER_
 * UPDATE subscribers once RECONCILING actually clears -- otherwise the
 * scheduler's own copy of this worker's status (fed purely from whatever
 * WORKERS_PUSH it last received; core_events.c's core_upsert_worker does a
 * bare `worker->status = w->status` with no other refresh path) stays
 * RECONCILING forever, permanently freezing that worker's resInUse
 * (sched_reset_worker_/sched_update_account's by-design bug-1799/1801
 * freeze) even after every job on it has long since finished DONE --
 * live-observed via a real scheduler-replay capture: a 22-slot worker
 * stuck at used=22.00 for the rest of the run with zero RUN jobs left.
 * Mirrors bug-1679's identical job-side gap and its
 * test_reconcile_pending_applies_releases_allocs_and_notifies spy-hook
 * proof above. */
static int g_twr_worker_update_hook_fires;

static int twr_worker_update_spy_hook(int type, int situation, int detail,
                                      void* data, va_list ap)
{
    (void)type; (void)data; (void)ap;
    if (situation == HOOK_AFTER_EVENT && detail == HOOK_DETAIL_WORKER_UPDATE)
        g_twr_worker_update_hook_fires++;
    return 0;
}

static void test_resync_resolve_round_notifies_worker_update(void)
{
    job_t*   j;
    worker_t w;
    int      rc;

    twr_setup();
    g_twr_worker_update_hook_fires = 0;
    TEST_ASSERT_EQUAL_INT(0,
        xce_add_hook(&g_twr.hooksys, XCE_CONTROLLER_HOOK_WORKER,
                    twr_worker_update_spy_hook, NULL));

    j = twr_seed_job(TWR_JOBID_RESYNC_SOLO, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_SOLO,
                                                  "resync-host-notify");

    memset(&w, 0, sizeof(w));
    w.host.name = (char*)"resync-host-notify";

    rc = resync_arm_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w.host.status));
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_worker_update_hook_fires,
        "arming a round must not itself fire HOOK_DETAIL_WORKER_UPDATE -- "
        "only resolving one does");

    resync_resolve_round(&g_twr, &w);
    TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w.host.status));
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_worker_update_hook_fires,
        "resync_resolve_round must notify HOOK_DETAIL_WORKER_UPDATE "
        "subscribers (scheduler/apiserver) once RECONCILING actually "
        "clears -- otherwise the scheduler never learns and stays frozen");

    /* Idempotent no-op: resolving an already-resolved round must NOT fire a
     * second, spurious notification. */
    resync_resolve_round(&g_twr, &w);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_worker_update_hook_fires,
        "resolving an already-resolved (NULL) round must not re-notify");

    twr_teardown();
    printf("test_resync_resolve_round_notifies_worker_update: PASS\n");
}

/* Req 5.3: the timeout resolves an outstanding round -- and does so via a
 * plain, synchronous function call with no sleep/wait of any kind, proving
 * the resolution itself never blocks (ha-async-io.md's delaymgr
 * thread-role contract). g_twr never starts a real delaymgr (matching
 * every other bare-cluster_t fixture in this file), so resync_arm_round's
 * own `!c->delaymgr.running` guard already took the no-real-timer branch
 * when the round above was armed; resync_test_fire_timeout below drives
 * the EXACT SAME internal resolution logic
 * (resync_on_timeout) a real timer firing would have. */
static void test_resync_timeout_resolves_round_without_blocking(void)
{
    job_t*    j;
    worker_t* w;
    int       rc;
    uint64_t  token;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_RESYNC_TIMEOUT, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_RESYNC_TIMEOUT,
                                                  "resync-host-timeout");

    w = twr_make_test_worker("resync-host-timeout");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    TEST_ASSERT_NOT_NULL(w->resync);
    TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));
    token = w->resync->round_token;

    /* resync_on_timeout (invoked via resync_test_fire_timeout) locks/
     * unlocks `w` itself -- this call must NOT be wrapped in an external
     * lock(w, ...), or it would deadlock against `w`'s own non-recursive
     * write lock. */
    resync_test_fire_timeout(&g_twr, w, token);
    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "the timeout must resolve (free/detach) the round");
    TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
        "the timeout must clear XCE_WORKER_RECONCILING so the worker is "
        "not withheld from dispatch indefinitely (Req 5.1, 5.2, 5.3)");

    /* A second, late fire for the SAME (now-stale) token must be a safe
     * no-op -- never a double-free/double-clear against an already-
     * resolved round. This is exactly the round-token staleness check
     * (Req 6.3) protecting against a real delaymgr task that could not be
     * dequeued/cancelled (delay.c exposes no cancel primitive) firing late
     * against a resync_state_t that is already gone or has already been
     * replaced by a newer round. */
    resync_test_fire_timeout(&g_twr, w, token);
    TEST_ASSERT_NULL(w->resync);
    TEST_ASSERT_FALSE(IS_WORKER_RECONCILING(w->host.status));

    twr_teardown();
    printf("test_resync_timeout_resolves_round_without_blocking: PASS\n");
}

/* ── resync_apply_report() tests (task 3.2) ──────────────────────────────
 *
 * Proves resync_apply_report() (resync.c) -- the report-apply glue wiring
 * together task 2.1's record_job_reconcile_pending_fact, task 2.2's
 * record_job_reconcile_assign_fact, and task 3.1's resync_arm_round/
 * resync_resolve_round/resync_round_is_current. Every test here uses a real
 * RTTI-managed worker_t (via twr_make_test_worker), since resync_apply_
 * report locks/unlocks `w` itself (unlike resync_arm_round/resync_resolve_
 * round/resync_round_is_current above, which require the caller to
 * pre-hold `w`'s lock and so tolerate a bare stack struct). */

#define TWR_JOBID_REPORT_FINOK    81108ULL
#define TWR_JOBID_REPORT_FINFAIL  81109ULL
#define TWR_JOBID_REPORT_RUNNING  81110ULL
#define TWR_JOBID_REPORT_UNREC    81111ULL
#define TWR_JOBID_REPORT_EXTRA    81112ULL
#define TWR_JOBID_REPORT_DUP_A    81113ULL
#define TWR_JOBID_REPORT_DUP_B    81114ULL
#define TWR_JOBID_REPORT_STALE    81115ULL
#define TWR_JOBID_REPORT_PARTIAL_A 81116ULL
#define TWR_JOBID_REPORT_PARTIAL_B 81117ULL
#define TWR_JOBID_REPORT_EXTRANOOP_Q 81118ULL
#define TWR_JOBID_REPORT_EXTRANOOP_E 81119ULL
#define TWR_JOBID_REPORT_FINISHTIME_REPLAY 81120ULL
#define TWR_JOBID_REPORT_LOSTDISPATCH_Q 81121ULL
#define TWR_JOBID_REPORT_LOSTDISPATCH 81122ULL

/* THE core proof (Req 4.1-4.5, 6.1): a single report covering all five
 * distinct outcome kinds in one pass -- finished-success, finished-failure,
 * still-running (queried), unrecognized (queried), and an unqueried extra --
 * leaves the job table in the correct state for each, routed through the
 * correct existing/new fact function, and resolves the round (outstanding
 * set empties -> XCE_WORKER_RECONCILING clears) once every QUERIED entry
 * has been answered.
 *
 * The unqueried-extra entry here is seeded TERMINAL (DONE), exercising
 * resync_apply_running_entry()'s branch (a) -- ALWAYS a no-op, regardless
 * of host (Req 4.4: a terminal job is never assign-eligible). This is
 * deliberate, not an oversight: round-2 review caught that this test
 * ORIGINALLY seeded j_extra TERMINAL and then asserted it became RUN --
 * i.e. it baked in, and certified as PASSING, the exact Req 4.4 regression
 * a since-corrected two-way guard introduced (calling record_job_
 * reconcile_assign_fact() on an already-terminal job, unconditionally
 * clobbering DONE/EXIT back to RUN). Branch (c) -- non-terminal AND
 * genuinely unassigned, the real "gets assigned" case -- is proven
 * separately, with full restart-replay rigor, by test_apply_report_
 * unqueried_extra_unassigned_pend_job_gets_assigned below; branch (b) --
 * non-terminal but already assigned (to this host or elsewhere) -- by
 * test_apply_report_unqueried_extra_already_tracked_is_noop. */
static void test_apply_report_all_five_outcomes_updates_job_table(void)
{
    job_t*    j_finok;
    job_t*    j_finfail;
    job_t*    j_running;
    job_t*    j_unrec;
    job_t*    j_extra;
    worker_t* w;
    int       rc;
    uint64_t  token;
    uint64_t  lsn_before, lsn_after;

    xce_job_state_entry_t  e_finok, e_finfail, e_running, e_unrec, e_extra;
    xce_job_state_entry_t* entries[5];
    xce_job_state_report_t report;

    twr_setup();

    j_finok = twr_seed_job(TWR_JOBID_REPORT_FINOK, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_finok);
    j_finok->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_REPORT_FINOK, "resync-report-host");

    j_finfail = twr_seed_job(TWR_JOBID_REPORT_FINFAIL, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_finfail);
    j_finfail->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_REPORT_FINFAIL, "resync-report-host");

    j_running = twr_seed_job(TWR_JOBID_REPORT_RUNNING, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_running);
    j_running->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_REPORT_RUNNING, "resync-report-host");

    j_unrec = twr_seed_job(TWR_JOBID_REPORT_UNREC, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_unrec);
    j_unrec->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_REPORT_UNREC, "resync-report-host");

    /* Seeded TERMINAL (DONE), so it is NOT part of the round's query set
     * (arm_round only collects non-terminal jobs), AND -- per Req 4.4 --
     * must be left COMPLETELY untouched when later reported running as an
     * unqueried extra: a terminal record is never assign-eligible, full
     * stop, regardless of host (resync_apply_running_entry()'s branch (a),
     * resync.c). */
    j_extra = twr_seed_job(TWR_JOBID_REPORT_EXTRA, JOB_STAT_DONE);
    TEST_ASSERT_NOT_NULL(j_extra);
    TEST_ASSERT_NULL(j_extra->job.allocs);

    w = twr_make_test_worker("resync-report-host");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT_MESSAGE(4, rc,
        "exactly the 4 non-terminal jobs on this host must be armed");
    TEST_ASSERT_NOT_NULL(w->resync);
    token = w->resync->round_token;

    e_finok = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_FINOK, .status = JobStateStatus_finishedOk,
        .exit_code = 0, .finish_time_ms = 1000, .queried = 1
    };
    e_finfail = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_FINFAIL,
        .status = JobStateStatus_finishedFail,
        .exit_code = 7, .finish_time_ms = 2000, .queried = 1
    };
    e_running = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_RUNNING, .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    e_unrec = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_UNREC, .status = JobStateStatus_unrecognized,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    e_extra = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_EXTRA, .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 0
    };

    entries[0] = &e_finok;
    entries[1] = &e_finfail;
    entries[2] = &e_running;
    entries[3] = &e_unrec;
    entries[4] = &e_extra;

    report = (xce_job_state_report_t){
        .n_total = 5, .pos = 0, .count = 5, .entries = entries
    };

    lsn_before = twr_current_lsn();

    rc = resync_apply_report(&g_twr, w, token, &report);
    TEST_ASSERT_EQUAL_INT(0, rc);

    lsn_after = twr_current_lsn();

    /* record_job_finish_fact_at() (bug-1747 fix)/record_job_fail_fact
     * (called via on_job_fail, unchanged -- Req 4.1) follow the SAME
     * durable-append-only, DEFERRED pattern as any ordinary worker
     * finish/fail report: the in-memory state mutation happens later, when
     * apply_committed_entry() replays the committed fact -- NOT
     * synchronously here (unlike record_job_reconcile_pending_fact/assign_
     * fact's own EAGER pattern below, asserted synchronously exactly
     * because task 2.1/2.2 built those two functions to mutate eagerly).
     * This exact deferred behavior is already proven end-to-end (including
     * via a real restart replay) by test_wal_job_finish_fact.c's own
     * test_wjf_real_on_job_finish_survives_restart -- this task's own job
     * is only to prove routing (the correct fact TYPE, with the correct
     * worker-reported finish_time_ms for finishedOk, was durably appended
     * through the correct call), not to re-prove that already-tested apply
     * seam; test_apply_report_finished_success_uses_worker_reported_
     * finish_time below proves the finish_time_ms plumbing specifically,
     * via a real restart replay. */
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
        "finished-success routes through the DEFERRED record_job_finish_"
        "fact pattern -- state mutation happens later via apply_committed_"
        "entry, not synchronously here");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finfail->job.state,
        "finished-failure routes through the DEFERRED record_job_fail_fact "
        "pattern -- state mutation happens later via apply_committed_entry, "
        "not synchronously here");
    TEST_ASSERT_TRUE_MESSAGE(lsn_after >= lsn_before + 3,
        "at least 3 durable facts must be appended for this report: "
        "finished-success (WAL_JOB_FINISH), finished-failure (WAL_JOB_"
        "FAIL), and unrecognized (WAL_JOB_UPDATE) -- the still-running "
        "entry and the TERMINAL unqueried extra (Req 4.4 no-op) record "
        "none");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
        "a queried still-running job must be left exactly as-is -- no fact "
        "recorded");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j_unrec->job.state,
        "unrecognized must apply through record_job_reconcile_pending_fact "
        "(-> PEND)");
    TEST_ASSERT_NULL_MESSAGE(j_unrec->job.allocs,
        "bug-1631: reconciled-to-pending must release the allocation");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j_extra->job.state,
        "Req 4.4: a TERMINAL job reported running as an unqueried extra "
        "must be left COMPLETELY untouched -- never routed through "
        "record_job_reconcile_assign_fact, regardless of host");
    TEST_ASSERT_NULL_MESSAGE(j_extra->job.allocs,
        "the terminal extra's allocation (already NULL) must stay "
        "untouched too");

    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "once every queried job (4 of them) has a reconciled outcome, the "
        "round must resolve");
    TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
        "XCE_WORKER_RECONCILING must clear once the round resolves");

    twr_teardown();
    printf("test_apply_report_all_five_outcomes_updates_job_table: PASS\n");
}

/* Req 6.2: applying the identical report a second time (same round token)
 * must produce the same result as applying it once -- no further job-table
 * change, no additional WAL append. Since the first application already
 * fully resolves the round (both entries here are queried), the round
 * token is no longer current by the time the second, identical call
 * arrives -- resync_apply_report's own Req 6.1/6.3 stale-round discard is
 * exactly the mechanism that makes the duplicate a pure no-op. */
static void test_apply_report_duplicate_reapplication_is_noop(void)
{
    job_t*    ja;
    job_t*    jb;
    worker_t* w;
    int       rc;
    uint64_t  token;
    uint64_t  lsn_before_first, lsn_after_first, lsn_after_second;

    xce_job_state_entry_t  ea, eb;
    xce_job_state_entry_t* entries[2];
    xce_job_state_report_t report;

    twr_setup();

    ja = twr_seed_job(TWR_JOBID_REPORT_DUP_A, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(ja);
    ja->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_DUP_A,
        "resync-dup-host");

    jb = twr_seed_job(TWR_JOBID_REPORT_DUP_B, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(jb);
    jb->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_DUP_B,
        "resync-dup-host");

    w = twr_make_test_worker("resync-dup-host");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(2, rc);
    token = w->resync->round_token;

    ea = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_DUP_A, .status = JobStateStatus_finishedOk,
        .exit_code = 0, .finish_time_ms = 500, .queried = 1
    };
    eb = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_DUP_B, .status = JobStateStatus_unrecognized,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    entries[0] = &ea;
    entries[1] = &eb;
    report = (xce_job_state_report_t){
        .n_total = 2, .pos = 0, .count = 2, .entries = entries
    };

    lsn_before_first = twr_current_lsn();

    rc = resync_apply_report(&g_twr, w, token, &report);
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* ea (finishedOk) routes through the DEFERRED record_job_finish_fact_at
     * pattern (bug-1747 fix) -- state mutation happens later via
     * apply_committed_entry, not synchronously here (see the "all five
     * outcomes" test's own comment for the full rationale); ja's state is
     * asserted unchanged (still RUN) below instead. */
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, ja->job.state);
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, jb->job.state);
    TEST_ASSERT_NULL(jb->job.allocs);
    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "the round must resolve after the first, complete application");
    lsn_after_first = twr_current_lsn();
    TEST_ASSERT_TRUE_MESSAGE(lsn_after_first >= lsn_before_first + 2,
        "the first application must durably append a fact for each of the "
        "2 queried entries");

    /* Second, IDENTICAL application -- same token, same report contents. */
    rc = resync_apply_report(&g_twr, w, token, &report);
    TEST_ASSERT_EQUAL_INT(0, rc);
    lsn_after_second = twr_current_lsn();

    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, ja->job.state,
        "a duplicate re-application must change nothing further");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, jb->job.state,
        "a duplicate re-application must change nothing further");
    TEST_ASSERT_NULL(jb->job.allocs);
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_after_first, lsn_after_second,
        "a duplicate re-application for an already-resolved round must "
        "append no further WAL fact (Req 6.2)");
    TEST_ASSERT_NULL(w->resync);

    twr_teardown();
    printf("test_apply_report_duplicate_reapplication_is_noop: PASS\n");
}

/* Req 6.1, 6.3: a report carrying a round token already superseded by a
 * newer round for the SAME worker is dropped in its entirety -- the job
 * table must be completely untouched, and the (unrelated, current) newer
 * round must be left exactly as it was. */
static void test_apply_report_stale_round_token_is_discarded(void)
{
    job_t*    j;
    worker_t* w;
    int       rc;
    uint64_t  token1, token2;
    uint64_t  lsn_before, lsn_after;

    xce_job_state_entry_t  e;
    xce_job_state_entry_t* entries[1];
    xce_job_state_report_t report;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_REPORT_STALE, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_STALE,
        "resync-stale-host");

    w = twr_make_test_worker("resync-stale-host");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    token1 = w->resync->round_token;

    /* Simulated disconnect-then-rejoin: a fresh round supersedes the first
     * before its report ever arrives. */
    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    token2 = w->resync->round_token;
    TEST_ASSERT_NOT_EQUAL(token1, token2);

    e = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_STALE, .status = JobStateStatus_finishedOk,
        .exit_code = 0, .finish_time_ms = 999, .queried = 1
    };
    entries[0] = &e;
    report = (xce_job_state_report_t){
        .n_total = 1, .pos = 0, .count = 1, .entries = entries
    };

    lsn_before = twr_current_lsn();

    /* A report carrying the now-stale token1 must be dropped entirely. */
    rc = resync_apply_report(&g_twr, w, token1, &report);
    TEST_ASSERT_EQUAL_INT(0, rc);

    lsn_after = twr_current_lsn();

    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
        "a stale-round report must be dropped without touching the job "
        "table");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
        "a stale-round report must append no WAL fact");
    TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
        "the CURRENT (newer) round must be left outstanding, untouched by "
        "the stale report");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(token2, w->resync->round_token,
        "the current round's own token must be unaffected");
    TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);

    resync_resolve_round(&g_twr, w);
    twr_teardown();
    printf("test_apply_report_stale_round_token_is_discarded: PASS\n");
}

/* Req 5.1, 5.2: a partial report (not all queried jobs answered yet) must
 * NOT resolve the round -- the outstanding set stays non-empty and
 * XCE_WORKER_RECONCILING stays set -- until a LATER call with the
 * remaining entries completes it, proving resolution is genuinely
 * incremental against the outstanding set, not "resolve unconditionally
 * after any report." */
static void test_apply_report_partial_report_does_not_resolve_round(void)
{
    job_t*    ja;
    job_t*    jb;
    worker_t* w;
    int       rc;
    uint64_t  token;
    uint64_t  lsn_before, lsn_after;

    xce_job_state_entry_t  ea, eb;
    xce_job_state_entry_t* entries_a[1];
    xce_job_state_entry_t* entries_b[1];
    xce_job_state_report_t report_a, report_b;

    twr_setup();

    ja = twr_seed_job(TWR_JOBID_REPORT_PARTIAL_A, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(ja);
    ja->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_PARTIAL_A,
        "resync-partial-host");

    jb = twr_seed_job(TWR_JOBID_REPORT_PARTIAL_B, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(jb);
    jb->job.allocs = twr_make_test_alloc_for_host(TWR_JOBID_REPORT_PARTIAL_B,
        "resync-partial-host");

    w = twr_make_test_worker("resync-partial-host");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(2, rc);
    token = w->resync->round_token;

    ea = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_PARTIAL_A,
        .status = JobStateStatus_finishedOk,
        .exit_code = 0, .finish_time_ms = 111, .queried = 1
    };
    entries_a[0] = &ea;
    report_a = (xce_job_state_report_t){
        .n_total = 1, .pos = 0, .count = 1, .entries = entries_a
    };

    lsn_before = twr_current_lsn();

    rc = resync_apply_report(&g_twr, w, token, &report_a);
    TEST_ASSERT_EQUAL_INT(0, rc);

    lsn_after = twr_current_lsn();

    /* ea (finishedOk) routes through the DEFERRED record_job_finish_fact
     * pattern (via on_job_finish, unchanged) -- state mutation happens
     * later via apply_committed_entry, not synchronously here (see the
     * "all five outcomes" test's own comment for the full rationale); a
     * durable append is asserted instead. */
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, ja->job.state);
    TEST_ASSERT_TRUE_MESSAGE(lsn_after > lsn_before,
        "job A's finished-success entry must durably append a fact");
    TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
        "job B has not been answered yet -- the round must stay "
        "outstanding");
    TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
    TEST_ASSERT_TRUE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
        "XCE_WORKER_RECONCILING must remain set while the round is still "
        "partially outstanding");

    eb = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_PARTIAL_B, .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    entries_b[0] = &eb;
    report_b = (xce_job_state_report_t){
        .n_total = 1, .pos = 0, .count = 1, .entries = entries_b
    };

    rc = resync_apply_report(&g_twr, w, token, &report_b);
    TEST_ASSERT_EQUAL_INT(0, rc);
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_RUN, jb->job.state);
    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "the round must resolve once the LAST outstanding queried job "
        "(B) is answered");
    TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
        "XCE_WORKER_RECONCILING must clear once the round fully resolves");

    twr_teardown();
    printf("test_apply_report_partial_report_does_not_resolve_round: "
          "PASS\n");
}

/* Req 3.6/4.7/4.4 (the crux branching this task's own brief flagged as
 * needing careful resolution against job.c's record_job_reconcile_assign_
 * fact doc comment; round-2 review strengthened this test after finding it
 * could not actually distinguish "correctly no-op'd" from "incorrectly
 * reassigned" -- see below): an unqueried extra reported RUNNING that the
 * controller ALREADY has a non-terminal record of (here: assigned to a
 * DIFFERENT host -- resync_apply_running_entry()'s branch (b), non-terminal
 * AND job->job.allocs != NULL) must NOT be routed through record_job_
 * reconcile_assign_fact -- it is already correctly represented by simply
 * existing and being reported running.
 *
 * Proven three ways, precisely enough to actually catch a regression back
 * to either of the two review-caught defects (round 1's Req 4.8-axis bug;
 * round 2's Req 4.4-axis bug):
 *   1. allocs survives UNTOUCHED -- record_job_reconcile_assign_fact always
 *      sets job->job.allocs to NULL (Req 4.8), so a surviving allocation
 *      proves the no-op branch fired instead of the assign branch.
 *   2. start_time survives UNTOUCHED at its seeded sentinel value --
 *      record_job_reconcile_assign_fact ALSO unconditionally stamps
 *      job->job.start_time = time(NULL) (bug-1697), even though the net
 *      STATE effect (RUN -> RUN) looks harmless and is invisible to (1) or
 *      to a bare state-equality assertion; this is exactly what round 1's
 *      version of this test could not catch, since it never checked
 *      start_time and the job was already RUN before the call either way.
 *   3. no additional WAL fact is appended (LSN unchanged) -- record_job_
 *      reconcile_assign_fact durably appends a fresh WAL_JOB_DISPATCH fact
 *      unconditionally; a no-op must append nothing (the paired queried
 *      "running" entry records nothing either, so the WHOLE report's LSN
 *      delta must be exactly zero). */
static void test_apply_report_unqueried_extra_already_tracked_is_noop(void)
{
    job_t*    j_queried;
    job_t*    j_extra;
    worker_t* w;
    int       rc;
    uint64_t  token;
    uint64_t  lsn_before, lsn_after;
    time_t    seeded_start_time;

    xce_job_state_entry_t  e_queried, e_extra;
    xce_job_state_entry_t* entries[2];
    xce_job_state_report_t report;

    twr_setup();

    /* Keeps the round non-empty (arm_round needs >=1 non-terminal job on
     * w's own host to arm at all) -- unrelated to the extra under test. */
    j_queried = twr_seed_job(TWR_JOBID_REPORT_EXTRANOOP_Q, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_queried);
    j_queried->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_REPORT_EXTRANOOP_Q, "resync-extranoop-host");

    /* Non-terminal (RUN), already assigned -- to a DIFFERENT host, so
     * arm_round's own query-set construction (non-terminal jobs assigned
     * to w's specific host) never includes it; it arrives only as an
     * unqueried extra in the report below. */
    j_extra = twr_seed_job(TWR_JOBID_REPORT_EXTRANOOP_E, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_extra);
    j_extra->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_REPORT_EXTRANOOP_E, "resync-extranoop-OTHER-host");

    /* A deliberately-distinguishable sentinel, far from "now" -- so a
     * wrongful record_job_reconcile_assign_fact() call (which stamps
     * time(NULL)) is caught precisely, not masked by coincidence. */
    seeded_start_time      = (time_t)424242;
    j_extra->job.start_time = seeded_start_time;

    w = twr_make_test_worker("resync-extranoop-host");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
        "only j_queried is assigned to w's own host");
    token = w->resync->round_token;

    e_queried = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_EXTRANOOP_Q,
        .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    e_extra = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_REPORT_EXTRANOOP_E,
        .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 0
    };
    entries[0] = &e_queried;
    entries[1] = &e_extra;
    report = (xce_job_state_report_t){
        .n_total = 2, .pos = 0, .count = 2, .entries = entries
    };

    lsn_before = twr_current_lsn();

    rc = resync_apply_report(&g_twr, w, token, &report);
    TEST_ASSERT_EQUAL_INT(0, rc);

    lsn_after = twr_current_lsn();

    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_extra->job.state,
        "an already-non-terminal extra must be left exactly as-is");
    TEST_ASSERT_NOT_NULL_MESSAGE(j_extra->job.allocs,
        "record_job_reconcile_assign_fact must NOT have been called -- its "
        "own contract always sets allocs to NULL, so a surviving "
        "allocation proves the no-op branch fired instead");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)seeded_start_time,
        (uint64_t)j_extra->job.start_time,
        "record_job_reconcile_assign_fact must NOT have been called -- it "
        "unconditionally stamps start_time = time(NULL) (bug-1697), so a "
        "start_time still equal to the seeded sentinel (not \"now\") "
        "proves the no-op branch fired, catching what a bare state/allocs "
        "check alone cannot (round-2 review finding)");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(lsn_before, lsn_after,
        "neither entry in this report may append a WAL fact: the queried "
        "running entry never does, and the already-tracked unqueried "
        "extra must not durably re-append a WAL_JOB_DISPATCH fact for a "
        "job that was never actually lost/unassigned");

    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "the round's only queried job (j_queried) was answered -- it must "
        "resolve regardless of the unqueried extra");

    twr_teardown();
    printf("test_apply_report_unqueried_extra_already_tracked_is_noop: "
          "PASS\n");
}

/* bug-1747 fix (Req 4.2): a finished-success reconciliation entry's
 * WORKER-reported finish_time_ms must end up as the job's recorded
 * finish_time after a real restart replay -- NOT reconciliation-processing
 * wall-clock time. Mirrors test_reconcile_assign_survives_restart_replay's
 * own Phase 1 (real cluster + disk persist, drive the real function under
 * test, flush) / Phase 2 (fresh persist_t + persist_recover(), assert on
 * the recovered job_t) shape, and test_wal_job_finish_fact.c's own
 * test_wjf_real_on_job_finish_survives_restart precedent for proving a
 * finish_time round-trip through a real WAL append + replay, not a
 * hand-built payload. Uses its own local cluster_t/persist_t/tmpdir (Phase
 * 1's disk state must survive past Phase 1's own teardown) -- only
 * g_twr.workermgr (via twr_make_test_worker) is shared with the rest of
 * this file's fixture, exactly as test_reconcile_assign_survives_restart_
 * replay already establishes is safe (resync_apply_report's own locking
 * only ever touches `w` and the `c` passed to it, never workermgr). */
static void
test_apply_report_finished_success_uses_worker_reported_finish_time(void)
{
    char     tmp[256];
    uint64_t worker_reported_finish_time_s;

    strcpy(tmp, "/tmp/xce_twr24_finishtime_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    twr_setup();

    /* A finish time deliberately, clearly DIFFERENT from "now" -- one full
     * day in the past -- so a reconciliation-processing-time bug
     * (time(NULL)) cannot accidentally produce a matching value. */
    worker_reported_finish_time_s = (uint64_t)time(NULL) - 86400;

    /* Phase 1: real cluster + real disk persist. Seed a durable
     * WAL_JOB_SUBMIT fact directly (so replay can reconstruct the job --
     * this job was never dispatched through any controller in this log)
     * and the in-memory job_t the same way (bypassing dispatch/replay),
     * already RUN and assigned to the reconciling worker's host, then
     * drive a real resync_apply_report() call carrying the worker-reported
     * finish_time_ms. */
    {
        cluster_t              c;
        kvmgr_params_t         int_params = { .key_type = KVMGR_KEY_INT,
                                              .nslots = 32, .delta = 16 };
        persist_config_t       cfg;
        job_t*                 j;
        worker_t*               w;
        int                     rc;
        uint64_t                token;
        xce_job_state_entry_t   e;
        xce_job_state_entry_t*  entries[1];
        xce_job_state_report_t  report;

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";

            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = TWR_JOBID_REPORT_FINISHTIME_REPLAY;
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;

            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);
        }

        {
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";

            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = TWR_JOBID_REPORT_FINISHTIME_REPLAY;
            jdef.state  = JOB_STAT_RUN;
            jdef.submit = &submit;
            j = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(j);
        }
        j->job.allocs = twr_make_test_alloc_for_host(
            TWR_JOBID_REPORT_FINISHTIME_REPLAY, "resync-finishtime-host");

        w = twr_make_test_worker("resync-finishtime-host");
        TEST_ASSERT_NOT_NULL(w);

        rc = resync_arm_round(&c, w);
        TEST_ASSERT_EQUAL_INT(1, rc);
        token = w->resync->round_token;

        e = (xce_job_state_entry_t){
            .jobid = TWR_JOBID_REPORT_FINISHTIME_REPLAY,
            .status = JobStateStatus_finishedOk,
            .exit_code = 0,
            .finish_time_ms = worker_reported_finish_time_s * 1000ULL,
            .queried = 1
        };
        entries[0] = &e;
        report = (xce_job_state_report_t){
            .n_total = 1, .pos = 0, .count = 1, .entries = entries
        };

        rc = resync_apply_report(&c, w, token, &report);
        TEST_ASSERT_EQUAL_INT(0, rc);
        TEST_ASSERT_NULL_MESSAGE(w->resync,
            "the round's only queried job was answered -- it must resolve");

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
     * recover from disk -- simulates a controller restart. */
    {
        persist_t* p2 = twr_make_fresh_persist(tmp);
        cluster_t  c2;
        job_t*     j2 = NULL;
        int        rc;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 32, .delta = 16 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        key_find(c2.jobmgr, (void*)TWR_JOBID_REPORT_FINISHTIME_REPLAY,
                 (void**)&j2);
        unlock(c2.jobmgr);
        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
            "restart replay must reconstruct the job via WAL_JOB_SUBMIT");
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2->job.state,
            "the finished-success fact must apply to DONE on replay");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(worker_reported_finish_time_s,
            (uint64_t)j2->job.finish_time,
            "bug-1747 (Req 4.2): the recovered finish_time must match the "
            "WORKER-reported finish_time_ms, not reconciliation-processing "
            "wall-clock time");

        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    twr_teardown();
    printf("test_apply_report_finished_success_uses_worker_reported_"
          "finish_time: PASS\n");
}

/* Review-caught regression fix (Req 4.8): a job the controller currently
 * shows PEND (non-terminal, allocs == NULL, unassigned to any host) --
 * exactly design.md's own "Documented scope decision" scenario, e.g. its
 * WAL_JOB_DISPATCH fact never reached durable majority commit before a
 * controller failover, even though dispatch_a_job_() already sent
 * xce_startjob_req() to the worker and the worker really is running it --
 * must NOT be left stuck at PEND when reported running as an unqueried
 * extra. It must reconcile via record_job_reconcile_assign_fact() to RUN,
 * matching the reporting worker's host, with allocs staying NULL (there is
 * none to fabricate, Req 4.8's own documented scope decision). Mirrors
 * test_reconcile_assign_survives_restart_replay's rigor: real WAL_JOB_
 * SUBMIT seed + real resync_apply_report() call + real WAL flush, followed
 * by a genuine persist_recover() restart replay -- not just an in-memory
 * assertion -- so this proves the fix survives a real controller restart,
 * not merely the live eager-mutation path. A second, unrelated queried job
 * on the same host keeps the round non-empty (arm_round's own query set
 * construction only ever finds jobs it already believes are non-terminal
 * AND assigned to the host -- by definition NOT this PEND/unassigned job,
 * which is exactly why it only ever arrives as an unqueried extra). */
static void
test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned(void)
{
    char tmp[256];

    strcpy(tmp, "/tmp/xce_twr25_lostdispatch_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    twr_setup();

    /* Phase 1: real cluster + real disk persist. */
    {
        cluster_t              c;
        kvmgr_params_t         int_params = { .key_type = KVMGR_KEY_INT,
                                              .nslots = 32, .delta = 16 };
        persist_config_t       cfg;
        job_t*                 jq;
        job_t*                 jlost;
        worker_t*               w;
        int                     rc;
        uint64_t                token;
        xce_job_state_entry_t   eq, elost;
        xce_job_state_entry_t*  entries[2];
        xce_job_state_report_t  report;

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        /* Durable WAL_JOB_SUBMIT seeds -- jq at RUN (assigned, keeps the
         * round non-empty), jlost at PEND (unassigned -- the job whose
         * WAL_JOB_DISPATCH fact never durably committed). */
        {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";

            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = TWR_JOBID_REPORT_LOSTDISPATCH_Q;
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);

            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = TWR_JOBID_REPORT_LOSTDISPATCH;
            wal_jdef.state  = JOB_STAT_PEND;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);
        }

        /* Seed the in-memory job_t's the same way, bypassing dispatch/
         * replay. */
        {
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";

            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = TWR_JOBID_REPORT_LOSTDISPATCH_Q;
            jdef.state  = JOB_STAT_RUN;
            jdef.submit = &submit;
            jq = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(jq);
            jq->job.allocs = twr_make_test_alloc_for_host(
                TWR_JOBID_REPORT_LOSTDISPATCH_Q, "resync-lostdispatch-host");

            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = TWR_JOBID_REPORT_LOSTDISPATCH;
            jdef.state  = JOB_STAT_PEND;
            jdef.submit = &submit;
            jlost = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(jlost);
            TEST_ASSERT_NULL_MESSAGE(jlost->job.allocs,
                "the lost-dispatch-fact job starts PEND and unassigned -- "
                "no allocation was ever durably recorded for it");
        }

        w = twr_make_test_worker("resync-lostdispatch-host");
        TEST_ASSERT_NOT_NULL(w);

        rc = resync_arm_round(&c, w);
        TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
            "only jq (RUN, assigned) is in the controller's own query set "
            "-- jlost (PEND, unassigned) is invisible to arm_round's own "
            "scan, exactly why it can only ever arrive as an unqueried "
            "extra");
        token = w->resync->round_token;

        eq = (xce_job_state_entry_t){
            .jobid = TWR_JOBID_REPORT_LOSTDISPATCH_Q,
            .status = JobStateStatus_running,
            .exit_code = 0, .finish_time_ms = 0, .queried = 1
        };
        elost = (xce_job_state_entry_t){
            .jobid = TWR_JOBID_REPORT_LOSTDISPATCH,
            .status = JobStateStatus_running,
            .exit_code = 0, .finish_time_ms = 0, .queried = 0
        };
        entries[0] = &eq;
        entries[1] = &elost;
        report = (xce_job_state_report_t){
            .n_total = 2, .pos = 0, .count = 2, .entries = entries
        };

        rc = resync_apply_report(&c, w, token, &report);
        TEST_ASSERT_EQUAL_INT(0, rc);

        /* Eager pattern (record_job_reconcile_assign_fact) -- observable
         * synchronously, before any commit round-trip. */
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jlost->job.state,
            "a PEND, unassigned job reported running as an unqueried extra "
            "must reconcile to RUN via record_job_reconcile_assign_fact -- "
            "not be left stuck at PEND");
        TEST_ASSERT_NULL_MESSAGE(jlost->job.allocs,
            "Req 4.8: no allocation is fabricated for a reconciliation-"
            "assigned job");
        TEST_ASSERT_NULL_MESSAGE(w->resync,
            "the round's only QUERIED job (jq) was answered -- it must "
            "resolve regardless of the unqueried extra");

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t, recover from disk --
     * proves the fix survives a real controller restart, not just the live
     * eager-mutation path. */
    {
        persist_t* p2 = twr_make_fresh_persist(tmp);
        cluster_t  c2;
        job_t*     j2 = NULL;
        int        rc;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 32, .delta = 16 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        key_find(c2.jobmgr, (void*)TWR_JOBID_REPORT_LOSTDISPATCH,
                 (void**)&j2);
        unlock(c2.jobmgr);
        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
            "restart replay must reconstruct the job via WAL_JOB_SUBMIT");
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2->job.state,
            "restart replay must recover the job as RUNNING from the "
            "WAL_JOB_DISPATCH fact appended by record_job_reconcile_"
            "assign_fact()'s real call -- the fix must survive a real "
            "controller restart, not just the live eager-mutation path");
        TEST_ASSERT_NULL_MESSAGE(j2->job.allocs,
            "Req 4.8: replay must NOT fabricate a resource allocation for "
            "a reconciliation-assigned job");

        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    twr_teardown();
    printf("test_apply_report_unqueried_extra_unassigned_pend_job_gets_"
          "assigned: PASS\n");
}

/* ══════════════════════════════════════════════════════════════════════
 * Task 6.2 (Req 8.1, 8.3): resync_handle_report_page() -- multi-page wire
 * reassembly of an incoming OPCode_jobStateReport, ahead of resync_apply_
 * report() (task 3.2, proven above). Every test here proves the exact
 * gap this task's own brief called out: not just that the correct END
 * STATE results, but that resync_apply_report() is invoked EXACTLY ONCE
 * per report regardless of page count (resync_apply_report_call_count_
 * for_test(), resync.h) -- job.c's own idempotent fact guards would make
 * even a buggy once-per-page implementation LOOK correct on a happy-path
 * report, exactly the class of bug task 4.3's own round-1 review missed
 * with a purely behavioral test (tasks.md Implementation Notes).
 * ══════════════════════════════════════════════════════════════════════ */

#define TWR_JOBID_MPAGE_FINOK   81201ULL
#define TWR_JOBID_MPAGE_UNREC   81202ULL
#define TWR_JOBID_MPAGE_RUNNING 81203ULL
#define TWR_JOBID_MPAGE_STALE   81210ULL
#define TWR_JOBID_MPAGE_BAD     81220ULL
#define TWR_JOBID_MPAGE_BAD2    81221ULL

/* Baseline: a single-page report (pos=0, count==n_total) delivered through
 * resync_handle_report_page() -- not resync_apply_report() directly -- is
 * reassembled trivially (one page IS the whole report) and applied exactly
 * once. Establishes the expected end state the multi-page test below must
 * independently reproduce via genuine reassembly. */
static void test_handle_report_page_single_page_baseline(void)
{
    job_t*    j_unrec;
    worker_t* w;
    int       rc;
    uint64_t  token;
    int       calls_before, calls_after;

    xce_job_state_entry_t  e_unrec;
    xce_job_state_entry_t* entries[1];
    xce_job_state_report_t page;

    twr_setup();

    j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_unrec);
    j_unrec->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-1");

    w = twr_make_test_worker("mpage-host-1");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    token = w->resync->round_token;

    e_unrec = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    entries[0] = &e_unrec;
    page = (xce_job_state_report_t){
        .n_total = 1, .pos = 0, .count = 1, .entries = entries
    };

    resync_apply_report_call_count_reset_for_test();
    calls_before = resync_apply_report_call_count_for_test();

    rc = resync_handle_report_page(&g_twr, w, &page);

    calls_after = resync_apply_report_call_count_for_test();

    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
        "a single page whose count == n_total completes the report "
        "immediately");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
        "resync_apply_report must be invoked exactly once");
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j_unrec->job.state);
    TEST_ASSERT_NULL(j_unrec->job.allocs);
    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "the round must resolve once its only queried job is answered");

    (void)token;
    twr_teardown();
    printf("test_handle_report_page_single_page_baseline: PASS\n");
}

/* THE core proof (task 6.2's own Observable Completion Criterion): the SAME
 * report, split across two wire pages, must (a) leave the job table
 * COMPLETELY untouched after the first (non-final) page -- no partial
 * application -- (b) converge to EXACTLY the same end state the single-page
 * baseline above reached, once the final page arrives, and (c) invoke
 * resync_apply_report() EXACTLY ONCE for the whole report, not once per
 * page (the structural proof a behavioral-only assertion cannot provide --
 * see this section's own header comment). */
static void test_handle_report_page_multi_page_reassembles_and_applies_once(
    void)
{
    job_t*    j_finok;
    job_t*    j_unrec;
    job_t*    j_running;
    worker_t* w;
    int       rc;
    int       calls_before, calls_mid, calls_after;

    xce_job_state_entry_t  e_finok, e_unrec, e_running;
    xce_job_state_entry_t* entries_p1[2];
    xce_job_state_entry_t* entries_p2[1];
    xce_job_state_report_t page1, page2;

    twr_setup();

    j_finok = twr_seed_job(TWR_JOBID_MPAGE_FINOK, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_finok);
    j_finok->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_FINOK, "mpage-host-2");

    j_unrec = twr_seed_job(TWR_JOBID_MPAGE_UNREC, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_unrec);
    j_unrec->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_UNREC, "mpage-host-2");

    j_running = twr_seed_job(TWR_JOBID_MPAGE_RUNNING, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j_running);
    j_running->job.allocs = twr_make_test_alloc_for_host(
        TWR_JOBID_MPAGE_RUNNING, "mpage-host-2");

    w = twr_make_test_worker("mpage-host-2");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT_MESSAGE(3, rc,
        "all 3 non-terminal jobs on this host must be armed");

    e_finok = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_FINOK, .status = JobStateStatus_finishedOk,
        .exit_code = 0, .finish_time_ms = 4242, .queried = 1
    };
    e_unrec = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_UNREC, .status = JobStateStatus_unrecognized,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    e_running = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_RUNNING, .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };

    /* Page 1 of 2: entries [0..1] (finok, unrec). Page 2 of 2: entry [2]
     * (running). Mirrors xce_job_state_report_req()'s own {pos,count,
     * n_total} pagination shape (request.c) -- this is exactly what a real
     * multi-page OPCode_jobStateReport looks like on the wire. */
    entries_p1[0] = &e_finok;
    entries_p1[1] = &e_unrec;
    page1 = (xce_job_state_report_t){
        .n_total = 3, .pos = 0, .count = 2, .entries = entries_p1
    };

    entries_p2[0] = &e_running;
    page2 = (xce_job_state_report_t){
        .n_total = 3, .pos = 2, .count = 1, .entries = entries_p2
    };

    resync_apply_report_call_count_reset_for_test();
    calls_before = resync_apply_report_call_count_for_test();

    rc = resync_handle_report_page(&g_twr, w, &page1);

    calls_mid = resync_apply_report_call_count_for_test();

    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
        "page 1 of 2 alone must NOT complete the report");
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, calls_mid - calls_before,
        "resync_apply_report must NOT be called after a non-final page -- "
        "the structural 'not once per page' proof");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
        "no partial application: job A must be untouched after page 1");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_unrec->job.state,
        "no partial application: job B must be untouched after page 1 -- "
        "still RUN, NOT yet reconciled to PEND");
    TEST_ASSERT_NOT_NULL_MESSAGE(j_unrec->job.allocs,
        "no partial application: job B's allocation must not be released "
        "until the report is fully assembled and applied");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
        "job C (only referenced in page 2, not yet arrived) is untouched");
    TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
        "the round must still be outstanding after only a partial report");

    rc = resync_handle_report_page(&g_twr, w, &page2);

    calls_after = resync_apply_report_call_count_for_test();

    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
        "page 2 of 2 completes the report");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_mid,
        "resync_apply_report must be invoked EXACTLY ONCE, on the final "
        "page, for the whole 3-entry report -- not once per page");

    /* Equivalence (task 6.2's own Observable Completion Criterion): the SAME
     * end state test_handle_report_page_single_page_baseline reached for an
     * equivalent single-page report. */
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_finok->job.state,
        "finished-success stays in the DEFERRED record_job_finish_fact_at "
        "pattern -- state mutates later via apply_committed_entry, same as "
        "an equivalent single-page report (test_apply_report_all_five_"
        "outcomes_updates_job_table proves this deferred path directly)");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j_unrec->job.state,
        "job B reconciles to PEND -- identical to the single-page baseline");
    TEST_ASSERT_NULL_MESSAGE(j_unrec->job.allocs,
        "job B's allocation is released -- identical to the single-page "
        "baseline");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j_running->job.state,
        "job C (queried running) is left exactly as-is -- no fact "
        "recorded, identical to the single-page baseline");
    TEST_ASSERT_NULL_MESSAGE(w->resync,
        "once every queried job across BOTH pages has a reconciled "
        "outcome, the round must resolve -- identical to the single-page "
        "baseline");
    TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
        "XCE_WORKER_RECONCILING must clear once the round resolves");

    twr_teardown();
    printf("test_handle_report_page_multi_page_reassembles_and_applies_once:"
          " PASS\n");
}

/* Req 6.1/6.3 at the reassembly layer: a report that STARTS arriving under
 * one round but finishes assembling only after that round has been
 * superseded by a fresh resync_arm_round() call (simulating a disconnect-
 * then-immediate-rejoin racing the tail of a still-in-flight report) must
 * be discarded in its entirety -- not matched against whichever round
 * happens to be current once the LAST page arrives. Proves the round_token
 * CAPTURE point (page 1, resync_report_accum_t.round_token) is what makes
 * this correct, per resync.h's own doc comment on why that capture is not
 * an arbitrary choice. */
static void test_handle_report_page_stale_round_discarded_when_superseded(
    void)
{
    job_t*    j;
    worker_t* w;
    int       rc;
    int       calls_before, calls_after;

    xce_job_state_entry_t  e1, e2;
    xce_job_state_entry_t* entries_p1[1];
    xce_job_state_entry_t* entries_p2[1];
    xce_job_state_report_t page1, page2;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_MPAGE_STALE, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_STALE, "mpage-host-3");

    w = twr_make_test_worker("mpage-host-3");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);

    e1 = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedOk,
        .exit_code = 0, .finish_time_ms = 1, .queried = 1
    };
    entries_p1[0] = &e1;
    page1 = (xce_job_state_report_t){
        .n_total = 2, .pos = 0, .count = 1, .entries = entries_p1
    };

    rc = resync_handle_report_page(&g_twr, w, &page1);
    TEST_ASSERT_EQUAL_INT(0, rc);

    /* Simulated disconnect-then-rejoin racing the tail of the in-flight
     * report above: a fresh round supersedes the one page1 started under,
     * BEFORE page 2 ever arrives. */
    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);
    TEST_ASSERT_NOT_NULL(w->resync);

    /* A second entry, unrelated jobid, just to complete the (now-stale)
     * report's page count -- its content is immaterial since the whole
     * report must be discarded regardless. */
    e2 = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_STALE, .status = JobStateStatus_finishedFail,
        .exit_code = 0, .finish_time_ms = 2, .queried = 1
    };
    entries_p2[0] = &e2;
    page2 = (xce_job_state_report_t){
        .n_total = 2, .pos = 1, .count = 1, .entries = entries_p2
    };

    resync_apply_report_call_count_reset_for_test();
    calls_before = resync_apply_report_call_count_for_test();

    rc = resync_handle_report_page(&g_twr, w, &page2);

    calls_after = resync_apply_report_call_count_for_test();

    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
        "the report completes (page 2 arrived) and IS handed to "
        "resync_apply_report -- which then discards it internally as "
        "stale, per its own Req 6.1/6.3 contract");
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
        "resync_apply_report is still called exactly once -- discarding "
        "happens INSIDE it (round-token mismatch), not by skipping the "
        "call");
    TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j->job.state,
        "the stale report must never touch the job table");
    TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
        "the CURRENT (superseding) round must be left outstanding, "
        "untouched by the stale report");
    TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);

    resync_resolve_round(&g_twr, w);
    twr_teardown();
    printf("test_handle_report_page_stale_round_discarded_when_superseded:"
          " PASS\n");
}

/* Robustness: a page whose [pos, pos+count) range falls outside the
 * report's own declared n_total is rejected (-EINVAL) and discards
 * whatever partial accumulation was in progress -- proven by immediately
 * starting a FRESH, valid report afterward and confirming it applies
 * correctly, i.e. the corrupt attempt left no residue behind to corrupt a
 * later, legitimate report on the same worker. */
static void test_handle_report_page_malformed_bounds_rejected_and_recovers(
    void)
{
    job_t*    j;
    worker_t* w;
    int       rc;

    xce_job_state_entry_t  e_bad, e_good;
    xce_job_state_entry_t* entries_bad[1];
    xce_job_state_entry_t* entries_good[1];
    xce_job_state_report_t page_bad, page_good;

    twr_setup();

    j = twr_seed_job(TWR_JOBID_MPAGE_BAD, JOB_STAT_RUN);
    TEST_ASSERT_NOT_NULL(j);
    j->job.allocs
        = twr_make_test_alloc_for_host(TWR_JOBID_MPAGE_BAD, "mpage-host-4");

    w = twr_make_test_worker("mpage-host-4");
    TEST_ASSERT_NOT_NULL(w);

    rc = resync_arm_round(&g_twr, w);
    TEST_ASSERT_EQUAL_INT(1, rc);

    /* pos(1) + count(1) == 2 > n_total(1): out of bounds. */
    e_bad = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_BAD2, .status = JobStateStatus_running,
        .exit_code = 0, .finish_time_ms = 0, .queried = 0
    };
    entries_bad[0] = &e_bad;
    page_bad = (xce_job_state_report_t){
        .n_total = 1, .pos = 1, .count = 1, .entries = entries_bad
    };

    rc = resync_handle_report_page(&g_twr, w, &page_bad);
    TEST_ASSERT_EQUAL_INT_MESSAGE(-EINVAL, rc,
        "an out-of-bounds page must be rejected");

    /* A fresh, valid, single-page report for the SAME worker right after --
     * proves the malformed attempt above left no corrupted accumulator
     * state behind. */
    e_good = (xce_job_state_entry_t){
        .jobid = TWR_JOBID_MPAGE_BAD, .status = JobStateStatus_unrecognized,
        .exit_code = 0, .finish_time_ms = 0, .queried = 1
    };
    entries_good[0] = &e_good;
    page_good = (xce_job_state_report_t){
        .n_total = 1, .pos = 0, .count = 1, .entries = entries_good
    };

    rc = resync_handle_report_page(&g_twr, w, &page_good);
    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
        "a subsequent valid report must apply normally");
    TEST_ASSERT_EQUAL_UINT32(JOB_STAT_PEND, j->job.state);
    TEST_ASSERT_NULL(j->job.allocs);
    TEST_ASSERT_NULL(w->resync);

    twr_teardown();
    printf("test_handle_report_page_malformed_bounds_rejected_and_recovers:"
          " PASS\n");
}

/* ══════════════════════════════════════════════════════════════════════
 * Task 7.1 (Req 1.1-1.3, 2.1-2.2, 3.1-3.3, 3.6, 4.1-4.8, 7.1-7.3): full
 * rejoin-reconciliation round trip against a live controller+worker pair.
 *
 * design.md's own Testing Strategy files this under "Integration Tests" as
 * "against a live controller+worker pair, mirroring test_worker_join.c's
 * fixture conventions (setup_test_cluster/teardown_test_cluster, direct
 * calls, one test per behavior)" -- NOT separate OS processes or real TCP
 * daemons. test_jobstatereport_dispatch.c's own header comment (task 6.2)
 * additionally confirms a genuine wire dispatch through myreq_handler /
 * handle_jobstatereport() cannot resolve a REAL worker_t in THIS binary at
 * all: get_ep_worker() (misc.c) is not linked into controller_unit_test --
 * a fixed test double defined in test_commit_barrier_reply.c stands in for
 * it, always returning NULL/a fixture-controlled stub, for every test file
 * in this binary, this one included. So this test drives the exact same
 * REAL production entry points test_worker_join.c's own task 6.1 tests use
 * directly -- rejoin_worker() (the real join path) to arm the round and
 * worker_send_resync_query() to send the real wire query -- then feeds a
 * REAL wire-encoded-then-decoded OPCode_jobStateReport (via task 1.4's own
 * xce_job_state_report_req()/xce_decode() codec) into resync_handle_report_
 * page() (task 6.2's own dispatch/reassembly entry point -- never resync_
 * apply_report() directly), exactly mirroring how a real worker's answer
 * would arrive and be reassembled/dispatched, without needing get_ep_
 * worker()'s (unavailable) wire-to-worker_t resolution.
 *
 * Proves COMPOSITION, not per-outcome correctness -- each individual outcome
 * (still-running no-op, finished-success, finished-failure, unrecognized)
 * is already exhaustively unit-tested in isolation by tasks 2.1/2.2/3.2 (see
 * test_apply_report_all_five_outcomes_updates_job_table above, and task
 * 6.1's own test_worker_join.c tests for arm+query-send in isolation). This
 * test's own value-add is proving all of join -> arm -> real wire query ->
 * real wire report -> dispatch/reassembly -> apply -> scheduler-notify
 * compose correctly TOGETHER, for a MIX of four different job outcomes, in
 * ONE round, in ONE test run -- and that the scheduler notify path is
 * reached for every actual state change produced, across BOTH of this
 * feature's two distinct notify patterns: the EAGER path (unrecognized ->
 * pending, task 2.1's pattern -- notified synchronously within the same
 * call, asserted in Phase 1 below) and the DEFERRED path (finished-success/
 * failure, task 3.2's pattern -- notified only once the durable fact is
 * later applied). This bare-persist RETAIN-mode fixture has no live commit-
 * engine apply pump to do that second half automatically, so this test
 * proves it the same way test_apply_report_finished_success_uses_worker_
 * reported_finish_time and test_reconcile_assign_survives_restart_replay
 * already establish as this file's own precedent for that exact situation:
 * a genuine two-phase restart-replay (Phase 1: real cluster + disk persist,
 * drive the real chain, flush, close; Phase 2: fresh persist_t + fresh
 * cluster_t, persist_recover() -- the SAME apply_committed_entry() seam a
 * live controller's commit-apply pump would invoke). */

#define TWR_JOBID_RT_RUNNING 81301ULL
#define TWR_JOBID_RT_FINOK   81302ULL
#define TWR_JOBID_RT_FINFAIL 81303ULL
#define TWR_JOBID_RT_UNREC   81304ULL
#define TWR_RT_N_RES         9

/* Mirrors test_worker_join.c's own new_test_endpoint/pop_and_decode/assert_
 * queue_empty helpers exactly -- real (heap) endpoint_t via endpoint_get()
 * (bug-378: never a hand-rolled/zeroed stack stand-in for anything actually
 * sent/received over), for genuine wire capture of both the query this test
 * sends TO the worker and the report it simulates receiving FROM one. */
static endpoint_t* twr_rt_new_test_endpoint(void)
{
    endpoint_t* ep = endpoint_get();
    TEST_ASSERT_NOT_NULL(ep);
    return ep;
}

static void twr_rt_pop_and_decode(endpoint_t* ep, xce_request_t* out)
{
    data_item_t* it = endpoint_deq(ep);
    TEST_ASSERT_NOT_NULL(it);

    io_packet_t* pkt = ITPTR_TO_PACKET(it);

    memset(out, 0, sizeof(*out));
    TEST_ASSERT_EQUAL_INT(0, xce_decode(pkt, out)); /* consumes/frees pkt */
}

static void test_full_rejoin_reconciliation_round_trip(void)
{
    char     tmp[256];
    uint64_t worker_reported_finish_time_s;
    int32_t  worker_reported_exit_code = 42;

    strcpy(tmp, "/tmp/xce_twr71_roundtrip_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    /* Deliberately far from "now" so a reconciliation-processing-time bug
     * (time(NULL)) could never accidentally produce a matching value --
     * same defensive idiom test_apply_report_finished_success_uses_worker_
     * reported_finish_time already establishes. */
    worker_reported_finish_time_s = (uint64_t)time(NULL) - 3600;

    /* Phase 1: real cluster (workermgr + resmgr + jobmgr + hooksys) + real
     * disk persist -- drives the REAL join/query-send/report-apply chain.
     * Own local cluster_t (not the shared g_twr fixture), mirroring every
     * other restart-replay test in this file, since Phase 1's disk state
     * must survive past Phase 1's own teardown into Phase 2 below. */
    {
        cluster_t        c;
        kvmgr_params_t   str_params = { .key_type = KVMGR_KEY_STRING,
                                        .nslots = 64, .delta = 16 };
        kvmgr_params_t   int_params = { .key_type = KVMGR_KEY_INT,
                                        .nslots = 32, .delta = 16 };
        persist_config_t cfg;
        worker_t*        w;
        job_t*           jobs[4];
        uint64_t         jobids[4] = { TWR_JOBID_RT_RUNNING,
                                       TWR_JOBID_RT_FINOK,
                                       TWR_JOBID_RT_FINFAIL,
                                       TWR_JOBID_RT_UNREC };
        hostattrs_t       attrs, new_attrs;
        xce_hostconfig_t  hconfig;
        endpoint_t        stack_ep;
        endpoint_t*       ep;
        xce_request_t     decoded_query, decoded_report;
        int               saw[4] = { 0, 0, 0, 0 };
        int               i, rc;
        uint64_t          lsn_before, lsn_after;

        static const char* res_names[TWR_RT_N_RES] = {
            "type", "model", "cpuf", "maxcpus", "maxmem",
            "maxswap", "maxtmp", "ndisks", "slots"
        };
        static const int res_valtypes[TWR_RT_N_RES] = {
            XCE_VAL_STRING, XCE_VAL_STRING,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC
        };

        if (makeString((char*)"twr71-probe") == NULL)
            xstring_table_init(64);
        if (!g_twr_jobobj_ready) {
            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
            g_twr_jobobj_ready = 1;
        }

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));

        c.workermgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.workermgr);
        c.resmgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.resmgr);
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        for (i = 0; i < TWR_RT_N_RES; i++) {
            xce_resource_t rdef;
            memset(&rdef, 0, sizeof(rdef));
            rdef.name    = (char*)res_names[i];
            rdef.valtype = res_valtypes[i];
            rdef.flags   = 0;
            rdef.idx     = -1;
            TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
        }

        c.myhost_info.name = (char*)"twr71-controller";
        c.myhost_info.key  = (char*)"twr71-key";
        c.myhost           = &c.myhost_info;

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        /* Seed a durable WAL_JOB_SUBMIT fact + a matching in-memory job_t
         * for all 4 jobs -- all RUN, all assigned to the SAME worker host,
         * exactly what the controller believes right before this rejoin
         * (bypassing dispatch/replay, mirroring every sibling restart-
         * replay test in this file). All 4 are non-terminal, so all 4 are
         * expected in the round's query set (Req 2.1) -- "unrecognized" is
         * not a DIFFERENT starting state from the controller's own point of
         * view, only a different worker ANSWER. */
        for (i = 0; i < 4; i++) {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";
            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = jobids[i];
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";
            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = jobids[i];
            jdef.state  = JOB_STAT_RUN;
            jdef.submit = &submit;
            jobs[i] = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(jobs[i]);
            jobs[i]->job.allocs = twr_make_test_alloc_for_host(
                jobids[i], "resync-roundtrip-host");
            TEST_ASSERT_NOT_NULL(jobs[i]->job.allocs);
        }

        /* ── Join: rejoin_worker (the REAL join entry point) arms the round,
         * worker_send_resync_query sends the REAL wire query (Req 1.1-1.3,
         * 2.1, 2.2). */
        memset(&attrs, 0, sizeof(attrs));
        attrs.hosttype  = makeString("LINUX");
        attrs.hostmodel = makeString("x86");
        attrs.cpufactor = 1.0f;
        attrs.maxcpus   = 4;
        attrs.maxmem    = 16;
        attrs.maxswap   = 32;
        attrs.maxtmp    = 64;
        attrs.ndisks    = 1;

        {
            xce_worker_t host;
            memset(&host, 0, sizeof(host));
            host.name  = (char*)"resync-roundtrip-host";
            host.attrs = &attrs;
            w = makeWorker(c.workermgr, &host);
        }
        TEST_ASSERT_NOT_NULL(w);

        memset(&new_attrs, 0, sizeof(new_attrs));
        new_attrs.hosttype  = makeString("LINUX");
        new_attrs.hostmodel = makeString("x86");
        new_attrs.cpufactor = 1.0f;
        new_attrs.maxcpus   = 4;
        new_attrs.maxmem    = 16;
        new_attrs.maxswap   = 32;
        new_attrs.maxtmp    = 64;
        new_attrs.ndisks    = 1;

        memset(&hconfig, 0, sizeof(hconfig));
        hconfig.hosttype  = (char*)"!";
        hconfig.hostmodel = (char*)"!";
        hconfig.mxj       = 4;

        memset(&stack_ep, 0, sizeof(stack_ep));
        rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
        TEST_ASSERT_GREATER_OR_EQUAL(0, rc);

        TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
            "a rejoin with 4 known non-terminal jobs on this host must arm "
            "a reconciliation round (Req 1.1)");
        TEST_ASSERT_EQUAL_INT_MESSAGE(4, w->resync->n_queried_jobids,
            "the round's query set must match exactly the host's 4 "
            "non-terminal jobs (Req 2.1)");
        TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));

        ep = twr_rt_new_test_endpoint();
        TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&c, w, ep));

        twr_rt_pop_and_decode(ep, &decoded_query);
        TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded_query.opcode);
        TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
                              decoded_query.body_which);
        TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(4,
            decoded_query.body.jobstatequerymsg->n_total,
            "the REAL wire query (Req 2.1, 2.2) must list exactly the 4 "
            "armed jobids");
        for (i = 0; i < decoded_query.body.jobstatequerymsg->count; i++) {
            uint64_t id = decoded_query.body.jobstatequerymsg->jobids[i];
            if (id == TWR_JOBID_RT_RUNNING) saw[0] = 1;
            if (id == TWR_JOBID_RT_FINOK)   saw[1] = 1;
            if (id == TWR_JOBID_RT_FINFAIL) saw[2] = 1;
            if (id == TWR_JOBID_RT_UNREC)   saw[3] = 1;
        }
        TEST_ASSERT_TRUE_MESSAGE(saw[0] && saw[1] && saw[2] && saw[3],
            "the wire query must cover all 4 jobs by unique ID (Req 2.2)");
        xce_free_request(&decoded_query);

        /* ── Worker's answer: a REAL wire-encoded-then-decoded OPCode_
         * jobStateReport (task 1.4's own xce_job_state_report_req()/xce_
         * decode() codec), covering all 4 outcomes in one page, fed into
         * resync_handle_report_page() -- task 6.2's own dispatch/
         * reassembly entry point, never resync_apply_report() directly
         * (Req 3.1-3.3, 3.6, 8.1-8.3). */
        {
            xce_job_state_entry_t  e_running, e_finok, e_finfail, e_unrec;
            xce_job_state_entry_t* entries[4];
            xce_job_state_report_t report;

            e_running = (xce_job_state_entry_t){
                .jobid = TWR_JOBID_RT_RUNNING,
                .status = JobStateStatus_running,
                .exit_code = 0, .finish_time_ms = 0, .queried = 1
            };
            e_finok = (xce_job_state_entry_t){
                .jobid = TWR_JOBID_RT_FINOK,
                .status = JobStateStatus_finishedOk,
                .exit_code = 0,
                .finish_time_ms = worker_reported_finish_time_s * 1000ULL,
                .queried = 1
            };
            e_finfail = (xce_job_state_entry_t){
                .jobid = TWR_JOBID_RT_FINFAIL,
                .status = JobStateStatus_finishedFail,
                .exit_code = worker_reported_exit_code,
                .finish_time_ms = 0, .queried = 1
            };
            e_unrec = (xce_job_state_entry_t){
                .jobid = TWR_JOBID_RT_UNREC,
                .status = JobStateStatus_unrecognized,
                .exit_code = 0, .finish_time_ms = 0, .queried = 1
            };
            entries[0] = &e_running;
            entries[1] = &e_finok;
            entries[2] = &e_finfail;
            entries[3] = &e_unrec;
            report = (xce_job_state_report_t){
                .n_total = 4, .pos = 0, .count = 4, .entries = entries
            };

            TEST_ASSERT_EQUAL_INT(0,
                xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
                                         0));
        }

        twr_rt_pop_and_decode(ep, &decoded_report);
        TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport, decoded_report.opcode);
        TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
                              decoded_report.body_which);
        TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        pthread_mutex_lock(&c.persist.lock);
        lsn_before = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        rc = resync_handle_report_page(&c, w,
                                       decoded_report.body.jobstatereportmsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
            "a single-page report (count == n_total) must complete the "
            "report immediately");

        pthread_mutex_lock(&c.persist.lock);
        lsn_after = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        xce_free_request(&decoded_report);

        /* ── End-to-end assertions (Phase 1 -- everything observable before
         * any commit round trip, mirroring record_job_reconcile_pending_
         * fact's own eager-mutation precedent). */
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jobs[0]->job.state,
            "still-running (queried) must be left exactly as-is -- no fact "
            "recorded (Req 4.1)");
        TEST_ASSERT_NOT_NULL(jobs[0]->job.allocs);

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jobs[1]->job.state,
            "finished-success stays in the DEFERRED record_job_finish_"
            "fact_at pattern -- state mutates later via apply_committed_"
            "entry (proven below, Phase 2), not synchronously here, exactly "
            "as test_apply_report_all_five_outcomes_updates_job_table's own "
            "established precedent documents");
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, jobs[2]->job.state,
            "finished-failure stays in the DEFERRED on_job_fail/record_job_"
            "fail_fact pattern -- same deferred precedent as finished-"
            "success");

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, jobs[3]->job.state,
            "unrecognized reconciles EAGERLY through record_job_reconcile_"
            "pending_fact (Req 4.5, 4.6) -- PEND synchronously, before any "
            "commit round trip");
        TEST_ASSERT_NULL_MESSAGE(jobs[3]->job.allocs,
            "bug-1631: reconciled-to-pending must release the allocation "
            "synchronously");

        TEST_ASSERT_NULL_MESSAGE(w->resync,
            "once every one of the 4 queried jobs has a reconciled outcome, "
            "the round must resolve (Req 5.1, 5.2)");
        TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
            "XCE_WORKER_RECONCILING must clear once the round resolves");

        TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
            "Req 7.1, 7.2, 7.3: the EAGER unrecognized-to-pending "
            "transition must reach the scheduler's/apiserver's notify path "
            "SYNCHRONOUSLY, within this same call -- the two DEFERRED "
            "outcomes (finished-success/failure) have not applied yet at "
            "this point, so they must NOT have notified yet either "
            "(confirmed to notify later, in Phase 2 below)");

        TEST_ASSERT_TRUE_MESSAGE(lsn_after >= lsn_before + 3,
            "at least 3 durable facts must be appended for this report: "
            "finished-success (WAL_JOB_FINISH), finished-failure (WAL_JOB_"
            "FAIL), and unrecognized (WAL_JOB_UPDATE) -- the still-running "
            "entry records none");

        xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        if (c.workermgr) dereference(c.workermgr);
        if (c.resmgr)    dereference(c.resmgr);
        if (c.jobmgr)    dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);

        dereference(new_attrs.hosttype);
        dereference(new_attrs.hostmodel);
        endpoint_fini(ep);
        free(ep);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
     * recover from disk -- simulates a controller restart, the SAME apply_
     * committed_entry() seam a live controller's own commit-apply pump
     * would invoke to settle the two DEFERRED outcomes above. Mirrors
     * test_apply_report_finished_success_uses_worker_reported_finish_
     * time's own established two-phase idiom exactly. */
    {
        persist_t* p2 = twr_make_fresh_persist(tmp);
        cluster_t  c2;
        job_t*     j2_running = NULL;
        job_t*     j2_finok   = NULL;
        job_t*     j2_finfail = NULL;
        job_t*     j2_unrec   = NULL;
        int        rc;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 32, .delta = 16 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        key_find(c2.jobmgr, (void*)TWR_JOBID_RT_RUNNING, (void**)&j2_running);
        key_find(c2.jobmgr, (void*)TWR_JOBID_RT_FINOK, (void**)&j2_finok);
        key_find(c2.jobmgr, (void*)TWR_JOBID_RT_FINFAIL, (void**)&j2_finfail);
        key_find(c2.jobmgr, (void*)TWR_JOBID_RT_UNREC, (void**)&j2_unrec);
        unlock(c2.jobmgr);

        TEST_ASSERT_NOT_NULL(j2_running);
        TEST_ASSERT_NOT_NULL(j2_finok);
        TEST_ASSERT_NOT_NULL(j2_finfail);
        TEST_ASSERT_NOT_NULL(j2_unrec);

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, j2_running->job.state,
            "still-running: no fact was ever recorded for it, so replay "
            "leaves it exactly as WAL_JOB_SUBMIT alone reconstructs it "
            "(Req 4.1)");

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2_finok->job.state,
            "finished-success: the DEFERRED record_job_finish_fact_at fact "
            "must apply to DONE on replay (Req 4.1)");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(worker_reported_finish_time_s,
            (uint64_t)j2_finok->job.finish_time,
            "Req 4.2, 3.5: the recovered finish_time must be the WORKER-"
            "reported finish_time_ms, not reconciliation-processing "
            "wall-clock time");

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT, j2_finfail->job.state,
            "finished-failure: the DEFERRED on_job_fail/record_job_fail_"
            "fact fact must apply to EXIT on replay (Req 4.1)");
        /* Req 4.2 remains DOCUMENTED-unsatisfiable for the finished-
         * FAILURE finish_time specifically (bug-1747) -- not asserted here.
         * The worker-reported EXIT CODE, however, is durably recorded via
         * WAL_JOB_FAIL's code trailer since bug-1754. */
        TEST_ASSERT_NOT_NULL_MESSAGE(j2_finfail->job.exit_info,
            "bug-1754: a finished-failure's worker-reported exit code must "
            "survive restart replay in job.exit_info");
        TEST_ASSERT_EQUAL_INT(worker_reported_exit_code,
                              j2_finfail->job.exit_info->exitcode);

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND, j2_unrec->job.state,
            "unrecognized: the EAGER record_job_reconcile_pending_fact's "
            "durable WAL_JOB_UPDATE fact must ALSO reconstruct correctly on "
            "replay -- convergence via either the live eager path (Phase 1 "
            "above) or a restart, not just the live path (Req 4.5, 4.6)");
        TEST_ASSERT_NULL(j2_unrec->job.allocs);

        TEST_ASSERT_EQUAL_INT_MESSAGE(3, g_twr_update_hook_fires,
            "Req 7.1-7.3: replaying the durable facts this reconciliation "
            "round produced must reach the scheduler's/apiserver's notify "
            "path (HOOK_DETAIL_JOB_UPDATE) for EVERY actual state change -- "
            "finished-success, finished-failure, and unrecognized-to-"
            "pending -- exactly 3 times (still-running recorded no fact, "
            "and WAL_JOB_SUBMIT fires HOOK_DETAIL_JOB_NEW, not JOB_UPDATE, "
            "so it does not add to this count). Together with Phase 1's own "
            "single synchronous fire, this confirms EVERY actual state "
            "change this round produced -- both the eager and the deferred "
            "ones -- reaches the same notify path, not just some of them");

        xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);
        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    printf("test_full_rejoin_reconciliation_round_trip: PASS\n");
}

/* ── Task 7.2: Failover-during-job recovery -- the feature's own core
 * motivating scenario (bug-1741; requirements.md's Introduction; design.md
 * "Testing Strategy" > "Integration Tests": "Failover-during-job: a job's
 * finish notify is lost across a primary failover; the new primary's
 * rejoin reconciliation recovers it (this is the feature's core bug-1741
 * regression scenario)") ────────────────────────────────────────────────
 *
 * Proves the SIMPLEST version of bug-1741's own reported failure: one job,
 * one lost finish report, one failover, one rejoin. bug-1741 itself was a
 * burst of 7 near-simultaneous finish reports losing 6 of them; that
 * burst/many-jobs shape is Requirement 8's own separate coverage (already
 * proven by the multi-page tests above and by test_full_rejoin_
 * reconciliation_round_trip's four-outcome round trip) -- this test's own
 * job is only to name and prove the single-job case in isolation, not to
 * re-prove the general round-trip machinery task 7.1 already covers.
 *
 * Per requirements.md's Introduction, today a worker's finish report is a
 * single fire-and-forget XCECLI_FF_CALL send with no retry -- if it is
 * lost (the worker briefly disconnects, or the controller it was talking
 * to fails over to a new primary mid-flight), the job's true outcome is
 * never recorded anywhere and its state (RUN) sticks forever. This test
 * models that starting condition directly: a fresh persist state in which
 * the job is durably recorded as RUN, assigned to a worker host, with NO
 * finish fact of any kind ever appended for it -- exactly as if the real
 * XCECLI_FF_CALL send had simply vanished before this feature ever
 * existed. No second controller process or literal primary-to-primary
 * handoff mechanism is simulated -- the OBSERVABLE starting condition (RUN
 * job, no finish fact, worker has since actually finished it) is what
 * this feature's own recovery mechanism must correct, regardless of how
 * that condition came to exist; per Requirement 1.3, the controller's own
 * trigger does not distinguish a first join, an ordinary reconnect, or a
 * post-failover reconnect to a new primary -- the same rejoin path
 * (`rejoin_worker`) is exercised below exactly as it would be for any of
 * the three.
 *
 * Drives the SAME real rejoin -> query-send -> report-apply chain task
 * 7.1's own round-trip test established (`rejoin_worker`, the REAL join
 * entry point; `worker_send_resync_query`'s REAL wire OPCode_jobStateQuery;
 * a REAL worker answer via `xce_job_state_report_req`'s REAL wire OPCode_
 * jobStateReport; `resync_handle_report_page`, never `resync_apply_report`
 * directly) -- narrowed to ONE job, ONE outcome (finished-success),
 * because this test's whole point is to isolate the bug-1741 scenario, not
 * to duplicate the four-outcome coverage task 7.1 already built.
 *
 * finished-success stays on record_job_finish_fact_at's DEFERRED pattern
 * (task 3.2/bug-1747) -- so, like test_full_rejoin_reconciliation_round_
 * trip's own Phase 1/Phase 2 split, this test needs a real restart-replay
 * (Phase 2, the SAME apply_committed_entry() seam a live controller's own
 * commit-apply pump -- or another controller instance recovering the same
 * durable log -- would invoke) to observe the job actually reach DONE;
 * this bare-persist test fixture has no live commit-engine apply pump of
 * its own, exactly the situation task 7.1's own Implementation Notes
 * document. */

#define TWR_JOBID_FAILOVER_LOST 81401ULL

static void test_failover_during_job_recovers_lost_finish(void)
{
    char     tmp[256];
    uint64_t worker_reported_finish_time_s;

    strcpy(tmp, "/tmp/xce_twr72_failover_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    /* Deliberately far in the past -- the job genuinely finished well
     * before this reconciliation round runs, so a reconciliation-
     * processing-time bug (time(NULL)) could never accidentally produce a
     * matching value. Same defensive idiom as this file's other finish-
     * time tests. */
    worker_reported_finish_time_s = (uint64_t)time(NULL) - 5400;

    /* Phase 1: the (new) primary's own persist state -- job RUN, no
     * finish fact anywhere in it (the lost-notify starting condition) --
     * then the REAL rejoin/query/report chain. */
    {
        cluster_t        c;
        kvmgr_params_t   str_params = { .key_type = KVMGR_KEY_STRING,
                                        .nslots = 64, .delta = 16 };
        kvmgr_params_t   int_params = { .key_type = KVMGR_KEY_INT,
                                        .nslots = 32, .delta = 16 };
        persist_config_t cfg;
        worker_t*        w;
        job_t*           job;
        hostattrs_t       attrs, new_attrs;
        xce_hostconfig_t  hconfig;
        endpoint_t        stack_ep;
        endpoint_t*       ep;
        xce_request_t     decoded_query, decoded_report;
        int               rc;
        uint64_t          lsn_before, lsn_after;
        int               i;

        static const char* res_names[TWR_RT_N_RES] = {
            "type", "model", "cpuf", "maxcpus", "maxmem",
            "maxswap", "maxtmp", "ndisks", "slots"
        };
        static const int res_valtypes[TWR_RT_N_RES] = {
            XCE_VAL_STRING, XCE_VAL_STRING,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC
        };

        if (makeString((char*)"twr72-probe") == NULL)
            xstring_table_init(64);
        if (!g_twr_jobobj_ready) {
            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
            g_twr_jobobj_ready = 1;
        }

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));

        c.workermgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.workermgr);
        c.resmgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.resmgr);
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        for (i = 0; i < TWR_RT_N_RES; i++) {
            xce_resource_t rdef;
            memset(&rdef, 0, sizeof(rdef));
            rdef.name    = (char*)res_names[i];
            rdef.valtype = res_valtypes[i];
            rdef.flags   = 0;
            rdef.idx     = -1;
            TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
        }

        c.myhost_info.name = (char*)"twr72-controller";
        c.myhost_info.key  = (char*)"twr72-key";
        c.myhost           = &c.myhost_info;

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        /* Seed a durable WAL_JOB_SUBMIT fact + a matching in-memory job_t,
         * RUN, assigned to the worker host -- and NOTHING ELSE. No
         * WAL_JOB_START/DISPATCH/FINISH is ever appended for this job in
         * this log: the controller's own durable state simply has no
         * record the job ever finished, exactly bug-1741's own reported
         * gap. */
        {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";
            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = TWR_JOBID_FAILOVER_LOST;
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";
            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = TWR_JOBID_FAILOVER_LOST;
            jdef.state  = JOB_STAT_RUN;
            jdef.submit = &submit;
            job = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(job);
            job->job.allocs = twr_make_test_alloc_for_host(
                TWR_JOBID_FAILOVER_LOST, "resync-failover-host");
            TEST_ASSERT_NOT_NULL(job->job.allocs);
        }

        /* ── Worker rejoin: the REAL join entry point. Requirement 1.3 --
         * the controller does not need to know or care whether this is
         * the host's first-ever join, an ordinary network reconnect, or
         * (this scenario) a reconnect to a new primary after a controller
         * failover; the same trigger fires because the controller still
         * holds one non-terminal job for this host. */
        memset(&attrs, 0, sizeof(attrs));
        attrs.hosttype  = makeString("LINUX");
        attrs.hostmodel = makeString("x86");
        attrs.cpufactor = 1.0f;
        attrs.maxcpus   = 4;
        attrs.maxmem    = 16;
        attrs.maxswap   = 32;
        attrs.maxtmp    = 64;
        attrs.ndisks    = 1;

        {
            xce_worker_t host;
            memset(&host, 0, sizeof(host));
            host.name  = (char*)"resync-failover-host";
            host.attrs = &attrs;
            w = makeWorker(c.workermgr, &host);
        }
        TEST_ASSERT_NOT_NULL(w);

        memset(&new_attrs, 0, sizeof(new_attrs));
        new_attrs.hosttype  = makeString("LINUX");
        new_attrs.hostmodel = makeString("x86");
        new_attrs.cpufactor = 1.0f;
        new_attrs.maxcpus   = 4;
        new_attrs.maxmem    = 16;
        new_attrs.maxswap   = 32;
        new_attrs.maxtmp    = 64;
        new_attrs.ndisks    = 1;

        memset(&hconfig, 0, sizeof(hconfig));
        hconfig.hosttype  = (char*)"!";
        hconfig.hostmodel = (char*)"!";
        hconfig.mxj       = 4;

        memset(&stack_ep, 0, sizeof(stack_ep));
        rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
        TEST_ASSERT_GREATER_OR_EQUAL(0, rc);

        TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
            "a rejoin finding this host's one lost-finish job still "
            "recorded as RUN must arm a reconciliation round (Req 1.1, "
            "1.3)");
        TEST_ASSERT_EQUAL_INT(1, w->resync->n_queried_jobids);
        TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));

        ep = twr_rt_new_test_endpoint();
        TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&c, w, ep));

        twr_rt_pop_and_decode(ep, &decoded_query);
        TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded_query.opcode);
        TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
                              decoded_query.body_which);
        TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(1,
            decoded_query.body.jobstatequerymsg->n_total,
            "the REAL wire query must ask about exactly the one job the "
            "controller still believes is running (Req 2.1)");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(TWR_JOBID_FAILOVER_LOST,
            decoded_query.body.jobstatequerymsg->jobids[0],
            "the query must identify the job unambiguously by its ID "
            "(Req 2.2)");
        xce_free_request(&decoded_query);

        /* ── The worker's REAL answer: it actually finished this job
         * successfully, long before this rejoin -- the controller simply
         * never heard about it (the lost XCECLI_FF_CALL). */
        {
            xce_job_state_entry_t  e;
            xce_job_state_entry_t* entries[1];
            xce_job_state_report_t report;

            e = (xce_job_state_entry_t){
                .jobid = TWR_JOBID_FAILOVER_LOST,
                .status = JobStateStatus_finishedOk,
                .exit_code = 0,
                .finish_time_ms = worker_reported_finish_time_s * 1000ULL,
                .queried = 1
            };
            entries[0] = &e;
            report = (xce_job_state_report_t){
                .n_total = 1, .pos = 0, .count = 1, .entries = entries
            };

            TEST_ASSERT_EQUAL_INT(0,
                xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
                                         0));
        }

        twr_rt_pop_and_decode(ep, &decoded_report);
        TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport, decoded_report.opcode);
        TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
                              decoded_report.body_which);
        TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        pthread_mutex_lock(&c.persist.lock);
        lsn_before = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        rc = resync_handle_report_page(&c, w,
                                       decoded_report.body.jobstatereportmsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
            "a single-page report (count == n_total) must complete the "
            "report immediately");

        pthread_mutex_lock(&c.persist.lock);
        lsn_after = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        xce_free_request(&decoded_report);

        /* ── Phase 1 assertions: the round resolves and a durable fact
         * capturing the worker's TRUE outcome is appended -- but the job
         * is NOT yet observably DONE in this process's own memory. That
         * is expected and correct: finished-success reconciles through
         * record_job_finish_fact_at's DEFERRED pattern (task 3.2/bug-
         * 1747), same as an ordinary finish report -- the state mutation
         * happens later, when the fact is applied (Phase 2 below), never
         * synchronously here. */
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN, job->job.state,
            "finished-success stays RUN synchronously -- the DEFERRED "
            "record_job_finish_fact_at pattern applies the state change "
            "later, via apply_committed_entry (Phase 2)");

        TEST_ASSERT_NULL_MESSAGE(w->resync,
            "the round's only queried job has a reconciled outcome -- it "
            "must resolve (Req 5.1, 5.2)");
        TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
            "XCE_WORKER_RECONCILING must clear once the round resolves");

        TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_update_hook_fires,
            "finished-success is DEFERRED -- no HOOK_DETAIL_JOB_UPDATE "
            "fire is expected synchronously here; that fire is proven "
            "below, in Phase 2, once the durable fact actually applies");

        TEST_ASSERT_TRUE_MESSAGE(lsn_after >= lsn_before + 1,
            "the worker's true finished-success outcome must be durably "
            "appended (WAL_JOB_FINISH) as part of this report, even "
            "though it has not applied to in-memory state yet");

        xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        if (c.workermgr) dereference(c.workermgr);
        if (c.resmgr)    dereference(c.resmgr);
        if (c.jobmgr)    dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);

        dereference(new_attrs.hosttype);
        dereference(new_attrs.hostmodel);
        endpoint_fini(ep);
        free(ep);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t, persist_recover() -- the
     * SAME apply_committed_entry() seam a live controller's own commit-
     * apply pump (or another controller instance recovering this same
     * durable log) would invoke. THIS is the test's core, observable
     * completion criterion: the job the controller's own durable state
     * showed stuck in RUN a moment ago, with its finish notify never
     * having reached any controller instance, now reconstructs as
     * genuinely finished -- DONE, with the worker's own true finish time,
     * not reconciliation-processing wall-clock time -- because the
     * worker rejoined and this feature's reconciliation round recovered
     * it (Req 4.1, 4.2). */
    {
        persist_t* p2 = twr_make_fresh_persist(tmp);
        cluster_t  c2;
        job_t*     j2 = NULL;
        int        rc;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 32, .delta = 16 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        key_find(c2.jobmgr, (void*)TWR_JOBID_FAILOVER_LOST, (void**)&j2);
        unlock(c2.jobmgr);

        TEST_ASSERT_NOT_NULL_MESSAGE(j2,
            "restart/rejoin-to-new-primary replay must reconstruct the "
            "job from its WAL_JOB_SUBMIT fact");

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2->job.state,
            "the feature's core completion criterion (bug-1741): a job "
            "whose finish notify never reached any controller instance "
            "ends up correctly finished, not stuck in RUN, once the "
            "worker rejoins the new primary (Req 4.1)");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(worker_reported_finish_time_s,
            (uint64_t)j2->job.finish_time,
            "the recovered finish_time must be the WORKER-reported "
            "finish_time_ms from the reconciliation report, not "
            "reconciliation-processing or replay wall-clock time "
            "(Req 4.2, 3.5)");

        TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_twr_update_hook_fires,
            "replaying the durable fact this reconciliation round "
            "produced must reach the scheduler's/apiserver's notify path "
            "(HOOK_DETAIL_JOB_UPDATE) exactly once for this one real "
            "state change (Req 7.1-7.3)");

        xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);
        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    printf("test_failover_during_job_recovers_lost_finish: PASS\n");
}

/* ── Task 7.3: Burst/array recovery unbounded by job count -- bug-1741's OWN
 * exact originally-reported shape (requirements.md's Introduction: "a burst
 * of near-simultaneous job completions on one worker host lost 6 of 7
 * finish reports, leaving those jobs stuck in RUN forever while their
 * siblings all completed normally") ────────────────────────────────────────
 *
 * Task 7.1 proved the general four-outcome round trip (one job per
 * outcome); task 7.2 proved the single-job lost-finish recovery in
 * isolation. Neither proves what THIS task is for: bug-1741's own reported
 * shape was SEVERAL jobs at once, MOST lost, with recovery required to
 * scale correctly across all of them together in ONE round -- not merely
 * repeat the single-job case N times with no cross-job interaction check
 * (tasks.md's own Observable Completion Criterion for this task).
 *
 * 7 jobs total on ONE worker host (bug-1741's own literal numbers):
 *  - 1 job (TWR_JOBID_BURST_ALREADY_DONE) already finished successfully
 *    BEFORE this rejoin -- a durable WAL_JOB_SUBMIT + WAL_JOB_FINISH pair
 *    is already on disk (the FINISH appended directly via persist_submit_
 *    job_finish_fact(), the same primitive record_job_finish_fact_at()
 *    itself calls -- modeling a finish report that DID reach and was
 *    durably committed by an earlier controller instance, before this
 *    rejoin), in-memory state already DONE. Being terminal, it must NOT be
 *    part of the round's query set (Req 1.1, 1.2, 2.1) -- but the worker's
 *    own retention window (Req 3.6, 3.7) would still recognize it as
 *    "recently finished" and report it AGAIN, as an unqueried extra, this
 *    time carrying a deliberately WRONG (bogus, recent) finish_time_ms.
 *    This is this task's own "none duplicated" Observable Completion
 *    Criterion, made concrete: this job's already-recorded outcome must
 *    survive completely unchanged -- both synchronously (Phase 1) and
 *    after a real restart replay (Phase 2) -- never overwritten by the
 *    bogus re-report, and with NO new WAL fact appended for it at all.
 *  - 6 jobs (TWR_JOBID_BURST_LOST_1..6) mirror bug-1741's own "6 of 7 lost"
 *    ratio exactly: each is durably recorded RUN with NO finish fact of
 *    any kind ever appended (the lost fire-and-forget XCECLI_FF_CALL, the
 *    same starting condition as task 7.2's single-job case) -- all 6 are
 *    non-terminal and assigned to the SAME host, so all 6 are armed into
 *    ONE round's query set together (Req 2.1, 8.1). Each carries its OWN
 *    distinct worker-reported finish_time_ms/exit_code (5 finished-success
 *    + 1 finished-failure, a genuine mixed-outcome burst) -- distinct per
 *    job specifically so a bug that mixes up entries across the loop (an
 *    off-by-one / shared-variable defect in resync_apply_report's
 *    per-entry routing) would surface as a WRONG value under a SPECIFIC
 *    jobid, never masked by every job sharing one identical expected
 *    outcome.
 *
 * Drives the SAME real rejoin -> query-send -> report-apply chain tasks
 * 7.1/7.2 already established (rejoin_worker, worker_send_resync_query's
 * real wire OPCode_jobStateQuery, a real worker answer via xce_job_state_
 * report_req's real wire OPCode_jobStateReport, resync_handle_report_page,
 * never resync_apply_report directly) -- one single-page report covering
 * all 7 entries together (6 queried + 1 unqueried extra). Deliberately NOT
 * forced across multiple wire pages: task 7.4's own separately-scoped job
 * is proving PAGINATION boundary correctness at a much larger count; this
 * task's job is proving RECOVERY LOGIC correctness at bug-1741's own
 * "several jobs at once" scale, per tasks.md's explicit boundary note
 * separating the two. */

#define TWR_JOBID_BURST_ALREADY_DONE 81501ULL
#define TWR_JOBID_BURST_LOST_1       81502ULL
#define TWR_JOBID_BURST_LOST_2       81503ULL
#define TWR_JOBID_BURST_LOST_3       81504ULL
#define TWR_JOBID_BURST_LOST_4       81505ULL
#define TWR_JOBID_BURST_LOST_5       81506ULL
#define TWR_JOBID_BURST_LOST_6       81507ULL /* finished-FAILURE, not OK */
#define TWR_BURST_N_LOST             6
#define TWR_BURST_N_TOTAL            7 /* 1 already-done + 6 lost -- bug-1741's own 7/6 numbers */

static void test_burst_recovery_unbounded_by_job_count(void)
{
    char     tmp[256];
    uint64_t already_done_finish_time_s;
    uint64_t bogus_reapply_finish_time_s;
    uint64_t lost_finish_time_s[TWR_BURST_N_LOST - 1]; /* LOST_1..LOST_5 (finished-ok) */
    int32_t  lost_fail_exit_code = 66;

    strcpy(tmp, "/tmp/xce_twr73_burst_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    /* Deliberately distinct/far-from-"now" values throughout -- so a
     * reconciliation-processing-time bug (time(NULL)) or a cross-job value
     * mix-up could never accidentally produce a matching value, same
     * defensive idiom this file's other finish-time tests already
     * establish. */
    already_done_finish_time_s  = (uint64_t)time(NULL) - 7200;
    bogus_reapply_finish_time_s = (uint64_t)time(NULL) - 30;
    {
        int k;
        for (k = 0; k < TWR_BURST_N_LOST - 1; k++)
            lost_finish_time_s[k]
                = (uint64_t)time(NULL) - 3600 - (uint64_t)(k * 137);
    }

    /* Phase 1: real cluster (workermgr + resmgr + jobmgr + hooksys) + real
     * disk persist -- drives the REAL join/query-send/report-apply chain
     * over all 7 jobs together. Own local cluster_t (not the shared g_twr
     * fixture), mirroring every other restart-replay test in this file. */
    {
        cluster_t        c;
        kvmgr_params_t   str_params = { .key_type = KVMGR_KEY_STRING,
                                        .nslots = 64, .delta = 16 };
        kvmgr_params_t   int_params = { .key_type = KVMGR_KEY_INT,
                                        .nslots = 32, .delta = 16 };
        persist_config_t cfg;
        worker_t*        w;
        job_t*           j_done;
        job_t*           j_lost[TWR_BURST_N_LOST];
        uint64_t         lost_ids[TWR_BURST_N_LOST] = {
            TWR_JOBID_BURST_LOST_1, TWR_JOBID_BURST_LOST_2,
            TWR_JOBID_BURST_LOST_3, TWR_JOBID_BURST_LOST_4,
            TWR_JOBID_BURST_LOST_5, TWR_JOBID_BURST_LOST_6
        };
        hostattrs_t       attrs, new_attrs;
        xce_hostconfig_t  hconfig;
        endpoint_t        stack_ep;
        endpoint_t*       ep;
        xce_request_t     decoded_query, decoded_report;
        int               saw[TWR_BURST_N_LOST];
        int               i, rc;
        uint64_t          lsn_before, lsn_after;

        static const char* res_names[TWR_RT_N_RES] = {
            "type", "model", "cpuf", "maxcpus", "maxmem",
            "maxswap", "maxtmp", "ndisks", "slots"
        };
        static const int res_valtypes[TWR_RT_N_RES] = {
            XCE_VAL_STRING, XCE_VAL_STRING,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC
        };

        if (makeString((char*)"twr73-probe") == NULL)
            xstring_table_init(64);
        if (!g_twr_jobobj_ready) {
            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
            g_twr_jobobj_ready = 1;
        }

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));

        c.workermgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.workermgr);
        c.resmgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.resmgr);
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        for (i = 0; i < TWR_RT_N_RES; i++) {
            xce_resource_t rdef;
            memset(&rdef, 0, sizeof(rdef));
            rdef.name    = (char*)res_names[i];
            rdef.valtype = res_valtypes[i];
            rdef.flags   = 0;
            rdef.idx     = -1;
            TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
        }

        c.myhost_info.name = (char*)"twr73-controller";
        c.myhost_info.key  = (char*)"twr73-key";
        c.myhost           = &c.myhost_info;

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        /* Seed the ALREADY-DONE job: durable WAL_JOB_SUBMIT + a REAL,
         * already-committed WAL_JOB_FINISH (appended directly via
         * persist_submit_job_finish_fact -- modeling a finish report that
         * reached and was durably recorded by an EARLIER controller
         * instance, before this rejoin) -- plus a matching in-memory job_t
         * already DONE, allocs already released, exactly what a genuinely
         * finished job looks like. */
        {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";
            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = TWR_JOBID_BURST_ALREADY_DONE;
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);

            TEST_ASSERT_EQUAL_INT(0,
                persist_submit_job_finish_fact(&c.persist,
                    TWR_JOBID_BURST_ALREADY_DONE, JOB_STAT_DONE,
                    already_done_finish_time_s, 0));

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";
            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = TWR_JOBID_BURST_ALREADY_DONE;
            jdef.state  = JOB_STAT_DONE;
            jdef.submit = &submit;
            j_done = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(j_done);
            j_done->job.finish_time = (time_t)already_done_finish_time_s;
            j_done->job.exit_info
                = (job_exitinfo_t*)calloc(1, sizeof(job_exitinfo_t));
            TEST_ASSERT_NOT_NULL(j_done->job.exit_info);
            j_done->job.exit_info->exitcode = 0;
            TEST_ASSERT_NULL(j_done->job.allocs);
        }

        /* Seed the 6 LOST jobs: durable WAL_JOB_SUBMIT only, RUN, all
         * assigned to the SAME host -- NO finish fact of any kind, the
         * lost-fire-and-forget-send starting condition, x6 (bug-1741's own
         * "6 of 7 lost" ratio). */
        for (i = 0; i < TWR_BURST_N_LOST; i++) {
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";
            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = lost_ids[i];
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";
            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = lost_ids[i];
            jdef.state  = JOB_STAT_RUN;
            jdef.submit = &submit;
            j_lost[i] = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(j_lost[i]);
            j_lost[i]->job.allocs = twr_make_test_alloc_for_host(
                lost_ids[i], "resync-burst-host");
            TEST_ASSERT_NOT_NULL(j_lost[i]->job.allocs);
        }

        /* ── Join: rejoin_worker (the REAL join entry point) arms the round
         * over exactly the 6 non-terminal (lost) jobs -- the 7th,
         * already-DONE job must NOT be part of the query set (Req 1.1,
         * 1.2, 2.1). */
        memset(&attrs, 0, sizeof(attrs));
        attrs.hosttype  = makeString("LINUX");
        attrs.hostmodel = makeString("x86");
        attrs.cpufactor = 1.0f;
        attrs.maxcpus   = 4;
        attrs.maxmem    = 16;
        attrs.maxswap   = 32;
        attrs.maxtmp    = 64;
        attrs.ndisks    = 1;

        {
            xce_worker_t host;
            memset(&host, 0, sizeof(host));
            host.name  = (char*)"resync-burst-host";
            host.attrs = &attrs;
            w = makeWorker(c.workermgr, &host);
        }
        TEST_ASSERT_NOT_NULL(w);

        memset(&new_attrs, 0, sizeof(new_attrs));
        new_attrs.hosttype  = makeString("LINUX");
        new_attrs.hostmodel = makeString("x86");
        new_attrs.cpufactor = 1.0f;
        new_attrs.maxcpus   = 4;
        new_attrs.maxmem    = 16;
        new_attrs.maxswap   = 32;
        new_attrs.maxtmp    = 64;
        new_attrs.ndisks    = 1;

        memset(&hconfig, 0, sizeof(hconfig));
        hconfig.hosttype  = (char*)"!";
        hconfig.hostmodel = (char*)"!";
        hconfig.mxj       = 4;

        memset(&stack_ep, 0, sizeof(stack_ep));
        rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
        TEST_ASSERT_GREATER_OR_EQUAL(0, rc);

        TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
            "a rejoin with 6 known non-terminal jobs on this host must arm "
            "a reconciliation round (Req 1.1)");
        TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_LOST,
            w->resync->n_queried_jobids,
            "the round's query set must match exactly the host's 6 "
            "non-terminal jobs -- the already-DONE 7th job must be "
            "excluded (Req 1.1, 1.2, 2.1)");
        TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));

        ep = twr_rt_new_test_endpoint();
        TEST_ASSERT_EQUAL_INT(0, worker_send_resync_query(&c, w, ep));

        twr_rt_pop_and_decode(ep, &decoded_query);
        TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery, decoded_query.opcode);
        TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
                              decoded_query.body_which);
        TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_LOST,
            decoded_query.body.jobstatequerymsg->n_total,
            "the REAL wire query (Req 2.1, 2.2, 8.1) must list exactly the "
            "6 lost jobs, regardless of the burst also containing a 7th, "
            "already-terminal one");
        for (i = 0; i < TWR_BURST_N_LOST; i++)
            saw[i] = 0;
        for (i = 0; i < decoded_query.body.jobstatequerymsg->count; i++) {
            uint64_t id = decoded_query.body.jobstatequerymsg->jobids[i];
            int      k;
            TEST_ASSERT_FALSE_MESSAGE(id == TWR_JOBID_BURST_ALREADY_DONE,
                "the already-terminal job must never appear in the query "
                "set (Req 1.2)");
            for (k = 0; k < TWR_BURST_N_LOST; k++) {
                if (id == lost_ids[k])
                    saw[k] = 1;
            }
        }
        for (i = 0; i < TWR_BURST_N_LOST; i++) {
            TEST_ASSERT_TRUE_MESSAGE(saw[i],
                "every one of the 6 lost jobs must be covered by the wire "
                "query, by unique ID (Req 2.2, 8.1)");
        }
        xce_free_request(&decoded_query);

        /* ── Worker's real answer: all 6 lost jobs report their TRUE
         * outcome (5 finished-success + 1 finished-failure, each with its
         * OWN distinct finish_time_ms/exit_code -- Req 3.1, 3.4, 3.5), plus
         * the 7th, already-DONE job reported AGAIN as an unqueried extra
         * (Req 3.6) -- carrying a deliberately WRONG finish_time_ms, to
         * prove reconciliation does not clobber/duplicate what was already
         * correctly recorded (this task's own "none duplicated" Observable
         * Completion Criterion). All 7 entries travel in ONE page (Req
         * 8.1, 8.2) -- this task is not about page-boundary mechanics
         * (task 7.4's own separate job). */
        {
            xce_job_state_entry_t  e_lost[TWR_BURST_N_LOST];
            xce_job_state_entry_t  e_already_done;
            xce_job_state_entry_t* entries[TWR_BURST_N_TOTAL];
            xce_job_state_report_t report;

            for (i = 0; i < TWR_BURST_N_LOST - 1; i++) {
                e_lost[i] = (xce_job_state_entry_t){
                    .jobid = lost_ids[i],
                    .status = JobStateStatus_finishedOk,
                    .exit_code = 0,
                    .finish_time_ms = lost_finish_time_s[i] * 1000ULL,
                    .queried = 1
                };
            }
            /* The 6th lost job: finished-FAILURE, not success -- a genuine
             * mixed-outcome burst (this task's own scope), not a re-proof
             * of 7.1's already-established per-outcome routing in
             * isolation. */
            e_lost[TWR_BURST_N_LOST - 1] = (xce_job_state_entry_t){
                .jobid = lost_ids[TWR_BURST_N_LOST - 1],
                .status = JobStateStatus_finishedFail,
                .exit_code = lost_fail_exit_code,
                .finish_time_ms = 0,
                .queried = 1
            };
            e_already_done = (xce_job_state_entry_t){
                .jobid = TWR_JOBID_BURST_ALREADY_DONE,
                .status = JobStateStatus_finishedOk,
                .exit_code = 0,
                .finish_time_ms = bogus_reapply_finish_time_s * 1000ULL,
                .queried = 0
            };

            for (i = 0; i < TWR_BURST_N_LOST; i++)
                entries[i] = &e_lost[i];
            entries[TWR_BURST_N_LOST] = &e_already_done;

            report = (xce_job_state_report_t){
                .n_total = TWR_BURST_N_TOTAL, .pos = 0,
                .count = TWR_BURST_N_TOTAL, .entries = entries
            };

            TEST_ASSERT_EQUAL_INT(0,
                xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
                                         0));
        }

        twr_rt_pop_and_decode(ep, &decoded_report);
        TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport, decoded_report.opcode);
        TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
                              decoded_report.body_which);
        TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_TOTAL,
            decoded_report.body.jobstatereportmsg->n_total,
            "the real wire report must carry all 7 entries in one page "
            "(Req 8.1, 8.2)");

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        pthread_mutex_lock(&c.persist.lock);
        lsn_before = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        rc = resync_handle_report_page(&c, w,
                                       decoded_report.body.jobstatereportmsg);
        TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc,
            "a single-page report (count == n_total) must complete the "
            "report immediately, regardless of the burst's job count "
            "(Req 8.1)");

        pthread_mutex_lock(&c.persist.lock);
        lsn_after = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        xce_free_request(&decoded_report);

        /* ── Phase 1 assertions: everything observable before any commit
         * round trip. */
        for (i = 0; i < TWR_BURST_N_LOST; i++) {
            TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
                j_lost[i]->job.state,
                "finished-success/failure both stay on the DEFERRED "
                "record_job_finish_fact_at/record_job_fail_fact pattern -- "
                "state mutates later via apply_committed_entry (Phase 2), "
                "not synchronously here, for every one of the 6 lost "
                "jobs");
        }

        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j_done->job.state,
            "the already-DONE job must be left completely untouched by "
            "its own bogus re-report (Req 4.3, 4.4 -- 'none duplicated')");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(already_done_finish_time_s,
            (uint64_t)j_done->job.finish_time,
            "the already-DONE job's ORIGINAL finish_time must survive "
            "unchanged -- NOT overwritten by the bogus finish_time_ms "
            "carried in its own unqueried re-report");
        TEST_ASSERT_NOT_NULL(j_done->job.exit_info);
        TEST_ASSERT_EQUAL_INT(0, j_done->job.exit_info->exitcode);
        TEST_ASSERT_NULL(j_done->job.allocs);

        TEST_ASSERT_NULL_MESSAGE(w->resync,
            "once every one of the 6 queried jobs has a reconciled "
            "outcome, the round must resolve regardless of the burst's "
            "count (Req 5.1, 5.2, 8.1)");
        TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
            "XCE_WORKER_RECONCILING must clear once the round resolves");

        TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_twr_update_hook_fires,
            "no EAGER outcome is present in this burst (no unrecognized, "
            "no genuinely-unassigned extra) -- the 6 deferred finished-"
            "success/failure facts have not applied yet, and the "
            "already-DONE job's re-report is correctly a pure no-op with "
            "no notify at all (Req 4.3, 4.4)");

        TEST_ASSERT_EQUAL_UINT64_MESSAGE(
            lsn_before + (uint64_t)TWR_BURST_N_LOST, lsn_after,
            "EXACTLY one durable fact per lost job (5x WAL_JOB_FINISH + 1x "
            "WAL_JOB_FAIL) must be appended for this report -- the "
            "already-DONE job's duplicate re-report must append NONE, so "
            "an exact count (not merely a lower bound) is the genuine "
            "'none duplicated' proof: a regression that re-appended a "
            "fact for the already-DONE job would silently pass a "
            "'>=' check but fails this exact one");

        xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        if (c.workermgr) dereference(c.workermgr);
        if (c.resmgr)    dereference(c.resmgr);
        if (c.jobmgr)    dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);

        dereference(new_attrs.hosttype);
        dereference(new_attrs.hostmodel);
        endpoint_fini(ep);
        free(ep);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
     * recover from disk -- simulates a controller restart, the SAME apply_
     * committed_entry() seam a live controller's own commit-apply pump
     * would invoke to settle the 6 DEFERRED outcomes above. This is the
     * test's core, observable completion criterion: every one of the 6
     * lost jobs reaches its correct final state, individually and by its
     * OWN distinct worker-reported value (not some other job's from the
     * same burst), and the 7th job's already-correct outcome is confirmed
     * to have survived completely untouched -- through a real restart
     * replay, not just the live path. */
    {
        persist_t* p2 = twr_make_fresh_persist(tmp);
        cluster_t  c2;
        job_t*     j2_done                     = NULL;
        job_t*     j2_lost[TWR_BURST_N_LOST]    = { NULL };
        uint64_t   lost_ids[TWR_BURST_N_LOST] = {
            TWR_JOBID_BURST_LOST_1, TWR_JOBID_BURST_LOST_2,
            TWR_JOBID_BURST_LOST_3, TWR_JOBID_BURST_LOST_4,
            TWR_JOBID_BURST_LOST_5, TWR_JOBID_BURST_LOST_6
        };
        int        rc, i;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 32, .delta = 16 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        key_find(c2.jobmgr, (void*)TWR_JOBID_BURST_ALREADY_DONE,
                 (void**)&j2_done);
        for (i = 0; i < TWR_BURST_N_LOST; i++)
            key_find(c2.jobmgr, (void*)lost_ids[i], (void**)&j2_lost[i]);
        unlock(c2.jobmgr);

        TEST_ASSERT_NOT_NULL(j2_done);
        for (i = 0; i < TWR_BURST_N_LOST; i++) {
            TEST_ASSERT_NOT_NULL_MESSAGE(j2_lost[i],
                "every one of the 6 lost jobs must reconstruct from its "
                "own WAL_JOB_SUBMIT fact");
        }

        /* The core completion criterion (bug-1741, 'none left stuck'):
         * every one of the 6 lost jobs, individually, by its own jobid,
         * reaches its correct final state AND its OWN distinct
         * worker-reported value -- proving no cross-job mixing across the
         * burst (Req 8.1). */
        for (i = 0; i < TWR_BURST_N_LOST - 1; i++) {
            TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE,
                j2_lost[i]->job.state,
                "each lost finished-success job must recover to DONE, "
                "regardless of the burst's count (Req 8.1)");
            TEST_ASSERT_EQUAL_UINT64_MESSAGE(lost_finish_time_s[i],
                (uint64_t)j2_lost[i]->job.finish_time,
                "each lost job must recover ITS OWN worker-reported "
                "finish_time -- not another job's from the same burst "
                "(cross-job mixing regression check, Req 4.2, 3.5)");
        }
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT,
            j2_lost[TWR_BURST_N_LOST - 1]->job.state,
            "the one finished-FAILURE job in the burst must recover to "
            "EXIT, not DONE, distinctly from its 5 finished-success "
            "siblings in the SAME round");
        /* Req 4.2 remains documented-unsatisfiable for FAIL's own
         * finish_time specifically (bug-1747). Its exit code is recorded
         * since bug-1754. */
        TEST_ASSERT_NOT_NULL(j2_lost[TWR_BURST_N_LOST - 1]->job.exit_info);
        TEST_ASSERT_EQUAL_INT(lost_fail_exit_code,
            j2_lost[TWR_BURST_N_LOST - 1]->job.exit_info->exitcode);

        /* The 'none duplicated' completion criterion, confirmed to
         * survive a restart replay too: the already-DONE job's ORIGINAL
         * outcome, not the bogus re-reported one. */
        TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE, j2_done->job.state,
            "the already-DONE job must still be DONE after replay");
        TEST_ASSERT_EQUAL_UINT64_MESSAGE(already_done_finish_time_s,
            (uint64_t)j2_done->job.finish_time,
            "the already-DONE job's finish_time after replay must still "
            "be its ORIGINAL value -- the bogus re-report's "
            "finish_time_ms must never have been durably recorded at "
            "all");

        TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_BURST_N_TOTAL,
            g_twr_update_hook_fires,
            "replaying the durable facts in this log must reach the "
            "scheduler's/apiserver's notify path exactly once per REAL "
            "state change -- 6 for the 6 lost jobs' newly-recorded facts, "
            "PLUS 1 for the already-DONE job's own ORIGINALLY-seeded "
            "WAL_JOB_FINISH fact (recorded before this round even ran, "
            "modeling an earlier controller instance's already-committed "
            "finish): this bare-persist fixture runs no live apply pump, "
            "so THIS replay is the first time apply_committed_entry ever "
            "applies that pre-existing fact too, exactly like every other "
            "seeded WAL_JOB_SUBMIT in this test -- NOT a second, "
            "duplicate application of it. The genuine 'none duplicated' "
            "proof is not this count (which is 7 either way) but that the "
            "already-DONE job's finish_time below is still its ORIGINAL "
            "value, and that Phase 1's own exact LSN-delta check already "
            "proved no fact was durably appended for its bogus re-report "
            "(Req 7.1-7.3)");

        xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);
        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    printf("test_burst_recovery_unbounded_by_job_count: PASS\n");
}

/* ── Task 7.4: Large job-count pagination correctness (Req 8.1-8.3) ──────
 *
 * design.md's own "Performance/Load" testing-strategy line: "Reconciliation
 * exchange for a worker running an order of magnitude more jobs than a
 * single message can carry completes correctly across multiple pages, not
 * truncated (Requirement 8.1-8.3)." Tasks 7.1-7.3 all deliberately stayed
 * within ONE wire page each (task 7.3's own Implementation Notes:
 * "Deliberately kept to a single wire page (page-boundary correctness is
 * task 7.4's separate concern)") -- THIS is that separate concern: proving
 * BOTH directions of the exchange (controller->worker OPCode_jobStateQuery,
 * worker->controller OPCode_jobStateReport) genuinely span MULTIPLE wire
 * pages for a large job count, and that resync_handle_report_page()'s
 * accumulator (task 6.2) reassembles all of them into exactly one applied
 * report -- not merely re-proving task 7.1's four-outcome/task 7.3's burst
 * recovery logic at a bigger N.
 *
 * Real production page-size constants (found by reading the actual call
 * sites, not assumed):
 *   - REPORT direction (worker -> controller): xcecd's real production call
 *     site (controller_connector.c's handle_jobstatequery ->
 *     xcecd_resync_handle_query_page -> xcecd_resync_answer_query) passes
 *     XCECD_RESYNC_REPORT_PKTLIMIT == 100 (controller_connector.c) into
 *     xce_job_state_report_req() (task 1.4, request.c) -- a REAL, wired,
 *     production page cap.
 *   - QUERY direction (controller -> worker): the controller's own real
 *     production call site, worker_send_resync_query() (worker.c:730),
 *     hardcodes pktlimit=0 in its xce_job_state_query_req() call --
 *     xce_job_state_query_req()'s own pktlimit<=0 convention means
 *     "pktlimit = nrows", i.e. ALWAYS exactly ONE wire page, however many
 *     jobs are queried. There is no real production constant to cite for
 *     this direction -- see this task's own CONCERNS (bug-1755). To
 *     genuinely exercise the QUERY side's already-implemented pagination
 *     loop (task 1.4) at scale, this test calls xce_job_state_query_req()
 *     directly (bypassing worker_send_resync_query()'s hardcoded 0),
 *     supplying the SAME page size as the worker's own real report cap for
 *     symmetry -- the exact same production function, exercised with a
 *     real bound instead of an effectively-disabled one.
 *
 * TWR_LARGE_N_TOTAL (below) is an order of magnitude larger than
 * TWR_LARGE_PKTLIMIT (100), yielding a double-digit page count in EACH
 * direction, with a genuinely PARTIAL final page (the total, and each of
 * the four outcome-group sizes, are deliberately NOT multiples of the page
 * limit) -- so several wire pages straddle two different outcome groups, a
 * genuine page-boundary-falls-mid-group case, and the very last page of
 * each direction is short. Covers all four outcome kinds (running,
 * finished-success, finished-failure, unrecognized) so this is not a
 * degenerate single-outcome test.
 */

#define TWR_JOBID_LARGE_BASE 900000ULL
#define TWR_LARGE_N_RUNNING  260
#define TWR_LARGE_N_FINOK    245
#define TWR_LARGE_N_FINFAIL  235
#define TWR_LARGE_N_UNREC    302
#define TWR_LARGE_N_TOTAL    1042 /* 260+245+235+302 -- not a multiple of
                                   * TWR_LARGE_PKTLIMIT, so the final wire
                                   * page in each direction is genuinely
                                   * partial (42 entries), not just N full
                                   * pages */

/* Shared page-size cap for both directions in this test -- == the REAL
 * production XCECD_RESYNC_REPORT_PKTLIMIT for the report direction; the
 * query direction has no real production constant (see header comment
 * above / CONCERNS, bug-1755), so this test uses the same value for
 * symmetry while directly exercising xce_job_state_query_req()'s already-
 * implemented pagination loop. */
#define TWR_LARGE_PKTLIMIT 100
#define TWR_LARGE_N_PAGES \
    ((TWR_LARGE_N_TOTAL + TWR_LARGE_PKTLIMIT - 1) / TWR_LARGE_PKTLIMIT)

static uint64_t twr_large_jobid(int i)
{
    return TWR_JOBID_LARGE_BASE + (uint64_t)i;
}

/* 0 = running, 1 = finished-success, 2 = finished-failure, 3 =
 * unrecognized -- see this test's own header comment on why the group
 * boundaries are deliberately not page-aligned. */
static int twr_large_group(int i)
{
    if (i < TWR_LARGE_N_RUNNING)
        return 0;
    if (i < TWR_LARGE_N_RUNNING + TWR_LARGE_N_FINOK)
        return 1;
    if (i < TWR_LARGE_N_RUNNING + TWR_LARGE_N_FINOK + TWR_LARGE_N_FINFAIL)
        return 2;
    return 3;
}

static void test_large_job_count_pagination_correctness(void)
{
    char     tmp[256];
    uint64_t finok_anchor_s;

    strcpy(tmp, "/tmp/xce_twr74_large_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));

    /* Deliberately far from "now" -- same defensive idiom as every other
     * finish-time test in this file. */
    finok_anchor_s = (uint64_t)time(NULL) - 3600;

    /* Phase 1: real cluster (workermgr + resmgr + jobmgr + hooksys) + real
     * disk persist -- drives the REAL rejoin/query-construction chain over
     * TWR_LARGE_N_TOTAL jobs, then a REAL, explicitly-bounded wire query
     * send and a REAL, real-production-bounded wire report send, both
     * genuinely multi-page. Own local cluster_t (not the shared g_twr
     * fixture), mirroring every other restart-replay test in this file. */
    {
        cluster_t       c;
        kvmgr_params_t  str_params = { .key_type = KVMGR_KEY_STRING,
                                       .nslots = 64, .delta = 16 };
        kvmgr_params_t  int_params = { .key_type = KVMGR_KEY_INT,
                                       .nslots = 1024, .delta = 256 };
        persist_config_t cfg;
        worker_t*         w;
        job_t**           jobs;
        hostattrs_t       attrs, new_attrs;
        xce_hostconfig_t  hconfig;
        endpoint_t        stack_ep;
        endpoint_t*       ep;
        xce_job_state_entry_t*  storage;
        xce_job_state_entry_t** entries;
        xce_job_state_query_t   query;
        xce_job_state_report_t  report;
        int      i, rc;
        uint64_t lsn_before, lsn_after;
        int      calls_before, calls_after;

        static const char* res_names[TWR_RT_N_RES] = {
            "type", "model", "cpuf", "maxcpus", "maxmem",
            "maxswap", "maxtmp", "ndisks", "slots"
        };
        static const int res_valtypes[TWR_RT_N_RES] = {
            XCE_VAL_STRING, XCE_VAL_STRING,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC, XCE_VAL_NUMERIC, XCE_VAL_NUMERIC,
            XCE_VAL_NUMERIC
        };

        if (makeString((char*)"twr74-probe") == NULL)
            xstring_table_init(64);
        if (!g_twr_jobobj_ready) {
            TEST_ASSERT_EQUAL_INT(0, job_obj_init(32, 32));
            g_twr_jobobj_ready = 1;
        }

        memset(&c, 0, sizeof(c));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c.hooksys, XCE_CONTROLLER_HOOK_MAX));

        c.workermgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.workermgr);
        c.resmgr = makeInstance(&xkvmgr_type, &str_params);
        TEST_ASSERT_NOT_NULL(c.resmgr);
        c.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c.jobmgr);

        for (i = 0; i < TWR_RT_N_RES; i++) {
            xce_resource_t rdef;
            memset(&rdef, 0, sizeof(rdef));
            rdef.name    = (char*)res_names[i];
            rdef.valtype = res_valtypes[i];
            rdef.flags   = 0;
            rdef.idx     = -1;
            TEST_ASSERT_NOT_NULL(makeResource(c.resmgr, &rdef));
        }

        c.myhost_info.name = (char*)"twr74-controller";
        c.myhost_info.key  = (char*)"twr74-key";
        c.myhost           = &c.myhost_info;

        cfg             = persist_config_defaults();
        cfg.wal_sync_ms = 5000;
        persist_set_workdir(&c.persist, tmp, "ctrl-test");
        TEST_ASSERT_EQUAL_INT(0, persist_init(&c.persist, &cfg, NULL));
        TEST_ASSERT_EQUAL_INT(0, persist_start(&c.persist));

        jobs = (job_t**)malloc(sizeof(job_t*) * TWR_LARGE_N_TOTAL);
        TEST_ASSERT_NOT_NULL(jobs);

        /* Seed a durable WAL_JOB_SUBMIT fact + a matching in-memory job_t
         * for all TWR_LARGE_N_TOTAL jobs -- all RUN, all assigned to the
         * SAME worker host, exactly what the controller believes right
         * before this rejoin (bypassing dispatch/replay, mirroring every
         * sibling restart-replay test in this file). */
        for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
            uint64_t          jobid = twr_large_jobid(i);
            job_submit_data_t wal_submit;
            xce_job_t         wal_jdef;
            uint8_t*          buf = NULL;
            size_t            sz  = 0;
            job_submit_data_t submit;
            xce_job_t         jdef;

            memset(&wal_submit, 0, sizeof(wal_submit));
            wal_submit.queue = (char*)"q";
            wal_submit.user  = (char*)"tester";
            memset(&wal_jdef, 0, sizeof(wal_jdef));
            wal_jdef.jobid  = jobid;
            wal_jdef.state  = JOB_STAT_RUN;
            wal_jdef.submit = &wal_submit;
            TEST_ASSERT_EQUAL_INT(0, twr_encode_job(&wal_jdef, &buf, &sz));
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(&c.persist, WAL_JOB_SUBMIT, buf,
                                  (uint32_t)sz));
            free(buf);

            memset(&submit, 0, sizeof(submit));
            submit.queue = (char*)"q";
            submit.user  = (char*)"tester";
            memset(&jdef, 0, sizeof(jdef));
            jdef.jobid  = jobid;
            jdef.state  = JOB_STAT_RUN;
            jdef.submit = &submit;
            jobs[i] = makeJob(c.jobmgr, &jdef);
            TEST_ASSERT_NOT_NULL(jobs[i]);
            jobs[i]->job.allocs
                = twr_make_test_alloc_for_host(jobid, "resync-large-host");
            TEST_ASSERT_NOT_NULL(jobs[i]->job.allocs);
        }

        /* ── Join: rejoin_worker (the REAL join entry point) arms the round
         * over all TWR_LARGE_N_TOTAL non-terminal jobs -- Req 1.1, 2.1, and
         * the "no artificial cap" proof (Req 8.1) that resync_collect_
         * query_jobids' realloc-doubling growth (resync.c) genuinely has no
         * fixed-size ceiling. */
        memset(&attrs, 0, sizeof(attrs));
        attrs.hosttype  = makeString("LINUX");
        attrs.hostmodel = makeString("x86");
        attrs.cpufactor = 1.0f;
        attrs.maxcpus   = 4;
        attrs.maxmem    = 16;
        attrs.maxswap   = 32;
        attrs.maxtmp    = 64;
        attrs.ndisks    = 1;

        {
            xce_worker_t host;
            memset(&host, 0, sizeof(host));
            host.name  = (char*)"resync-large-host";
            host.attrs = &attrs;
            w = makeWorker(c.workermgr, &host);
        }
        TEST_ASSERT_NOT_NULL(w);

        memset(&new_attrs, 0, sizeof(new_attrs));
        new_attrs.hosttype  = makeString("LINUX");
        new_attrs.hostmodel = makeString("x86");
        new_attrs.cpufactor = 1.0f;
        new_attrs.maxcpus   = 4;
        new_attrs.maxmem    = 16;
        new_attrs.maxswap   = 32;
        new_attrs.maxtmp    = 64;
        new_attrs.ndisks    = 1;

        memset(&hconfig, 0, sizeof(hconfig));
        hconfig.hosttype  = (char*)"!";
        hconfig.hostmodel = (char*)"!";
        hconfig.mxj       = 4;

        memset(&stack_ep, 0, sizeof(stack_ep));
        rc = rejoin_worker(&c, w, &stack_ep, &new_attrs, &hconfig);
        TEST_ASSERT_GREATER_OR_EQUAL(0, rc);

        TEST_ASSERT_NOT_NULL_MESSAGE(w->resync,
            "a rejoin with TWR_LARGE_N_TOTAL known non-terminal jobs on "
            "this host must arm a reconciliation round (Req 1.1)");
        TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_TOTAL,
            w->resync->n_queried_jobids,
            "the round's query set must match exactly the host's non-"
            "terminal jobs, with no artificial cap on how many resync_"
            "arm_round will collect (Req 1.1, 2.1, 8.1)");
        TEST_ASSERT_TRUE(IS_WORKER_RECONCILING(w->host.status));

        ep = twr_rt_new_test_endpoint();

        /* ── QUERY direction: a REAL wire-encoded xce_job_state_query_req()
         * send, called directly (not via worker_send_resync_query(), which
         * hardcodes pktlimit=0 -- see this test's own header comment and
         * CONCERNS) with an explicit page size, genuinely producing
         * multiple wire pages for this job count. */
        query = (xce_job_state_query_t){
            .n_total = w->resync->n_queried_jobids,
            .pos     = 0,
            .count   = w->resync->n_queried_jobids,
            .jobids  = w->resync->queried_jobids
        };
        TEST_ASSERT_EQUAL_INT(0,
            xce_job_state_query_req(ep, c.myhost, &query, NULL, NULL,
                                    TWR_LARGE_PKTLIMIT));

        {
            bool seen_query[TWR_LARGE_N_TOTAL];
            int  running_total = 0;
            int  query_pages   = 0;

            memset(seen_query, 0, sizeof(seen_query));

            while (running_total < TWR_LARGE_N_TOTAL) {
                xce_request_t decoded_query;
                int           j;

                twr_rt_pop_and_decode(ep, &decoded_query);
                TEST_ASSERT_EQUAL_INT(OPCode_jobStateQuery,
                                      decoded_query.opcode);
                TEST_ASSERT_EQUAL_INT(Request_body_jobstatequerymsg,
                                      decoded_query.body_which);
                TEST_ASSERT_NOT_NULL(decoded_query.body.jobstatequerymsg);
                TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_TOTAL,
                    decoded_query.body.jobstatequerymsg->n_total,
                    "Req 2.1: every query page must carry the SAME full "
                    "n_total, not a per-page total");
                TEST_ASSERT_EQUAL_INT_MESSAGE(running_total,
                    decoded_query.body.jobstatequerymsg->pos,
                    "Req 8.3: each page's pos must pick up exactly where "
                    "the previous page's count left off -- no gap, no "
                    "overlap");

                for (j = 0; j < decoded_query.body.jobstatequerymsg->count;
                     j++) {
                    uint64_t jobid
                        = decoded_query.body.jobstatequerymsg->jobids[j];
                    uint64_t idx = jobid - TWR_JOBID_LARGE_BASE;

                    TEST_ASSERT_TRUE_MESSAGE(
                        idx < (uint64_t)TWR_LARGE_N_TOTAL,
                        "a jobid appearing on the wire query must be one "
                        "of the armed round's own jobids");
                    TEST_ASSERT_FALSE_MESSAGE(seen_query[idx],
                        "Req 8.3: no jobid may be sent twice across query "
                        "pages");
                    seen_query[idx] = true;
                }

                running_total
                    += decoded_query.body.jobstatequerymsg->count;
                query_pages++;
                xce_free_request(&decoded_query);
            }

            TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_PAGES, query_pages,
                "Req 8.1-8.3: an order-of-magnitude-larger-than-one-page "
                "query must genuinely span many wire pages, including a "
                "partial final page, not silently collapse to one (the "
                "real production worker_send_resync_query() call site "
                "would collapse to one -- see CONCERNS, bug-1755 -- this "
                "assertion is against xce_job_state_query_req()'s own "
                "pagination mechanism, exercised directly)");

            for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
                TEST_ASSERT_TRUE_MESSAGE(seen_query[i],
                    "Req 8.3: every armed jobid must appear exactly once "
                    "across all query pages -- none silently dropped past "
                    "a page boundary");
            }
        }

        /* ── Worker's answer: a REAL wire-encoded-then-decoded OPCode_
         * jobStateReport, sent at the REAL production report page size
         * (XCECD_RESYNC_REPORT_PKTLIMIT, controller_connector.c) --
         * genuinely many wire pages -- covering all four outcome kinds,
         * fed page-by-page into resync_handle_report_page() (task 6.2's
         * real reassembly entry point), never resync_apply_report()
         * directly. */
        storage = (xce_job_state_entry_t*)malloc(
            sizeof(xce_job_state_entry_t) * TWR_LARGE_N_TOTAL);
        entries = (xce_job_state_entry_t**)malloc(
            sizeof(xce_job_state_entry_t*) * TWR_LARGE_N_TOTAL);
        TEST_ASSERT_NOT_NULL(storage);
        TEST_ASSERT_NOT_NULL(entries);

        for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
            xce_job_state_entry_t* e = &storage[i];

            memset(e, 0, sizeof(*e));
            e->jobid   = twr_large_jobid(i);
            e->queried = 1;

            switch (twr_large_group(i)) {
            case 0:
                e->status = JobStateStatus_running;
                break;
            case 1: {
                int      local = i - TWR_LARGE_N_RUNNING;
                uint64_t fin_s = finok_anchor_s - (uint64_t)(local * 3);
                e->status         = JobStateStatus_finishedOk;
                e->finish_time_ms = fin_s * 1000ULL;
                break;
            }
            case 2: {
                int local = i - TWR_LARGE_N_RUNNING - TWR_LARGE_N_FINOK;
                e->status    = JobStateStatus_finishedFail;
                e->exit_code = 200 + local;
                break;
            }
            default:
                e->status = JobStateStatus_unrecognized;
                break;
            }

            entries[i] = e;
        }

        report = (xce_job_state_report_t){
            .n_total = TWR_LARGE_N_TOTAL,
            .pos     = 0,
            .count   = TWR_LARGE_N_TOTAL,
            .entries = entries
        };

        TEST_ASSERT_EQUAL_INT(0,
            xce_job_state_report_req(ep, c.myhost, &report, NULL, NULL,
                                     TWR_LARGE_PKTLIMIT));

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        pthread_mutex_lock(&c.persist.lock);
        lsn_before = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        resync_apply_report_call_count_reset_for_test();
        calls_before = resync_apply_report_call_count_for_test();

        {
            int running_total = 0;
            int report_pages  = 0;

            while (running_total < TWR_LARGE_N_TOTAL) {
                xce_request_t decoded_report;
                int           rc_page;

                twr_rt_pop_and_decode(ep, &decoded_report);
                TEST_ASSERT_EQUAL_INT(OPCode_jobStateReport,
                                      decoded_report.opcode);
                TEST_ASSERT_EQUAL_INT(Request_body_jobstatereportmsg,
                                      decoded_report.body_which);
                TEST_ASSERT_NOT_NULL(decoded_report.body.jobstatereportmsg);

                running_total
                    += decoded_report.body.jobstatereportmsg->count;
                report_pages++;

                rc_page = resync_handle_report_page(
                    &c, w, decoded_report.body.jobstatereportmsg);

                if (running_total < TWR_LARGE_N_TOTAL) {
                    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_page,
                        "a non-final report page must not complete the "
                        "report");
                } else {
                    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rc_page,
                        "the final (partial) report page must complete "
                        "the report");
                }

                xce_free_request(&decoded_report);
            }

            TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_PAGES, report_pages,
                "Req 8.1-8.3: at the real production per-page limit, this "
                "job count must genuinely span many wire pages, including "
                "a partial final page");
        }

        calls_after = resync_apply_report_call_count_for_test();
        TEST_ASSERT_EQUAL_INT_MESSAGE(1, calls_after - calls_before,
            "resync_apply_report must be invoked EXACTLY ONCE for the "
            "whole large, many-page report -- not once per page (task "
            "6.2's own structural proof, generalized from 2 pages to "
            "double digits)");

        pthread_mutex_lock(&c.persist.lock);
        lsn_after = c.persist.current_lsn;
        pthread_mutex_unlock(&c.persist.lock);

        TEST_ASSERT_EQUAL_UINT64_MESSAGE(
            lsn_before
                + (uint64_t)(TWR_LARGE_N_FINOK + TWR_LARGE_N_FINFAIL
                              + TWR_LARGE_N_UNREC),
            lsn_after,
            "EXACTLY one durable fact per non-running job (finished-"
            "success x WAL_JOB_FINISH + finished-failure x WAL_JOB_FAIL + "
            "unrecognized x WAL_JOB_UPDATE), and NONE for the still-"
            "running jobs -- an exact count (not merely a lower bound) is "
            "the genuine 'none dropped, none duplicated across page "
            "boundaries' proof");

        TEST_ASSERT_EQUAL_INT_MESSAGE(TWR_LARGE_N_UNREC,
            g_twr_update_hook_fires,
            "Req 7.1-7.3: the EAGER unrecognized-to-pending transition "
            "must reach the scheduler's/apiserver's notify path "
            "synchronously, exactly once per unrecognized job, regardless "
            "of which page it happened to land on");

        TEST_ASSERT_NULL_MESSAGE(w->resync,
            "once every queried job across every report page has a "
            "reconciled outcome, the round must resolve (Req 5.1, 5.2)");
        TEST_ASSERT_FALSE_MESSAGE(IS_WORKER_RECONCILING(w->host.status),
            "XCE_WORKER_RECONCILING must clear once the round resolves");

        /* Phase 1 end-to-end per-job assertions (Req 8.1-8.3: EVERY job
         * accounted for, not a sample) -- everything observable before any
         * commit round-trip. */
        for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
            switch (twr_large_group(i)) {
            case 0:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
                    jobs[i]->job.state,
                    "still-running (queried) must be left exactly as-is");
                TEST_ASSERT_NOT_NULL(jobs[i]->job.allocs);
                break;
            case 1:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
                    jobs[i]->job.state,
                    "finished-success stays DEFERRED (record_job_finish_"
                    "fact_at) -- mutates later via apply_committed_entry, "
                    "not synchronously here");
                break;
            case 2:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
                    jobs[i]->job.state,
                    "finished-failure stays DEFERRED (on_job_fail/record_"
                    "job_fail_fact), same as finished-success");
                break;
            default:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND,
                    jobs[i]->job.state,
                    "unrecognized reconciles EAGERLY -- PEND "
                    "synchronously, before any commit round-trip");
                TEST_ASSERT_NULL_MESSAGE(jobs[i]->job.allocs,
                    "bug-1631: reconciled-to-pending must release the "
                    "allocation synchronously, at scale, across page "
                    "boundaries");
                break;
            }
        }

        xce_disable_hook(&c.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);

        TEST_ASSERT_EQUAL_INT(0, persist_flush(&c.persist));

        persist_stop(&c.persist);
        persist_fini(&c.persist);
        if (c.workermgr) dereference(c.workermgr);
        if (c.resmgr)    dereference(c.resmgr);
        if (c.jobmgr)    dereference(c.jobmgr);
        xce_hook_fini(&c.hooksys);

        dereference(new_attrs.hosttype);
        dereference(new_attrs.hostmodel);
        endpoint_fini(ep);
        free(ep);

        free(jobs);
        free(storage);
        free(entries);
    }

    /* Phase 2: fresh persist_t + fresh cluster_t (jobmgr/hooksys only),
     * recover from disk -- simulates a controller restart, settling the two
     * DEFERRED outcomes (finished-success/failure) across every page they
     * happened to land on, and re-confirming the EAGER unrecognized->
     * pending outcome also round-trips through a real WAL replay, not just
     * the live eager-mutation path -- for EVERY one of the TWR_LARGE_N_
     * TOTAL jobs, individually, not a sample. */
    {
        persist_t*     p2 = twr_make_fresh_persist(tmp);
        cluster_t      c2;
        job_t**        jobs2;
        int            rc, i;
        kvmgr_params_t int_params = { .key_type = KVMGR_KEY_INT,
                                      .nslots = 1024, .delta = 256 };

        TEST_ASSERT_NOT_NULL(p2);

        memset(&c2, 0, sizeof(c2));
        TEST_ASSERT_EQUAL_INT(0,
            xce_hook_init(&c2.hooksys, XCE_CONTROLLER_HOOK_MAX));
        c2.jobmgr = makeInstance(&xkvmgr_type, &int_params);
        TEST_ASSERT_NOT_NULL(c2.jobmgr);

        g_twr_update_hook_fires = 0;
        TEST_ASSERT_EQUAL_INT(0,
            xce_add_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                        twr_job_update_spy_hook, NULL));

        p2->cluster = &c2;
        rc          = persist_recover(p2);
        TEST_ASSERT_EQUAL_INT(0, rc);

        jobs2 = (job_t**)malloc(sizeof(job_t*) * TWR_LARGE_N_TOTAL);
        TEST_ASSERT_NOT_NULL(jobs2);

        lock(c2.jobmgr, RTTI_LOCK_READ);
        for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
            jobs2[i] = NULL;
            key_find(c2.jobmgr, (void*)twr_large_jobid(i),
                     (void**)&jobs2[i]);
        }
        unlock(c2.jobmgr);

        for (i = 0; i < TWR_LARGE_N_TOTAL; i++) {
            TEST_ASSERT_NOT_NULL_MESSAGE(jobs2[i],
                "restart replay must reconstruct every job via its own "
                "WAL_JOB_SUBMIT fact");

            switch (twr_large_group(i)) {
            case 0:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_RUN,
                    jobs2[i]->job.state,
                    "still-running: no fact was ever recorded, so replay "
                    "leaves it exactly as WAL_JOB_SUBMIT alone "
                    "reconstructs it");
                break;
            case 1: {
                int      local = i - TWR_LARGE_N_RUNNING;
                uint64_t fin_s = finok_anchor_s - (uint64_t)(local * 3);
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_DONE,
                    jobs2[i]->job.state,
                    "finished-success: the DEFERRED record_job_finish_"
                    "fact_at fact must apply to DONE on replay");
                TEST_ASSERT_EQUAL_UINT64_MESSAGE(fin_s,
                    (uint64_t)jobs2[i]->job.finish_time,
                    "each finished-success job must recover ITS OWN "
                    "worker-reported finish_time -- no cross-job mixing "
                    "across a page boundary");
                break;
            }
            case 2:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_EXIT,
                    jobs2[i]->job.state,
                    "finished-failure: the DEFERRED on_job_fail/record_"
                    "job_fail_fact fact must apply to EXIT on replay");
                break;
            default:
                TEST_ASSERT_EQUAL_UINT32_MESSAGE(JOB_STAT_PEND,
                    jobs2[i]->job.state,
                    "unrecognized: the EAGER record_job_reconcile_"
                    "pending_fact's durable WAL_JOB_UPDATE fact must ALSO "
                    "reconstruct correctly on replay");
                TEST_ASSERT_NULL(jobs2[i]->job.allocs);
                break;
            }
        }

        TEST_ASSERT_EQUAL_INT_MESSAGE(
            TWR_LARGE_N_FINOK + TWR_LARGE_N_FINFAIL + TWR_LARGE_N_UNREC,
            g_twr_update_hook_fires,
            "Req 7.1-7.3: replaying the durable facts this reconciliation "
            "round produced must reach the scheduler's/apiserver's notify "
            "path for EVERY actual state change -- finished-success, "
            "finished-failure, and unrecognized-to-pending -- exactly "
            "once each, across every job (still-running recorded no fact, "
            "and WAL_JOB_SUBMIT fires HOOK_DETAIL_JOB_NEW, not JOB_UPDATE, "
            "so neither adds to this count)");

        xce_disable_hook(&c2.hooksys, XCE_CONTROLLER_HOOK_JOB,
                         twr_job_update_spy_hook);
        xce_hook_fini(&c2.hooksys);
        if (c2.jobmgr) dereference(c2.jobmgr);
        twr_teardown_persist(p2);
        free(jobs2);
    }

    {
        char cmd[512];
        snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
        if (system(cmd) != 0)
            fprintf(stderr, "cleanup warning: rm -rf %s failed\n", tmp);
    }

    printf("test_large_job_count_pagination_correctness: PASS\n");
}

/* ── Test group entry point ───────────────────────────────────────────── */

void test_worker_resync(void)
{
    UnitySetTestFile(__FILE__);
    RUN_TEST(test_reconcile_pending_applies_releases_allocs_and_notifies);
    RUN_TEST(test_reconcile_pending_noop_on_already_pending_job);
    RUN_TEST(test_reconcile_pending_rejects_terminal_job);
    RUN_TEST(test_reconcile_assign_applies_running_no_allocs_and_notifies);
    RUN_TEST(test_reconcile_assign_survives_restart_replay);
    RUN_TEST(test_resync_arm_round_query_set_matches_exactly);
    RUN_TEST(test_resync_arm_round_no_jobs_is_noop);
    RUN_TEST(test_resync_arm_round_fresh_token_supersedes_prior);
    RUN_TEST(test_resync_reconciling_flag_set_after_arm_cleared_after_resolve);
    RUN_TEST(test_resync_resolve_round_notifies_worker_update);
    RUN_TEST(test_resync_timeout_resolves_round_without_blocking);
    RUN_TEST(test_apply_report_all_five_outcomes_updates_job_table);
    RUN_TEST(test_apply_report_duplicate_reapplication_is_noop);
    RUN_TEST(test_apply_report_stale_round_token_is_discarded);
    RUN_TEST(test_apply_report_partial_report_does_not_resolve_round);
    RUN_TEST(test_apply_report_unqueried_extra_already_tracked_is_noop);
    RUN_TEST(
        test_apply_report_finished_success_uses_worker_reported_finish_time);
    RUN_TEST(
        test_apply_report_unqueried_extra_unassigned_pend_job_gets_assigned);
    RUN_TEST(test_handle_report_page_single_page_baseline);
    RUN_TEST(test_handle_report_page_multi_page_reassembles_and_applies_once);
    RUN_TEST(test_handle_report_page_stale_round_discarded_when_superseded);
    RUN_TEST(test_handle_report_page_malformed_bounds_rejected_and_recovers);
    RUN_TEST(test_full_rejoin_reconciliation_round_trip);
    RUN_TEST(test_failover_during_job_recovers_lost_finish);
    RUN_TEST(test_burst_recovery_unbounded_by_job_count);
    RUN_TEST(test_large_job_count_pagination_correctness);
}