fix bug-1060 bug-1067

2026-06-24 17:05:43 +08:00 · 2026-06-24 17:05:43 +08:00 · 61e1391cd0
commit 61e1391cd0
parent 6067ba6c94
28 changed files with 7021 additions and 2926 deletions
--- a/.claude/skills/xce-test-ha/SKILL.md
+++ b/.claude/skills/xce-test-ha/SKILL.md
@ -108,8 +108,24 @@ XCE_HA_ROUNDS=10 XCE_HA_TOP=/tmp/my-ha-run nix develop ./src --command bash "$SK
   restart the node downed in the *previous* round (so the cluster returns to 3
   members) → wait for a new primary → `verify_no_loss` (expect-job every tracked
   JobId) → submit another batch → `verify_no_loss` again → `walcompare`.
-12. **Verdict** — exits `0` (PASS) only if every round verified zero job loss;
+   Each elected primary is recorded for the leadership-coverage check.
-    any missing job, failed election, or unreachable primary exits non-zero (FAIL).
+11b. **Leadership coverage** — every node must serve as primary at least once.
   The natural kill loop only ever elects the two highest-priority nodes
   (`ctrl-1`/`ctrl-2`): the election timeout is **priority-staggered**
   (`my_priority * election_priority_step_ms` — p0 immediate, p1 +500ms,
   p2 +1000ms), so the lowest-priority node (`ctrl-3`) never wins while a
   lower-index node survives alongside it. After the loop, the driver restores
   3-of-3 and **forces any never-yet-primary node to leadership** via a
   deterministic, loss-free maneuver that exploits the RequestVote freshness
   gate (a behind node cannot win): kill the spoiler standby → commit a batch so
   it falls behind → kill the primary (target holds all commits) → restart the
   stale spoiler so the freshness-fenced quorum elects the target → verify no
   loss → reconverge. A node staying behind cannot catch up while there is no
   primary, so the target wins **deterministically**, not by timing.
 12. **Verdict** — exits `0` (PASS) only if every round verified zero job loss
    **and all three nodes (`ctrl-1`, `ctrl-2`, `ctrl-3`) acted as primary**;
    any missing job, failed election, unreachable primary, or a node that never
    led exits non-zero (FAIL). The PASS line names the nodes that led.
 ## Step 3 — Report the result
--- a/.claude/skills/xce-test-ha/run-ha-soak.sh
+++ b/.claude/skills/xce-test-ha/run-ha-soak.sh
@ -54,12 +54,27 @@ PEERS="ctrl-1=localhost:9991,ctrl-2=localhost:9992,ctrl-3=localhost:9993"
 MASTERS="localhost:9991,localhost:9992,localhost:9993"
 ALL_JOBIDS=()           # every JobId we have ever successfully submitted
 declare -A SEEN_PRIMARY  # node name -> 1 once it has served as primary
 log()  { echo "[xce-test-ha] $*"; }
 fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
 cleanup() { pkill -9 -x xce_controller 2>/dev/null; sleep 0.3; }
 # Record that a node has acted as primary at least once (leadership-coverage
 # tracking — the run asserts all three nodes lead, see the verdict).
 record_primary() { [ -n "$1" ] && SEEN_PRIMARY["$1"]=1; }
 # kill -9 a controller by node name. Match on BINARY + --myname so the pattern
 # cannot hit this driver's own shell snapshot (its argv carries "--myname ctrl-N").
 kill_node() {
    local node="$1" pid
    pid="$(pgrep -f -- "xce_controller --myname $node ")"
    [ -n "$pid" ] || return 1
    kill -9 $pid 2>/dev/null
    return 0
 }
 # ── Preconditions ────────────────────────────────────────────────────────────
 [ -x "$BIN" ] || { echo "[xce-test-ha] FAIL: controller binary missing: $BIN (run /xce-build controller)" >&2; exit 2; }
 [ -x "$XT" ] || { echo "[xce-test-ha] FAIL: xce_test missing: $XT (run /xce-build controller)" >&2; exit 2; }
@ -285,6 +300,66 @@ walcheck() {
    fi
 }
 # Drive a SPECIFIC node to primary — used to exercise leadership on the
 # lowest-priority node (ctrl-3), which the natural single-failover loop NEVER
 # elects: the election timeout is priority-staggered (my_priority * step_ms, p0
 # immediate / p1 +500ms / p2 +1000ms), so whenever ctrl-3 is in the surviving
 # pair a lower-index node out-races it.
 #
 # Mechanism — exploits the RequestVote freshness gate (a behind node cannot win):
 #   precondition: cluster 3-of-3 converged, target T caught up, T not primary.
 #   P = current primary; S = the third node (not P, not T).
 #   1. kill S (a standby) so it stops receiving entries.
 #   2. submit a batch → it commits on {P, T}; S is now provably BEHIND.
 #   3. kill P → only T alive (brief, deliberate no-quorum). T holds EVERY
 #      committed entry (all commits were on {P,T}), so nothing is lost.
 #   4. restart S (still behind — it cannot catch up while there is no primary,
 #      so it stays fenced for the whole election): quorum {T fresh, S stale};
 #      the freshness gate denies S, and T wins DETERMINISTICALLY (not by timing).
 #   5. verify no loss, restart P, reconverge 3-of-3.
 # Fails the run loudly if T does not win.
 force_elect_node() {
    local T="$1" P S n
    P="$(discover_primary)"
    [ -n "$P" ] || { log "force-elect $T: no serving primary"; return 1; }
    if [ "$P" = "$T" ]; then record_primary "$T"; log "force-elect: $T already primary"; return 0; fi
    S=""
    for n in "${NODES[@]}"; do [ "$n" != "$P" ] && [ "$n" != "$T" ] && S="$n"; done
    [ -n "$S" ] || { log "force-elect $T: could not pick spoiler"; return 1; }
    log "force-elect $T: primary=$P spoiler=$S (freshness-fence $S so $T wins)"
    local PREV_TERM; PREV_TERM="$(latest_primary_term)"; PREV_TERM="${PREV_TERM:-0}"
    # 1. Kill the spoiler standby so it falls behind.
    kill_node "$S" || fail "force-elect $T: could not kill spoiler $S"
    sleep 2
    # 2. Commit a batch on {P,T} — guarantees S is behind and T is caught up.
    submit_jobs "$NJOBS" || fail "force-elect $T: submit on {$P,$T} failed"
    verify_no_loss || fail "force-elect $T: loss after spoiler-down submit"
    # 3. Kill the primary. Only T is alive (transient no-quorum); T has all commits.
    kill_node "$P" || fail "force-elect $T: could not kill primary $P"
    sleep 2
    # 4. Restart the (behind) spoiler to restore quorum; T must win on freshness.
    log "force-elect $T: restarting behind-spoiler $S — expecting $T to win"
    start_node "$S"
    NEWP="$(wait_for_new_primary "$PREV_TERM")"
    [ -n "$NEWP" ] || fail "force-elect $T: no new primary after restoring quorum"
    [ "$NEWP" = "$T" ] || fail "force-elect $T: expected $T to win, got $NEWP (freshness-fence failed)"
    record_primary "$T"
    PRIMARY="$(discover_primary)"
    [ -n "$PRIMARY" ] || fail "force-elect $T: $T elected but client cannot reach it"
    verify_no_loss || fail "force-elect $T: committed job LOST during maneuver"
    log "force-elect $T: SUCCESS — $T is primary (term=$(latest_primary_term))"
    # 5. Restart P and reconverge to a healthy 3-of-3.
    start_node "$P"
    wait_for_full_convergence || fail "force-elect $T: cluster did not reconverge 3-of-3 after $T led"
    return 0
 }
 # ── Step 2: deploy ───────────────────────────────────────────────────────────
 log "deploying 3-node cluster to $TOP"
 cleanup
@ -314,6 +389,7 @@ log "waiting for primary election (cold start ~15s)"
 wait_for_primary_log || fail "no primary elected within 45s (check $TOP/logs)"
 PRIMARY="$(discover_primary)"
 [ -n "$PRIMARY" ] || fail "primary elected in logs but client could not reach it"
 record_primary "$PRIMARY"
 log "primary is $PRIMARY"
 # ── Step 5: initial submit + verify + waldump ────────────────────────────────
@ -371,6 +447,7 @@ for round in $(seq 1 "$ROUNDS"); do
    NEWP="$(wait_for_new_primary "$PREV_TERM")"
    [ -n "$NEWP" ] || fail "round $round: no new primary elected after kill (no CTRL_PRIMARY term>$PREV_TERM)"
    log "round $round: new primary elected: $NEWP (term=$(latest_primary_term))"
    record_primary "$NEWP"
    PRIMARY="$(discover_primary)"
    [ -n "$PRIMARY" ] || fail "round $round: new primary $NEWP elected but client cannot reach it (bug-854 class)"
    verify_no_loss || fail "round $round: a committed job was LOST after failover"
@ -383,7 +460,35 @@ for round in $(seq 1 "$ROUNDS"); do
    walcheck "$((1 + 2 * ${#ALL_JOBIDS[@]}))"
 done
-log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss"
+# ── Leadership coverage: EVERY node must have served as primary ──────────────
 # The natural single-failover loop only ever elects the two highest-priority
 # nodes (ctrl-1/ctrl-2) — the priority-staggered election timeout means ctrl-3
 # never wins while a lower-index node survives alongside it. Drive any
 # never-yet-primary node to leadership explicitly so all three are exercised.
 # First restore the cluster to 3-of-3 (the loop leaves the last-killed primary
 # down).
 if [ -n "$DOWN" ]; then
    log "leadership-coverage: restarting $DOWN to restore 3-of-3 before maneuver"
    start_node "$DOWN"
    wait_for_full_convergence || fail "leadership-coverage: cluster did not reconverge before maneuver"
    DOWN=""
 fi
 for n in "${NODES[@]}"; do
    if [ -z "${SEEN_PRIMARY[$n]:-}" ]; then
        log "leadership-coverage: $n has never been primary — forcing election"
        force_elect_node "$n"
    fi
 done
 # Assert all three nodes acted as primary at some point in the run.
 MISSING=""
 for n in "${NODES[@]}"; do
    [ -z "${SEEN_PRIMARY[$n]:-}" ] && MISSING="$MISSING $n"
 done
 [ -z "$MISSING" ] || fail "not all nodes acted as primary — never led:$MISSING"
 LED="$(printf '%s\n' "${!SEEN_PRIMARY[@]}" | sort | tr '\n' ',' | sed 's/,$//')"
 log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss; all 3 nodes led ($LED)"
 cleanup
 trap - EXIT INT TERM
 exit 0
--- a/.wolf/anatomy.md
+++ b/.wolf/anatomy.md
@ -1,10 +1,21 @@
 # anatomy.md
-> Auto-maintained by OpenWolf. Last scanned: 2026-06-23T04:47:56.802Z
+> Auto-maintained by OpenWolf. Last scanned: 2026-06-24T08:33:10.048Z
-> Files: 551 tracked | Anatomy hits: 0 | Misses: 0
+> Files: 570 tracked | Anatomy hits: 0 | Misses: 0
 ## ../../../../../../tmp/xce-ha-verify/
 - `test.sh` (~291 tok)
 ## ../../../../../claude/.claude/plans/
 - `piped-mixing-glade.md` — Fix bug-854 (client primary-discovery) and bug-1028 (post-failover availability) (~1921 tok)
 - `task-notification-task-id-bkkk2llpr-tas-zany-locket.md` — XCE HA Failover Soak — Result (~343 tok)
 ## ../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/
 - `project_bug1060_leader_completeness.md` (~538 tok)
 - `project_bug1067_ae_two_terms.md` (~716 tok)
 - `project_raft_commit_safety_done.md` (~603 tok)
 - `project_raft_log_replication_done.md` (~507 tok)
@ -19,6 +30,8 @@
 - `package-lock.json` — npm lock file (~1039 tok)
 - `package.json` — Node.js package manifest (~1 tok)
 - `README.md` — Project documentation (~612 tok)
 - `repro_joblist.sh` — Restart the kept failover-e2e cluster and query joblist via the client to see (~201 tok)
 - `repro854.sh` — Reproduce bug-854 client discovery against the kept failover-e2e cluster. (~422 tok)
 - `uninstall-code-review-graph.sh` — Fully uninstall `code-review-graph` (the Python/uv MCP tool). (~672 tok)
 ## .claude/
@ -203,6 +216,11 @@
 - `SKILL.md` — XCE HA Cluster Deployment (~1345 tok)
 ## .claude/skills/xce-test-ha/
 - `run-ha-soak.sh` — run-ha-soak.sh — driver for /xce-test-ha: deploy a 3-node XCE HA cluster, then (~6047 tok)
 - `SKILL.md` — XCE HA Failover Soak Test (~2511 tok)
 ## .codegraph/
 - `.gitignore` — Git ignore rules (~47 tok)
@ -764,13 +782,14 @@
 ## src/controller/
- `cluster.c` — include "config.h" (~16351 tok)
+- `CLAUDE.md` — XCE Controller (~3309 tok)
 - `cluster.c` — include "config.h" (~16451 tok)
 - `commit.c` — Declares persist_t (~5043 tok)
- `config_repl.c` — define _GNU_SOURCE (~24927 tok)
+- `config_repl.c` — define _GNU_SOURCE (~25407 tok)
- `config_repl.h` — pragma once (~5622 tok)
+- `config_repl.h` — pragma once (~6008 tok)
 - `ha_convergence.c` — Declares char (~4868 tok)
- `ha.c` — define _GNU_SOURCE (~77256 tok)
+- `ha.c` — define _GNU_SOURCE (~78897 tok)
- `ha.h` — ifndef HA_H (~21010 tok)
+- `ha.h` — ifndef HA_H (~21515 tok)
 - `meson.build` (~1352 tok)
 - `myapp.c` — define _GNU_SOURCE (~4691 tok)
 - `persist.c` — define _GNU_SOURCE (~33172 tok)
@ -784,21 +803,23 @@
 - `clean_failover_preservation.sh` — clean_failover_preservation.sh — ha-term-safety task 5.2 live integration test. (~4972 tok)
 - `controller_test.c` — include <argp.h> (~1623 tok)
 - `test_coldstart_lsn0.c` — Declares char (~4405 tok)
 - `test_commit_barrier_rollout.c` — Declares explicitly (~19506 tok)
- `test_config_repl.c` — define _GNU_SOURCE (~34816 tok)
+- `test_config_repl.c` — define _GNU_SOURCE (~35905 tok)
 - `test_demotion_convergence.c` — define _GNU_SOURCE (~9602 tok)
 - `test_e2e_capnp.c` — include <stdint.h> (~2760 tok)
- `test_ha_election.c` — Declares char (~12140 tok)
+- `test_ha_election.c` — Declares char (~12969 tok)
 - `test_ha_prevote.c` — Declares char (~11053 tok)
 - `test_ha_vote_gate.c` — define _GNU_SOURCE (~13251 tok)
 - `test_persist_recovery.c` — define _GNU_SOURCE (~3654 tok)
 - `test_repl_apply.c` — include <dirent.h> (~5744 tok)
 - `test_repl_unit.c` — include <assert.h> (~6231 tok)
- `test_term_safety_adapter.c` — define _GNU_SOURCE (~25125 tok)
+- `test_term_safety_adapter.c` — define _GNU_SOURCE (~26786 tok)
 - `test_walcompare.c` — define _GNU_SOURCE (~3795 tok)
 - `test.c` — include <argp.h> (~27479 tok)
 - `walcompare.c` — define _GNU_SOURCE (~2485 tok)
 - `walcompare.h` — pragma once (~839 tok)
- `waldump.c` — define _GNU_SOURCE (~4586 tok)
+- `waldump.c` — define _GNU_SOURCE (~4758 tok)
 - `waldump.h` — pragma once (~1006 tok)
 ## src/controller/testsuite/clean_failover_preservation.sh (~330 lines, ~3k tokens)
@ -806,14 +827,19 @@
 ## src/include/
 - `netio.h` — if !defined(_NETIO_H_) (~1293 tok)
 - `persist.h` — pragma once (~9539 tok)
 - `repl.h` — pragma once (~2163 tok)
- `typemap.h` — if !defined(_TYPEMAP_H_) (~4480 tok)
+- `typemap.h` — if !defined(_TYPEMAP_H_) (~4632 tok)
 - `wal_types.h` — pragma once (~1400 tok)
 ## src/lib/
- `request.c` — include <errno.h> (~15710 tok)
+- `client.c` — include <errno.h> (~6440 tok)
 - `hook.c` — include <errno.h> (~1197 tok)
 - `netio.c` — include <arpa/inet.h> (~6688 tok)
 - `request.c` — include <errno.h> (~15862 tok)
 - `xcectx.c` — include <errno.h> (~3042 tok)
 ## src/lib/raft/
@ -824,8 +850,10 @@
 ## src/lib/testsuite/
 - `test_config.c` — include <stdio.h> (~490 tok)
 - `test_hooks.c` — include <errno.h> (~1416 tok)
 - `test_raft.c` — include <errno.h> (~68779 tok)
 - `test_wal_wire.c` — include <errno.h> (~2970 tok)
 ## src/schema/
- `controller.capnp` — Declares OPCode (~8927 tok)
+- `controller.capnp` — Declares OPCode (~9156 tok)
--- a/.wolf/buglog.json
+++ b/.wolf/buglog.json
--- a/.wolf/cerebrum.md
+++ b/.wolf/cerebrum.md
@ -9,6 +9,8 @@
 <!-- How the user likes things done. Code style, tools, patterns, communication. -->
 ## Key Learnings
 - (2026-06-23, bug-1028 FIXED) There MUST be exactly ONE authoritative source for a WAL entry's Raft term. The leader's library send path (`ha_read_wal_entry_at`, ha.c:1472-1476) and the standby's `logOk`/`on_get_entry_term` both source terms from `persist_entry_term` (in-memory index, legacy-0→WAL_LEGACY_TERM). The heartbeat catch-up streamer (`ha_stream_wal_to_standby`, config_repl.c) — the third consumer — was sourcing terms from raw on-disk `hdr.term` (0 for legacy entries). A standby's `prev_log_term=1` (normalised) ≠ streamer's `prev_log_term=0` (raw) → `logOk` rejects the new-term NOOP forever → `applied_lsn` stuck → primary re-streams every heartbeat cycle. Fix: all three `hdr.term` reads in the streamer replaced with `persist_entry_term(&persist, hdr.lsn)`; function gained a `const persist_t*` parameter. This is the pattern: every source of the per-LSN (term,index) tuple MUST go through `persist_entry_term`; NEVER read `hdr.term` raw from disk — it is not normalised.
 - (2026-06-23, bug-854 FIXED) The `XCECLI_FAILOVER_CALL` retry loop classifies a status-0 reply from ANY reachable node as SUCCESS and returns — it only rotates on failure. But ctrlstatus is answerable in every role, so a standby always answers 0, making `xcecli_ctrlstatus` pin to the first reachable node forever. The outer `xcecli_ctrlstatus_primary` loop captures a redirect hint from the non-authoritative reply but never forces rotation to the next master — so the configured master list is never traversed. Fix: `xcecli_ctrlstatus_primary` now calls `xce_failover_rotate(ctx)` + `xcecli_ctx_recover(ctx)` under `failover.lock` on every non-authoritative reply; the floor of `max_attempts` was raised to `nmasters+1` to cover the full ring. General pattern: any outer retry loop that queries a role-agnostic status must **force endpoint rotation** on a non-authoritative reply — do not rely on the inner failover wrapper, because status-0 preempts it.
 - (2026-06-22, ha-term-safety task 3.1) The single primary-append seam is `ha_primary_log_append(cluster_t*,type,payload,len,&out_lsn)` (ha.c, decl ha.h ~333): packs the `raft_entry_blob_t` envelope and calls `raft_log_append(&c->ha.raft,...)` so the LIBRARY stamps `s->term` + assigns LSN + fires `on_persist_entry`. CRITICAL: `raft_log_append` returns `-EPERM` unless `s->role==RAFT_LEADER` — and `ha_do_promote`/the non-raft `ha_promote_to_primary` road sets `ha->role=CTRL_PRIMARY` but NOT `ha->raft.role` (only the raft election road via on_become_leader sets RAFT_LEADER). So `ha_primary_log_append` MUST have a `-EPERM`→legacy `persist_wal_async` fallback (stamping `ha->term` via persist_set_term first) or init-time/non-raft/engine-less promotion breaks. Retired the hand-rolled bypass by routing persist.c `wal_hook_append_classified` (RETAIN job/queue/worker hooks) + `persist_wal_sync_append` (WAL_JOB_SUBMIT WAIT) through `ha_primary_log_append` GUARDED on `p->cluster != NULL` — standalone persist (lib-level tests, `persist_init(p,&cfg,NULL)`) keeps the direct persist_wal_async path. `p->cluster` is the back-ref (persist.h, set in persist_init); ha.h is reachable in persist.c via `xce/cluster_def.h`. Co-tracking invariant `p->current_lsn == s->log_len + 1` holds because raft_init (2.5) seeds log_len from durable floor and every primary append advances both by 1. To make a controller TEST drive raft_log_append: `ha_init_raft` then set `c->ha.raft.role=RAFT_LEADER` + `c->ha.raft.term=T` directly (the raft-sim convention). on_persist_entry calls persist_wal_async (NOT the hooks) so no recursion. `ha_arm_own_term_gate` is static → test seam `ha_arm_own_term_gate_for_test`. Validation: pre-existing `test_wal_catchup_*` (7, bug-924 overlayfs) fail IDENTICALLY on baseline (git stash + rebuild proves it); every other group 0 fail.
 - (2026-06-20, waldump tooling) The `xce_test` binary (`src/controller/meson.build`) builds from `testsuite/test.c` ONLY + `libxce_dep` — it does NOT link `persist.c`/`job.c` (those are in the `controller_unit_test` target). The investigation doc's claim that xce_test "links persist.c/job.c" was WRONG. BUT capnp `read_Job` is generated into `controller.capnp.c` which IS compiled into `libxce`, so it's reachable transitively. To USE capnp/zlib symbols from test.c you must add `c_capnproto_dep` + `zlib_dep` to `xce_test_deps` (static-lib deps don't auto-propagate through `declare_dependency(link_with: libxce)`). For a zero-alloc jobid decode read `struct Job.jobid` directly after `read_Job` — skip the allocating `decode_Job`. WAL CRC reader template = `persist_test_durable_lsn_on_disk` (persist.c ~888): seg files `seg-%016llx.wal` under `<node>/data/wal`, 40B `wal_entry_hdr_t`, crc32 over `WAL_HDR_CRC_SIZE`(=32) header bytes + payload, prev_crc chain, all-zero hdr = clean EOF (pre-alloc). Offline `waldump` lives in `testsuite/waldump.{c,h}` + `cmd_waldump` in test.c top_commands.
 - (2026-06-18, ha-commit-durability task 5.2 VALIDATION) Deterministic E2E on the FIXED build = GREEN, stable 3/3 runs (9/9 probes PASS), in-window primary fdatasync=2 on EVERY probe (vs 0/RED on the unfixed 1.3 build) — the RED→GREEN flip is the deterministic acceptance and is confirmed. Run via `nix develop ./src --command bash -c 'bash src/tools/test-commit-durability.sh --deterministic <deploy>'` background+poll worktree log; cold election ~15s, 3 probes, exit 0. The ROLLING SIGKILL SOAK is host-verification-needed in-sandbox: rounds 1-2 PASS (jobs survive failover), then round 3 onward "no new primary elected within 40s" and every "DATA LOSS" report is a query against a cluster with NO live primary (same 5 jobs frozen, never re-acked). ROOT CAUSE of the collapse is the durability fix's OWN promotion fence working AS DESIGNED: ctrl-1 won PreVote for term=5, computed safe_committed_lsn=13, then `persist_recover_fatal_if_unrecoverable` FATAL-terminated ("recovery-truncated log ... no healthy peer reachable ... durable_floor_lsn=13 < hwm=13 ... terminating rather than serving divergent state, Req 2.4") — refusing to let a structurally-truncated node lead and overwrite the good timeline (exactly the bug-661 failure mode the fix prevents). Under aggressive rolling-kill+immediate-restart the fence repeatedly self-terminates recovering nodes → cluster drops below quorum → liveness collapse, NOT durable loss. PROOF it's not loss: committed entries still on disk after the run (ctrl-3 WAL has seg-1 + seg-15; ctrl-2 seg-1) and 0 live controllers at end. So the soak's exit=1 / "DATA LOSS" verdict is a HARNESS ARTIFACT (it equates empty-joblist-with-no-primary to loss) — the soak harness conflates election-stall liveness failure with durability loss. The deterministic flip is the must-have and passed; the live multi-round soak needs a real host (slower election timing, no sandbox reaping) to distinguish fence-driven safety self-termination from a true regression. Did NOT patch production (5.2 is validation-only).
@ -88,6 +90,8 @@
 - (2026-06-22, ha-term-safety task 3.3 remediation) Do NOT run `clang-format -i` on whole XCE controller source files (ha.c/ha.h/test files): these files are NOT clang-format-clean against the repo's config, so a full reformat rewrites ~1700 lines and BURIES the targeted change (and violates a "preserve the existing diff" instruction). I then made it worse with `git checkout HEAD -- <files>` to undo the format — but the uncommitted task-3.3 work lived ONLY in the working tree, so that wiped it (no stash, unrecoverable from git). Recovery worked only because the full original content was already in my context. Lesson: (1) only format the exact changed regions or skip formatting; (2) NEVER `git checkout HEAD -- <file>` to discard formatting noise when the file also holds uncommitted feature work — instead revert just the format hunks, or stash first. The reflog showed branch HEAD was already at task-3.2 (754e586), and the initial-context "recent commit 2c5d298" was a stale snapshot — the uncommitted 3.3 diff sat on top of 3.2.
 - (2026-06-22, ACK peer-index, bug-960) The walAck branch of `repl_ack_decode_and_dispatch` (ha.c) fires for TWO endpoint types whose stored `peer_idx` means DIFFERENT things: promoted-vote eps (`vote_ep_ctx_t`, `ctx!=NULL`) carry a self-EXCLUDED *votes* index (relative); heartbeat standby eps (`standby_ep_ctx_t`, `ctx==NULL`) carry an ALREADY-ABSOLUTE candidate index. The authoritative branch is ha.c:6187-6189 (`sctx->ctx != NULL ? votes_idx : i`). The pre-existing ACK consumer `ha_walack_on_ack(c, vctx->peer_idx, …)` uses `peer_idx` DIRECTLY (no remap) — proof the ACK path's index is already absolute. Any code consuming the ACK `peer_idx` for the library (match_index[]/next_index[], which raft.c writes UNCHECKED) MUST branch on `vctx->ctx != NULL` before remapping, or it double-maps the heartbeat path → wrong-peer false quorum / OOB write. `self_idx == ha->my_priority` (raft_cfg.self_idx = my_priority everywhere).
 - (2026-06-24, bug-1060 — RECURRENCE of the 2026-06-22 entry above) Ran `clang-format -i` on whole ha.c/ha.h/test_ha_election.c AGAIN → spurious whole-file reformat (3268/969/635 lines) that buried a ~180-line fix. Confirmed spurious by checking the repo's HEAD ha.c is NOT idempotent under the nix `clang-format` (a `.clang-format` exists at src/.clang-format but the pinned formatter version differs from whatever produced HEAD). Recovery THIS time succeeded because: (a) I had already printed the full pre-format `git diff ha.c` (5 hunks: bug-1028/1042/1043 pre-existing + my bug-1060) into context; (b) ha.h's only genuine new symbol vs HEAD was `ha_sync_raft_log_from_recovery` (found via `comm` on `grep -oE '\bha_[a-z_]+\b'` of HEAD-vs-current — everything else was pointer-style/comment reflow); (c) test_ha_election.c was NOT in the session-start `M` list so it held only my changes. Then `git checkout HEAD -- <3 files>` + re-applied the logical hunks by Edit. LESSON STANDS: do NOT clang-format whole files here; format only changed regions or skip. The `git checkout HEAD --` rescue is ONLY safe after you have positively accounted for EVERY uncommitted change in those files (capture the pre-format `git diff` FIRST, and confirm which files had pre-existing work via the session-start `git status`).
 <!-- Mistakes made and corrected. Each entry prevents the same mistake recurring. -->
 <!-- Format: [YYYY-MM-DD] Description of what went wrong and what to do instead. -->
@ -245,6 +249,17 @@ Pure library core (no I/O/threads) + controller callback adapters keeps it simul
 - FEATURE GATE: raft_commit_index==0 means INERT (library source not yet wired by tasks 3.x) → barrier keeps autonomous durable-majority behavior. This is what stops 2.4 from regressing the whole test_commit_engine suite (which sets no raft index). The ceiling only engages at commit_index>=1.
 - Unit-test runner: `./xce.build/controller/controller_unit_test -t <group> [-t <group2>]` runs only named groups (argp `--run`/`-t`). Far faster than the full binary which times out (bug-924). Full clean build still `nix develop --command bash tools/xce-build.sh controller`.
 ## Key Learning (2026-06-24, bug-1060) — a new leader MUST NEVER truncate its own log (Raft §5.4 Leader Completeness)
 - The post-election `safe_committed_lsn` (median of granted-voter LSNs) is a valid commit LOWER-BOUND and the promotion-gate HWM, but it is NOT a safe WAL-truncation cutoff: it can sit BELOW entries the new leader itself holds durably. Those entries were acked by this node as a standby ⇒ committed by the prior leader. `raft_cb_become_leader` calling `persist_truncate_wal_above(safe_lsn)` destroyed them (granted `{self=47, peer=44}` → median 44 cut the leader's own committed 45..47), and `persist_truncate_wal_above` ALSO sets `committed_lsn=cutoff` (a commitIndex DECREASE) + clamps durable down. The leadership NOP then landed at LSN 45 (the rewound tip) under term+1, diverging vs peers still holding the old job at 45 → contradictory freshness denials (one node fresher by index, the other by term) → permanent prevote deadlock + committed-job loss.
 - FIX shape: clamp the truncation cutoff to `max(safe_lsn, persist_durable_floor)` via pure helper `ha_promotion_truncate_floor` so truncate is a no-op on the leader's durable log. Do NOT clamp `safe_lsn` itself inside `ha_compute_safe_committed_lsn` — `promotion_safe_lsn` is also the behind-node fence HWM; inflating it to self's tip would make every node trivially pass its own gate. The deadlock was a SYMPTOM; the freshness gate is correct (it refused to elect a leader that would lose committed data). Same class as bug-795/bug-828 ("new leader truncates own committed entries"); bug-828 fixed grant-RECORDING so the median reflects the majority, bug-1060 fixes the remaining case where the median is below the leader's OWN tip.
 - `persist_truncate_wal_above(cutoff)` is a NO-OP when `cutoff+1 >= current_lsn` (returns early, does NOT touch committed_lsn). Only when it actually cuts does it set committed_lsn=cutoff + clamp durable. So clamping the cutoff to >= durable tip makes it inert on a recovered leader (current_lsn == durable_tip+1).
 ## Key Learning (2026-06-24, bug-1067) — AppendEntries has TWO terms; never conflate them
 - Raft's AppendEntries carries (a) the HEADER/leader term — `term` arg of raft_on_append_entries_request, used for UpdateTerm + the stale-reject `if term < s->term → ae_reject_stale` — and (b) each ENTRY's STORED term (`entry_term` arg), used for logOk + on_persist_entry. The XCE library `raft_on_append_entries_request` already takes them as SEPARATE args; the controller adapter `ha_consume_append_entries` was setting `ae_term = entry_term` (the stored term). Correct for a LIVE append (leader appends at its current term ⇒ equal), but for CATCH-UP of a HISTORICAL entry written by a PRIOR leader, entry_term < current leader term ⇒ stale-reject of every historical entry ⇒ cross-term catch-up wedges forever (standby stuck, streamer retransmits). FIX: carry the leader's current term out-of-band per entry (`wal_catchup_entry_t.leader_current_term`, wire `WalCatchupEntry.leaderCurrentTerm @8`), stamped by `ha_stream_wal_to_standby` = ha->term, used as ae_term with fallback to entry_term when 0.
 - DO NOT try to fix this receiver-side by using `s->term` as the AE term: a genuinely STALE leader (leader term < s->term) could then push divergent entries through (the stale-check is the ONLY thing rejecting it). The leader's current term MUST travel on the wire and be checked. Test both: cross-term ACCEPT + stale-leader REJECT.
 - `wal_catchup_entry_t` is HAND-DEFINED in src/include/typemap.h, NOT generated — but `capnp compile` GENERATES encode/decode (controller.capnp.c) AGAINST it via `$C.mapname`. So adding a schema field requires editing BOTH controller.capnp (wire) AND typemap.h (the C struct) or the build fails "no member". All scalar mapname fields auto-decode (request.c xce_decode only hand-extracts the Data payload); the leader term rides on each sub-entry and reaches ha_consume via ae.entries[0]=e by reference (no extra plumbing through ha_apply_term_entry_via_library).
 - `ha_stream_wal_to_standby` has MANY callers incl. several in test_config_repl.c (2589/2630/2664/2740) and test_coldstart_lsn0.c (339/387) — grep ALL of them before changing the signature, not just the production caller.
 ## Do-Not-Repeat (2026-06-21)
 - Don't trust a green "baseline" reading blindly: the lib suite was actually red on HEAD via a stale test_config.c assertion (DEFAULT_SYNC_TIMEOUT_MS 50 vs committed 500 from commit ca1d9ae). Verify with the exact meson test command, not assumptions.
@ -257,3 +272,28 @@ Pure library core (no I/O/threads) + controller callback adapters keeps it simul
 - conf_save_config bumps myconf->config_version_tag (the advertised tag) WITHOUT decoding/activating (cluster_config unchanged). Only load_primary_config decodes into mc->cluster_config. So the activation witness in tests is the cluster_config POINTER (load_primary_config frees+reallocates it), NOT config_version_tag.
 - In the single-member commit fixture (commit_engine_init n_peers=1), majority_candidate_locked returns persist_durable_lsn => committed_lsn auto-advances to the durable WAL tip on any commit re-eval (incl. conf_save_config). To hold a flushed marker "uncommitted" in a test, SAVE first then atomic_store committed_lsn back below the marker LSN (models a standby whose committed prefix lags its local WAL tip).
 - Off-iomgr scheduling pattern: ha_convergence.c is the canonical example — delay_task_ms(&c->delaymgr, 1, 1, entry, ctx) enqueues (ordq_put only, non-blocking) and the delaymgr worker runs the body off the iomgr ACK thread. Tests drive the queued task by calling its entry fn directly (the fixture leaves delaymgr unstarted). Guard scheduling on c->delaymgr.running to avoid leaking tasks into an undrained queue in the unit-test fixture.
 ## Key Learnings (2026-06-23, /xce-test-ha HA soak — 3 failover bugs)
 - (bug-854 dead-first residual) The client `client`/`xcecli_start` initial connect was hardcoded to `endpoint_connect(masters[0])`; if masters[0] is a KILLED node (connection-refused), the whole session fails BEFORE the failover loop (which only runs on an already-connected ctx) can rotate. FIX: loop `xcecli_start` over ALL masters, re-init the endpoint between attempts (endpoint_fini + endpoint_init + endpoint_set_peer + rebind ctx.iomgr — same as xce_ctx_reconnect), set failover.cur_master to the connected index. Verified: dead-first masters list discovers primary in 46ms (was: hang). Distinct from the earlier bug-854 fix (xcecli_ctrlstatus_primary force-rotation) which only covered the all-live standby-first case.
 - (bug-1042 rejoin never converges) ha_init_raft (myapp.c, BEFORE election) sources the raft library `initial_log_len` from persist_durable_floor — but on a standby-REJOIN boot persist_recover runs LATER (cluster_init_phase2). So a restarted node's library comes up log_len=0 and its logOk rejects EVERY AppendEntries (prevLogIndex>=1 vs empty log) → permanently stuck non-converging standby; primary logs "standby[i] lsn=N < committed_lsn=M -> stream WAL batches" forever. FIX: ha_sync_raft_log_from_recovery(ha,p) after persist_recover (cluster.c), raising raft.log_len/commit_index/term to recovered values (monotonic, clamp commit<=log_len). Mirrors bug-848's applied_lsn sync. Diagnostic tell: `ae_reject_logmismatch` at prevIdx==recovered_floor on the rejoined node.
 - (bug-1043 catch-up apply gap) The term-safe library catch-up route (ha_apply_term_entry_via_library → ha_consume_append_entries → on_persist_entry → persist_wal_write_replicated) writes the WAL but NEVER calls apply_wal_entry/restore_job — so a member that catches up the missing suffix by REPLICATION holds entries on disk but NOT in the queryable job table. On promotion joblist/expect-job miss them → looks like committed-job loss but walcompare is byte-identical (it's an apply gap, not data loss). The LEGACY ha_apply_single_entry path called apply_wal_entry after persist; the term-safe route (ha-term-safety 3.4) dropped it. FIX: re-add apply_wal_entry on the accepted path in ha_apply_term_entry_via_library.
 - (HA soak harness design) /xce-test-ha convergence gate must NOT key on "standby[i] converged" log (only re-emitted for a standby that was NOT-converged under the current primary term — a standby restarted already-caught-up never logs it). Use "no NEW 'stream WAL batches' line on the primary for ~6s" == all standbys converged (HA_REPL_CONVERGED is a silent no-op). Detect new-primary election via the INFO line "HA FSM (raft): → CTRL_PRIMARY (term=N)" with term STRICTLY > pre-kill term (user guidance). Only SINGLE-failover is guaranteed — the soak MUST wait full 3-of-3 reconvergence before the next kill, else it tests an unsupported double-fault window and reports false loss.
 ## Do-Not-Repeat (2026-06-23, shell portability)
 - This deploy env's `realpath` is BUSYBOX (`/sbin/realpath`): it does NOT support `-m` (treats it as a filename, errors to stderr, and on a relative path whose parent exists still prints a resolved line to stdout → with `|| echo "$TOP"` you get MULTI-LINE garbage in the var). `readlink -f` also fails on nonexistent paths here. Don't rely on `realpath -m`/`readlink -f` for path normalization in scripts that run under nix/this host. Use pure-bash absolutization instead: `case "$P" in /*) : ;; *) P="$PWD/$P" ;; esac`. (Caught while adding the [dir] arg to run-ha-soak.sh.)
 ## Key Learnings (2026-06-23, /xce-test-ha [dir] arg)
 - run-ha-soak.sh deploy dir now resolves: positional `$1` > `$XCE_HA_TOP` env > default `/home/claude/tmp/xce-ha-deploy`. The skill maps `/xce-test-ha [rounds] [dir]` → rounds to `XCE_HA_ROUNDS` env, dir to the positional arg. Because the deploy is `rm -rf`-destructive and the dir is now caller-chosen, a sentinel `.xce-ha-deploy` marker is written at deploy time and a guard refuses to wipe a non-empty dir lacking it (fail-loud, protects a typo'd home/source path). Empty/nonexistent dir is always safe.
 ## Key Learnings (2026-06-23, bug-845 WAL double-append fixed)
 - The hook system (src/lib/hook.c) had NO dedup: xce_add_hook appended a new slot for a repeat (fn,data); xce_run_hooks fires ALL slots; xce_disable_hook NULLs only ONE. So registering the same hook twice = fires twice + survives one disable. Every WAL_JOB_SUBMIT was double-appended (byte-identical, adjacent LSN) because persist_start() (boot, unconditional) AND ha_register_persist_hooks() (PRIMARY promotion) both register persist_job_hook_fun with the same persist_t. The hooks_registered guard only stops ha_register from re-adding ITSELF — it's blind to persist_start's registration.
 - FIX = make xce_add_hook idempotent on exact (fn,data): re-adding an active (fn,data) is a no-op; same fn + DISTINCT data still creates a 2nd slot (legitimate multi-closure, preserved by test). Chosen over "remove persist_start's registration" because MANY controller unit tests (test_e2e_checkpoint, test_ha_role_lifecycle, test_commit_barrier_reply:387 "persist_start registers the production hooks") DEPEND on persist_start registering the hooks and do NOT call ha_register — removing it would break that contract.
 - ack gating: handle_jobsubmit (job.c) acks the client iff xce_run_hooks(AFTER_EVENT JOB_NEW) returns 0, and xce_run_hooks returns the LAST hook's rc. With the double-register the ack was (accidentally) gated on the 2nd append's repl_wait_majority_ack (LSN+1) — an extra majority round-trip that MASKED a latent failover durability race. Single-append (correct) removed that margin: one post-fix soak then hit the KNOWN intermittent committed-job-loss (bug-795/828 class — LSN-N cross-node divergence: killed primary had the entry, the 2 survivors did not). Confirmed pre-existing/timing (baseline also passes-or-fails on timing; 3/3 post-fix soaks then passed), NOT caused by the dedup. Validation recipe for HA soak: rebuild controller/xce_controller+xce_test, then run .claude/skills/xce-test-ha/run-ha-soak.sh; judge by waldump single-vs-double entries, not the leaky ASAN exit code.
 ## Do-Not-Repeat (2026-06-23, isolate before blaming your own change)
 - When a soak regresses right after your edit, DO NOT assume your edit caused it: the working tree here had pre-existing UNCOMMITTED HA changes (cluster.c/ha.c/persist.c/config_repl.c) that the stale prebuilt binaries (dated 15:01) had NEVER compiled. The first rebuild pulls BOTH your change AND those deltas. Isolate by `git stash push -- <your files>`, rebuild, re-run. Here it proved the WAL-dup fix was clean and the job-loss was the known intermittent durability race.
 ## Key Learnings (2026-06-23, /xce-test-ha leadership coverage — ctrl-3 never primary)
 - WHY ctrl-3 never led: HA election timeout is PRIORITY-STAGGERED. my_priority = index in --peers list (ctrl-1=0, ctrl-2=1, ctrl-3=2; lower = higher priority), and the election delay = my_priority * election_priority_step_ms (ha.h:138, default 500ms → p0 immediate / p1 +500 / p2 +1000). The soak only ever kills ONE node (single-failover), so the surviving pair always contains a lower-index node that out-races ctrl-3. Confirmed: ctrl-3 had 0 CTRL_PRIMARY announcements but DID start 6 elections (it tries, always loses the race). No leadership-transfer/stepdown command exists in xce_test or ha.c, so leadership can't be handed over directly.
 - HOW to force a specific (lowest-priority) node T to lead, DETERMINISTICALLY and loss-free (run-ha-soak.sh force_elect_node): exploit the RequestVote freshness gate (a behind node cannot win). P=current primary, S=third node. (1) kill S so it stops receiving entries; (2) submit a batch → commits on {P,T}, S now provably behind; (3) kill P → only T alive briefly (T holds ALL commits — they were on {P,T} — so nothing is lost); (4) restart the STILL-BEHIND S → quorum {T fresh, S stale}; S cannot catch up while there is no primary, so it stays freshness-fenced for the whole election → T wins deterministically (NOT by timing); (5) verify no loss, restart P, reconverge. KEY INSIGHT: a behind node can't catch up without a primary, and there's no primary during the election, so the spoiler is guaranteed fenced — this removes all timing-race flakiness.
 - The soak now tracks SEEN_PRIMARY per node and the verdict ASSERTS all 3 led (PASS line: "all 3 nodes led (ctrl-1,ctrl-2,ctrl-3)"); a node that never leads is a FAIL. force_elect_node is generic (works for any never-led node), run post-loop for each missing node after restoring 3-of-3.
--- a/.wolf/hooks/_session.json
+++ b/.wolf/hooks/_session.json
--- a/.wolf/memory.md
+++ b/.wolf/memory.md
@ -2579,3 +2579,221 @@
 | 21:47 | Edited .kiro/steering/persistence.md | inline fix | ~126 |
 | 21:48 | Session end: 294 writes across 37 files (test_term_safety_adapter.c, commit.h, commit.c, ha.c, ha.h) | 66 reads | ~607176 tok |
 | 00:04 | Session end: 294 writes across 37 files (test_term_safety_adapter.c, commit.h, commit.c, ha.c, ha.h) | 66 reads | ~607176 tok |
 ## Session: 2026-06-23 00:14
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 00:29 | Created ../../../../../claude/.claude/plans/piped-mixing-glade.md | — | ~2049 |
 | 00:34 | Edited src/controller/config_repl.h | expanded (+10 lines) | ~391 |
 | 00:34 | Edited src/controller/config_repl.h | modified struct() | ~91 |
 | 00:35 | Edited src/controller/config_repl.c | modified ha_stream_wal_to_standby() | ~64 |
 | 00:35 | Edited src/controller/config_repl.c | added error handling | ~285 |
 | 00:35 | Edited src/controller/config_repl.c | modified persist_entry_term() | ~140 |
 | 00:36 | Edited src/controller/config_repl.c | modified persist_entry_term() | ~110 |
 | 00:36 | Edited src/controller/config_repl.c | 3→3 lines | ~47 |
 | 00:36 | Edited src/controller/ha.c | modified if() | ~202 |
 | 00:37 | Edited src/controller/testsuite/test_config_repl.c | 4→4 lines | ~66 |
 | 00:37 | Edited src/controller/testsuite/test_config_repl.c | 4→4 lines | ~66 |
 | 00:37 | Edited src/controller/testsuite/test_config_repl.c | 4→4 lines | ~67 |
 | 00:38 | Edited src/controller/testsuite/test_coldstart_lsn0.c | 3→3 lines | ~50 |
 | 00:38 | Edited src/controller/testsuite/test_coldstart_lsn0.c | 3→3 lines | ~50 |
 | 00:39 | Edited src/controller/testsuite/test_config_repl.c | added error handling | ~1034 |
 | 00:40 | Edited src/controller/testsuite/test_config_repl.c | 4→5 lines | ~76 |
 | 00:44 | Edited src/controller/config_repl.c | 3→3 lines | ~48 |
 | 00:45 | Edited src/controller/config_repl.c | modified persist_entry_term() | ~49 |
 | 00:46 | Edited src/lib/client.c | added 1 condition(s) | ~188 |
 | 00:47 | Edited src/lib/client.c | added 1 condition(s) | ~487 |
 | 01:03 | Created repro854.sh | — | ~416 |
 | 01:05 | Edited repro854.sh | 3→3 lines | ~45 |
 | 01:22 | Created repro_joblist.sh | — | ~201 |
 | 08:30 | bug-1028: streamer term-source FIXED (persist_entry_term); bug-854: client rotation FIXED (xce_failover_rotate on non-auth reply); both RED→GREEN, full build 0, unit :FAIL:=0, live committed-prefix identical + wait-primary 92ms | config_repl.c/.h, ha.c, client.c, test_config_repl.c | bugs CLOSED | ~28k |
 | 01:29 | Session end: 23 writes across 9 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 16 reads | ~250791 tok |
 | 01:33 | Session end: 23 writes across 9 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 16 reads | ~250791 tok |
 | 01:44 | Edited src/lib/testsuite/test_wal_wire.c | modified heartbeat() | ~150 |
 | 01:46 | Session end: 24 writes across 10 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 18 reads | ~266661 tok |
 | 02:15 | Session end: 24 writes across 10 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 19 reads | ~275693 tok |
 | 02:44 | Edited src/controller/cluster.c | added 1 condition(s) | ~660 |
 | 02:53 | Edited src/lib/xcectx.c | 1→2 lines | ~26 |
 | 02:54 | Edited src/lib/xcectx.c | added 2 condition(s) | ~286 |
 | 02:58 | Session end: 27 writes across 12 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 22 reads | ~281425 tok |
 | 03:07 | Session end: 27 writes across 12 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 22 reads | ~281425 tok |
 | 03:25 | Edited src/lib/netio.c | modified iomgr_monitor() | ~240 |
 | 03:27 | Session end: 28 writes across 13 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 22 reads | ~282052 tok |
 | 03:31 | Created ../../../../../../tmp/xce-ha-verify/test.sh | — | ~291 |
 | 03:37 | Edited src/lib/netio.c | modified handle_ssl_handshake() | ~276 |
 | 03:40 | Edited src/controller/cluster.c | expanded (+13 lines) | ~282 |
 | 03:43 | Edited src/lib/netio.c | inline fix | ~12 |
 | 03:44 | Edited src/lib/netio.c | inline fix | ~7 |
 | 03:46 | Edited src/include/netio.h | 2→3 lines | ~60 |
 | 03:48 | Edited src/controller/cluster.c | inline fix | ~12 |
 | 03:49 | Edited src/controller/cluster.c | modified if() | ~290 |
 | 03:51 | Edited src/lib/netio.c | modified iomgr_monitor() | ~161 |
 | 03:52 | Edited src/lib/netio.c | modified handle_ssl_handshake() | ~202 |
 | 03:53 | Edited src/include/netio.h | 3→2 lines | ~37 |
 | 03:54 | Edited src/lib/netio.c | inline fix | ~6 |
 | 03:58 | Session end: 40 writes across 15 files (piped-mixing-glade.md, config_repl.h, config_repl.c, ha.c, test_config_repl.c) | 23 reads | ~290775 tok |
 ## Session: 2026-06-23 06:20
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 06:24 | Created .claude/skills/xce-test-ha/run-ha-soak.sh | — | ~2292 |
 | 06:25 | Created .claude/skills/xce-test-ha/SKILL.md | — | ~1430 |
 | 06:27 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified xtest() | ~112 |
 | 06:27 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified code() | ~263 |
 | 06:28 | Edited .claude/skills/xce-test-ha/SKILL.md | 7→12 lines | ~194 |
 | 06:30 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified discover_primary() | ~199 |
 | 06:37 | Edited src/controller/testsuite/waldump.c | 3→4 lines | ~69 |
 | 06:37 | Edited src/controller/testsuite/waldump.c | added 1 condition(s) | ~227 |
 | 06:37 | Edited src/controller/testsuite/waldump.c | 2→3 lines | ~50 |
 | 06:42 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | 7→10 lines | ~158 |
 | 06:59 | Edited src/lib/client.c | added 1 condition(s) | ~550 |
 | 07:01 | Edited src/lib/client.c | added 2 condition(s) | ~451 |
 | 07:04 | Edited .claude/skills/xce-test-ha/SKILL.md | expanded (+33 lines) | ~516 |
 | 07:08 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified wait_for_primary_log() | ~503 |
 | 07:08 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified 10() | ~770 |
 | 07:15 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified logs() | ~790 |
 | 07:15 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | expanded (+8 lines) | ~467 |
 | 07:23 | Edited src/controller/ha.c | added error handling | ~746 |
 | 07:23 | Edited src/controller/ha.h | expanded (+11 lines) | ~214 |
 | 07:24 | Edited src/controller/cluster.c | 2→7 lines | ~135 |
 | 07:25 | Edited src/controller/testsuite/test_persist_recovery.c | modified test_recovered_member_syncs_raft_log_len() | ~955 |
 | 07:39 | Edited src/controller/ha.c | 3→3 lines | ~37 |
 | 07:43 | Edited src/controller/ha.c | modified if() | ~516 |
 | 07:48 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified wait_for_full_convergence() | ~537 |
 | 14:52 | /xce-test-ha skill + 3 failover bug fixes (bug-854 dead-first, bug-1042 rejoin log_len, bug-1043 catch-up apply); 5-round soak PASS zero loss | client.c, ha.c, cluster.c, ha.h, waldump.c, skill | 3 bugs FIXED + skill created | ~60k |
 | 07:53 | Edited .claude/skills/xce-test-ha/SKILL.md | expanded (+8 lines) | ~175 |
 | 07:54 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified submit_jobs() | ~426 |
 | 08:00 | Session end: 26 writes across 8 files (run-ha-soak.sh, SKILL.md, waldump.c, client.c, ha.c) | 13 reads | ~186275 tok |
 ## Session: 2026-06-23 08:01
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 ## Session: 2026-06-23 08:15
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 08:31 | Created ../../../../../claude/.claude/plans/task-notification-task-id-bkkk2llpr-tas-zany-locket.md | — | ~366 |
 | 08:31 | Session end: 1 writes across 1 files (task-notification-task-id-bkkk2llpr-tas-zany-locket.md) | 1 reads | ~392 tok |
 | 08:47 | Session end: 1 writes across 1 files (task-notification-task-id-bkkk2llpr-tas-zany-locket.md) | 1 reads | ~392 tok |
 | 09:01 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | expanded (+13 lines) | ~379 |
 | 09:02 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | expanded (+9 lines) | ~286 |
 | 09:02 | Edited .claude/skills/xce-test-ha/SKILL.md | "End-to-end HA failover so" → "End-to-end HA failover so" | ~62 |
 | 09:02 | Edited .claude/skills/xce-test-ha/SKILL.md | expanded (+8 lines) | ~213 |
 | 09:03 | Edited .claude/skills/xce-test-ha/SKILL.md | expanded (+9 lines) | ~153 |
 | 09:04 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | 5→9 lines | ~136 |
 | 22:11 | enhance /xce-test-ha: add [dir] arg (positional $1 > XCE_HA_TOP > default) + rm-rf safety guard (.xce-ha-deploy marker, fail-loud) | run-ha-soak.sh, SKILL.md | verified: guard refuses unmarked non-empty dir; custom-dir 5-round soak PASS 18 jobs zero loss | ~3k |
 | 15:11 | Session end: 7 writes across 3 files (task-notification-task-id-bkkk2llpr-tas-zany-locket.md, run-ha-soak.sh, SKILL.md) | 3 reads | ~7892 tok |
 | 15:13 | Session end: 7 writes across 3 files (task-notification-task-id-bkkk2llpr-tas-zany-locket.md, run-ha-soak.sh, SKILL.md) | 3 reads | ~7892 tok |
 | 15:50 | Edited src/lib/testsuite/test_hooks.c | modified AND() | ~682 |
 | 15:52 | Edited src/lib/hook.c | added 4 condition(s) | ~377 |
 | 23:10 | fix bug-845 WAL double-append: xce_add_hook idempotent on (fn,data) | src/lib/hook.c, test_hooks.c | duplicates gone (waldump single-entry), 5/5 hook tests, 343 ctrl tests 0 new fails, 3/3 soaks pass; 1 earlier soak hit known intermittent bug-795/828 loss (pre-existing) | ~9k |
 | 16:10 | Session end: 9 writes across 5 files (task-notification-task-id-bkkk2llpr-tas-zany-locket.md, run-ha-soak.sh, SKILL.md, test_hooks.c, hook.c) | 10 reads | ~120437 tok |
 ## Session: 2026-06-24 10:37
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 19:58 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | modified log() | ~227 |
 | 10:58 | Session end: 1 writes across 1 files (run-ha-soak.sh) | 3 reads | ~5079 tok |
 | 19:58 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | 3→4 lines | ~44 |
 | 19:58 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | 4→5 lines | ~91 |
 | 20:00 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | added error handling | ~890 |
 | 20:00 | Edited .claude/skills/xce-test-ha/run-ha-soak.sh | expanded (+28 lines) | ~369 |
 | 20:01 | Edited .claude/skills/xce-test-ha/SKILL.md | modified nodes() | ~447 |
 | 03:04 | enhance /xce-test-ha: force lowest-priority ctrl-3 to lead (freshness-fence maneuver) + assert all 3 nodes led | run-ha-soak.sh, SKILL.md | verified: ctrl-3 elected term 8, zero loss, PASS "all 3 nodes led" | ~7k |
 | 20:21 | Session end: 6 writes across 2 files (run-ha-soak.sh, SKILL.md) | 4 reads | ~9263 tok |
 ## Session: 2026-06-24 11:28
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 11:35 | /xce-test-ha 10 /tmp/xce-test-ha: 10 rounds, 36 jobs, zero job loss, all 3 nodes led | xce-ha-soak.log | PASS (exit 0) | ~50 |
 ## Session: 2026-06-24 11:50
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 12:05 | Diagnosed prevote deadlock after soak test: ctrl-3 incorrectly truncated committed entries 45-47 (term=12) on becoming primary (safe_committed_lsn=44 from voter quorum, not honoring own ack knowledge); ctrl-2 received term=13 NOOP at LSN=45 (conflicts with ctrl-1 term=12 job); 2-node deadlock permanent — restart ctrl-3 to resolve | ctrl-*.log, WALs | ANALYZED: bug-795 class | ~1200 |
 ## Session: 2026-06-24 21:15
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 21:20 | Edited src/controller/testsuite/test_ha_election.c | added error handling | ~841 |
 | 21:20 | Edited src/controller/testsuite/test_ha_election.c | 3→6 lines | ~78 |
 | 21:21 | Edited src/controller/ha.h | expanded (+20 lines) | ~362 |
 | 21:21 | Edited src/controller/ha.c | modified ha_promotion_truncate_floor() | ~195 |
 | 21:21 | Edited src/controller/ha.c | modified 1060() | ~356 |
 | 21:30 | Edited src/controller/ha.c | expanded (+15 lines) | ~308 |
 | 21:30 | Edited src/controller/ha.c | 6→10 lines | ~188 |
 | 21:31 | Edited src/controller/ha.c | added error handling | ~738 |
 | 21:31 | Edited src/controller/ha.c | modified ha_promotion_truncate_floor() | ~195 |
 | 21:31 | Edited src/controller/ha.c | modified 1060() | ~356 |
 | 21:32 | Edited src/controller/ha.h | expanded (+11 lines) | ~235 |
 | 21:32 | Edited src/controller/ha.h | expanded (+19 lines) | ~383 |
 | 21:33 | Edited src/controller/testsuite/test_ha_election.c | added error handling | ~841 |
 | 21:33 | Edited src/controller/testsuite/test_ha_election.c | 3→6 lines | ~78 |
 | 21:36 | Edited src/controller/CLAUDE.md | inline fix | ~239 |
 | 21:37 | Edited src/controller/CLAUDE.md | inline fix | ~147 |
 | 21:37 | Edited src/controller/CLAUDE.md | inline fix | ~63 |
 ## Session: 2026-06-24 12:38
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 18:05 | fix bug-1060: new leader truncated its own committed entries (Raft Leader Completeness violation) → divergent WALs + prevote deadlock + job loss | src/controller/ha.c, ha.h, testsuite/test_ha_election.c | added ha_promotion_truncate_floor=max(safe_lsn,durable_tip); raft_cb_become_leader truncates at clamped floor (no-op on own log); unit test GREEN, 0 failures | ~60k |
 | 18:06 | steering + bookkeeping for bug-1060 | src/controller/CLAUDE.md, .wolf/buglog.json, .wolf/cerebrum.md | updated Leader Completeness/Post-Election/checklist; bug-1060 marked FIXED; Key Learning + Do-Not-Repeat (clang-format whole-file recurrence) | ~8k |
 | 21:39 | Created ../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1060_leader_completeness.md | — | ~553 |
 | 21:39 | Session end: 1 writes across 1 files (project_bug1060_leader_completeness.md) | 0 reads | ~593 tok |
 ## Session: 2026-06-24 14:50
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 ## Session: 2026-06-24 15:03
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 ## Session: 2026-06-24 15:36
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 01:17 | Edited src/controller/testsuite/test_term_safety_adapter.c | modified test_intake_crossterm_catchup_uses_leader_current_term() | ~1730 |
 | 01:17 | Edited src/controller/testsuite/test_term_safety_adapter.c | 2→4 lines | ~68 |
 | 01:17 | Edited src/schema/controller.capnp | added error handling | ~260 |
 | 01:17 | Edited src/lib/request.c | added error handling | ~139 |
 | 01:18 | Edited src/lib/request.c | 4→7 lines | ~112 |
 | 01:18 | Edited src/controller/config_repl.c | modified ha_stream_wal_to_standby() | ~70 |
 | 01:18 | Edited src/controller/config_repl.c | added error handling | ~190 |
 | 01:18 | Edited src/controller/config_repl.c | modified if() | ~121 |
 | 01:18 | Edited src/controller/ha.c | modified if() | ~292 |
 | 01:19 | Edited src/controller/config_repl.h | added error handling | ~166 |
 | 01:19 | Edited src/controller/testsuite/test_config_repl.c | 3→5 lines | ~95 |
 | 01:20 | Edited src/controller/ha.c | added error handling | ~256 |
 | 01:21 | Edited src/include/typemap.h | added error handling | ~222 |
 | 01:22 | Edited src/controller/testsuite/test_config_repl.c | 3→4 lines | ~63 |
 | 01:22 | Edited src/controller/testsuite/test_config_repl.c | 3→4 lines | ~62 |
 | 01:22 | Edited src/controller/testsuite/test_config_repl.c | 3→4 lines | ~67 |
 | 01:23 | Edited src/controller/testsuite/test_coldstart_lsn0.c | 2→3 lines | ~48 |
 | 01:32 | Edited src/controller/CLAUDE.md | added error handling | ~136 |
 | 20:10 | fix bug-1067: cross-term WAL catch-up stalled — AE header term conflated with entry's stored term → ae_reject_stale on every historical entry | src/schema/controller.capnp, src/include/typemap.h, src/lib/request.c, src/controller/config_repl.c, ha.c, config_repl.h | added leader_current_term field (wire @8 + C struct); streamer stamps ha->term; receiver uses it as AE header term w/ fallback. 2 new tests GREEN (cross-term accept + stale-leader reject); 185+80+6 PASS, 0 fail | ~95k |
 | 20:11 | bookkeeping for bug-1067 | .wolf/buglog.json, .wolf/cerebrum.md, src/controller/CLAUDE.md | bug-1067 FIXED; Key Learning (AE two-term distinction; capnp struct hand-defined in typemap.h); steering checklist item added | ~6k |
 | 01:33 | Created ../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1067_ae_two_terms.md | — | ~743 |
 | 01:34 | Session end: 19 writes across 11 files (test_term_safety_adapter.c, controller.capnp, request.c, config_repl.c, ha.c) | 13 reads | ~244235 tok |
 ## Session: 2026-06-24 16:35
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
 | 16:45 | xce-test-ha soak regression: 10 rounds, 36 jobs, zero loss, all 3 nodes led; bug-1060 (leader-completeness truncation fix) + bug-1067 (cross-term AE header fix) both GREEN | src/controller/ha.c, config_repl.c, request.c, typemap.h | PASS | ~50 |
 ## Session: 2026-06-24 02:00
 | Time | Action | File(s) | Outcome | ~Tokens |
 |------|--------|---------|---------|--------|
--- a/.wolf/token-ledger.json
+++ b/.wolf/token-ledger.json
--- a/src/controller/CLAUDE.md
+++ b/src/controller/CLAUDE.md
@ -70,7 +70,7 @@ The file `raft/raft.tla` contains Diego Ongaro's TLA+ specification of the Raft
 **Election Safety** (≤1 leader per term): Enforced by `votedFor` — a server grants at most one vote per term (`raft_on_vote_request` line 145-151). A candidate needs majority votes; each server votes at most once per term.
-**Leader Completeness** (committed entries survive): Freshness gate in `RequestVote` — candidate must have `lastLogTerm ≥ voter` AND `lastLogIndex ≥ voter`. Since committed entries are on a majority, the new leader must have all committed entries. `persist_truncate_wal_above(safe_committed_lsn)` removes any divergent entries above the safe commit point.
+**Leader Completeness** (committed entries survive): Freshness gate in `RequestVote` — candidate must have `lastLogTerm ≥ voter` AND `lastLogIndex ≥ voter`. Since committed entries are on a majority, the new leader must have all committed entries. The post-election truncation removes only a divergent tail **above the leader's own durable log**: the cutoff is `ha_promotion_truncate_floor(safe_committed_lsn, durable_tip) = max(safe_committed_lsn, persist_durable_floor)`, so `persist_truncate_wal_above` is a no-op on the leader's own durable entries. **A new leader MUST NEVER truncate its own log** — entries it holds durably were acked by it (as a standby) and so were committed by a prior leader; truncating to the bare majority element destroys them (bug-1060: granted `{self=47, peer=44}` → `safe=44` cut the leader's own committed 45..47 → divergent WALs + permanent prevote deadlock).
 **State Machine Safety** (no divergent entries): Monotonic `committed_lsn` + truncation on promotion. Entries at or below `committed_lsn` are identical across all servers.
@ -88,7 +88,7 @@ After winning election, the new primary determines `safe_committed_lsn` from vot
 safe_committed_lsn = max(lsn : count of voters with peerLastLogLsn >= lsn) + 1 >= majority)
 ```
-The `RequestVoteReply` message already carries `peerConfigTag` and `peerLastLogLsn` from each voter. The winning candidate uses these (from `ha_elect_ctx_t.votes[]`) to compute `safe_committed_lsn` immediately in `raft_cb_become_leader`, then calls `persist_truncate_wal_above(safe_committed_lsn)`.
+The `RequestVoteReply` message already carries `peerConfigTag` and `peerLastLogLsn` from each voter. The winning candidate uses these (from `ha_elect_ctx_t.votes[]`) to compute `safe_committed_lsn` immediately in `raft_cb_become_leader`, then truncates at `ha_promotion_truncate_floor(safe_committed_lsn, persist_durable_floor)` — i.e. `max(safe_committed_lsn, own durable tip)`, so it never truncates the leader's own durable log (bug-1060 Leader Completeness; `safe_committed_lsn` is still used as the commit lower-bound and the promotion-gate HWM).
 **Never** use `last_known_committed_lsn` from the previous primary's ctrlhello for truncation — it may be stale (the old primary may have committed entries after the last ctrlhello was sent).
@ -100,8 +100,9 @@ When modifying HA protocol code (election, replication, heartbeat), verify:
 - [ ] `commitIndex` never decreases (monotonic)
 - [ ] A leader only commits entries from its current term (or equivalent via per-entry majority ack)
 - [ ] `safe_committed_lsn` is derived from voter replies, not from stale ctrlhello
- [ ] WAL truncation on promotion only removes entries above the safe commit point
+- [ ] WAL truncation on promotion never falls below the leader's OWN durable tip — cutoff = `max(safe_committed_lsn, persist_durable_floor)` (`ha_promotion_truncate_floor`); a new leader never truncates its own committed log (bug-1060)
 - [ ] Freshness gate in RequestVote: candidate must have `lastLogTerm >= voter` AND `lastLogIndex >= voter`
 - [ ] AppendEntries carries TWO distinct terms: the HEADER term = leader's CURRENT term (UpdateTerm/stale-check) and each ENTRY's STORED term (logOk/persist). For catch-up of historical entries they DIFFER — the AE header term must be the leader's current term (`wal_catchup_entry_t.leader_current_term`), never the entry's stored term, or the library stale-rejects every cross-term entry (bug-1067)
 - [ ] `committed_lsn` advances only as a contiguous LSN-ordered prefix — no out-of-order or per-entry commit
 - [ ] Durability flushing makes a contiguous prefix durable; `durable_lsn` is a prefix watermark (never a per-entry set)
 - [ ] Causal dependencies are appended in order (cause before effect, e.g. submit before start)
--- a/src/controller/cluster.c
+++ b/src/controller/cluster.c
@ -1372,6 +1372,11 @@ int cluster_init_phase2(cluster_t* c)
     * receiver overwrites already-consistent records instead of fetching only
     * the missing suffix (Raft incremental catch-up). */
    ha_sync_applied_lsn_from_recovery(&c->ha, &c->persist);
    /* bug-1042: also raise the raft LIBRARY's log_len/commit_index to the
     * recovered position. ha_init_raft ran BEFORE this recovery on the standby-
     * rejoin boot, so without this the library stays at log_len=0 and rejects
     * every AppendEntries (logOk fail) — wedging the rejoined member forever. */
    ha_sync_raft_log_from_recovery(&c->ha, &c->persist);
    INFO("HA: persist_recover done — recovered durable_floor=%" PRIu64
         " current_lsn=%" PRIu64 "; applied_lsn=%" PRIu64
         " advertised for catch-up",
--- a/src/controller/config_repl.c
+++ b/src/controller/config_repl.c
@ -2014,6 +2014,7 @@ static int send_wal_batch(endpoint_t *ep,
 }
 int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path,
                              const persist_t *p, uint64_t leader_term,
                              uint64_t standby_lsn, uint64_t committed_lsn)
 {
    if (!wal_path)
@ -2102,9 +2103,11 @@ int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path,
                        batch_bytes = 0;
                    }
                    /* Advance the prev tracker even on the OOM-skip so a later
-                     * entry's prev tuple stays correct. */
+                     * entry's prev tuple stays correct. bug-1028: term from the
                     * authoritative index, not raw hdr.term (see main tracker). */
                    prev_streamed_lsn  = hdr.lsn;
-                    prev_streamed_term = hdr.term;
+                    prev_streamed_term = p ? persist_entry_term(p, hdr.lsn)
                                           : hdr.term;
                    off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
                    continue;
                }
@ -2119,9 +2122,18 @@ int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path,
                 * the (lsn,term) of the immediately preceding WAL entry (0/0
                 * before the first), which IS the predecessor in contiguous LSN
                 * order. */
-                entry->term           = hdr.term;
+                /* bug-1028: own term from the authoritative per-LSN index, not
                 * raw hdr.term. prev_streamed_term is already normalised above. */
                entry->term           = p ? persist_entry_term(p, hdr.lsn)
                                          : hdr.term;
                entry->prev_log_index = prev_streamed_lsn;
                entry->prev_log_term  = prev_streamed_term;
                /* bug-1067: stamp the leader's CURRENT term as the AppendEntries
                 * header term. entry->term above is the entry's STORED term (may
                 * be a PRIOR leader's term for a historical catch-up entry); the
                 * standby intake must check the leader's current term, not the
                 * stored term, or it stale-rejects every cross-term entry. */
                entry->leader_current_term = leader_term;
                if (hdr.payload_len > 0) {
                    uint8_t *pbuf = malloc(hdr.payload_len);
@ -2157,9 +2169,16 @@ int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path,
             * at/below standby_lsn that are not sent — so the first streamed
             * entry's prev tuple references the entry the standby already holds
             * (prev_log_index == standby_lsn). (Skips on the OOM path above set
-             * it inline before continuing.) */
+             * it inline before continuing.)
             *
             * bug-1028: source the term from the AUTHORITATIVE per-LSN term index
             * (persist_entry_term), NOT the raw on-disk hdr.term. The standby's
             * logOk and the library send path both normalise via persist_entry_term
             * (legacy-0 → WAL_LEGACY_TERM); sending raw hdr.term (0 for a legacy
             * entry) makes prev_log_term diverge from the standby's term → logOk
             * rejects the catch-up forever. p may be NULL only in count-only tests. */
            prev_streamed_lsn  = hdr.lsn;
-            prev_streamed_term = hdr.term;
+            prev_streamed_term = p ? persist_entry_term(p, hdr.lsn) : hdr.term;
            off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
        }
        close(fd);
@ -2240,9 +2259,14 @@ static void catchup_stream_range(cluster_t *c, repl_ep_t ep,
        return;
    }
 #endif
-    if (c && c->persist.wal_path[0])
+    if (c && c->persist.wal_path[0]) {
        /* bug-1067: pass the leader's CURRENT term so historical catch-up entries
         * carry the AE header term, not their (lower) stored term. */
        uint64_t leader_term =
            atomic_load_explicit(&c->ha.term, memory_order_acquire);
        ha_stream_wal_to_standby((endpoint_t *)ep, c->persist.wal_path,
-                                 standby_lsn, target);
+                                 &c->persist, leader_term, standby_lsn, target);
    }
 }
 /* ha_catchup_continuation_ctx_t — heap context for the one-shot delaymgr task.
--- a/src/controller/config_repl.h
+++ b/src/controller/config_repl.h
@ -11,6 +11,12 @@
 #include "request.h"
 #include "wal_types.h"   /* WAL_CONFIG_MARKER, WAL_CONFIG_MARKER_PAYLOAD_LEN */
 /* Forward decl: ha_stream_wal_to_standby takes a persist handle by pointer only
 * (for the authoritative per-LSN term index via persist_entry_term). Avoids a
 * heavy persist.h include in this header. */
 struct persist_s;
 typedef struct persist_s persist_t;
 /* Max config blob size — reject before allocating */
 #define CONF_MAX_SIZE (1 * 1024 * 1024)   /* 1 MiB */
@ -362,16 +368,31 @@ int handle_upgrade_config(cluster_t *c, endpoint_t *ep, xce_request_t *req);
 * Parameters:
 *   ep           — outbound endpoint to the standby
 *   wal_path     — directory containing WAL segment files (seg-*.wal)
 *   p            — persist handle for the AUTHORITATIVE per-LSN term index
 *                  (persist_entry_term). Each streamed entry's term and the
 *                  AppendEntries prev_log_term MUST come from this index, NOT
 *                  the raw on-disk hdr.term — the standby's logOk and the
 *                  library send path both use persist_entry_term, which maps a
 *                  legacy-0 entry to WAL_LEGACY_TERM. Sourcing raw hdr.term (0)
 *                  for a legacy entry diverges from the standby's normalized
 *                  term (1) → logOk rejects forever (bug-1028). May be NULL only
 *                  in count-only unit tests that do not assert terms.
 *   standby_lsn  — standby's last applied LSN (stream starts at standby_lsn+1)
 *   committed_lsn— primary's committed LSN (stream ends at committed_lsn inclusive)
 *
 * Returns 0 on success; negative on internal error (e.g., NULL ep).
- * Requirements: 5.1, 5.2, 5.4, 5.5 (ha-repl-hardening spec)
+ * Requirements: 5.1, 5.2, 5.4, 5.5 (ha-repl-hardening spec); bug-1028.
 * Boundary: WAL Streaming (config_repl.c)
 */
 #define WAL_STREAM_BATCH_ENTRIES 256
 #define WAL_STREAM_BATCH_BYTES   65536
 /* bug-1067: leader_term is the streaming leader's CURRENT term — stamped onto
 * each streamed entry's leader_current_term so the standby intake uses it as the
 * AppendEntries header term (UpdateTerm/stale-check), distinct from each entry's
 * stored term (logOk/persist). Historical catch-up entries have a stored term
 * BELOW leader_term; using the stored term as the AE term stale-rejected them. */
 int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path,
                              const persist_t *p, uint64_t leader_term,
                              uint64_t standby_lsn, uint64_t committed_lsn);
 #define SNAPSHOT_CHUNK_SIZE 65536
--- a/src/controller/ha.c
+++ b/src/controller/ha.c
@ -568,6 +568,18 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
         " (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
 }
 /* ── bug-1060: never truncate the new leader's own durable log ───────────────
 *
 * The post-election WAL truncation cutoff must never fall below the leader's own
 * durable tip, or it destroys entries committed by a prior leader (which this
 * node had acked as a standby, and so are committed) — a Raft §5.4 Leader
 * Completeness violation that diverges WALs and deadlocks the next election.
 * See ha.h for the full rationale. */
 uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
 {
    return safe_lsn > durable_tip ? safe_lsn : durable_tip;
 }
 /* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
 *
 * ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6).  Build the Boot
@ -939,9 +951,22 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
            promotion_safe_lsn = safe_lsn;
            /* Store results for use in WAL truncation and post-election commit.
             * safe_lsn is the highest LSN on a majority; entries above it
-             * may not have been committed on all nodes and must be truncated. */
+             * may not have been committed on all nodes and must be truncated.
             *
             * bug-1060 (Leader Completeness, Raft §5.4): the truncation cutoff is
             * clamped to the leader's OWN durable tip so it never destroys entries
             * this node holds durably — those were acked by this node as a standby
             * and so were committed by the prior leader.  Truncating to the bare
             * majority element (which can sit BELOW the leader's durable tip when
             * a slower peer is in the granting majority) diverged WALs and dead-
             * locked the next election.  With the clamp, persist_truncate_wal_above
             * is a no-op on the leader's durable log; safe_lsn only governs removal
             * of a non-durable divergent tail strictly above the durable tip. */
            if (c) {
-                persist_truncate_wal_above(&c->persist, safe_lsn);
+                uint64_t durable_tip = persist_durable_floor(&c->persist);
                uint64_t trunc_floor =
                    ha_promotion_truncate_floor(safe_lsn, durable_tip);
                persist_truncate_wal_above(&c->persist, trunc_floor);
            }
            /* Task 4.2 (ha-repl-hardening): set committed_lsn = safe_lsn.
             * persist_truncate_wal_above already sets c->persist.committed_lsn
@ -2128,6 +2153,21 @@ static int ha_apply_term_entry_via_library(ha_state_t *ha, persist_t *p,
         * e->lsn + 2 and SKIPS the real next-needed entry e->lsn + 1, wedging
         * logOk and stalling commit. */
        (void)before_term;
        /* bug-1043: the library route (on_persist_entry → persist_wal_write_
         * replicated) writes the entry to the WAL but does NOT apply it to the
         * in-memory entity state (job/queue/worker tables). The legacy direct-
         * apply path (ha_apply_single_entry) called apply_wal_entry after the
         * persist; the term-safe library route dropped it. So a standby that
         * caught up entries 8..N via replication held them on disk but NOT in the
         * queryable job table — on promotion `joblist`/`expect-job` could not
         * find a committed job (it IS on disk; walcompare passes), looking like a
         * committed-job loss. Re-apply here, mirroring the legacy path. Idempotent
         * on a re-stream (restore_job upserts by jobid). WAL_NOOP / WAL_CONFIG_
         * MARKER carry no entity payload — apply_wal_entry no-ops on them. */
        apply_wal_entry(p, (uint32_t)e->entry_type, e->lsn,
                        e->raw_payload, (uint32_t)e->raw_payload_len);
        ha_advance_applied_lsn(ha, e->lsn);
        ha_send_wal_ack_reply(ha, ep, c, e->lsn);
        return 1;
@ -2323,9 +2363,16 @@ int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry,
        const void *payload = e->raw_payload;
        uint32_t plen = (uint32_t)e->raw_payload_len;
        entry_term = e->term;
-        /* ae_term for UpdateTerm is the entry's term (the leader appends at its
+        /* bug-1067: the AppendEntries HEADER term (UpdateTerm/stale-check) is the
-         * current term, so entry_term == leader term for a live append). */
+         * leader's CURRENT term, NOT the entry's stored term.  For a LIVE append
-        ae_term = entry_term;
+         * the two are equal (the leader appends at its current term), but for a
         * CATCH-UP entry written by a PRIOR leader the stored term is LOWER than
         * the current leader's term — using it as the AE term made the library
         * stale-reject (term < s->term) every historical entry, wedging cross-term
         * catch-up forever.  The streamer stamps leader_current_term = ha->term;
         * fall back to the stored term only when it is unset (legacy sender). */
        ae_term = (e->leader_current_term != 0) ? e->leader_current_term
                                                : entry_term;
        blob_len = sizeof(raft_entry_blob_t) + plen;
        uint8_t *buf = (blob_len <= sizeof(stackbuf)) ? stackbuf
@ -2680,10 +2727,19 @@ int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
        if (c && c->persist.wal_path[0]) {
            extern int ha_stream_wal_to_standby(endpoint_t *ep,
                                                const char *wal_path,
                                                const persist_t *p,
                                                uint64_t leader_term,
                                                uint64_t standby_lsn,
                                                uint64_t committed_lsn);
-            ha_stream_wal_to_standby(ep, c->persist.wal_path,
+            /* bug-1028: pass &c->persist so streamed entry terms come from the
-                                     standby_lsn, committed_lsn);
+             * authoritative per-LSN index (persist_entry_term), matching the
             * standby's logOk + the library send path.
             * bug-1067: pass the leader's CURRENT term so historical entries
             * carry the AE header term (not their lower stored term). */
            uint64_t leader_term =
                atomic_load_explicit(&c->ha.term, memory_order_acquire);
            ha_stream_wal_to_standby(ep, c->persist.wal_path, &c->persist,
                                     leader_term, standby_lsn, committed_lsn);
        }
        break;
@ -3521,6 +3577,50 @@ void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p)
                              memory_order_release);
 }
 /* bug-1042: re-sync the raft library's log_len / commit_index / term from the
 * recovered WAL AFTER persist_recover.
 *
 * Init-ordering hazard: ha_init_raft (myapp.c, before the election) sources
 * raft.initial_log_len from persist_durable_floor — but on a STANDBY-REJOIN boot
 * persist_recover has not run yet at that point, so the library comes up with
 * log_len=0. After cluster_init_phase2 runs persist_recover (recovering e.g.
 * durable_floor=7), the library still thinks its log is EMPTY. Every incoming
 * AppendEntries then fails logOk (prevLogIndex>=1 against log_len=0 → reject),
 * so the rejoined member is PERMANENTLY stuck as a non-converging standby and
 * the primary re-streams [tip+1..committed] every heartbeat forever.
 *
 * Fix: after recovery, raise the library's log_len to the recovered WAL tip and
 * its commit_index to the recovered durable floor (clamped to log_len, honoring
 * the commit_index<=log_len invariant), and align raft.term with the persisted
 * term so the per-LSN (term,index) reconcile reads consistent state. Monotonic:
 * never lowers an already-higher library position (e.g. if catch-up advanced it
 * before this runs). The library stores no entry bytes — on_get_entry_term reads
 * the recovered per-LSN term index — so raising log_len is sufficient for logOk. */
 void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p)
 {
    if (!ha || ha->raft_c == NULL || !p)
        return;
    uint64_t recovered_tip = persist_durable_floor(p);
    if (recovered_tip > ha->raft.log_len)
        ha->raft.log_len = recovered_tip;
    /* commit_index advances to the recovered durable floor but never past the
     * log tip (library invariant commit_index <= log_len), and never decreases. */
    uint64_t recovered_commit = recovered_tip;
    if (recovered_commit > ha->raft.log_len)
        recovered_commit = ha->raft.log_len;
    if (recovered_commit > ha->raft.commit_index)
        ha->raft.commit_index = recovered_commit;
    /* Align the library term with the persisted/restored HA term so the
     * (term,lsn) reconcile and own-term gate read a consistent value after a
     * rejoin (ha_restore_term already loaded ha->term before ha_init_raft). */
    uint64_t ha_term = atomic_load_explicit(&ha->term, memory_order_acquire);
    if (ha_term > ha->raft.term)
        ha->raft.term = ha_term;
 }
 /* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ────────────────────────
 *
 * handle_request_vote: process an incoming OP_requestVote RPC.
--- a/src/controller/ha.h
+++ b/src/controller/ha.h
@ -599,6 +599,25 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
                                    uint64_t       *safe_lsn_out,
                                    uint64_t       *safe_tag_out);
 /* ── bug-1060: Leader Completeness guard for promotion-time WAL truncation ────
 *
 * ha_promotion_truncate_floor: clamp the post-election WAL truncation cutoff so
 * a newly-elected leader NEVER truncates entries it holds durably.
 *
 * Returns max(safe_lsn, durable_tip).  safe_lsn is the majority-threshold commit
 * point from the vote replies (ha_compute_safe_committed_lsn); durable_tip is the
 * leader's own durable WAL tip (persist_durable_floor).  Because the cutoff is
 * never below the leader's durable tip, persist_truncate_wal_above is a no-op on
 * the leader's own durable log — preserving every entry committed by a prior
 * leader (Raft §5.4 Leader Completeness).  safe_lsn only governs removal of a
 * divergent, non-durable tail STRICTLY ABOVE the leader's durable tip.
 *
 * Root cause it fixes: truncating to the bare majority element destroyed
 * committed entries the new leader had acked as a standby (granted voters
 * {self=47, peer=44} → safe_lsn=44 cut the leader's own committed 45..47),
 * producing divergent WALs and a permanent prevote deadlock. */
 uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip);
 /* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ──────────
 *
 * ha_promotion_gate_decision: PURE pre-promotion eligibility decision (Req 2.2,
@ -957,6 +976,17 @@ uint64_t ha_candidate_advertise_lsn(persist_t *p);
 * already-higher applied_lsn (monotonic). */
 void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p);
 /* ha_sync_raft_log_from_recovery (bug-1042) — after local WAL recovery on a
 * (re)start, raise the raft LIBRARY's log_len / commit_index / term to the
 * recovered position. ha_init_raft (before the election) sources the library's
 * initial_log_len from persist_durable_floor, but on a standby-rejoin boot
 * persist_recover runs LATER (cluster_init_phase2), so the library would
 * otherwise stay at log_len=0 and reject every incoming AppendEntries (logOk
 * fails for prevLogIndex>=1), wedging the rejoined member as a permanently non-
 * converging standby. Call this right after persist_recover. Monotonic + NULL-
 * safe; clamps commit_index<=log_len. */
 void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p);
 /*
 * ha_elect_evaluate_and_signal: evaluate accumulated ha_elect_ctx_t votes.
 * Sets ctx->result (1=primary, 0=standby) and posts ctx->done semaphore.
--- a/src/controller/testsuite/test_coldstart_lsn0.c
+++ b/src/controller/testsuite/test_coldstart_lsn0.c
@ -336,7 +336,8 @@ static void test_h4_empty_standby_receives_lsn1_and_lsn2_no_hole(void)
    /* Empty fresh standby = standby_lsn 0; primary committed_lsn = 2 (holds
     * [1,2]). Drive the REAL streamer. */
-    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
+    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
                                      /*leader_term=*/7,
                                      /*standby_lsn=*/0,
                                      /*committed_lsn=*/2);
    xce_wal_stream_request_hook = NULL;
@ -384,7 +385,8 @@ static void test_h4_control_standby_with_lsn1_gets_only_lsn2(void)
    h4_capture_reset();
    /* standby_lsn = 1 (already holds the first entry) catching up to committed 2. */
-    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
+    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
                                      /*leader_term=*/7,
                                      /*standby_lsn=*/1,
                                      /*committed_lsn=*/2);
    xce_wal_stream_request_hook = NULL;
--- a/src/controller/testsuite/test_config_repl.c
+++ b/src/controller/testsuite/test_config_repl.c
@ -2586,7 +2586,8 @@ static void test_wal_stream_500_entries_two_batches(void)
    g_wal_stream_last_n_entries = 0;
    /* Stream entries [1..500]: standby_lsn=0, committed_lsn=500 */
-    rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
+    rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
                                   /*leader_term=*/7,
                                   /*standby_lsn=*/0,
                                   /*committed_lsn=*/500);
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed");
@ -2627,7 +2628,8 @@ static void test_wal_stream_partial_batch(void)
    g_wal_stream_last_n_entries = 0;
    /* Stream entries [6..14]: standby_lsn=5, committed_lsn=14 */
-    rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
+    rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
                                   /*leader_term=*/7,
                                   /*standby_lsn=*/5,
                                   /*committed_lsn=*/14);
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed");
@ -2661,7 +2663,8 @@ static void test_wal_stream_empty_range_no_batches(void)
    g_wal_stream_total_entries  = 0;
    /* standby_lsn == committed_lsn: nothing to stream */
-    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir,
+    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, tmpdir, /*p=*/NULL,
                                       /*leader_term=*/7,
                                       /*standby_lsn=*/100,
                                       /*committed_lsn=*/100);
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "Empty range must return 0");
@ -2676,6 +2679,89 @@ static void test_wal_stream_empty_range_no_batches(void)
    printf("test_wal_stream_empty_range_no_batches: PASS\n");
 }
 /*
 * bug-1028 — the catch-up streamer must source each streamed entry's term and
 * the AppendEntries prev_log_term from the AUTHORITATIVE per-LSN term index
 * (persist_entry_term), NOT the raw on-disk hdr.term. A legacy-0 entry has
 * hdr.term==0 on disk but persist_entry_term maps it to WAL_LEGACY_TERM(1) —
 * which is exactly what the standby's logOk and the library send path compute.
 * If the streamer sends the raw 0, prev_log_term(0) != standby's term(1) → logOk
 * rejects the new-term entry forever (the "stream WAL batches" wedge).
 *
 * Fixture: LSN 1 appended at term 0 (legacy), LSN 2 at term 3. Streaming
 * (standby_lsn=1, committed=2] must emit LSN 2 with term=3 and prev_log_term=1
 * (NOT 0). RED before the fix (streamer sent hdr.term=0 for the predecessor).
 */
 static uint64_t g_cap_term_for_lsn2     = 0;
 static uint64_t g_cap_prevterm_for_lsn2 = 0;
 static int      g_cap_saw_lsn2          = 0;
 static int capture_terms_wal_stream_hook(endpoint_t *ep, xce_request_t *req)
 {
    (void)ep;
    if (!req || !req->body.walcatchupentrymsg) return 0;
    wal_catchup_entry_t *batch =
        (wal_catchup_entry_t *)req->body.walcatchupentrymsg;
    for (int i = 0; i < batch->n_entries; i++) {
        wal_catchup_entry_t *e = batch->entries[i];
        if (e && e->lsn == 2) {
            g_cap_saw_lsn2          = 1;
            g_cap_term_for_lsn2     = e->term;
            g_cap_prevterm_for_lsn2 = e->prev_log_term;
        }
    }
    return 0;
 }
 static void test_wal_stream_term_from_index_not_raw_hdr(void)
 {
    char tmp[256];
    strcpy(tmp, "/tmp/xce_cr_streamterm_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
    cluster_t *c = t42_make_cluster(tmp);
    TEST_ASSERT_NOT_NULL(c);
    /* LSN 1: legacy entry (term 0 on disk → persist_entry_term maps to 1). */
    persist_set_term(&c->persist, 0);
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c->persist, WAL_JOB_UPDATE,
                                               NULL, 0));
    /* LSN 2: real new-term entry (term 3). */
    persist_set_term(&c->persist, 3);
    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(&c->persist, WAL_JOB_UPDATE,
                                               NULL, 0));
    /* Sanity: the authoritative index normalises legacy-0 → 1, keeps 3. */
    TEST_ASSERT_EQUAL_UINT64(WAL_LEGACY_TERM, persist_entry_term(&c->persist, 1));
    TEST_ASSERT_EQUAL_UINT64(3, persist_entry_term(&c->persist, 2));
    xce_wal_stream_request_hook = capture_terms_wal_stream_hook;
    g_cap_saw_lsn2 = 0; g_cap_term_for_lsn2 = 0; g_cap_prevterm_for_lsn2 = 0;
    /* Standby holds LSN 1, catching up to committed 2 — streams just LSN 2,
     * whose prev tuple references LSN 1 (the legacy entry). */
    int rc = ha_stream_wal_to_standby(/*ep=*/NULL, c->persist.wal_path,
                                      &c->persist,
                                      /*leader_term=*/atomic_load_explicit(
                                          &c->ha.term, memory_order_acquire),
                                      /*standby_lsn=*/1, /*committed_lsn=*/2);
    TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "ha_stream_wal_to_standby failed");
    xce_wal_stream_request_hook = NULL;
    TEST_ASSERT_TRUE_MESSAGE(g_cap_saw_lsn2, "LSN 2 must be streamed");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(3, g_cap_term_for_lsn2,
        "streamed entry term must come from persist_entry_term (3)");
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(WAL_LEGACY_TERM, g_cap_prevterm_for_lsn2,
        "bug-1028: prev_log_term must be the index-normalised legacy term (1), "
        "NOT raw on-disk hdr.term (0) — else the standby's logOk rejects forever");
    t42_teardown(c);
    char cmd[512];
    snprintf(cmd, sizeof(cmd), "rm -rf %s", tmp);
    system(cmd);
    printf("test_wal_stream_term_from_index_not_raw_hdr: PASS\n");
 }
 /* ── Task 3.5: Snapshot chunk transfer ──────────────────────── */
 static snapshot_transfer_t g_snaptest_chunks[16];
@ -3363,6 +3449,7 @@ void test_config_repl(void)
    RUN_TEST(test_wal_stream_500_entries_two_batches);
    RUN_TEST(test_wal_stream_partial_batch);
    RUN_TEST(test_wal_stream_empty_range_no_batches);
    RUN_TEST(test_wal_stream_term_from_index_not_raw_hdr); /* bug-1028 */
    /* Task 3.5: snapshot chunk transfer (sender side) */
    RUN_TEST(test_snapshot_transfer_two_chunks);
    RUN_TEST(test_snapshot_transfer_small_less_than_one_chunk);
--- a/src/controller/testsuite/test_ha_election.c
+++ b/src/controller/testsuite/test_ha_election.c
@ -1001,6 +1001,60 @@ static void test_safe_commit_reflects_granting_voter_lsn(void)
    printf("test_safe_commit_reflects_granting_voter_lsn: PASS\n");
 }
 /* ── bug-1060: a new leader must NEVER truncate its own committed entries ─────
 *
 * Leader Completeness (Raft §5.4): a server that wins an election must already
 * hold every entry committed by a prior leader, and it must never truncate its
 * own log to "catch down" to a slower majority.
 *
 * Production root cause (bug-1060): raft_cb_become_leader truncated its WAL down
 * to ha_compute_safe_committed_lsn()'s majority-threshold element.  When the new
 * leader durably holds entries ABOVE that element (it acked them as a standby
 * under the prior primary — so they were committed), truncating to the majority
 * element physically destroyed those committed entries and rewound commitIndex.
 *   e.g. granted voters {self=47, peer=44} → median safe_lsn=44, but self holds
 *   committed entries 45..47 → truncate(44) deletes 45..47.  The leader then
 *   wrote its leadership NOP at LSN 45 (term+1) while the diverged peers still
 *   held the old job at LSN 45 → permanent prevote deadlock + committed-job loss.
 *
 * Invariant under test: the promotion truncation floor is clamped to the new
 * leader's own durable tip, so persist_truncate_wal_above is a no-op on the
 * leader's durable log.  safe_lsn only governs removal of entries STRICTLY ABOVE
 * the leader's durable tip (non-durable, divergent tail).
 *
 * RED before the fix: ha_promotion_truncate_floor does not exist (undefined
 * reference).
 */
 static void test_promotion_never_truncates_own_committed_entries(void)
 {
    /* The bug-1060 case: majority element (44) is BELOW the leader's own durable
     * tip (47).  The truncation floor must clamp UP to the durable tip so the
     * leader's committed entries 45..47 are never cut. */
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(
        47ULL, ha_promotion_truncate_floor(/*safe_lsn=*/44ULL, /*durable_tip=*/47ULL),
        "truncation floor must never fall below the leader's own durable tip "
        "(Leader Completeness: a new leader never truncates its own committed log)");
    /* Equal: floor stays at the durable tip (no truncation). */
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(
        47ULL, ha_promotion_truncate_floor(47ULL, 47ULL),
        "equal safe_lsn and durable tip → floor at the tip (no-op truncation)");
    /* safe_lsn ABOVE the durable tip: only a divergent tail strictly above the
     * leader's durable log can be removed — safe_lsn governs there. */
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(
        50ULL, ha_promotion_truncate_floor(50ULL, 47ULL),
        "when safe_lsn exceeds the durable tip, the floor follows safe_lsn "
        "(removes only the non-durable divergent tail above the leader's log)");
    /* Single-node / fresh boot: durable tip 0 → floor follows safe_lsn. */
    TEST_ASSERT_EQUAL_UINT64_MESSAGE(
        0ULL, ha_promotion_truncate_floor(0ULL, 0ULL),
        "empty log: floor 0 (nothing to truncate)");
    printf("test_promotion_never_truncates_own_committed_entries: PASS\n");
 }
 /* ── Test suite entry point ───────────────────────────────────────────────── */
 void test_ha_election(void)
@ -1059,4 +1113,7 @@ void test_ha_election(void)
    /* bug-828 — safe-commit-from-votes must reflect the granting voter LSN */
    RUN_TEST(test_safe_commit_reflects_granting_voter_lsn);
    /* bug-1060 — a new leader must never truncate its own committed entries */
    RUN_TEST(test_promotion_never_truncates_own_committed_entries);
 }
--- a/src/controller/testsuite/test_persist_recovery.c
+++ b/src/controller/testsuite/test_persist_recovery.c
@ -309,6 +309,75 @@ static void test_recovered_member_advertises_real_applied_lsn(void)
    printf("test_recovered_member_advertises_real_applied_lsn: PASS\n");
 }
 /* ------------------------------------------------------------------ */
 /* bug-1042: after recovery, the raft LIBRARY's log_len/commit_index   */
 /* must be raised to the recovered position. ha_init_raft runs BEFORE  */
 /* persist_recover on a standby-rejoin boot, leaving the library at    */
 /* log_len=0; without ha_sync_raft_log_from_recovery the library then  */
 /* rejects every AppendEntries (logOk fail for prevLogIndex>=1) and    */
 /* the rejoined member never converges.                                */
 /* ------------------------------------------------------------------ */
 static void test_recovered_member_syncs_raft_log_len(void)
 {
    char tmp[256];
    strcpy(tmp, "/tmp/xce_rec1042_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
    /* Phase 1: write WAL entries, flush, stop. */
    {
        persist_t* p = make_fresh_persist(tmp);
        TEST_ASSERT_NOT_NULL(p);
        for (int i = 0; i < 5; i++)
            TEST_ASSERT_EQUAL_INT(0,
                persist_wal_async(p, WAL_JOB_UPDATE, "z", 1));
        TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
        teardown_persist(p);
    }
    /* Phase 2: restart — recover the WAL, then mimic the init-ordering hazard:
     * the raft library was init'd with log_len=0 BEFORE recovery ran. The sync
     * must raise the library's log_len to the recovered durable tip. */
    {
        persist_t* p2 = make_fresh_persist(tmp);
        TEST_ASSERT_NOT_NULL(p2);
        TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
        uint64_t recovered_floor = persist_durable_floor(p2);
        TEST_ASSERT_TRUE(recovered_floor > 0);
        ha_state_t ha;
        memset(&ha, 0, sizeof(ha));
        /* Library came up empty (ha_init_raft before recovery). */
        ha.raft.log_len      = 0;
        ha.raft.commit_index = 0;
        ha.raft.term         = 0;
        atomic_store_explicit(&ha.term, 4ULL, memory_order_release);
        /* The helper requires a non-NULL raft_c back-ref (production wiring). */
        ha.raft_c = (cluster_t*)0x1; /* non-NULL sentinel; helper never derefs it */
        ha_sync_raft_log_from_recovery(&ha, p2);
        /* The bug: log_len used to stay 0 → logOk rejects every AppendEntries. */
        TEST_ASSERT_EQUAL_UINT64(recovered_floor, ha.raft.log_len);
        /* commit_index advanced to the durable floor, clamped to log_len. */
        TEST_ASSERT_EQUAL_UINT64(recovered_floor, ha.raft.commit_index);
        TEST_ASSERT_TRUE(ha.raft.commit_index <= ha.raft.log_len);
        /* term aligned to the persisted/restored HA term. */
        TEST_ASSERT_EQUAL_UINT64(4ULL, ha.raft.term);
        /* Monotonic: never lowers an already-higher library position. */
        ha.raft.log_len      = recovered_floor + 100;
        ha.raft.commit_index = recovered_floor + 50;
        ha_sync_raft_log_from_recovery(&ha, p2);
        TEST_ASSERT_EQUAL_UINT64(recovered_floor + 100, ha.raft.log_len);
        TEST_ASSERT_EQUAL_UINT64(recovered_floor + 50, ha.raft.commit_index);
        teardown_persist(p2);
    }
    printf("test_recovered_member_syncs_raft_log_len: PASS\n");
 }
 void test_persist_recovery(void)
 {
    RUN_TEST(test_recover_from_empty_state);
@ -317,4 +386,5 @@ void test_persist_recovery(void)
    RUN_TEST(test_recover_rejects_corrupt_snapshot);
    RUN_TEST(test_recover_picks_latest_snapshot);
    RUN_TEST(test_recovered_member_advertises_real_applied_lsn);
    RUN_TEST(test_recovered_member_syncs_raft_log_len);
 }
--- a/src/controller/testsuite/test_term_safety_adapter.c
+++ b/src/controller/testsuite/test_term_safety_adapter.c
@ -1529,6 +1529,141 @@ static void test_catchup_applied_lsn_is_last_held_no_skip(void)
    printf("test_catchup_applied_lsn_is_last_held_no_skip: PASS\n");
 }
 /* ================================================================== */
 /* bug-1067 — cross-term catch-up: the AppendEntries header term is the  */
 /*            leader's CURRENT term, NOT the streamed entry's stored term */
 /* ================================================================== */
 /* Root cause: ha_consume_append_entries used the entry's stored term (entry_term)
 * as the AppendEntries header term. For LIVE appends that is the leader's current
 * term (leader appends at its term), but for CATCH-UP streaming of HISTORICAL
 * entries written by a PRIOR leader, entry_term < the current leader's term. The
 * library's stale-term check (term < s->term → ae_reject_stale) then permanently
 * rejected every historical entry, so the lagging standby could never catch up
 * (standby_reported_lsn never advanced; the streamer retransmitted forever).
 *
 * The fix carries the leader's current term out-of-band per entry
 * (wal_catchup_entry_t.leader_current_term, WalCatchupEntry.leaderCurrentTerm @8),
 * set by the streamer to ha->term, and uses THAT as the AE header term while the
 * entry's stored term remains the per-entry term for logOk / on_persist_entry.
 *
 * RED before the fix: the leader_current_term field does not exist (compile
 * error); once added but before the receiver reads it, ae_term == entry_term (4) <
 * s->term (5) → stale-reject → LSN 2 never persists. */
 static void test_intake_crossterm_catchup_uses_leader_current_term(void)
 {
    char tmp[256];
    strcpy(tmp, "/tmp/xce_ts_xterm_catchup_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
    cluster_t* c = make_cluster(tmp);
    TEST_ASSERT_NOT_NULL(c);
    /* Standby has already adopted the current leader's term (5) from heartbeats. */
    make_follower(c, /*term*/5, /*n_peers*/3);
    /* Local log holds a HISTORICAL entry LSN1@term4 (written by the prior leader
     * at term 4, replicated to this standby). */
    uint8_t blob[64];
    size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
    TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n));
    c->ha.raft.log_len = 1;
    TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1));
    /* The CURRENT leader (term 5) streams the historical entry LSN2@term4 as a
     * catch-up sub-entry: the entry's STORED term is 4, but the AppendEntries
     * header term is the leader's CURRENT term 5 (leader_current_term). prev tuple
     * (1,4) matches the local log so logOk holds. */
    uint8_t lpay[1] = { 'b' };
    wal_catchup_entry_t sub;
    memset(&sub, 0, sizeof(sub));
    sub.lsn                = 2;
    sub.entry_type         = WAL_JOB_SUBMIT;
    sub.term               = 4;   /* the entry's STORED term (prior leader)      */
    sub.leader_current_term = 5;  /* the streaming leader's CURRENT term         */
    sub.prev_log_index     = 1;
    sub.prev_log_term      = 4;
    sub.raw_payload        = lpay;
    sub.raw_payload_len    = sizeof(lpay);
    wal_catchup_entry_t* sub_arr[1] = { &sub };
    wal_catchup_entry_t batch;
    memset(&batch, 0, sizeof(batch));
    batch.n_entries           = 1;
    batch.entries             = sub_arr;
    batch.term                = 4;
    batch.leader_current_term = 5;
    batch.prev_log_index      = 1;
    batch.prev_log_term       = 4;
    int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c);
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* THE bug-1067 assertion: the historical entry is ACCEPTED (not stale-rejected)
     * and persisted at its stored term 4. Pre-fix it was rejected (absent). */
    TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 2));
    /* The agreed prefix (LSN1@term4) is untouched. */
    TEST_ASSERT_EQUAL_UINT64(4, persist_entry_term(&c->persist, 1));
    /* The leader's current term (5) equals our term — no spurious term change. */
    TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
    teardown_cluster(c);
    printf("test_intake_crossterm_catchup_uses_leader_current_term: PASS\n");
 }
 /* Safety guard: a GENUINELY stale leader (leader_current_term < our term) must
 * STILL be rejected — the fix must not open a hole letting a deposed leader's
 * divergent entries through. */
 static void test_intake_stale_leader_term_still_rejected(void)
 {
    char tmp[256];
    strcpy(tmp, "/tmp/xce_ts_stale_leader_XXXXXX");
    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
    cluster_t* c = make_cluster(tmp);
    TEST_ASSERT_NOT_NULL(c);
    make_follower(c, /*term*/5, /*n_peers*/3);
    uint8_t blob[64];
    size_t n = pack_entry(blob, WAL_JOB_SUBMIT, "a", 1);
    TEST_ASSERT_EQUAL_INT(0, ha_cb_persist_entry_for_test(&c->ha, 1, 4, blob, n));
    c->ha.raft.log_len = 1;
    /* A deposed leader at term 3 (< our term 5) tries to stream. Even though the
     * entry's stored term and prev tuple look locally plausible, the AE header
     * term (leader_current_term=3) is stale → reject. */
    uint8_t lpay[1] = { 'z' };
    wal_catchup_entry_t sub;
    memset(&sub, 0, sizeof(sub));
    sub.lsn                 = 2;
    sub.entry_type          = WAL_JOB_SUBMIT;
    sub.term                = 3;
    sub.leader_current_term = 3;   /* STALE leader term < our term 5            */
    sub.prev_log_index      = 1;
    sub.prev_log_term       = 4;
    sub.raw_payload         = lpay;
    sub.raw_payload_len     = sizeof(lpay);
    wal_catchup_entry_t* sub_arr[1] = { &sub };
    wal_catchup_entry_t batch;
    memset(&batch, 0, sizeof(batch));
    batch.n_entries           = 1;
    batch.entries             = sub_arr;
    batch.term                = 3;
    batch.leader_current_term = 3;
    batch.prev_log_index      = 1;
    batch.prev_log_term       = 4;
    int rc = ha_apply_wal_catchup_entry(&c->ha, &c->persist, &batch, /*ep*/NULL, c);
    TEST_ASSERT_EQUAL_INT(0, rc);
    /* Stale leader's entry was REJECTED — LSN 2 absent, our term unchanged. */
    TEST_ASSERT_EQUAL_UINT64(0, persist_entry_term(&c->persist, 2));
    TEST_ASSERT_EQUAL_UINT64(5, c->ha.raft.term);
    teardown_cluster(c);
    printf("test_intake_stale_leader_term_still_rejected: PASS\n");
 }
 /* ================================================================== */
 /* Task 3.3 — drive raft_on_append_entries_reply + raft_advance_commit */
 /* on the replication ACK path                                         */
@ -2065,6 +2200,8 @@ void test_term_safety_adapter(void)
    RUN_TEST(test_intake_logmismatch_rejected_no_false_convergence);
    RUN_TEST(test_live_broadcast_receive_persists_leader_term);
    RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);
    RUN_TEST(test_intake_crossterm_catchup_uses_leader_current_term);
    RUN_TEST(test_intake_stale_leader_term_still_rejected);
    RUN_TEST(test_repl_ack_currentterm_quorum_advances_commit);
    RUN_TEST(test_repl_ack_priorterm_quorum_holds_commit);
    RUN_TEST(test_repl_ack_staleterm_reply_dropped);
--- a/src/controller/testsuite/waldump.c
+++ b/src/controller/testsuite/waldump.c
@ -14,6 +14,7 @@
 #include "config.h"           /* WAL_HDR_CRC_SIZE, WAL_CRC_NONE, MAX_WAL_PAYLOAD_BYTES */
 #include "controller.capnp.h" /* Job decode for WAL_JOB_SUBMIT */
 #include "utils.h"            /* fmt_jobid — human-readable JobId formatting */
 #include "wal_types.h"
 #define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
@ -303,8 +304,9 @@ static void scan_segment(const char* seg_path, FILE* out,
            }
        }
-        /* Per-entry line, with jobid/subtime for submits. */
+        /* Per-entry line, with jobid/subtime for submits. Sized to hold the
-        char info[96]     = "";
+         * formatted JobId (fmt_jobid, up to ~40 chars) + subtime + state. */
        char info[192]    = "";
        int  is_dup       = 0;
        int  dup_identical = 0;
        if (hdr.type == WAL_JOB_SUBMIT) {
@ -312,9 +314,16 @@ static void scan_segment(const char* seg_path, FILE* out,
            uint64_t      jobid = 0;
            if (decode_submit_info(payload, hdr.payload_len, &si) == 0) {
                jobid = si.jobid;
                /* Render the JobId in the same human-readable form the client
                 * (joblist / jobsub) prints — fmt_jobid (utils.c) — so a waldump
                 * jobid can be eyeballed against an `expect-job <id>` / joblist
                 * entry without converting the raw uint64. */
                char jidStr[128];
                if (fmt_jobid(jidStr, sizeof(jidStr), si.jobid) < 0)
                    snprintf(jidStr, sizeof(jidStr), "%" PRIu64, si.jobid);
                snprintf(info, sizeof(info),
-                         "jobid=%" PRIu64 " subtime=%" PRIu64 " state=%u",
+                         "jobid=%s subtime=%" PRIu64 " state=%u",
-                         si.jobid, si.subtime, si.state);
+                         jidStr, si.subtime, si.state);
            } else {
                snprintf(info, sizeof(info), "jobid=?");
            }
--- a/src/include/typemap.h
+++ b/src/include/typemap.h
@ -364,6 +364,14 @@ struct wal_catchup_entry {
     * the AppendEntries so the standby intake passes it into
     * raft_on_append_entries_request. Wire field leaderCommit @7. */
    uint64_t                   leader_commit;
    /* bug-1067: the streaming leader's CURRENT term — the Raft AppendEntries
     * HEADER term, DISTINCT from `term` above (the entry's STORED term). Equal for
     * a live append; for a HISTORICAL catch-up entry the stored term is lower than
     * the leader's current term. The standby intake uses this as the AE header
     * term (UpdateTerm/stale-check) so cross-term catch-up is not stale-rejected;
     * 0 = unset (legacy sender) → fall back to the stored term. Wire field
     * leaderCurrentTerm @8. */
    uint64_t                   leader_current_term;
 };
 /* ── Persistence Cap'n Proto mapped types (P0-9) ─────────────────────
--- a/src/lib/client.c
+++ b/src/lib/client.c
@ -123,30 +123,59 @@ int xcecli_try_wakeup(xcecli_ctx_t c, int status)
 int xcecli_start(xcecli_ctx_t c)
 {
-    int        rc;
+    int        rc = -ECONNREFUSED;
    api_ctx_t* ctx = (api_ctx_t*)c;
    int        i;
    int        connected_idx = -1;
    if (c == NULL) {
        return -EINVAL;
    }
-    rc = endpoint_connect(&(ctx->ep), ctx->conf->masters[0],
+    /* Initial connect rotates through ALL configured masters, not just
-                          ctx->conf->master_ports[0]);
+     * masters[0] (bug-854 dead-first-master): if the first master is down
-    if (rc < 0) {
+     * (connection-refused after a failover that killed it), a hardcoded
     * masters[0] connect fails the whole `client` session before the failover
     * loop — which only runs on an ALREADY-connected ctx — ever gets a chance
     * to rotate. Try each master in order; the first that connects becomes the
     * sticky cursor start. The subsequent ctrlstatus/op failover loop then
     * rotates onward to the authoritative primary as needed.
     *
     * ctx->ep was endpoint_init'd + bound to the iomgr in xcecli_init. A failed
     * endpoint_connect leaves the socket fd unusable, so before retrying the
     * NEXT master we must rebuild the endpoint exactly as xce_ctx_reconnect
     * does: endpoint_fini → endpoint_init (fresh fd) → re-set peer + re-bind the
     * iomgr (endpoint_connect needs ep->iomgr->ssl_ctx). */
    for (i = 0; i < ctx->conf->nmasters; i++) {
        if (i > 0) {
            endpoint_fini(&(ctx->ep));
            if (endpoint_init(&(ctx->ep), AF_INET, SOCK_STREAM, 0, 0) < 0)
                continue;
            endpoint_set_peer(&(ctx->ep), &(ctx->ctx.myinfo));
            ctx->ep.iomgr = &(ctx->ctx.iomgr);
        }
        rc = endpoint_connect(&(ctx->ep), ctx->conf->masters[i],
                              ctx->conf->master_ports[i]);
        if (rc >= 0) {
            connected_idx = i;
            break;
        }
    }
    if (connected_idx < 0) {
        /* No master reachable at all — surface the last transport error. */
        endpoint_fini(&(ctx->ep));
        return rc;
    }
-    /* Initialize per-context failover state: sticky primary starts at
+    /* Initialize per-context failover state: sticky primary starts at the master
-     * masters[0] (the initial connect target above), retry tunables default
+     * we actually connected to (not always index 0), retry tunables default to
-     * to XCE_RETRY_DEFAULTS, and no rejoin hook is installed for pure clients.
+     * XCE_RETRY_DEFAULTS, and no rejoin hook is installed for pure clients. */
     */
    rc = mylock_init(&(ctx->failover.lock));
    if (rc < 0) {
        endpoint_fini(&(ctx->ep));
        return rc;
    }
-    ctx->failover.cur_master = 0;
+    ctx->failover.cur_master = connected_idx;
    ctx->failover.retry      = XCE_RETRY_DEFAULTS;
    ctx->failover.rejoin_cb  = NULL;
    ctx->failover.rejoin_arg = NULL;
@ -414,11 +443,17 @@ int xcecli_ctrlstatus_primary(xcecli_ctx_t c, xce_ctrlstatus_t** status)
    /* Bound the redirect loop by the configured failover retry budget so we
     * never spin unbounded chasing a never-appearing primary (each inner
-     * xcecli_ctrlstatus already runs its own full reach-some-node failover). */
+     * xcecli_ctrlstatus already runs its own full reach-some-node failover).
     * Floor it at the master-ring size so a standby-first / dead-first master
     * list is always traversable in full before giving up (bug-854): one extra
     * attempt lets the rotation that lands on the primary actually query it. */
    max_attempts = ctx->failover.retry.max_retries;
    if (max_attempts < 1) {
        max_attempts = 1;
    }
    if (ctx->conf != NULL && ctx->conf->nmasters + 1 > max_attempts) {
        max_attempts = ctx->conf->nmasters + 1;
    }
    for (attempt = 0; attempt < max_attempts; attempt++) {
        hint = NULL;
@ -436,21 +471,34 @@ int xcecli_ctrlstatus_primary(xcecli_ctx_t c, xce_ctrlstatus_t** status)
            return 0;
        }
-        /* Non-authoritative HINT (standby/electing/init). Direct the next query
+        /* Non-authoritative reply (standby/electing/init). The inner
-         * at the reported primary if we can map it to an address; otherwise let
+         * xcecli_ctrlstatus SUCCEEDS against ANY reachable node — ctrlstatus is
-         * the next xcecli_ctrlstatus rotate the configured master list. The
+         * answerable in every role — so it classifies the standby's status-0
-         * redirect hint is consumed one-shot by xce_failover_next_target. */
+         * reply as success and NEVER rotates off it on its own. We must advance
         * the endpoint here, or the loop re-queries the same non-primary every
         * attempt and exhausts the budget without ever reaching the primary
         * (bug-854). Capture a redirect hint when the responder named the primary
         * (1-hop direct redirect), then FORCE rotation: xce_failover_rotate
         * consumes the hint when set, else advances to the next configured
         * master; recover repairs the ctx so the next attempt's send proceeds.
         * Mirror xcecli_retry_next: only recover when the rotate succeeded — a
         * failed rotate leaves the ctx for the next inner call's own transport-
         * error failover to heal. All under failover.lock (the inner call takes
         * and releases it itself, so it is not held here). */
        {
            char     phost[XCE_MAXHOSTNAME_LEN];
            uint16_t pport = 0;
            mylock_lock(&ctx->failover.lock);
            if (xcecli_ctrlstatus_primary_addr(hint, phost, sizeof(phost),
                                               &pport)
                == 0) {
                mylock_lock(&ctx->failover.lock);
                xce_ctx_capture_redirect_hint(&ctx->ctx, phost, pport);
                mylock_unlock(&ctx->failover.lock);
            }
            if (xce_failover_rotate(ctx) == 0) {
                xcecli_ctx_recover(ctx);
            }
            mylock_unlock(&ctx->failover.lock);
        }
        /* Free the non-authoritative interim reply before the next attempt — no
--- a/src/lib/hook.c
+++ b/src/lib/hook.c
@ -42,9 +42,18 @@ int xce_add_hook(hooksys_t* s, int typ, hook_fn_t fn, void* data)
        return -EINVAL;
    }
-    /* Scan for a disabled (NULL-fn) slot of the same type to reuse */
+    /* Single scan of the type's slots:
     *   - if (fn,data) is ALREADY registered and active, adding again is a
     *     no-op (idempotent). xce_disable_hook removes only ONE matching slot
     *     and xce_run_hooks fires ALL slots, so a duplicate (fn,data) would
     *     fire twice and survive a single disable — the bug-845 double-append
     *     (persist_start + ha_register_persist_hooks both register the same
     *     persist_job_hook_fun closure). Dedup is keyed on (fn,data): the same
     *     fn with a DISTINCT closure stays a legitimate second registration.
     *   - otherwise remember the first disabled (NULL-fn) slot to reuse. */
    {
        xlist_t*     l     = &s->hooks[typ];
        hook_item_t* reuse = NULL;
        int          i;
        for (i = 0; i < l->count; i++) {
            hook_item_t* slot = (hook_item_t*)l->data[i];
@ -52,10 +61,16 @@ int xce_add_hook(hooksys_t* s, int typ, hook_fn_t fn, void* data)
            hook_fn_t cur =
                atomic_load_explicit(&slot->fn, memory_order_acquire);
            if (cur == NULL) {
-                slot->data = data;
+                if (!reuse) reuse = slot;
-                atomic_store_explicit(&slot->fn, fn, memory_order_release);
+                continue;
                return 0;
            }
            if (cur == fn && slot->data == data)
                return 0; /* already registered — no duplicate */
        }
        if (reuse != NULL) {
            reuse->data = data;
            atomic_store_explicit(&reuse->fn, fn, memory_order_release);
            return 0;
        }
    }
--- a/src/lib/request.c
+++ b/src/lib/request.c
@ -486,6 +486,11 @@ static void wal_entry_to_wire(struct capn_segment *cs,
    w->prevLogIndex = e->prev_log_index;
    w->prevLogTerm  = e->prev_log_term;
    w->leaderCommit = e->leader_commit;
    /* bug-1067: carry the streaming leader's CURRENT term per sub-entry (distinct
     * from `term` above, the entry's stored term). The standby intake uses this as
     * the AppendEntries header term so historical (entry_term < leader_term)
     * catch-up entries are not stale-rejected. */
    w->leaderCurrentTerm = e->leader_current_term;
    if (e->raw_payload && e->raw_payload_len > 0) {
        capn_list8 pl = capn_new_list8(cs, (int)e->raw_payload_len);
        capn_setv8(pl, 0, e->raw_payload, (int)e->raw_payload_len);
@ -539,6 +544,9 @@ int xce_wal_batch_encode_raw(wal_catchup_entry_t **entries, int n_entries,
    top.prevLogIndex  = entries[0]->prev_log_index;
    top.prevLogTerm   = entries[0]->prev_log_term;
    top.leaderCommit  = entries[0]->leader_commit;
    /* bug-1067: the AppendEntries header term (leader's CURRENT term), distinct
     * from top.term (entries[0]'s stored term). Mirrors the per-entry field. */
    top.leaderCurrentTerm = entries[0]->leader_current_term;
    /* Batch container: entries list carries the per-entry payload(s) (empty for
     * a heartbeat). Each sub-entry carries lsn/entryType/term/payload. */
--- a/src/lib/testsuite/test_hooks.c
+++ b/src/lib/testsuite/test_hooks.c
@ -100,6 +100,59 @@ static void test_add_hook_reuses_null_slot(void)
    printf("test_add_hook_reuses_null_slot: PASS\n");
 }
 /* bug-845: re-registering the SAME (fn,data) on a type must be idempotent.
 * xce_disable_hook removes only ONE matching slot and xce_run_hooks fires ALL
 * slots, so a duplicate (fn,data) registration would (a) fire twice and (b)
 * survive a single disable.  This is exactly how the controller double-appended
 * every WAL_JOB_SUBMIT: persist_start() and ha_register_persist_hooks() both
 * register persist_job_hook_fun with the same persist_t closure. */
 static void test_add_hook_dedups_same_fn_and_data(void)
 {
    hooksys_t hs;
    int       a = 0;
    TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&hs, 2));
    /* First add registers; the second add with the SAME fn AND data must be a
     * no-op (still rc=0, still exactly one slot). */
    TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &a));
    TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &a));
    TEST_ASSERT_EQUAL_INT(1, hs.hooks[HOOK_AFTER_EVENT].count);
    /* Fires ONCE per event, never twice. */
    xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
    TEST_ASSERT_EQUAL_INT(1, a);
    /* A single disable fully removes it — no duplicate slot lingers. */
    TEST_ASSERT_EQUAL_INT(0, xce_disable_hook(&hs, HOOK_AFTER_EVENT, counting_fn));
    xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
    TEST_ASSERT_EQUAL_INT(1, a); /* unchanged */
    xce_hook_fini(&hs);
    printf("test_add_hook_dedups_same_fn_and_data: PASS\n");
 }
 /* The dedup is keyed on (fn,data): the SAME fn with DISTINCT closures is a
 * legitimate pair of registrations and BOTH must fire. */
 static void test_add_hook_allows_same_fn_distinct_data(void)
 {
    hooksys_t hs;
    int       a = 0, b = 0;
    TEST_ASSERT_EQUAL_INT(0, xce_hook_init(&hs, 2));
    TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &a));
    TEST_ASSERT_EQUAL_INT(0, xce_add_hook(&hs, HOOK_AFTER_EVENT, counting_fn, &b));
    TEST_ASSERT_EQUAL_INT(2, hs.hooks[HOOK_AFTER_EVENT].count);
    xce_run_hooks(&hs, HOOK_AFTER_EVENT, 0, 0);
    TEST_ASSERT_EQUAL_INT(1, a);
    TEST_ASSERT_EQUAL_INT(1, b);
    xce_hook_fini(&hs);
    printf("test_add_hook_allows_same_fn_distinct_data: PASS\n");
 }
 /* --- suite entry point --- */
 void test_hooks(void)
@ -109,4 +162,6 @@ void test_hooks(void)
    RUN_TEST(test_disable_hook_stops_invocation);
    RUN_TEST(test_disable_hook_returns_enoent_if_not_found);
    RUN_TEST(test_add_hook_reuses_null_slot);
    RUN_TEST(test_add_hook_dedups_same_fn_and_data);
    RUN_TEST(test_add_hook_allows_same_fn_distinct_data);
 }
--- a/src/lib/testsuite/test_wal_wire.c
+++ b/src/lib/testsuite/test_wal_wire.c
@ -191,8 +191,11 @@ static void test_wal_batch_encode_rejects_invalid_args(void)
    TEST_ASSERT_EQUAL_INT(-EINVAL,
        xce_wal_batch_encode_raw(NULL, 1, &test_peer, 0, &pkt));
    /* n_entries == 0 is a valid heartbeat (ha-term-safety task 3.2): a 0-entry AE
     * carries the AppendEntries metadata (term, prev tuple, leader_commit) on the
     * top-level container to drive UpdateTerm/logOk. Only n_entries < 0 is invalid. */
    TEST_ASSERT_EQUAL_INT(-EINVAL,
-        xce_wal_batch_encode_raw(entries, 0, &test_peer, 0, &pkt));
+        xce_wal_batch_encode_raw(entries, -1, &test_peer, 0, &pkt));
    TEST_ASSERT_EQUAL_INT(-EINVAL,
        xce_wal_batch_encode_raw(entries, 1, &test_peer, 0, NULL));
--- a/src/lib/xcectx.c
+++ b/src/lib/xcectx.c
@ -7,6 +7,7 @@
 #include "netio.h"
 #include "typemap.h"
 #include "utils.h"
 #include "controller.capnp.h" /* free_CtrlStatus_ptr, free_ClusterConf_ptr */
 #include "xcectx.h"
 static int handle_get_reply(xce_ctx_t* ctx, xce_reply_t* reply)
@ -71,6 +72,14 @@ static int handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
    case OPCode_clusterconf:
        rc = -1;
        if (reply->data_which == RequestReply_data_clusterconf) {
            /* Free a prior unconsumed reply before storing the new one.
             * The consumer xce_clusterconf() MOVE_PTRs out, so in the normal
             * single-request-per-ctx path this is a no-op; on a re-stolen slot
             * (reply decode on a stale fd after client rotation) it prevents
             * a permanent heap leak of the prior occupant. */
            if (ctx->reply.u.clusterconf) {
                free_ClusterConf_ptr(&ctx->reply.u.clusterconf);
            }
            MOVE_PTR(ctx->reply.u.clusterconf, reply->data.clusterconf);
            rc = 1;
        }
@ -78,6 +87,9 @@ static int handle_reply(xce_ctx_t* ctx, xce_reply_t* reply)
    case OPCode_ctrlstatus:
        rc = -1;
        if (reply->data_which == RequestReply_data_ctrlstatus) {
            if (ctx->reply.u.ctrlstatus) {
                free_CtrlStatus_ptr(&ctx->reply.u.ctrlstatus);
            }
            MOVE_PTR(ctx->reply.u.ctrlstatus, reply->data.ctrlstatus);
            rc = 1;
        }
--- a/src/schema/controller.capnp
+++ b/src/schema/controller.capnp
@ -898,6 +898,17 @@ struct WalCatchupEntry $C.mapname("wal_catchup_entry_t") {
  # the standby intake passes it into raft_on_append_entries_request. Append-only
  # field number (@7) — no collision with the @4/@5/@6 term fields above.
  leaderCommit @7 :UInt64 $C.mapname("leader_commit");
  # bug-1067: the streaming leader's CURRENT term — the Raft AppendEntries header
  # term (`term` arg of raft_on_append_entries_request), DISTINCT from `term` @4
  # which is the entry's STORED term. For a live append the two are equal; for
  # catch-up of a HISTORICAL entry written by a prior leader, the entry's stored
  # term is LOWER than the current leader's term. Using the stored term as the AE
  # header term made the library stale-reject every historical entry
  # (ae_reject_stale), wedging cross-term catch-up. The standby intake uses this
  # as the AE header term (UpdateTerm/stale-check) and keeps `term` @4 as the
  # per-entry term (logOk / on_persist_entry). 0 = unset (legacy sender) → the
  # intake falls back to the entry's stored term (the prior, live-only behaviour).
  leaderCurrentTerm @8 :UInt64 $C.mapname("leader_current_term");
 }
 struct CtrlHelloRequest $C.mapname("ctrl_hello_req_t") {