diff --git a/.claude/skills/xce-test-ha/SKILL.md b/.claude/skills/xce-test-ha/SKILL.md index 59a7fbf8..431ab862 100644 --- a/.claude/skills/xce-test-ha/SKILL.md +++ b/.claude/skills/xce-test-ha/SKILL.md @@ -510,6 +510,32 @@ invariant is asserted with an immediate `fail()` inside `run_config_upgrade_wal_trim_round()`, so a violation aborts the run before this line is ever reached. +### `--truncated-fence` mode: promotion fence on a truncated-boot node (bug-1919) + +Drives a node through the whole ha-commit-durability fence lifecycle on +purpose instead of waiting for a kill to land mid-write: ctrl-1 (priority 0, +first to start a PreVote) is made a standby, killed, has its last WAL entry's +CRC broken, misses a batch of commits, and is restarted (boots +`RECOVER_TRUNCATED`, proven from its log). The primary is then killed until +ctrl-1 wins an election (up to 3 attempts) and its promotion is `REFUSED`; +after commits on the new primary the node's log must show `healed by +catch-up` + `promotion fence RELEASED` within 120 s, the cluster must +reconverge, and ctrl-1 must then be force-elected and lead with zero job loss. +Single dedicated round, not a loop. + +```bash +nix develop ./src --command bash .claude/skills/xce-test-ha/run-ha-soak.sh --truncated-fence /tmp/ha-fence +``` + +### WAL segment-layout gate (sequential, chaotic, truncated-fence) + +Before declaring PASS the driver runs `wal-layout-check.py `: +every controller's WAL segment `seg-.wal` must hold one contiguous +LSN run starting at that LSN. A misfiled LSN (bug-1920: a replicated write +before boot recovery truncated and reused `seg-...0001`) is never streamed by +catch-up, so this fails the run even when no standby happened to stall on it. +The checker also runs standalone on any deploy dir. + ### Scheduler replay-parity gate (sequential + chaotic, on by default) Whenever sequential/chaotic mode has the real worker + scheduler pair live, diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index 97522d23..32e707a9 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -222,7 +222,7 @@ # # Env overrides: # XCE_HA_TOP deploy dir (default /home/claude/tmp/xce-ha-deploy) -# XCE_HA_MODE sequential | chaotic | force-snapshot (default sequential) +# XCE_HA_MODE sequential | chaotic | force-snapshot | truncated-fence (default sequential) # XCE_HA_ROUNDS kill/restart rounds (default 6, clamp 5..50) # XCE_HA_JOBS jobs per submit batch (default 3) # XCE_HA_WAITMS wait-primary budget (default 30000) @@ -353,6 +353,7 @@ while [ $# -gt 0 ]; do --chaotic) MODE="chaotic" ;; --sequential) MODE="sequential" ;; --force-snapshot) MODE="force-snapshot" ;; + --truncated-fence) MODE="truncated-fence" ;; --factstream) MODE="factstream" ;; --array-soak) MODE="array-soak" ;; --array-chaos-retry) MODE="array-chaos-retry" ;; @@ -474,6 +475,19 @@ V_SIMVERIFY_UNCLEAN=0 # recordings abandoned without a clean record stop log() { echo "[xce-test-ha] $*"; } fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; } +# bug-1920: every WAL segment must hold one contiguous LSN run starting at the +# LSN in its file name -- the catch-up streamer orders segments by name, so a +# misfiled LSN is never streamed and a standby that needs it stalls. Checked +# on the settled cluster before declaring PASS (wal-layout-check.py). +verify_wal_layout() { + local out + if ! out="$(python3 "$SELF_DIR/wal-layout-check.py" "$TOP" 2>&1)"; then + printf '%s\n' "$out" | sed 's/^/[xce-test-ha] /' >&2 + return 1 + fi + log "$(printf '%s\n' "$out" | tail -1)" +} + # ── Cross-run collision guard (bug-1436) ───────────────────────────────────── # Every pgrep/pkill in this script MUST be scoped to THIS run's own --workdir, # not just the binary name / --myname. Two run-ha-soak.sh invocations (e.g. a @@ -1686,6 +1700,111 @@ run_chaotic_rounds() { # # This does NOT loop or share state with run_chaotic_rounds/the sequential # loop — it is one dedicated, deterministic scenario, run standalone. +# ── truncated-fence: bug-1919 promotion-fence release, driven on purpose ───── +# A node that boots RECOVER_TRUNCATED and then wins an election must be refused +# promotion and fenced (ha-commit-durability Req 2.2), then -- once it has +# caught up from the live primary -- have the fence released (Req 2.6, +# bug-1919) and be able to lead. Normal soaks hit this only by chance (a kill +# landing mid-write); this round makes it deterministic: +# 1. make ctrl-1 (priority 0: it starts its PreVote first) a standby; +# 2. kill it, break its WAL tail CRC, commit jobs without it, restart it +# (boots RECOVER_TRUNCATED) and let it catch up; +# 3. kill the primary until ctrl-1 wins an election -> promotion REFUSED; +# 4. commit jobs on the new primary; ctrl-1's fence must be healed and +# RELEASED within the re-check window; +# 5. reconverge, force ctrl-1 to lead, verify zero job loss. +# Every step is proven from the node's own log, not assumed. +corrupt_wal_tail() { + python3 - "$TOP/controller/$1/data/wal" <<'PY' +import glob, os, struct, sys +HDR = struct.Struct(" fence healed + released + submit_jobs "$NJOBS" || fail "truncated-fence: submit on $newp failed" + verify_no_loss || fail "truncated-fence: loss after $T's refused promotion" + local waited=0 + until tail -n +"$lines" "$TOP/logs/$T.log" | grep -q "promotion fence RELEASED"; do + [ "$waited" -ge 120 ] && fail "truncated-fence: $T's promotion fence was not released within 120 s of catching up (bug-1919)" + sleep 5; waited=$((waited + 5)) + done + tail -n +"$lines" "$TOP/logs/$T.log" | grep -q "healed by catch-up" \ + || fail "truncated-fence: fence released without the truncation being healed" + log "truncated-fence: $T's truncation healed and fence RELEASED after ${waited}s" + + # 5. reconverge, then the healed node must be able to lead + start_node "$P" + wait_for_full_convergence || fail "truncated-fence: no 3-of-3 after the fence release" + force_elect_node "$T" || fail "truncated-fence: healed $T could not be elected" + verify_no_loss || fail "truncated-fence: committed job LOST" + log "truncated-fence: healed $T led the cluster (term=$(latest_primary_term))" +} + run_force_snapshot_round() { local target other1 other2 n before_lsn snap_log @@ -3245,6 +3364,16 @@ walcheck "${#ALL_JOBIDS[@]}" # Opt-in (--force-snapshot / XCE_HA_MODE=force-snapshot). See # run_force_snapshot_round's own header comment. Single round, not a loop — # emits a verdict and exits, like chaotic mode. +if [ "$MODE" = "truncated-fence" ]; then + log "MODE=truncated-fence — driving a RECOVER_TRUNCATED node through refused promotion, fence release and leadership (bug-1919)" + run_truncated_fence_round + verify_wal_layout || fail "truncated-fence: WAL segment layout inconsistent (bug-1920 signature)" + log "PASS (truncated-fence): ${#ALL_JOBIDS[@]} jobs, zero job loss; truncated boot -> refused promotion -> fence released -> led" + cleanup + trap - EXIT INT TERM + exit 0 +fi + if [ "$MODE" = "force-snapshot" ]; then log "MODE=force-snapshot — forcing one full-checkpoint catch-up round (max-wal-records=$MAX_WAL_RECORDS wal-segment-bytes=$WAL_SEGMENT_BYTES)" run_force_snapshot_round @@ -3468,6 +3597,12 @@ if [ "$MODE" = "chaotic" ]; then trap - EXIT INT TERM exit 1 fi + if [ "$VERDICT_RC" -eq 0 ] && ! verify_wal_layout; then + echo "[xce-test-ha] FAIL: chaotic: WAL segment layout inconsistent (bug-1920 signature)" >&2 + cleanup + trap - EXIT INT TERM + exit 1 + fi if [ "$VERDICT_RC" -eq 0 ]; then log "PASS (chaotic): $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs, zero committed divergence, zero committed job loss (bug854=$V_BUG854 bug924=$V_BUG924 excluded)$([ "$WORKER_ENABLED" -eq 1 ] && echo ", ${#WORKER_RESYNC_JOBIDS[@]} worker-resync jobs reconciled")" cleanup @@ -3625,6 +3760,8 @@ done [ -z "$MISSING" ] || fail "not all nodes acted as primary — never led:$MISSING" LED="$(printf '%s\n' "${!SEEN_PRIMARY[@]}" | sort | tr '\n' ',' | sed 's/,$//')" +verify_wal_layout || fail "WAL segment layout inconsistent (bug-1920 signature)" + log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss; all 3 nodes led ($LED)$([ "$WORKER_ENABLED" -eq 1 ] && echo "; $ROUNDS worker-resync jobs reconciled")" cleanup trap - EXIT INT TERM diff --git a/.claude/skills/xce-test-ha/wal-layout-check.py b/.claude/skills/xce-test-ha/wal-layout-check.py new file mode 100755 index 00000000..cf3c0493 --- /dev/null +++ b/.claude/skills/xce-test-ha/wal-layout-check.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""wal-layout-check.py -- check every controller's WAL segment layout (bug-1920). + +Each WAL segment file is named seg-.wal and must hold one +contiguous LSN run starting at that LSN. The catch-up streamer +(ha_stream_wal_to_standby) orders segments by filename and skips a segment +whose successor starts at or below the standby's position, so an LSN filed +under the wrong segment name (bug-1920: a replicated write before boot +recovery truncated and reused seg-...0001) is never streamed and a standby +that needs it stalls. + +Usage: wal-layout-check.py (scans /controller/*/data/wal) +Exit 0 when every segment is consistent, 1 otherwise (one line per problem). +""" +import glob +import os +import struct +import sys + +HDR = struct.Struct("