diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index b249290c..bab90fc7 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -553,6 +553,65 @@ wait_for_full_convergence() { return 1 } +# Config version ("tag") the given node currently has loaded for cluster +# membership — 0 is the cold-start default before the elected primary's +# post-election config-push (ctrlhello) has reached and been applied. Same +# direct single-node client-probe technique as node_probe (bug-1449: the +# config-push marker is only visible via the buffered logfile otherwise, same +# lag problem as everything else in this section). Echoes the numeric +# configVersion, or empty with rc!=0 if the node is unreachable. +node_config_tag() { + local node="$1" addr out tag + addr="localhost:${PORT[$node]}" + out="$(printf 'set masters %s\nset myname conv\nclient\nctrlstatus\nquit\n' "$addr" | xtest)" + tag="$(printf '%s\n' "$out" | sed -n 's/^configVersion: \([0-9][0-9]*\)$/\1/p' | head -1)" + [ -n "$tag" ] || return 1 + echo "$tag" +} + +# True (rc 0) iff every one of the 3 nodes reports a non-zero config tag — the +# elected primary's config-push has reached and been applied by both standbys. +# This is the "3-of-3 initial config convergence" signal for round 1's first +# kill (bug-1449): a standby still at tag=0 is VOTER ONLY +# (ha-config-bootstrap Req 2.8's ha_on_primary_ctrl_fd_rdhup guard correctly +# refuses to let it self-elect), so killing the primary before this converges +# can leave the cluster with no candidate at all — even though the guard +# itself is working exactly as designed. +cluster_config_converged() { + local n tag + for n in "${NODES[@]}"; do + tag="$(node_config_tag "$n")" || return 1 + [ "$tag" != "0" ] || return 1 + done + return 0 +} + +# Wait until all 3 nodes report a non-zero config tag (bug-1449). Mirrors +# wait_for_full_convergence's polling primitive exactly (alive-count guard, +# STABLE_TICKS consecutive converged probes, ~90s bound) but gates on +# cluster_config_converged (config-push landed) instead of +# cluster_committed_converged (committed-prefix caught up) — the two are +# independent signals, and round 1 needs the former specifically: nothing else +# in this script waits for it before the very first kill. +wait_for_initial_config_convergence() { + local i alive stable=0 + local STABLE_TICKS=2 # 2 consecutive converged probes ⇒ settled + for i in $(seq 1 30); do + alive="$(pgrep -c -f -- "$(run_pattern)")" + if [ "$alive" -lt 3 ]; then + stable=0; sleep 2; continue + fi + if cluster_config_converged; then + stable=$((stable + 1)) + [ "$stable" -ge "$STABLE_TICKS" ] && return 0 + else + stable=0 + fi + sleep 2 + done + return 1 +} + # Submit jobs through the live primary until N are cleanly acked, appending each # acked JobId to ALL_JOBIDS. A FRESHLY-promoted primary is briefly not write-able # (it appends its own-term NOOP but cannot commit until a standby reconnects and @@ -1894,6 +1953,24 @@ if [ "$MODE" = "chaotic" ]; then exit 1 fi +# ── Round-1 config-convergence gate (bug-1449) ──────────────────────────────── +# The sequential loop below (Step 10) blocks for full 3-of-3 reconvergence +# between rounds, but round 1 has no equivalent wait before its FIRST kill — it +# falls straight from Step 5 (initial submit+verify) into killing whatever +# primary Step 4 found. Cold-start election + config-push can occasionally take +# longer than Steps 4-5's budget to land on both standbys: if round 1's kill +# fires first, the standbys are still at config tag=0 and +# ha_on_primary_ctrl_fd_rdhup's voter-only guard (ha-config-bootstrap Req 2.8, +# working exactly as designed) correctly refuses to let either self-elect, +# producing a false "no primary elected" round-1 failure (hit once in ~11 +# Group-1 kiro-validate-impl soaks, cleared on immediate retry). Close that gap +# the same way every later round is already protected: block here for 3-of-3 +# initial config convergence before the loop's first kill. +log "waiting for 3-of-3 initial config convergence before round 1" +wait_for_initial_config_convergence \ + || fail "initial config convergence: standbys did not reach a non-zero config tag within budget (bug-1449 guard)" +log "3-of-3 initial config convergence confirmed" + # ── Steps 7–10 looped: kill primary, verify, submit, restart, reconverge ───── # # Each round kills the CURRENT primary, then restarts the node downed in the