fix(test-harness): bug-1449 wait for initial config convergence before round-1 kill

run-ha-soak.sh's sequential mode waited for full 3-of-3 reconvergence
between rounds 2+ but had no equivalent wait before round 1's first
kill. A slow cold-start election could let the kill land before
config-push reached both standbys, leaving them at config tag=0 --
ha_on_primary_ctrl_fd_rdhup's voter-only guard then correctly refuses
to let an unconfigured standby self-elect, producing a false "no
primary elected" round-1 failure. Adds node_config_tag/
cluster_config_converged/wait_for_initial_config_convergence,
mirroring the existing reconvergence-wait's polling style, and gates
sequential mode's round 1 on it.
This commit is contained in:
Rongsong Shen 2026-07-18 13:27:48 +00:00
parent 7512dc83f0
commit 4d2c90389f

View file

@ -553,6 +553,65 @@ wait_for_full_convergence() {
return 1
}
# Config version ("tag") the given node currently has loaded for cluster
# membership — 0 is the cold-start default before the elected primary's
# post-election config-push (ctrlhello) has reached and been applied. Same
# direct single-node client-probe technique as node_probe (bug-1449: the
# config-push marker is only visible via the buffered logfile otherwise, same
# lag problem as everything else in this section). Echoes the numeric
# configVersion, or empty with rc!=0 if the node is unreachable.
node_config_tag() {
local node="$1" addr out tag
addr="localhost:${PORT[$node]}"
out="$(printf 'set masters %s\nset myname conv\nclient\nctrlstatus\nquit\n' "$addr" | xtest)"
tag="$(printf '%s\n' "$out" | sed -n 's/^configVersion: \([0-9][0-9]*\)$/\1/p' | head -1)"
[ -n "$tag" ] || return 1
echo "$tag"
}
# True (rc 0) iff every one of the 3 nodes reports a non-zero config tag — the
# elected primary's config-push has reached and been applied by both standbys.
# This is the "3-of-3 initial config convergence" signal for round 1's first
# kill (bug-1449): a standby still at tag=0 is VOTER ONLY
# (ha-config-bootstrap Req 2.8's ha_on_primary_ctrl_fd_rdhup guard correctly
# refuses to let it self-elect), so killing the primary before this converges
# can leave the cluster with no candidate at all — even though the guard
# itself is working exactly as designed.
cluster_config_converged() {
local n tag
for n in "${NODES[@]}"; do
tag="$(node_config_tag "$n")" || return 1
[ "$tag" != "0" ] || return 1
done
return 0
}
# Wait until all 3 nodes report a non-zero config tag (bug-1449). Mirrors
# wait_for_full_convergence's polling primitive exactly (alive-count guard,
# STABLE_TICKS consecutive converged probes, ~90s bound) but gates on
# cluster_config_converged (config-push landed) instead of
# cluster_committed_converged (committed-prefix caught up) — the two are
# independent signals, and round 1 needs the former specifically: nothing else
# in this script waits for it before the very first kill.
wait_for_initial_config_convergence() {
local i alive stable=0
local STABLE_TICKS=2 # 2 consecutive converged probes ⇒ settled
for i in $(seq 1 30); do
alive="$(pgrep -c -f -- "$(run_pattern)")"
if [ "$alive" -lt 3 ]; then
stable=0; sleep 2; continue
fi
if cluster_config_converged; then
stable=$((stable + 1))
[ "$stable" -ge "$STABLE_TICKS" ] && return 0
else
stable=0
fi
sleep 2
done
return 1
}
# Submit jobs through the live primary until N are cleanly acked, appending each
# acked JobId to ALL_JOBIDS. A FRESHLY-promoted primary is briefly not write-able
# (it appends its own-term NOOP but cannot commit until a standby reconnects and
@ -1894,6 +1953,24 @@ if [ "$MODE" = "chaotic" ]; then
exit 1
fi
# ── Round-1 config-convergence gate (bug-1449) ────────────────────────────────
# The sequential loop below (Step 10) blocks for full 3-of-3 reconvergence
# between rounds, but round 1 has no equivalent wait before its FIRST kill — it
# falls straight from Step 5 (initial submit+verify) into killing whatever
# primary Step 4 found. Cold-start election + config-push can occasionally take
# longer than Steps 4-5's budget to land on both standbys: if round 1's kill
# fires first, the standbys are still at config tag=0 and
# ha_on_primary_ctrl_fd_rdhup's voter-only guard (ha-config-bootstrap Req 2.8,
# working exactly as designed) correctly refuses to let either self-elect,
# producing a false "no primary elected" round-1 failure (hit once in ~11
# Group-1 kiro-validate-impl soaks, cleared on immediate retry). Close that gap
# the same way every later round is already protected: block here for 3-of-3
# initial config convergence before the loop's first kill.
log "waiting for 3-of-3 initial config convergence before round 1"
wait_for_initial_config_convergence \
|| fail "initial config convergence: standbys did not reach a non-zero config tag within budget (bug-1449 guard)"
log "3-of-3 initial config convergence confirmed"
# ── Steps 7–10 looped: kill primary, verify, submit, restart, reconverge ─────
#
# Each round kills the CURRENT primary, then restarts the node downed in the