fix(test-harness): bug-1449 wait for initial config convergence before round-1 kill
run-ha-soak.sh's sequential mode waited for full 3-of-3 reconvergence between rounds 2+ but had no equivalent wait before round 1's first kill. A slow cold-start election could let the kill land before config-push reached both standbys, leaving them at config tag=0 -- ha_on_primary_ctrl_fd_rdhup's voter-only guard then correctly refuses to let an unconfigured standby self-elect, producing a false "no primary elected" round-1 failure. Adds node_config_tag/ cluster_config_converged/wait_for_initial_config_convergence, mirroring the existing reconvergence-wait's polling style, and gates sequential mode's round 1 on it.
This commit is contained in:
parent
7512dc83f0
commit
4d2c90389f
1 changed files with 77 additions and 0 deletions
|
|
@ -553,6 +553,65 @@ wait_for_full_convergence() {
|
|||
return 1
|
||||
}
|
||||
|
||||
# Config version ("tag") the given node currently has loaded for cluster
|
||||
# membership — 0 is the cold-start default before the elected primary's
|
||||
# post-election config-push (ctrlhello) has reached and been applied. Same
|
||||
# direct single-node client-probe technique as node_probe (bug-1449: the
|
||||
# config-push marker is only visible via the buffered logfile otherwise, same
|
||||
# lag problem as everything else in this section). Echoes the numeric
|
||||
# configVersion, or empty with rc!=0 if the node is unreachable.
|
||||
node_config_tag() {
|
||||
local node="$1" addr out tag
|
||||
addr="localhost:${PORT[$node]}"
|
||||
out="$(printf 'set masters %s\nset myname conv\nclient\nctrlstatus\nquit\n' "$addr" | xtest)"
|
||||
tag="$(printf '%s\n' "$out" | sed -n 's/^configVersion: \([0-9][0-9]*\)$/\1/p' | head -1)"
|
||||
[ -n "$tag" ] || return 1
|
||||
echo "$tag"
|
||||
}
|
||||
|
||||
# True (rc 0) iff every one of the 3 nodes reports a non-zero config tag — the
|
||||
# elected primary's config-push has reached and been applied by both standbys.
|
||||
# This is the "3-of-3 initial config convergence" signal for round 1's first
|
||||
# kill (bug-1449): a standby still at tag=0 is VOTER ONLY
|
||||
# (ha-config-bootstrap Req 2.8's ha_on_primary_ctrl_fd_rdhup guard correctly
|
||||
# refuses to let it self-elect), so killing the primary before this converges
|
||||
# can leave the cluster with no candidate at all — even though the guard
|
||||
# itself is working exactly as designed.
|
||||
cluster_config_converged() {
|
||||
local n tag
|
||||
for n in "${NODES[@]}"; do
|
||||
tag="$(node_config_tag "$n")" || return 1
|
||||
[ "$tag" != "0" ] || return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
# Wait until all 3 nodes report a non-zero config tag (bug-1449). Mirrors
|
||||
# wait_for_full_convergence's polling primitive exactly (alive-count guard,
|
||||
# STABLE_TICKS consecutive converged probes, ~90s bound) but gates on
|
||||
# cluster_config_converged (config-push landed) instead of
|
||||
# cluster_committed_converged (committed-prefix caught up) — the two are
|
||||
# independent signals, and round 1 needs the former specifically: nothing else
|
||||
# in this script waits for it before the very first kill.
|
||||
wait_for_initial_config_convergence() {
|
||||
local i alive stable=0
|
||||
local STABLE_TICKS=2 # 2 consecutive converged probes ⇒ settled
|
||||
for i in $(seq 1 30); do
|
||||
alive="$(pgrep -c -f -- "$(run_pattern)")"
|
||||
if [ "$alive" -lt 3 ]; then
|
||||
stable=0; sleep 2; continue
|
||||
fi
|
||||
if cluster_config_converged; then
|
||||
stable=$((stable + 1))
|
||||
[ "$stable" -ge "$STABLE_TICKS" ] && return 0
|
||||
else
|
||||
stable=0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Submit jobs through the live primary until N are cleanly acked, appending each
|
||||
# acked JobId to ALL_JOBIDS. A FRESHLY-promoted primary is briefly not write-able
|
||||
# (it appends its own-term NOOP but cannot commit until a standby reconnects and
|
||||
|
|
@ -1894,6 +1953,24 @@ if [ "$MODE" = "chaotic" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# ── Round-1 config-convergence gate (bug-1449) ────────────────────────────────
|
||||
# The sequential loop below (Step 10) blocks for full 3-of-3 reconvergence
|
||||
# between rounds, but round 1 has no equivalent wait before its FIRST kill — it
|
||||
# falls straight from Step 5 (initial submit+verify) into killing whatever
|
||||
# primary Step 4 found. Cold-start election + config-push can occasionally take
|
||||
# longer than Steps 4-5's budget to land on both standbys: if round 1's kill
|
||||
# fires first, the standbys are still at config tag=0 and
|
||||
# ha_on_primary_ctrl_fd_rdhup's voter-only guard (ha-config-bootstrap Req 2.8,
|
||||
# working exactly as designed) correctly refuses to let either self-elect,
|
||||
# producing a false "no primary elected" round-1 failure (hit once in ~11
|
||||
# Group-1 kiro-validate-impl soaks, cleared on immediate retry). Close that gap
|
||||
# the same way every later round is already protected: block here for 3-of-3
|
||||
# initial config convergence before the loop's first kill.
|
||||
log "waiting for 3-of-3 initial config convergence before round 1"
|
||||
wait_for_initial_config_convergence \
|
||||
|| fail "initial config convergence: standbys did not reach a non-zero config tag within budget (bug-1449 guard)"
|
||||
log "3-of-3 initial config convergence confirmed"
|
||||
|
||||
# ── Steps 7–10 looped: kill primary, verify, submit, restart, reconverge ─────
|
||||
#
|
||||
# Each round kills the CURRENT primary, then restarts the node downed in the
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue