fix(test-harness): bug-1449 wait for initial config convergence before round-1 kill
run-ha-soak.sh's sequential mode waited for full 3-of-3 reconvergence between rounds 2+ but had no equivalent wait before round 1's first kill. A slow cold-start election could let the kill land before config-push reached both standbys, leaving them at config tag=0 -- ha_on_primary_ctrl_fd_rdhup's voter-only guard then correctly refuses to let an unconfigured standby self-elect, producing a false "no primary elected" round-1 failure. Adds node_config_tag/ cluster_config_converged/wait_for_initial_config_convergence, mirroring the existing reconvergence-wait's polling style, and gates sequential mode's round 1 on it.
This commit is contained in:
parent
7512dc83f0
commit
4d2c90389f
1 changed files with 77 additions and 0 deletions
|
|
@ -553,6 +553,65 @@ wait_for_full_convergence() {
|
||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# Config version ("tag") the given node currently has loaded for cluster
|
||||||
|
# membership — 0 is the cold-start default before the elected primary's
|
||||||
|
# post-election config-push (ctrlhello) has reached and been applied. Same
|
||||||
|
# direct single-node client-probe technique as node_probe (bug-1449: the
|
||||||
|
# config-push marker is only visible via the buffered logfile otherwise, same
|
||||||
|
# lag problem as everything else in this section). Echoes the numeric
|
||||||
|
# configVersion, or empty with rc!=0 if the node is unreachable.
|
||||||
|
node_config_tag() {
|
||||||
|
local node="$1" addr out tag
|
||||||
|
addr="localhost:${PORT[$node]}"
|
||||||
|
out="$(printf 'set masters %s\nset myname conv\nclient\nctrlstatus\nquit\n' "$addr" | xtest)"
|
||||||
|
tag="$(printf '%s\n' "$out" | sed -n 's/^configVersion: \([0-9][0-9]*\)$/\1/p' | head -1)"
|
||||||
|
[ -n "$tag" ] || return 1
|
||||||
|
echo "$tag"
|
||||||
|
}
|
||||||
|
|
||||||
|
# True (rc 0) iff every one of the 3 nodes reports a non-zero config tag — the
|
||||||
|
# elected primary's config-push has reached and been applied by both standbys.
|
||||||
|
# This is the "3-of-3 initial config convergence" signal for round 1's first
|
||||||
|
# kill (bug-1449): a standby still at tag=0 is VOTER ONLY
|
||||||
|
# (ha-config-bootstrap Req 2.8's ha_on_primary_ctrl_fd_rdhup guard correctly
|
||||||
|
# refuses to let it self-elect), so killing the primary before this converges
|
||||||
|
# can leave the cluster with no candidate at all — even though the guard
|
||||||
|
# itself is working exactly as designed.
|
||||||
|
cluster_config_converged() {
|
||||||
|
local n tag
|
||||||
|
for n in "${NODES[@]}"; do
|
||||||
|
tag="$(node_config_tag "$n")" || return 1
|
||||||
|
[ "$tag" != "0" ] || return 1
|
||||||
|
done
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# Wait until all 3 nodes report a non-zero config tag (bug-1449). Mirrors
|
||||||
|
# wait_for_full_convergence's polling primitive exactly (alive-count guard,
|
||||||
|
# STABLE_TICKS consecutive converged probes, ~90s bound) but gates on
|
||||||
|
# cluster_config_converged (config-push landed) instead of
|
||||||
|
# cluster_committed_converged (committed-prefix caught up) — the two are
|
||||||
|
# independent signals, and round 1 needs the former specifically: nothing else
|
||||||
|
# in this script waits for it before the very first kill.
|
||||||
|
wait_for_initial_config_convergence() {
|
||||||
|
local i alive stable=0
|
||||||
|
local STABLE_TICKS=2 # 2 consecutive converged probes ⇒ settled
|
||||||
|
for i in $(seq 1 30); do
|
||||||
|
alive="$(pgrep -c -f -- "$(run_pattern)")"
|
||||||
|
if [ "$alive" -lt 3 ]; then
|
||||||
|
stable=0; sleep 2; continue
|
||||||
|
fi
|
||||||
|
if cluster_config_converged; then
|
||||||
|
stable=$((stable + 1))
|
||||||
|
[ "$stable" -ge "$STABLE_TICKS" ] && return 0
|
||||||
|
else
|
||||||
|
stable=0
|
||||||
|
fi
|
||||||
|
sleep 2
|
||||||
|
done
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
# Submit jobs through the live primary until N are cleanly acked, appending each
|
# Submit jobs through the live primary until N are cleanly acked, appending each
|
||||||
# acked JobId to ALL_JOBIDS. A FRESHLY-promoted primary is briefly not write-able
|
# acked JobId to ALL_JOBIDS. A FRESHLY-promoted primary is briefly not write-able
|
||||||
# (it appends its own-term NOOP but cannot commit until a standby reconnects and
|
# (it appends its own-term NOOP but cannot commit until a standby reconnects and
|
||||||
|
|
@ -1894,6 +1953,24 @@ if [ "$MODE" = "chaotic" ]; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# ── Round-1 config-convergence gate (bug-1449) ────────────────────────────────
|
||||||
|
# The sequential loop below (Step 10) blocks for full 3-of-3 reconvergence
|
||||||
|
# between rounds, but round 1 has no equivalent wait before its FIRST kill — it
|
||||||
|
# falls straight from Step 5 (initial submit+verify) into killing whatever
|
||||||
|
# primary Step 4 found. Cold-start election + config-push can occasionally take
|
||||||
|
# longer than Steps 4-5's budget to land on both standbys: if round 1's kill
|
||||||
|
# fires first, the standbys are still at config tag=0 and
|
||||||
|
# ha_on_primary_ctrl_fd_rdhup's voter-only guard (ha-config-bootstrap Req 2.8,
|
||||||
|
# working exactly as designed) correctly refuses to let either self-elect,
|
||||||
|
# producing a false "no primary elected" round-1 failure (hit once in ~11
|
||||||
|
# Group-1 kiro-validate-impl soaks, cleared on immediate retry). Close that gap
|
||||||
|
# the same way every later round is already protected: block here for 3-of-3
|
||||||
|
# initial config convergence before the loop's first kill.
|
||||||
|
log "waiting for 3-of-3 initial config convergence before round 1"
|
||||||
|
wait_for_initial_config_convergence \
|
||||||
|
|| fail "initial config convergence: standbys did not reach a non-zero config tag within budget (bug-1449 guard)"
|
||||||
|
log "3-of-3 initial config convergence confirmed"
|
||||||
|
|
||||||
# ── Steps 7–10 looped: kill primary, verify, submit, restart, reconverge ─────
|
# ── Steps 7–10 looped: kill primary, verify, submit, restart, reconverge ─────
|
||||||
#
|
#
|
||||||
# Each round kills the CURRENT primary, then restarts the node downed in the
|
# Each round kills the CURRENT primary, then restarts the node downed in the
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue