Rongsong Shen
294b9ce8eb
Verdict: raft-drive-async introduces NO new regression.
- Unit: controller_unit_test EXIT=0, xcelib_test 282/0 (all HA suites green;
dispatch/snapshot/configpush flags default OFF, inert).
- Live kill-based failover soak on a real fs (~/tmp), release + ASAN builds:
zero committed-JOB loss every completed round (jobs 3->12 monotonic).
- The 6-round CHAOTIC soak does not run clean to round 6 (fails ~round 4), but a
baseline comparison against pre-Phase-3 binaries (commit 21e26a1) fails round 4
IDENTICALLY (election stall / reconverge). The round-4 committed-prefix
divergence at LSN 9 is the documented pre-existing bug-851/795 family
(chaotic-failover silent divergence), present before Phase 3 and out of scope
for this spec. Logged as bug-1225 (baseline-confirmed).
Also: make the xce-test-ha soak driver's controller/xce_test paths
env-overridable (XCE_HA_BIN / XCE_HA_XT, backward-compatible defaults) — used to
run the baseline binaries through the same driver for the regression comparison.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S6fDCtXkkEgupu7rUah1y2
494 lines
23 KiB
Bash
Executable file
494 lines
23 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
#
|
||
# run-ha-soak.sh — driver for /xce-test-ha: deploy a 3-node XCE HA cluster, then
|
||
# repeatedly kill the primary, verify no committed job is ever lost, restart the
|
||
# downed node, and loop.
|
||
#
|
||
# Exit 0 == PASS (no job loss across all rounds); non-zero == FAIL.
|
||
#
|
||
# Must be invoked under `nix develop ./src --command bash ...` so the controller
|
||
# and xce_test runtime libs (openssl etc.) are on the library path.
|
||
#
|
||
# Usage:
|
||
# run-ha-soak.sh [DEPLOY_DIR]
|
||
# DEPLOY_DIR optional deploy directory (positional). Precedence:
|
||
# positional arg > $XCE_HA_TOP env > default. Relative paths
|
||
# are resolved to absolute.
|
||
#
|
||
# Env overrides:
|
||
# XCE_HA_TOP deploy dir (default /home/claude/tmp/xce-ha-deploy)
|
||
# XCE_HA_DATADIR config/certs source (default <repo>/ha/data)
|
||
# XCE_HA_ROUNDS kill/restart rounds (default 6, clamp 5..10)
|
||
# XCE_HA_JOBS jobs per submit batch (default 3)
|
||
# XCE_HA_WAITMS wait-primary budget (default 30000)
|
||
set -u
|
||
|
||
# ── Paths ────────────────────────────────────────────────────────────────────
|
||
REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||
# Deploy dir: explicit positional arg wins, then $XCE_HA_TOP, then the default.
|
||
TOP="${1:-${XCE_HA_TOP:-/home/claude/tmp/xce-ha-deploy}}"
|
||
# Make a relative dir absolute (against cwd) so it is unambiguous to deploy-ha.sh
|
||
# and the per-node scripts. Pure-bash — no realpath dependency (busybox realpath
|
||
# here lacks -m and would mangle nonexistent/relative paths).
|
||
case "$TOP" in
|
||
/*) : ;; # already absolute
|
||
*) TOP="$PWD/$TOP" ;; # resolve relative path against current directory
|
||
esac
|
||
# Sentinel written into a deploy we own; the guard before `rm -rf` only wipes a
|
||
# non-empty dir that carries this marker (see "Step 2: deploy" below).
|
||
MARKER_NAME=".xce-ha-deploy"
|
||
DATADIR="${XCE_HA_DATADIR:-$REPO/ha/data}"
|
||
BIN="${XCE_HA_BIN:-$REPO/src/xce.build/controller/xce_controller}"
|
||
XT="${XCE_HA_XT:-$REPO/src/xce.build/controller/xce_test}"
|
||
DEPLOY="$REPO/src/tools/deploy-ha.sh"
|
||
|
||
ROUNDS="${XCE_HA_ROUNDS:-6}"
|
||
[ "$ROUNDS" -lt 5 ] && ROUNDS=5
|
||
[ "$ROUNDS" -gt 10 ] && ROUNDS=10
|
||
NJOBS="${XCE_HA_JOBS:-3}"
|
||
WAITMS="${XCE_HA_WAITMS:-30000}"
|
||
|
||
NODES=(ctrl-1 ctrl-2 ctrl-3)
|
||
declare -A PORT=( [ctrl-1]=9991 [ctrl-2]=9992 [ctrl-3]=9993 )
|
||
PEERS="ctrl-1=localhost:9991,ctrl-2=localhost:9992,ctrl-3=localhost:9993"
|
||
MASTERS="localhost:9991,localhost:9992,localhost:9993"
|
||
|
||
ALL_JOBIDS=() # every JobId we have ever successfully submitted
|
||
declare -A SEEN_PRIMARY # node name -> 1 once it has served as primary
|
||
|
||
log() { echo "[xce-test-ha] $*"; }
|
||
fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
|
||
|
||
cleanup() { pkill -9 -x xce_controller 2>/dev/null; sleep 0.3; }
|
||
|
||
# Record that a node has acted as primary at least once (leadership-coverage
|
||
# tracking — the run asserts all three nodes lead, see the verdict).
|
||
record_primary() { [ -n "$1" ] && SEEN_PRIMARY["$1"]=1; }
|
||
|
||
# kill -9 a controller by node name. Match on BINARY + --myname so the pattern
|
||
# cannot hit this driver's own shell snapshot (its argv carries "--myname ctrl-N").
|
||
kill_node() {
|
||
local node="$1" pid
|
||
pid="$(pgrep -f -- "xce_controller --myname $node ")"
|
||
[ -n "$pid" ] || return 1
|
||
kill -9 $pid 2>/dev/null
|
||
return 0
|
||
}
|
||
|
||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||
[ -x "$BIN" ] || { echo "[xce-test-ha] FAIL: controller binary missing: $BIN (run /xce-build controller)" >&2; exit 2; }
|
||
[ -x "$XT" ] || { echo "[xce-test-ha] FAIL: xce_test missing: $XT (run /xce-build controller)" >&2; exit 2; }
|
||
[ -r "$DATADIR/cluster.bin" ] || { echo "[xce-test-ha] FAIL: cluster.bin missing under $DATADIR" >&2; exit 2; }
|
||
[ -r "$DATADIR/xce.crt" ] && [ -r "$DATADIR/xce.key" ] || { echo "[xce-test-ha] FAIL: certs missing under $DATADIR" >&2; exit 2; }
|
||
|
||
trap cleanup EXIT INT TERM
|
||
|
||
# ── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
# Run a batch of xce_test commands (stdin script) with a hard timeout.
|
||
# Disable ASAN leak detection: the debug/ASAN xce_test build reports harmless
|
||
# heap leaks at exit (non-zero exit code) that would otherwise be misread as a
|
||
# command failure. Job-loss is judged by the command OUTPUT (expect-job "not
|
||
# found"), not the leak-polluted process exit code.
|
||
xtest() { ASAN_OPTIONS=detect_leaks=0 timeout 60 "$XT" 2>&1; }
|
||
|
||
start_node() { bash "$TOP/scripts/start.sh" "$1" -vvv >/dev/null 2>&1 & }
|
||
|
||
# The INFO-level promotion marker each elected primary logs (ha.c: raft_cb_
|
||
# become_leader → "HA FSM (raft): → CTRL_PRIMARY (term=N, ...)"). Present even
|
||
# without -vvv. We track election by this line + its term, which is the most
|
||
# reliable "a new primary was elected" signal (independent of client-side
|
||
# discovery). NOTE: the arrow is a UTF-8 "→"; grep with a wildcard to avoid
|
||
# encoding pitfalls.
|
||
PRIMARY_LOG_RE='HA FSM .raft.: . CTRL_PRIMARY .term='
|
||
|
||
# Highest term any node has announced CTRL_PRIMARY for, across all logs.
|
||
latest_primary_term() {
|
||
grep -ohE "$PRIMARY_LOG_RE[0-9]+" "$TOP"/logs/*.log 2>/dev/null \
|
||
| grep -oE 'term=[0-9]+' | grep -oE '[0-9]+' | sort -n | tail -1
|
||
}
|
||
|
||
# The node name that most recently logged CTRL_PRIMARY (the elected primary).
|
||
# Echoes "ctrl-N" or "" — purely from the logs, no client round-trip.
|
||
primary_from_log() {
|
||
local n best_term="" best_node="" t
|
||
for n in "${NODES[@]}"; do
|
||
t="$(grep -oE "$PRIMARY_LOG_RE[0-9]+" "$TOP/logs/$n.log" 2>/dev/null \
|
||
| grep -oE 'term=[0-9]+' | grep -oE '[0-9]+' | sort -n | tail -1)"
|
||
[ -z "$t" ] && continue
|
||
if [ -z "$best_term" ] || [ "$t" -ge "$best_term" ]; then
|
||
best_term="$t"; best_node="$n"
|
||
fi
|
||
done
|
||
echo "$best_node"
|
||
}
|
||
|
||
# Block until a node announces CTRL_PRIMARY at a term STRICTLY GREATER than
|
||
# `$1` (the term before the kill), i.e. a genuinely NEW election has happened.
|
||
# Echoes the new primary's node name on success; returns non-zero on timeout.
|
||
wait_for_new_primary() {
|
||
local prev_term="${1:-0}" i t node
|
||
for i in $(seq 1 45); do
|
||
t="$(latest_primary_term)"
|
||
if [ -n "$t" ] && [ "$t" -gt "$prev_term" ]; then
|
||
node="$(primary_from_log)"
|
||
[ -n "$node" ] && { echo "$node"; return 0; }
|
||
fi
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Wait until SOME node logs CTRL_PRIMARY (cold-start election), up to ~45s.
|
||
wait_for_primary_log() {
|
||
local i
|
||
for i in $(seq 1 45); do
|
||
[ -n "$(latest_primary_term)" ] && return 0
|
||
sleep 1
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Identify the current primary by asking the cluster (authoritative ctrlstatus,
|
||
# which internally rotates to the serving primary). Echoes "ctrl-N" or "".
|
||
# Retries the whole wait-primary+ctrlstatus probe for up to ~60s so it spans a
|
||
# full failover election (liveness timeout ~10s + PreVote stagger + election).
|
||
# This is the SERVING check (the client can actually reach the primary), distinct
|
||
# from the log-based election check above.
|
||
discover_primary() {
|
||
local out name i
|
||
for i in $(seq 1 6); do
|
||
out="$(printf 'set masters %s\nset myname probe\nclient\nwait-primary %s\nctrlstatus\nquit\n' \
|
||
"$MASTERS" "$WAITMS" | xtest)"
|
||
name="$(echo "$out" | sed -n 's/^Primary: \(ctrl-[0-9]\)$/\1/p' | head -1)"
|
||
if [ -n "$name" ]; then
|
||
echo "$name"
|
||
return 0
|
||
fi
|
||
sleep 2
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Wait until the cluster is FULLY converged: all 3 controllers are alive AND the
|
||
# current primary is no longer actively streaming catch-up to any standby. This
|
||
# is a HARD precondition before killing the next primary — killing while a just-
|
||
# restarted node is still catching up tests an UNSUPPORTED double-fault-within-
|
||
# one-reconvergence window (a momentary 2-of-3 with a stale member), not the
|
||
# single-failover guarantee.
|
||
#
|
||
# Convergence signal: a lagging standby makes the primary log "stream WAL batches"
|
||
# every heartbeat (~3s); once every standby is caught up the primary STOPS
|
||
# emitting that line (action becomes HA_REPL_CONVERGED, a no-op). So "no new
|
||
# 'stream WAL batches' line on the primary for STABLE_S seconds" == all standbys
|
||
# converged. This is robust to the per-term re-emit quirk of the "standby[i]
|
||
# converged" transition log (which only fires for a standby that was previously
|
||
# NOT converged under the current primary term). Bounded ~90s.
|
||
wait_for_full_convergence() {
|
||
local i alive primary line_now line_prev stable=0
|
||
local STABLE_TICKS=3 # 3 ticks × 2s = 6s with no new stream line ⇒ converged
|
||
line_prev=""
|
||
for i in $(seq 1 45); do
|
||
alive="$(pgrep -c -f -- "xce_controller --myname ctrl-")"
|
||
if [ "$alive" -lt 3 ]; then
|
||
stable=0; sleep 2; continue
|
||
fi
|
||
primary="$(primary_from_log)"
|
||
[ -z "$primary" ] && { sleep 2; continue; }
|
||
# Most recent "stream WAL batches" line in the primary's log (empty if
|
||
# the primary has never streamed / has stopped). Compare across ticks:
|
||
# if it has not changed for STABLE_TICKS, no catch-up is in flight.
|
||
line_now="$(grep -n "stream WAL batches" "$TOP/logs/$primary.log" 2>/dev/null | tail -1)"
|
||
if [ "$line_now" = "$line_prev" ]; then
|
||
stable=$((stable + 1))
|
||
[ "$stable" -ge "$STABLE_TICKS" ] && return 0
|
||
else
|
||
stable=0
|
||
line_prev="$line_now"
|
||
fi
|
||
sleep 2
|
||
done
|
||
return 1
|
||
}
|
||
|
||
# Submit jobs through the live primary until N are cleanly acked, appending each
|
||
# acked JobId to ALL_JOBIDS. A FRESHLY-promoted primary is briefly not write-able
|
||
# (it appends its own-term NOOP but cannot commit until a standby reconnects and
|
||
# write-quorum is restored — log: "HA quorum lost/restored"), so a submit issued
|
||
# in that ~1-3s window fails its 500ms commit WAIT. That is NOT a job loss (the
|
||
# job was never acked, so it is correctly not tracked) — it is a transient that
|
||
# clears once quorum returns. Retry the SHORTFALL up to a few times.
|
||
submit_jobs() {
|
||
local want="$1" attempt acked=0 got script out jid
|
||
for attempt in 1 2 3 4 5; do
|
||
local need=$((want - acked))
|
||
[ "$need" -le 0 ] && break
|
||
script="set masters $MASTERS
|
||
set myname soak
|
||
client
|
||
wait-primary $WAITMS"
|
||
local i
|
||
for i in $(seq 1 "$need"); do
|
||
script+="
|
||
jobsub -u root -q normal -c /bin/true"
|
||
done
|
||
script+="
|
||
quit"
|
||
out="$(printf '%s\n' "$script" | xtest)"
|
||
got=0
|
||
while IFS= read -r jid; do
|
||
ALL_JOBIDS+=("$jid")
|
||
acked=$((acked + 1)); got=$((got + 1))
|
||
done < <(echo "$out" | sed -n 's/^jobsubmit ok: JobId: \(.*\)$/\1/p')
|
||
[ "$acked" -ge "$want" ] && break
|
||
# Shortfall: the primary likely just promoted and is regaining write-
|
||
# quorum. Wait a moment and retry the remainder.
|
||
log "submit attempt $attempt: $got/$need acked (total $acked/$want); waiting for write-quorum"
|
||
sleep 3
|
||
done
|
||
log "submitted $acked/$want jobs"
|
||
[ "$acked" -eq "$want" ]
|
||
}
|
||
|
||
# Verify EVERY JobId in ALL_JOBIDS is present via expect-job on the live primary.
|
||
# Judged by OUTPUT, not exit code (ASAN leaks pollute the exit code): require one
|
||
# "expect-job: ok" per tracked job and zero "not found". Returns non-zero on any
|
||
# missing job or short count.
|
||
verify_no_loss() {
|
||
local script out want oks
|
||
want="${#ALL_JOBIDS[@]}"
|
||
script="set masters $MASTERS
|
||
set myname verify
|
||
client
|
||
wait-primary $WAITMS"
|
||
local jid
|
||
for jid in "${ALL_JOBIDS[@]}"; do
|
||
script+="
|
||
expect-job $jid"
|
||
done
|
||
script+="
|
||
quit"
|
||
out="$(printf '%s\n' "$script" | xtest)"
|
||
if echo "$out" | grep -q "not found"; then
|
||
echo "$out" | grep "not found" >&2
|
||
return 1
|
||
fi
|
||
oks="$(echo "$out" | grep -c "expect-job: ok")"
|
||
if [ "$oks" -ne "$want" ]; then
|
||
echo "verify_no_loss: only $oks/$want jobs confirmed present" >&2
|
||
echo "$out" | tail -8 >&2
|
||
return 1
|
||
fi
|
||
log "verified all $want jobs present"
|
||
return 0
|
||
}
|
||
|
||
# Cross-check committed WAL prefixes across all live members are byte-identical
|
||
# (term+crc). Non-fatal warn-only on the known term-identity caveat; data loss
|
||
# (a member MISSING a committed entry) is the real failure surfaced by expect-job.
|
||
walcheck() {
|
||
local boundary="$1" d1 d2 d3 out
|
||
d1="$TOP/controller/ctrl-1/data"
|
||
d2="$TOP/controller/ctrl-2/data"
|
||
d3="$TOP/controller/ctrl-3/data"
|
||
out="$(printf 'walcompare --committed %s %s %s %s\nquit\n' \
|
||
"$boundary" "$d1" "$d2" "$d3" | xtest)"
|
||
if echo "$out" | grep -qiE "identical|OK"; then
|
||
log "walcompare: committed prefix consistent (boundary=$boundary)"
|
||
else
|
||
log "walcompare note (boundary=$boundary): $(echo "$out" | tail -2 | tr '\n' ' ')"
|
||
fi
|
||
}
|
||
|
||
# Drive a SPECIFIC node to primary — used to exercise leadership on the
|
||
# lowest-priority node (ctrl-3), which the natural single-failover loop NEVER
|
||
# elects: the election timeout is priority-staggered (my_priority * step_ms, p0
|
||
# immediate / p1 +500ms / p2 +1000ms), so whenever ctrl-3 is in the surviving
|
||
# pair a lower-index node out-races it.
|
||
#
|
||
# Mechanism — exploits the RequestVote freshness gate (a behind node cannot win):
|
||
# precondition: cluster 3-of-3 converged, target T caught up, T not primary.
|
||
# P = current primary; S = the third node (not P, not T).
|
||
# 1. kill S (a standby) so it stops receiving entries.
|
||
# 2. submit a batch → it commits on {P, T}; S is now provably BEHIND.
|
||
# 3. kill P → only T alive (brief, deliberate no-quorum). T holds EVERY
|
||
# committed entry (all commits were on {P,T}), so nothing is lost.
|
||
# 4. restart S (still behind — it cannot catch up while there is no primary,
|
||
# so it stays fenced for the whole election): quorum {T fresh, S stale};
|
||
# the freshness gate denies S, and T wins DETERMINISTICALLY (not by timing).
|
||
# 5. verify no loss, restart P, reconverge 3-of-3.
|
||
# Fails the run loudly if T does not win.
|
||
force_elect_node() {
|
||
local T="$1" P S n
|
||
P="$(discover_primary)"
|
||
[ -n "$P" ] || { log "force-elect $T: no serving primary"; return 1; }
|
||
if [ "$P" = "$T" ]; then record_primary "$T"; log "force-elect: $T already primary"; return 0; fi
|
||
S=""
|
||
for n in "${NODES[@]}"; do [ "$n" != "$P" ] && [ "$n" != "$T" ] && S="$n"; done
|
||
[ -n "$S" ] || { log "force-elect $T: could not pick spoiler"; return 1; }
|
||
|
||
log "force-elect $T: primary=$P spoiler=$S (freshness-fence $S so $T wins)"
|
||
local PREV_TERM; PREV_TERM="$(latest_primary_term)"; PREV_TERM="${PREV_TERM:-0}"
|
||
|
||
# 1. Kill the spoiler standby so it falls behind.
|
||
kill_node "$S" || fail "force-elect $T: could not kill spoiler $S"
|
||
sleep 2
|
||
|
||
# 2. Commit a batch on {P,T} — guarantees S is behind and T is caught up.
|
||
submit_jobs "$NJOBS" || fail "force-elect $T: submit on {$P,$T} failed"
|
||
verify_no_loss || fail "force-elect $T: loss after spoiler-down submit"
|
||
|
||
# 3. Kill the primary. Only T is alive (transient no-quorum); T has all commits.
|
||
kill_node "$P" || fail "force-elect $T: could not kill primary $P"
|
||
sleep 2
|
||
|
||
# 4. Restart the (behind) spoiler to restore quorum; T must win on freshness.
|
||
log "force-elect $T: restarting behind-spoiler $S — expecting $T to win"
|
||
start_node "$S"
|
||
NEWP="$(wait_for_new_primary "$PREV_TERM")"
|
||
[ -n "$NEWP" ] || fail "force-elect $T: no new primary after restoring quorum"
|
||
[ "$NEWP" = "$T" ] || fail "force-elect $T: expected $T to win, got $NEWP (freshness-fence failed)"
|
||
record_primary "$T"
|
||
PRIMARY="$(discover_primary)"
|
||
[ -n "$PRIMARY" ] || fail "force-elect $T: $T elected but client cannot reach it"
|
||
verify_no_loss || fail "force-elect $T: committed job LOST during maneuver"
|
||
log "force-elect $T: SUCCESS — $T is primary (term=$(latest_primary_term))"
|
||
|
||
# 5. Restart P and reconverge to a healthy 3-of-3.
|
||
start_node "$P"
|
||
wait_for_full_convergence || fail "force-elect $T: cluster did not reconverge 3-of-3 after $T led"
|
||
return 0
|
||
}
|
||
|
||
# ── Step 2: deploy ───────────────────────────────────────────────────────────
|
||
log "deploying 3-node cluster to $TOP"
|
||
cleanup
|
||
# Safety guard: the deploy is destructive (rm -rf). Since the deploy dir can now
|
||
# be any path the caller names, refuse to wipe a non-empty directory unless it
|
||
# carries the sentinel from a prior xce-test-ha run — protects an arbitrary
|
||
# user-named dir (e.g. a typo'd home/project path) from being nuked. An empty or
|
||
# non-existent dir is always safe to (re)deploy into.
|
||
if [ -d "$TOP" ] && [ -n "$(ls -A "$TOP" 2>/dev/null)" ] && [ ! -e "$TOP/$MARKER_NAME" ]; then
|
||
fail "refusing to wipe non-empty $TOP — not a prior xce-test-ha deploy (no $MARKER_NAME marker). Choose an empty/new dir or remove it yourself."
|
||
fi
|
||
rm -rf "$TOP"
|
||
mkdir -p "$TOP" || fail "could not create deploy dir $TOP"
|
||
touch "$TOP/$MARKER_NAME" # claim ownership so a later run may safely re-wipe it
|
||
bash "$DEPLOY" "$BIN" "$DATADIR" "$TOP" "localhost:9991,localhost:9992,localhost:9993" \
|
||
>"$TOP/deploy.log" 2>&1 || fail "deploy-ha.sh failed (see $TOP/deploy.log)"
|
||
|
||
# ── Step 3: start all nodes ──────────────────────────────────────────────────
|
||
log "starting ctrl-1 / ctrl-2 / ctrl-3 (-vvv)"
|
||
for n in "${NODES[@]}"; do
|
||
start_node "$n"
|
||
sleep 0.5
|
||
done
|
||
|
||
# ── Step 4: confirm a primary was elected ────────────────────────────────────
|
||
log "waiting for primary election (cold start ~15s)"
|
||
wait_for_primary_log || fail "no primary elected within 45s (check $TOP/logs)"
|
||
PRIMARY="$(discover_primary)"
|
||
[ -n "$PRIMARY" ] || fail "primary elected in logs but client could not reach it"
|
||
record_primary "$PRIMARY"
|
||
log "primary is $PRIMARY"
|
||
|
||
# ── Step 5: initial submit + verify + waldump ────────────────────────────────
|
||
submit_jobs "$NJOBS" || fail "initial job submission did not get $NJOBS clean acks"
|
||
verify_no_loss || fail "initial joblist verification failed"
|
||
walcheck "$((1 + 2 * ${#ALL_JOBIDS[@]}))"
|
||
|
||
# ── Steps 7–10 looped: kill primary, verify, submit, restart, reconverge ─────
|
||
#
|
||
# Each round kills the CURRENT primary, then restarts the node downed in the
|
||
# previous round, then BLOCKS until the cluster is fully 3-of-3 converged before
|
||
# the next round's kill. The convergence gate is essential: only SINGLE-failover
|
||
# is a supported guarantee — killing a second primary while a just-restarted node
|
||
# is still catching up is a double-fault-within-one-reconvergence window (a
|
||
# momentary 2-of-3 with a stale member) that no Raft cluster guarantees against.
|
||
DOWN="" # the node killed in this round, to be restarted+reconverged next round
|
||
for round in $(seq 1 "$ROUNDS"); do
|
||
log "=== round $round/$ROUNDS ==="
|
||
|
||
# Step 10 (from the previous round): restart the previously-downed node and
|
||
# WAIT for the cluster to return to a fully-converged 3-of-3 before we kill
|
||
# again. (Round 1 starts from the freshly-deployed converged cluster.)
|
||
if [ -n "$DOWN" ]; then
|
||
log "round $round: restarting previously-downed $DOWN, waiting for full convergence"
|
||
start_node "$DOWN"
|
||
wait_for_full_convergence \
|
||
|| fail "round $round: cluster did not reconverge to 3-of-3 after restarting $DOWN"
|
||
log "round $round: cluster reconverged (3-of-3)"
|
||
fi
|
||
|
||
# Re-discover the live primary (it rotates as we kill nodes).
|
||
PRIMARY="$(discover_primary)"
|
||
[ -n "$PRIMARY" ] || fail "round $round: no serving primary before kill"
|
||
log "round $round: current primary $PRIMARY"
|
||
|
||
# Record the election term BEFORE the kill so we can detect a STRICTLY-newer
|
||
# CTRL_PRIMARY announcement (a genuine new election) afterwards.
|
||
PREV_TERM="$(latest_primary_term)"
|
||
PREV_TERM="${PREV_TERM:-0}"
|
||
|
||
# Step 6: kill the current primary. Match on the BINARY + --myname so the
|
||
# pattern cannot also hit this driver's own shell snapshot (which carries the
|
||
# "--myname ctrl-N" substring in its argv and would otherwise be matched by a
|
||
# bare `pgrep -f -- "--myname ctrl-N"`, returning extra PIDs).
|
||
PID="$(pgrep -f -- "xce_controller --myname $PRIMARY ")"
|
||
[ -n "$PID" ] || fail "round $round: could not find pid for $PRIMARY"
|
||
log "round $round: killing primary $PRIMARY (term=$PREV_TERM, pid=$(echo $PID | tr '\n' ' '))"
|
||
kill -9 $PID 2>/dev/null
|
||
DOWN="$PRIMARY"
|
||
sleep 2
|
||
|
||
# Step 7: a NEW primary must be elected (CTRL_PRIMARY at a higher term in the
|
||
# logs) AND be reachable by the client; then joblist must show no loss.
|
||
log "round $round: waiting for new primary (term > $PREV_TERM)"
|
||
NEWP="$(wait_for_new_primary "$PREV_TERM")"
|
||
[ -n "$NEWP" ] || fail "round $round: no new primary elected after kill (no CTRL_PRIMARY term>$PREV_TERM)"
|
||
log "round $round: new primary elected: $NEWP (term=$(latest_primary_term))"
|
||
record_primary "$NEWP"
|
||
PRIMARY="$(discover_primary)"
|
||
[ -n "$PRIMARY" ] || fail "round $round: new primary $NEWP elected but client cannot reach it (bug-854 class)"
|
||
verify_no_loss || fail "round $round: a committed job was LOST after failover"
|
||
|
||
# Step 8: submit another batch on the new primary.
|
||
submit_jobs "$NJOBS" || fail "round $round: post-failover submit did not get $NJOBS acks"
|
||
|
||
# Step 9: confirm still no loss (old + new jobs).
|
||
verify_no_loss || fail "round $round: job loss after post-failover submit"
|
||
walcheck "$((1 + 2 * ${#ALL_JOBIDS[@]}))"
|
||
done
|
||
|
||
# ── Leadership coverage: EVERY node must have served as primary ──────────────
|
||
# The natural single-failover loop only ever elects the two highest-priority
|
||
# nodes (ctrl-1/ctrl-2) — the priority-staggered election timeout means ctrl-3
|
||
# never wins while a lower-index node survives alongside it. Drive any
|
||
# never-yet-primary node to leadership explicitly so all three are exercised.
|
||
# First restore the cluster to 3-of-3 (the loop leaves the last-killed primary
|
||
# down).
|
||
if [ -n "$DOWN" ]; then
|
||
log "leadership-coverage: restarting $DOWN to restore 3-of-3 before maneuver"
|
||
start_node "$DOWN"
|
||
wait_for_full_convergence || fail "leadership-coverage: cluster did not reconverge before maneuver"
|
||
DOWN=""
|
||
fi
|
||
for n in "${NODES[@]}"; do
|
||
if [ -z "${SEEN_PRIMARY[$n]:-}" ]; then
|
||
log "leadership-coverage: $n has never been primary — forcing election"
|
||
force_elect_node "$n"
|
||
fi
|
||
done
|
||
|
||
# Assert all three nodes acted as primary at some point in the run.
|
||
MISSING=""
|
||
for n in "${NODES[@]}"; do
|
||
[ -z "${SEEN_PRIMARY[$n]:-}" ] && MISSING="$MISSING $n"
|
||
done
|
||
[ -z "$MISSING" ] || fail "not all nodes acted as primary — never led:$MISSING"
|
||
LED="$(printf '%s\n' "${!SEEN_PRIMARY[@]}" | sort | tr '\n' ',' | sed 's/,$//')"
|
||
|
||
log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss; all 3 nodes led ($LED)"
|
||
cleanup
|
||
trap - EXIT INT TERM
|
||
exit 0
|