fix bug-1060 bug-1067
This commit is contained in:
parent
6067ba6c94
commit
61e1391cd0
28 changed files with 7021 additions and 2926 deletions
|
|
@ -108,8 +108,24 @@ XCE_HA_ROUNDS=10 XCE_HA_TOP=/tmp/my-ha-run nix develop ./src --command bash "$SK
|
|||
restart the node downed in the *previous* round (so the cluster returns to 3
|
||||
members) → wait for a new primary → `verify_no_loss` (expect-job every tracked
|
||||
JobId) → submit another batch → `verify_no_loss` again → `walcompare`.
|
||||
12. **Verdict** — exits `0` (PASS) only if every round verified zero job loss;
|
||||
any missing job, failed election, or unreachable primary exits non-zero (FAIL).
|
||||
Each elected primary is recorded for the leadership-coverage check.
|
||||
11b. **Leadership coverage** — every node must serve as primary at least once.
|
||||
The natural kill loop only ever elects the two highest-priority nodes
|
||||
(`ctrl-1`/`ctrl-2`): the election timeout is **priority-staggered**
|
||||
(`my_priority * election_priority_step_ms` — p0 immediate, p1 +500ms,
|
||||
p2 +1000ms), so the lowest-priority node (`ctrl-3`) never wins while a
|
||||
lower-index node survives alongside it. After the loop, the driver restores
|
||||
3-of-3 and **forces any never-yet-primary node to leadership** via a
|
||||
deterministic, loss-free maneuver that exploits the RequestVote freshness
|
||||
gate (a behind node cannot win): kill the spoiler standby → commit a batch so
|
||||
it falls behind → kill the primary (target holds all commits) → restart the
|
||||
stale spoiler so the freshness-fenced quorum elects the target → verify no
|
||||
loss → reconverge. A node staying behind cannot catch up while there is no
|
||||
primary, so the target wins **deterministically**, not by timing.
|
||||
12. **Verdict** — exits `0` (PASS) only if every round verified zero job loss
|
||||
**and all three nodes (`ctrl-1`, `ctrl-2`, `ctrl-3`) acted as primary**;
|
||||
any missing job, failed election, unreachable primary, or a node that never
|
||||
led exits non-zero (FAIL). The PASS line names the nodes that led.
|
||||
|
||||
## Step 3 — Report the result
|
||||
|
||||
|
|
|
|||
|
|
@ -53,13 +53,28 @@ declare -A PORT=( [ctrl-1]=9991 [ctrl-2]=9992 [ctrl-3]=9993 )
|
|||
PEERS="ctrl-1=localhost:9991,ctrl-2=localhost:9992,ctrl-3=localhost:9993"
|
||||
MASTERS="localhost:9991,localhost:9992,localhost:9993"
|
||||
|
||||
ALL_JOBIDS=() # every JobId we have ever successfully submitted
|
||||
ALL_JOBIDS=() # every JobId we have ever successfully submitted
|
||||
declare -A SEEN_PRIMARY # node name -> 1 once it has served as primary
|
||||
|
||||
log() { echo "[xce-test-ha] $*"; }
|
||||
fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
|
||||
|
||||
cleanup() { pkill -9 -x xce_controller 2>/dev/null; sleep 0.3; }
|
||||
|
||||
# Record that a node has acted as primary at least once (leadership-coverage
|
||||
# tracking — the run asserts all three nodes lead, see the verdict).
|
||||
record_primary() { [ -n "$1" ] && SEEN_PRIMARY["$1"]=1; }
|
||||
|
||||
# kill -9 a controller by node name. Match on BINARY + --myname so the pattern
|
||||
# cannot hit this driver's own shell snapshot (its argv carries "--myname ctrl-N").
|
||||
kill_node() {
|
||||
local node="$1" pid
|
||||
pid="$(pgrep -f -- "xce_controller --myname $node ")"
|
||||
[ -n "$pid" ] || return 1
|
||||
kill -9 $pid 2>/dev/null
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
[ -x "$BIN" ] || { echo "[xce-test-ha] FAIL: controller binary missing: $BIN (run /xce-build controller)" >&2; exit 2; }
|
||||
[ -x "$XT" ] || { echo "[xce-test-ha] FAIL: xce_test missing: $XT (run /xce-build controller)" >&2; exit 2; }
|
||||
|
|
@ -285,6 +300,66 @@ walcheck() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Drive a SPECIFIC node to primary — used to exercise leadership on the
|
||||
# lowest-priority node (ctrl-3), which the natural single-failover loop NEVER
|
||||
# elects: the election timeout is priority-staggered (my_priority * step_ms, p0
|
||||
# immediate / p1 +500ms / p2 +1000ms), so whenever ctrl-3 is in the surviving
|
||||
# pair a lower-index node out-races it.
|
||||
#
|
||||
# Mechanism — exploits the RequestVote freshness gate (a behind node cannot win):
|
||||
# precondition: cluster 3-of-3 converged, target T caught up, T not primary.
|
||||
# P = current primary; S = the third node (not P, not T).
|
||||
# 1. kill S (a standby) so it stops receiving entries.
|
||||
# 2. submit a batch → it commits on {P, T}; S is now provably BEHIND.
|
||||
# 3. kill P → only T alive (brief, deliberate no-quorum). T holds EVERY
|
||||
# committed entry (all commits were on {P,T}), so nothing is lost.
|
||||
# 4. restart S (still behind — it cannot catch up while there is no primary,
|
||||
# so it stays fenced for the whole election): quorum {T fresh, S stale};
|
||||
# the freshness gate denies S, and T wins DETERMINISTICALLY (not by timing).
|
||||
# 5. verify no loss, restart P, reconverge 3-of-3.
|
||||
# Fails the run loudly if T does not win.
|
||||
force_elect_node() {
|
||||
local T="$1" P S n
|
||||
P="$(discover_primary)"
|
||||
[ -n "$P" ] || { log "force-elect $T: no serving primary"; return 1; }
|
||||
if [ "$P" = "$T" ]; then record_primary "$T"; log "force-elect: $T already primary"; return 0; fi
|
||||
S=""
|
||||
for n in "${NODES[@]}"; do [ "$n" != "$P" ] && [ "$n" != "$T" ] && S="$n"; done
|
||||
[ -n "$S" ] || { log "force-elect $T: could not pick spoiler"; return 1; }
|
||||
|
||||
log "force-elect $T: primary=$P spoiler=$S (freshness-fence $S so $T wins)"
|
||||
local PREV_TERM; PREV_TERM="$(latest_primary_term)"; PREV_TERM="${PREV_TERM:-0}"
|
||||
|
||||
# 1. Kill the spoiler standby so it falls behind.
|
||||
kill_node "$S" || fail "force-elect $T: could not kill spoiler $S"
|
||||
sleep 2
|
||||
|
||||
# 2. Commit a batch on {P,T} — guarantees S is behind and T is caught up.
|
||||
submit_jobs "$NJOBS" || fail "force-elect $T: submit on {$P,$T} failed"
|
||||
verify_no_loss || fail "force-elect $T: loss after spoiler-down submit"
|
||||
|
||||
# 3. Kill the primary. Only T is alive (transient no-quorum); T has all commits.
|
||||
kill_node "$P" || fail "force-elect $T: could not kill primary $P"
|
||||
sleep 2
|
||||
|
||||
# 4. Restart the (behind) spoiler to restore quorum; T must win on freshness.
|
||||
log "force-elect $T: restarting behind-spoiler $S — expecting $T to win"
|
||||
start_node "$S"
|
||||
NEWP="$(wait_for_new_primary "$PREV_TERM")"
|
||||
[ -n "$NEWP" ] || fail "force-elect $T: no new primary after restoring quorum"
|
||||
[ "$NEWP" = "$T" ] || fail "force-elect $T: expected $T to win, got $NEWP (freshness-fence failed)"
|
||||
record_primary "$T"
|
||||
PRIMARY="$(discover_primary)"
|
||||
[ -n "$PRIMARY" ] || fail "force-elect $T: $T elected but client cannot reach it"
|
||||
verify_no_loss || fail "force-elect $T: committed job LOST during maneuver"
|
||||
log "force-elect $T: SUCCESS — $T is primary (term=$(latest_primary_term))"
|
||||
|
||||
# 5. Restart P and reconverge to a healthy 3-of-3.
|
||||
start_node "$P"
|
||||
wait_for_full_convergence || fail "force-elect $T: cluster did not reconverge 3-of-3 after $T led"
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Step 2: deploy ───────────────────────────────────────────────────────────
|
||||
log "deploying 3-node cluster to $TOP"
|
||||
cleanup
|
||||
|
|
@ -314,6 +389,7 @@ log "waiting for primary election (cold start ~15s)"
|
|||
wait_for_primary_log || fail "no primary elected within 45s (check $TOP/logs)"
|
||||
PRIMARY="$(discover_primary)"
|
||||
[ -n "$PRIMARY" ] || fail "primary elected in logs but client could not reach it"
|
||||
record_primary "$PRIMARY"
|
||||
log "primary is $PRIMARY"
|
||||
|
||||
# ── Step 5: initial submit + verify + waldump ────────────────────────────────
|
||||
|
|
@ -371,6 +447,7 @@ for round in $(seq 1 "$ROUNDS"); do
|
|||
NEWP="$(wait_for_new_primary "$PREV_TERM")"
|
||||
[ -n "$NEWP" ] || fail "round $round: no new primary elected after kill (no CTRL_PRIMARY term>$PREV_TERM)"
|
||||
log "round $round: new primary elected: $NEWP (term=$(latest_primary_term))"
|
||||
record_primary "$NEWP"
|
||||
PRIMARY="$(discover_primary)"
|
||||
[ -n "$PRIMARY" ] || fail "round $round: new primary $NEWP elected but client cannot reach it (bug-854 class)"
|
||||
verify_no_loss || fail "round $round: a committed job was LOST after failover"
|
||||
|
|
@ -383,7 +460,35 @@ for round in $(seq 1 "$ROUNDS"); do
|
|||
walcheck "$((1 + 2 * ${#ALL_JOBIDS[@]}))"
|
||||
done
|
||||
|
||||
log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss"
|
||||
# ── Leadership coverage: EVERY node must have served as primary ──────────────
|
||||
# The natural single-failover loop only ever elects the two highest-priority
|
||||
# nodes (ctrl-1/ctrl-2) — the priority-staggered election timeout means ctrl-3
|
||||
# never wins while a lower-index node survives alongside it. Drive any
|
||||
# never-yet-primary node to leadership explicitly so all three are exercised.
|
||||
# First restore the cluster to 3-of-3 (the loop leaves the last-killed primary
|
||||
# down).
|
||||
if [ -n "$DOWN" ]; then
|
||||
log "leadership-coverage: restarting $DOWN to restore 3-of-3 before maneuver"
|
||||
start_node "$DOWN"
|
||||
wait_for_full_convergence || fail "leadership-coverage: cluster did not reconverge before maneuver"
|
||||
DOWN=""
|
||||
fi
|
||||
for n in "${NODES[@]}"; do
|
||||
if [ -z "${SEEN_PRIMARY[$n]:-}" ]; then
|
||||
log "leadership-coverage: $n has never been primary — forcing election"
|
||||
force_elect_node "$n"
|
||||
fi
|
||||
done
|
||||
|
||||
# Assert all three nodes acted as primary at some point in the run.
|
||||
MISSING=""
|
||||
for n in "${NODES[@]}"; do
|
||||
[ -z "${SEEN_PRIMARY[$n]:-}" ] && MISSING="$MISSING $n"
|
||||
done
|
||||
[ -z "$MISSING" ] || fail "not all nodes acted as primary — never led:$MISSING"
|
||||
LED="$(printf '%s\n' "${!SEEN_PRIMARY[@]}" | sort | tr '\n' ',' | sed 's/,$//')"
|
||||
|
||||
log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss; all 3 nodes led ($LED)"
|
||||
cleanup
|
||||
trap - EXIT INT TERM
|
||||
exit 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue