chore(wolf): session trail + xce-test-ha cross-run collision guard
run-ha-soak.sh: scope every pgrep/pkill to this run's --workdir, not just binary name + --myname. An unscoped pkill -9 -x xce_controller (old cleanup()) killed every xce_controller on the machine, including a different concurrently-running soak's live primary — root cause of the first bug-1436 misdiagnosis (a killed-out-from-under-it primary produced a genuine, correctly-reported ECONNRESET that looked like a libxce bug). .wolf/*: anatomy/cerebrum/buglog updated for bug-1435/1436, session and token-ledger trail.
This commit is contained in:
parent
40c2351578
commit
b3219635b5
7 changed files with 2415 additions and 41 deletions
|
|
@ -180,17 +180,44 @@ FACTSTREAM_ALL_JOBIDS=() # every jobid ever driven through the fact stream, acro
|
|||
log() { echo "[xce-test-ha] $*"; }
|
||||
fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
|
||||
|
||||
cleanup() { pkill -9 -x xce_controller 2>/dev/null; sleep 0.3; }
|
||||
# ── Cross-run collision guard (bug-1436) ─────────────────────────────────────
|
||||
# Every pgrep/pkill in this script MUST be scoped to THIS run's own --workdir,
|
||||
# not just the binary name / --myname. Two run-ha-soak.sh invocations (e.g. a
|
||||
# sequential and a chaotic soak, or two soaks in different deploy dirs) share
|
||||
# the same xce_controller binary and the same hardcoded ctrl-1/ctrl-2/ctrl-3
|
||||
# node names — pgrep/pkill matched on binary name or "--myname ctrl-N" alone
|
||||
# cannot tell them apart. An unscoped `pkill -9 -x xce_controller` (the old
|
||||
# cleanup(), called unconditionally at the START of every deploy, at every
|
||||
# EXIT/INT/TERM via the trap, and on every fail()) kills EVERY xce_controller
|
||||
# process on the machine — including a DIFFERENT run's live, healthy,
|
||||
# freshly-elected primary. Root cause of bug-1436: a second soak's cleanup/
|
||||
# deploy step fired while an earlier soak's round was mid-verify, silently
|
||||
# SIGKILLing the primary the earlier run's `wait-primary` had just confirmed
|
||||
# moments before — the client's very next request then hit a genuine,
|
||||
# correctly-reported ECONNRESET (the process was really gone), not a libxce or
|
||||
# controller defect. `--workdir "$TOP/controller"` is present in every node's
|
||||
# argv (see start_node) and is unique per deploy dir, so scoping every match to
|
||||
# it makes each run's process lifecycle fully independent of any other
|
||||
# concurrently-running invocation (same or different deploy dir, same node
|
||||
# names). Two runs sharing a deploy dir AND therefore the same ports
|
||||
# (9991-9993) still cannot usefully coexist — the second run's start_node will
|
||||
# fail loudly with a bind error instead of silently killing the first run's
|
||||
# cluster, which is the correct "fail loud, not silently corrupt" behavior.
|
||||
node_pattern() { echo "xce_controller --myname $1 .*--workdir $TOP/controller"; }
|
||||
run_pattern() { echo "xce_controller .*--workdir $TOP/controller"; }
|
||||
|
||||
cleanup() { pkill -9 -f -- "$(run_pattern)" 2>/dev/null; sleep 0.3; }
|
||||
|
||||
# Record that a node has acted as primary at least once (leadership-coverage
|
||||
# tracking — the run asserts all three nodes lead, see the verdict).
|
||||
record_primary() { [ -n "$1" ] && SEEN_PRIMARY["$1"]=1; }
|
||||
|
||||
# kill -9 a controller by node name. Match on BINARY + --myname so the pattern
|
||||
# cannot hit this driver's own shell snapshot (its argv carries "--myname ctrl-N").
|
||||
# kill -9 a controller by node name. Match on BINARY + --myname + --workdir so
|
||||
# the pattern cannot hit this driver's own shell snapshot NOR a different
|
||||
# run-ha-soak.sh invocation's same-named node (bug-1436).
|
||||
kill_node() {
|
||||
local node="$1" pid
|
||||
pid="$(pgrep -f -- "xce_controller --myname $node ")"
|
||||
pid="$(pgrep -f -- "$(node_pattern "$node")")"
|
||||
[ -n "$pid" ] || return 1
|
||||
kill -9 $pid 2>/dev/null
|
||||
return 0
|
||||
|
|
@ -221,7 +248,7 @@ start_node() {
|
|||
# on the same --myname/port without ever killing the first. Guard here
|
||||
# (not by "fixing" DOWN's bookkeeping) so every call site is covered:
|
||||
# refuse to launch a duplicate if one is already running.
|
||||
if [ -n "$(pgrep -f -- "xce_controller --myname $1 ")" ]; then
|
||||
if [ -n "$(pgrep -f -- "$(node_pattern "$1")")" ]; then
|
||||
log "start_node: $1 already running — skipping duplicate launch (bug-1418)"
|
||||
return 0
|
||||
fi
|
||||
|
|
@ -435,7 +462,7 @@ wait_for_full_convergence() {
|
|||
local i alive stable=0
|
||||
local STABLE_TICKS=2 # 2 consecutive converged probes ⇒ settled
|
||||
for i in $(seq 1 30); do
|
||||
alive="$(pgrep -c -f -- "xce_controller --myname ctrl-")"
|
||||
alive="$(pgrep -c -f -- "$(run_pattern)")"
|
||||
if [ "$alive" -lt 3 ]; then
|
||||
stable=0; sleep 2; continue
|
||||
fi
|
||||
|
|
@ -754,7 +781,7 @@ run_chaotic_rounds() {
|
|||
submit_jobs "$NJOBS" || log "round $round: submit shortfall (transient — not a loss)"
|
||||
|
||||
# Kill the current primary, landing inside the next election window.
|
||||
pid="$(pgrep -f -- "xce_controller --myname $PRIMARY ")"
|
||||
pid="$(pgrep -f -- "$(node_pattern "$PRIMARY")")"
|
||||
if [ -n "$pid" ]; then
|
||||
log "round $round: killing primary $PRIMARY (term=$PREV_TERM, pid=$(echo $pid | tr '\n' ' '))"
|
||||
kill -9 $pid 2>/dev/null
|
||||
|
|
@ -1357,11 +1384,12 @@ for round in $(seq 1 "$ROUNDS"); do
|
|||
PREV_TERM="$(latest_primary_term)"
|
||||
PREV_TERM="${PREV_TERM:-0}"
|
||||
|
||||
# Step 6: kill the current primary. Match on the BINARY + --myname so the
|
||||
# pattern cannot also hit this driver's own shell snapshot (which carries the
|
||||
# "--myname ctrl-N" substring in its argv and would otherwise be matched by a
|
||||
# bare `pgrep -f -- "--myname ctrl-N"`, returning extra PIDs).
|
||||
PID="$(pgrep -f -- "xce_controller --myname $PRIMARY ")"
|
||||
# Step 6: kill the current primary. Match on the BINARY + --myname + --workdir
|
||||
# so the pattern cannot also hit this driver's own shell snapshot (which
|
||||
# carries the "--myname ctrl-N" substring in its argv and would otherwise be
|
||||
# matched by a bare `pgrep -f -- "--myname ctrl-N"`, returning extra PIDs)
|
||||
# NOR a different run-ha-soak.sh invocation's same-named node (bug-1436).
|
||||
PID="$(pgrep -f -- "$(node_pattern "$PRIMARY")")"
|
||||
[ -n "$PID" ] || fail "round $round: could not find pid for $PRIMARY"
|
||||
log "round $round: killing primary $PRIMARY (term=$PREV_TERM, pid=$(echo $PID | tr '\n' ' '))"
|
||||
kill -9 $PID 2>/dev/null
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue