chore(wolf): session trail + xce-test-ha cross-run collision guard

run-ha-soak.sh: scope every pgrep/pkill to this run's --workdir, not just
binary name + --myname. An unscoped pkill -9 -x xce_controller (old
cleanup()) killed every xce_controller on the machine, including a
different concurrently-running soak's live primary — root cause of the
first bug-1436 misdiagnosis (a killed-out-from-under-it primary produced
a genuine, correctly-reported ECONNRESET that looked like a libxce bug).

.wolf/*: anatomy/cerebrum/buglog updated for bug-1435/1436, session and
token-ledger trail.
This commit is contained in:
Rongsong Shen 2026-07-13 07:19:00 +00:00
parent 40c2351578
commit b3219635b5
7 changed files with 2415 additions and 41 deletions

View file

@ -180,17 +180,44 @@ FACTSTREAM_ALL_JOBIDS=() # every jobid ever driven through the fact stream, acro
log() { echo "[xce-test-ha] $*"; }
fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
cleanup() { pkill -9 -x xce_controller 2>/dev/null; sleep 0.3; }
# ── Cross-run collision guard (bug-1436) ─────────────────────────────────────
# Every pgrep/pkill in this script MUST be scoped to THIS run's own --workdir,
# not just the binary name / --myname. Two run-ha-soak.sh invocations (e.g. a
# sequential and a chaotic soak, or two soaks in different deploy dirs) share
# the same xce_controller binary and the same hardcoded ctrl-1/ctrl-2/ctrl-3
# node names — pgrep/pkill matched on binary name or "--myname ctrl-N" alone
# cannot tell them apart. An unscoped `pkill -9 -x xce_controller` (the old
# cleanup(), called unconditionally at the START of every deploy, at every
# EXIT/INT/TERM via the trap, and on every fail()) kills EVERY xce_controller
# process on the machine — including a DIFFERENT run's live, healthy,
# freshly-elected primary. Root cause of bug-1436: a second soak's cleanup/
# deploy step fired while an earlier soak's round was mid-verify, silently
# SIGKILLing the primary the earlier run's `wait-primary` had just confirmed
# moments before — the client's very next request then hit a genuine,
# correctly-reported ECONNRESET (the process was really gone), not a libxce or
# controller defect. `--workdir "$TOP/controller"` is present in every node's
# argv (see start_node) and is unique per deploy dir, so scoping every match to
# it makes each run's process lifecycle fully independent of any other
# concurrently-running invocation (same or different deploy dir, same node
# names). Two runs sharing a deploy dir AND therefore the same ports
# (9991-9993) still cannot usefully coexist — the second run's start_node will
# fail loudly with a bind error instead of silently killing the first run's
# cluster, which is the correct "fail loud, not silently corrupt" behavior.
node_pattern() { echo "xce_controller --myname $1 .*--workdir $TOP/controller"; }
run_pattern() { echo "xce_controller .*--workdir $TOP/controller"; }
cleanup() { pkill -9 -f -- "$(run_pattern)" 2>/dev/null; sleep 0.3; }
# Record that a node has acted as primary at least once (leadership-coverage
# tracking — the run asserts all three nodes lead, see the verdict).
record_primary() { [ -n "$1" ] && SEEN_PRIMARY["$1"]=1; }
# kill -9 a controller by node name. Match on BINARY + --myname so the pattern
# cannot hit this driver's own shell snapshot (its argv carries "--myname ctrl-N").
# kill -9 a controller by node name. Match on BINARY + --myname + --workdir so
# the pattern cannot hit this driver's own shell snapshot NOR a different
# run-ha-soak.sh invocation's same-named node (bug-1436).
kill_node() {
local node="$1" pid
pid="$(pgrep -f -- "xce_controller --myname $node ")"
pid="$(pgrep -f -- "$(node_pattern "$node")")"
[ -n "$pid" ] || return 1
kill -9 $pid 2>/dev/null
return 0
@ -221,7 +248,7 @@ start_node() {
# on the same --myname/port without ever killing the first. Guard here
# (not by "fixing" DOWN's bookkeeping) so every call site is covered:
# refuse to launch a duplicate if one is already running.
if [ -n "$(pgrep -f -- "xce_controller --myname $1 ")" ]; then
if [ -n "$(pgrep -f -- "$(node_pattern "$1")")" ]; then
log "start_node: $1 already running — skipping duplicate launch (bug-1418)"
return 0
fi
@ -435,7 +462,7 @@ wait_for_full_convergence() {
local i alive stable=0
local STABLE_TICKS=2 # 2 consecutive converged probes ⇒ settled
for i in $(seq 1 30); do
alive="$(pgrep -c -f -- "xce_controller --myname ctrl-")"
alive="$(pgrep -c -f -- "$(run_pattern)")"
if [ "$alive" -lt 3 ]; then
stable=0; sleep 2; continue
fi
@ -754,7 +781,7 @@ run_chaotic_rounds() {
submit_jobs "$NJOBS" || log "round $round: submit shortfall (transient — not a loss)"
# Kill the current primary, landing inside the next election window.
pid="$(pgrep -f -- "xce_controller --myname $PRIMARY ")"
pid="$(pgrep -f -- "$(node_pattern "$PRIMARY")")"
if [ -n "$pid" ]; then
log "round $round: killing primary $PRIMARY (term=$PREV_TERM, pid=$(echo $pid | tr '\n' ' '))"
kill -9 $pid 2>/dev/null
@ -1357,11 +1384,12 @@ for round in $(seq 1 "$ROUNDS"); do
PREV_TERM="$(latest_primary_term)"
PREV_TERM="${PREV_TERM:-0}"
# Step 6: kill the current primary. Match on the BINARY + --myname so the
# pattern cannot also hit this driver's own shell snapshot (which carries the
# "--myname ctrl-N" substring in its argv and would otherwise be matched by a
# bare `pgrep -f -- "--myname ctrl-N"`, returning extra PIDs).
PID="$(pgrep -f -- "xce_controller --myname $PRIMARY ")"
# Step 6: kill the current primary. Match on the BINARY + --myname + --workdir
# so the pattern cannot also hit this driver's own shell snapshot (which
# carries the "--myname ctrl-N" substring in its argv and would otherwise be
# matched by a bare `pgrep -f -- "--myname ctrl-N"`, returning extra PIDs)
# NOR a different run-ha-soak.sh invocation's same-named node (bug-1436).
PID="$(pgrep -f -- "$(node_pattern "$PRIMARY")")"
[ -n "$PID" ] || fail "round $round: could not find pid for $PRIMARY"
log "round $round: killing primary $PRIMARY (term=$PREV_TERM, pid=$(echo $PID | tr '\n' ' '))"
kill -9 $PID 2>/dev/null