test(xce-test-ha): add worker-rejoin-resync coverage to sequential/chaotic soak modes
Starts a real worker (auto-detects xcecd_virtual vs. real xcecd, mirroring xce-deploy-ha's own detection) + xce_scheduler alongside the controller cluster, dispatches a genuine job, kills the primary while it's RUN, and verifies the worker rejoins the new primary and the job recovers to DONE via real reconciliation -- closing the "no real daemon-to-daemon smoke test" gap flagged by worker-rejoin-resync's own feature validation. Live-verified end-to-end multiple times (sequential and chaotic, 2 rounds each) against a real 3-node cluster. Also logs two bugs discovered along the way (not fixed, out of scope for this test-infra change): bug-1759 (scheduler sim-mode dispatch/PEND inconsistency) and bug-1760 (scheduler has no periodic post-failover reconnect check, only a reactive one gated on its own next successful publish -- a chicken-and-egg deadlock, live-confirmed via 60s+ of stuck PEND with zero scheduler log activity). bug-1760 is worked around in this harness only by restarting the scheduler process after each new-primary election; the underlying gap is unfixed. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019ypUeKgbAqLWUkaSJ79poN
This commit is contained in:
parent
ea00879af0
commit
00fc0a43e0
2 changed files with 421 additions and 5 deletions
|
|
@ -109,6 +109,36 @@
|
|||
# --force-snapshot, hard-fails (not count-and-tolerate) on
|
||||
# any violated invariant, since these are the feature's core
|
||||
# correctness claims, not a chaotic/racy outcome class.
|
||||
# worker-rejoin-resync live coverage (--sequential / --chaotic, task 8.x):
|
||||
# when a real worker+scheduler binary pair exists alongside
|
||||
# the controller binary (same build dir -- run /xce-build
|
||||
# worker and /xce-build scheduler alongside /xce-build
|
||||
# controller), sequential/chaotic modes ALSO start a real
|
||||
# worker (start-worker.sh/start-scheduler.sh, same binaries
|
||||
# and auto-detection deploy-ha.sh already uses -- real
|
||||
# systemd present, or XCE_REAL_WORKER=1 forced, picks the
|
||||
# real production xcecd; otherwise xcecd_virtual, which
|
||||
# exercises byte-identical rejoin/reconciliation code with
|
||||
# job execution simulated instead of a real systemd unit)
|
||||
# and dispatch a REAL job onto it each round -- not
|
||||
# xce_test's scripted jobsub/jstart/jfinish stand-in, which
|
||||
# predates and cannot speak OPCode_jobStateQuery/Report at
|
||||
# all. Sequential mode hard-fails per round if the job does
|
||||
# not recover to a terminal state via a genuinely fresh
|
||||
# rejoin after the kill (proven via the worker's own log,
|
||||
# not just eventual re-delivery); chaotic mode tracks jobs
|
||||
# through the overlapping-kill loop and hard-fails only in
|
||||
# a final settled sweep after the loop, since "every lost
|
||||
# job eventually reconciles" is this feature's own core
|
||||
# correctness claim, not a chaotic/racy outcome class (same
|
||||
# standard --array-soak/--array-chaos-retry hold themselves
|
||||
# to). Soft-skipped with a warning, not fail(), when the
|
||||
# worker/scheduler binaries or their generated start
|
||||
# scripts are missing -- a controller-only build dir stays
|
||||
# fully backward compatible. Live-verified against a real
|
||||
# 3-node cluster + xcecd_virtual before being wired in here
|
||||
# (real dispatch to RUN, kill-during-RUN, confirmed-fresh
|
||||
# rejoin via the worker's own log, job recovered to DONE).
|
||||
# --array-chaos-retry job-array-single-wal-record task 6.2 (Req 8.2, 8.3):
|
||||
# submits a job ARRAY and kills the primary IMMEDIATELY
|
||||
# (before it is known whether the array's WAL_JOB_ARRAY_SUBMIT
|
||||
|
|
@ -133,6 +163,19 @@
|
|||
# --selftest run the verdict-parser self-test (soak-verdict-selftest.sh)
|
||||
# and exit — no cluster, no binaries needed.
|
||||
#
|
||||
# Worker-rejoin-resync live coverage env overrides:
|
||||
# XCE_REAL_WORKER 0|1 — forces xcecd_virtual/real xcecd choice, same
|
||||
# variable deploy-ha.sh's own generated start-worker.sh
|
||||
# already reads; unset auto-detects on /run/systemd/system
|
||||
# exactly as it always has. Nothing new to configure here.
|
||||
# XCE_HA_WORKER_RESYNC_SLEEP seconds the sequential loop's per-round
|
||||
# worker-dispatched job sleeps for (default 20) — sized
|
||||
# so the job's simulated finish lands strictly AFTER
|
||||
# that round's kill, guaranteeing its fire-and-forget
|
||||
# finish notify has nowhere live to land (a genuine
|
||||
# loss, not a timing coincidence) before the worker's
|
||||
# own reconnect logic rejoins the new primary.
|
||||
#
|
||||
# Env overrides:
|
||||
# XCE_HA_TOP deploy dir (default /home/claude/tmp/xce-ha-deploy)
|
||||
# XCE_HA_MODE sequential | chaotic | force-snapshot (default sequential)
|
||||
|
|
@ -341,6 +384,13 @@ V_BUG924=0 # overlayfs `fdatasync FAILED` log lines (environment defect)
|
|||
V_PHANTOM=0 # phantom fact reactions observed (see verify_factstream_no_phantom)
|
||||
FACTSTREAM_ALL_JOBIDS=() # every jobid ever driven through the fact stream, across rounds
|
||||
|
||||
# ── Worker-rejoin-resync live coverage (sequential/chaotic only) ────────────
|
||||
WORKER_NAME="soak-worker"
|
||||
WORKER_ENABLED=0 # 1 once start_worker_and_scheduler() confirms both joined
|
||||
WORKER_RESYNC_SLEEP="${XCE_HA_WORKER_RESYNC_SLEEP:-20}"
|
||||
WORKER_RESYNC_JOBIDS=() # chaotic mode: jobids tracked for the final settled sweep
|
||||
V_RESYNC_STUCK=0 # chaotic mode: worker-resync jobs still non-terminal at final sweep
|
||||
|
||||
log() { echo "[xce-test-ha] $*"; }
|
||||
fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
|
||||
|
||||
|
|
@ -370,7 +420,21 @@ fail() { echo "[xce-test-ha] FAIL: $*" >&2; cleanup; exit 1; }
|
|||
node_pattern() { echo "xce_controller --myname $1 .*--workdir $TOP/controller"; }
|
||||
run_pattern() { echo "xce_controller .*--workdir $TOP/controller"; }
|
||||
|
||||
cleanup() { pkill -9 -f -- "$(run_pattern)" 2>/dev/null; sleep 0.3; }
|
||||
# Same TOP-scoping discipline as node_pattern/run_pattern above, applied to
|
||||
# the worker-rejoin-resync worker/scheduler processes. `-l $TOP/logs` is
|
||||
# passed verbatim by generated start-worker.sh regardless of which binary
|
||||
# (xcecd_virtual or real xcecd) it targets, so one pattern covers both;
|
||||
# xce_scheduler is a cluster singleton so `--logfile $TOP/logs/scheduler.log`
|
||||
# alone is enough to scope it to this run.
|
||||
worker_pattern() { echo "xcecd.*-l $TOP/logs"; }
|
||||
sched_pattern() { echo "xce_scheduler .*--logfile $TOP/logs/scheduler.log"; }
|
||||
|
||||
cleanup() {
|
||||
pkill -9 -f -- "$(run_pattern)" 2>/dev/null
|
||||
pkill -9 -f -- "$(worker_pattern)" 2>/dev/null
|
||||
pkill -9 -f -- "$(sched_pattern)" 2>/dev/null
|
||||
sleep 0.3
|
||||
}
|
||||
|
||||
# Record that a node has acted as primary at least once (leadership-coverage
|
||||
# tracking — the run asserts all three nodes lead, see the verdict).
|
||||
|
|
@ -387,6 +451,211 @@ kill_node() {
|
|||
return 0
|
||||
}
|
||||
|
||||
# ── Worker-rejoin-resync live coverage (sequential/chaotic only) ────────────
|
||||
#
|
||||
# Real xcecd's log ($TOP/logs/xcecd.log, hardcoded regardless of -n) comes out
|
||||
# root-owned 640 (xce-deploy-ha SKILL.md, confirmed) -- a plain read as this
|
||||
# script's own user fails. xcecd_virtual's log ($TOP/logs/xcecd-$WORKER_NAME.log)
|
||||
# needs no such treatment. These three helpers are the single place that
|
||||
# knows which applies, so no other worker-resync code needs to care.
|
||||
worker_log_path() {
|
||||
[ -f "$TOP/logs/xcecd.log" ] && { echo "$TOP/logs/xcecd.log"; return; }
|
||||
echo "$TOP/logs/xcecd-$WORKER_NAME.log"
|
||||
}
|
||||
worker_log_grep() {
|
||||
local pat="$1" path; path="$(worker_log_path)"
|
||||
if [ "$path" = "$TOP/logs/xcecd.log" ]; then
|
||||
sudo -n grep "$pat" "$path" 2>/dev/null
|
||||
else
|
||||
grep "$pat" "$path" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
worker_log_count() {
|
||||
local pat="$1" path n; path="$(worker_log_path)"
|
||||
if [ "$path" = "$TOP/logs/xcecd.log" ]; then
|
||||
n="$(sudo -n grep -c "$pat" "$path" 2>/dev/null)"
|
||||
else
|
||||
n="$(grep -c "$pat" "$path" 2>/dev/null)"
|
||||
fi
|
||||
echo "${n:-0}"
|
||||
}
|
||||
|
||||
# Starts a real worker + scheduler alongside the controller cluster, using
|
||||
# the SAME generated start-worker.sh/start-scheduler.sh (and therefore the
|
||||
# SAME xcecd_virtual-vs-real-xcecd/XCE_REAL_WORKER auto-detection) deploy-ha.sh
|
||||
# already produces for xce-deploy-ha -- no separate detection logic here.
|
||||
# Soft-skips (log a warning, do not fail()) when the binaries/generated
|
||||
# scripts are missing, or when either process does not confirm joining
|
||||
# within budget, so a controller-only build dir stays fully backward
|
||||
# compatible and a flaky join never turns an otherwise-healthy HA soak into
|
||||
# a hard failure over an orthogonal feature's coverage.
|
||||
|
||||
# Pragmatic soak-harness workaround for bug-1760 (logged, not fixed): the
|
||||
# scheduler has NO periodic reconnect check analogous to the worker's
|
||||
# checking_connection() -- its only rejoin path is REACTIVE, firing only as
|
||||
# a side effect of its own next successful publish call
|
||||
# (sched_publish_check_rejoin(), scheduler.c). After a real controller
|
||||
# failover it has nothing queued to publish (no new job ever reaches its
|
||||
# picker, since the controller-side registration of "where the scheduler
|
||||
# is" does not appear to carry over to the newly-elected primary) -- a
|
||||
# chicken-and-egg deadlock, live-confirmed (60s+ of zero scheduler.log
|
||||
# activity, job stuck PEND). Restarting the scheduler process after each
|
||||
# new-primary election, the same way this harness already restarts a
|
||||
# downed CONTROLLER node, sidesteps it: a fresh process's sched_init() tries
|
||||
# all 3 candidates in order and joins whichever is actually reachable. Test-
|
||||
# infrastructure-only workaround -- does NOT fix the underlying gap.
|
||||
restart_scheduler_for_new_primary() {
|
||||
local before i
|
||||
before="$(wc -l < "$TOP/logs/scheduler.log" 2>/dev/null || echo 0)"
|
||||
pkill -9 -f -- "$(sched_pattern)" 2>/dev/null
|
||||
sleep 0.5
|
||||
setsid nohup bash "$TOP/scripts/start-scheduler.sh" -v \
|
||||
</dev/null >>"$TOP/logs/scheduler.stderr" 2>&1 &
|
||||
for i in $(seq 1 20); do
|
||||
tail -n +"$((before + 1))" "$TOP/logs/scheduler.log" 2>/dev/null \
|
||||
| grep -q "sched_init: joined cluster" && return 0
|
||||
sleep 1
|
||||
done
|
||||
log "worker-resync: scheduler restart did not confirm join within 20s (bug-1760 workaround) -- worker-resync coverage may fail this round"
|
||||
return 1
|
||||
}
|
||||
|
||||
start_worker_and_scheduler() {
|
||||
if [ ! -f "$TOP/scripts/start-worker.sh" ] || [ ! -f "$TOP/scripts/start-scheduler.sh" ]; then
|
||||
log "worker-resync: start-worker.sh/start-scheduler.sh not generated (worker and/or scheduler binary missing from this build dir) -- skipping worker-rejoin-resync coverage this run. Run /xce-build worker and /xce-build scheduler alongside /xce-build controller to enable it."
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "worker-resync: starting worker ($WORKER_NAME) + scheduler"
|
||||
setsid nohup bash "$TOP/scripts/start-worker.sh" "$WORKER_NAME" -vvv \
|
||||
</dev/null >>"$TOP/logs/$WORKER_NAME.stderr" 2>&1 &
|
||||
setsid nohup bash "$TOP/scripts/start-scheduler.sh" -v \
|
||||
</dev/null >>"$TOP/logs/scheduler.stderr" 2>&1 &
|
||||
|
||||
# Real xcecd started without systemd/dbus present still joins the cluster
|
||||
# fine (only job DISPATCH later fails) -- this check confirms liveness/
|
||||
# join, not "can actually run a job"; the first submit_worker_job() call
|
||||
# confirms that for real.
|
||||
local i
|
||||
for i in $(seq 1 20); do
|
||||
worker_log_grep "Successfully sent .* load indices" >/dev/null && break
|
||||
sleep 1
|
||||
done
|
||||
if ! worker_log_grep "Successfully sent .* load indices" >/dev/null; then
|
||||
log "worker-resync: worker did not confirm join within 20s ($(worker_log_path)) -- skipping worker-rejoin-resync coverage this run"
|
||||
return 0
|
||||
fi
|
||||
for i in $(seq 1 20); do
|
||||
grep -q "sched_init: joined cluster" "$TOP/logs/scheduler.log" 2>/dev/null && break
|
||||
sleep 1
|
||||
done
|
||||
if ! grep -q "sched_init: joined cluster" "$TOP/logs/scheduler.log" 2>/dev/null; then
|
||||
log "worker-resync: scheduler did not confirm join within 20s -- skipping worker-rejoin-resync coverage this run"
|
||||
return 0
|
||||
fi
|
||||
|
||||
WORKER_ENABLED=1
|
||||
log "worker-resync: worker + scheduler joined -- rejoin-resync coverage enabled for this run"
|
||||
}
|
||||
|
||||
# Submits ONE real job, dispatched through the live scheduler onto the live
|
||||
# worker -- NOT xce_test's scripted jobsub/jstart/jfinish stand-in. The
|
||||
# `sleep $1` command is what xcecd_worker_utils.c parses to drive the
|
||||
# virtual worker's own timer-based simulated execution (confirmed live: a
|
||||
# real xcecd_virtual genuinely reaches RUN via real scheduler dispatch,
|
||||
# independent of systemd/dbus). Echoes the jobid and returns 0 once RUN is
|
||||
# confirmed; returns 1 (echoing nothing) on a submit or RUN-timeout failure.
|
||||
submit_worker_job() {
|
||||
local sleep_secs="$1" out jid attempt st
|
||||
out="$(printf 'set masters %s\nset myname soak\nclient\nwait-primary %s\njobsub -u root -q normal -c "sleep %s"\nquit\n' \
|
||||
"$MASTERS" "$WAITMS" "$sleep_secs" | xtest)"
|
||||
jid="$(printf '%s\n' "$out" | sed -n 's/^jobsubmit ok: JobId: \(.*\)$/\1/p')"
|
||||
if [ -z "$jid" ]; then
|
||||
# log() writes to stdout, which this function's own callers capture
|
||||
# via $(...) to get the jobid back -- a diagnostic here MUST go to
|
||||
# stderr instead, or it silently becomes (part of) the "jobid".
|
||||
log "worker-resync: jobsub itself failed -- raw output: $(printf '%s' "$out" | tr '\n' ' ')" >&2
|
||||
return 1
|
||||
fi
|
||||
# The scheduler's own rejoin with a NEW primary is reactive, not
|
||||
# periodically polled like a worker's checking_connection() -- it only
|
||||
# re-issues xcecli_join() as a side effect of its NEXT successful
|
||||
# publish call (scheduler.c's sched_publish_check_rejoin(), fired from
|
||||
# sched_publish_retry_loop() only after that same call already
|
||||
# succeeded). Right after a controller failover, THIS job may be the
|
||||
# scheduler's first real activity since the kill, so give it a much
|
||||
# longer, generously-diagnosed budget than an ordinary RUN-wait --
|
||||
# confirmed necessary live (a 10s budget was not enough in testing).
|
||||
for attempt in $(seq 1 60); do
|
||||
st="$(jobinfo_of "$jid" 2000 | sed -n 's/^state: //p')"
|
||||
if [ "$st" = "RUN" ]; then
|
||||
echo "$jid"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
log "worker-resync: job $jid never reached RUN after 60s (last state: ${st:-unknown})" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
# Sequential-mode, per-round hard gate: verifies ONE worker-dispatched job
|
||||
# survived a real controller failover via genuine rejoin-reconciliation, not
|
||||
# eventual re-delivery or a lucky timing coincidence --
|
||||
# (a) the worker's OWN log shows a FRESH "rejoin after transparent
|
||||
# reconnect succeeded" line (count strictly greater than the baseline
|
||||
# captured before the kill) -- proof an actual disconnect+rejoin cycle
|
||||
# happened;
|
||||
# (b) the job ends up DONE on the new primary, never left stuck RUN.
|
||||
# fail()s (hard) on either miss -- this is the feature's own core
|
||||
# correctness claim (bug-1741's original motivating scenario), not a
|
||||
# chaotic/racy outcome class.
|
||||
verify_worker_rejoin_resync() {
|
||||
local jid="$1" pre_count="$2" round="$3" i post_count st
|
||||
for i in $(seq 1 30); do
|
||||
post_count="$(worker_log_count "rejoin after transparent reconnect succeeded")"
|
||||
[ "${post_count:-0}" -gt "${pre_count:-0}" ] && break
|
||||
sleep 1
|
||||
done
|
||||
[ "${post_count:-0}" -gt "${pre_count:-0}" ] \
|
||||
|| fail "round $round: worker-resync job $jid: worker never logged a fresh rejoin after the failover (pre=$pre_count post=$post_count)"
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
st="$(jobinfo_of "$jid" 2000 | sed -n 's/^state: //p')"
|
||||
[ "$st" = "DONE" ] && { log "round $round: worker-resync job $jid recovered to DONE via real rejoin-reconciliation"; return 0; }
|
||||
sleep 1
|
||||
done
|
||||
fail "round $round: worker-resync job $jid never reached DONE after rejoin (last state: ${st:-unknown}) -- stuck job, reconciliation did not recover it"
|
||||
}
|
||||
|
||||
# Chaotic-mode final sweep: unlike the sequential per-round gate, chaotic
|
||||
# mode deliberately never blocks mid-loop for a specific job's outcome (that
|
||||
# would contradict its whole overlapping-kill design). Every worker-resync
|
||||
# job submitted during the loop (WORKER_RESYNC_JOBIDS) is instead checked
|
||||
# ONCE, here, after the loop's own final "restore 3-of-3 and settle" step --
|
||||
# still a hard fail() on a genuinely stuck job, since "every lost job
|
||||
# eventually reconciles" is this feature's own core correctness claim, not a
|
||||
# chaotic/racy outcome class (same standard --array-soak/--array-chaos-retry
|
||||
# hold themselves to), just checked in one batched pass instead of per round.
|
||||
verify_worker_resync_jobs_final() {
|
||||
local jid i st stuck=()
|
||||
for jid in "${WORKER_RESYNC_JOBIDS[@]}"; do
|
||||
st=""
|
||||
for i in $(seq 1 30); do
|
||||
st="$(jobinfo_of "$jid" 2000 | sed -n 's/^state: //p')"
|
||||
[ "$st" = "DONE" ] && break
|
||||
sleep 1
|
||||
done
|
||||
if [ "$st" != "DONE" ]; then
|
||||
stuck+=("$jid(last=${st:-unknown})")
|
||||
V_RESYNC_STUCK=$((V_RESYNC_STUCK + 1))
|
||||
fi
|
||||
done
|
||||
[ "${#stuck[@]}" -eq 0 ] \
|
||||
|| fail "chaotic: ${#stuck[@]}/${#WORKER_RESYNC_JOBIDS[@]} worker-resync job(s) never reconciled after the loop settled: ${stuck[*]}"
|
||||
[ "${#WORKER_RESYNC_JOBIDS[@]}" -gt 0 ] \
|
||||
&& log "chaotic: all ${#WORKER_RESYNC_JOBIDS[@]} worker-resync job(s) reconciled to DONE"
|
||||
}
|
||||
|
||||
# ── Preconditions ────────────────────────────────────────────────────────────
|
||||
[ -x "$BIN" ] || { echo "[xce-test-ha] FAIL: controller binary missing: $BIN (run /xce-build controller)" >&2; exit 2; }
|
||||
[ -x "$XT" ] || { echo "[xce-test-ha] FAIL: xce_test missing: $XT (run /xce-build controller)" >&2; exit 2; }
|
||||
|
|
@ -1018,7 +1287,7 @@ scan_bug924() {
|
|||
# the prior node is restarted, so at most one node is down at a kill instant.
|
||||
# Every data outcome is COUNTED into the verdict, never hard-failed.
|
||||
run_chaotic_rounds() {
|
||||
local round PRIMARY PREV_TERM NEWP pid DOWN=""
|
||||
local round PRIMARY PREV_TERM NEWP pid DOWN="" resync_jid
|
||||
for round in $(seq 1 "$ROUNDS"); do
|
||||
log "=== chaotic round $round/$ROUNDS ==="
|
||||
|
||||
|
|
@ -1068,9 +1337,38 @@ run_chaotic_rounds() {
|
|||
log "round $round: NO new primary within window (possible election stall)"
|
||||
fi
|
||||
|
||||
# Count loss/divergence for this round (verdict, not a hard fail).
|
||||
# Count loss/divergence for this round (verdict, not a hard fail). Runs
|
||||
# BEFORE any worker-resync activity below on purpose -- this is the
|
||||
# EXISTING, already-validated check, timed exactly as it always has
|
||||
# been; worker-resync's own (much slower, up to ~80s combined)
|
||||
# scheduler-restart + job-submit-and-RUN-wait must never shift ITS
|
||||
# comparison window (confirmed live: doing this in the other order
|
||||
# produced spurious mid-run "committed divergence" hits that a
|
||||
# post-hoc walcompare on the SAME data proved were transient
|
||||
# replication-lag artifacts, not real loss -- the exact false-
|
||||
# positive class live_committed_boundary()'s own header comment
|
||||
# already documents, just newly exposed by worker-resync's added
|
||||
# per-round wall-clock time).
|
||||
verify_classify
|
||||
walcheck_count "${#ALL_JOBIDS[@]}"
|
||||
|
||||
# worker-rejoin-resync: best-effort, same tolerance as submit_jobs
|
||||
# above — chaotic mode never blocks mid-loop for one job's outcome
|
||||
# (that would contradict its whole overlapping-kill design). Every
|
||||
# job that DOES reach RUN is tracked and checked once, in a single
|
||||
# final sweep after the whole loop settles (verify_worker_resync_
|
||||
# jobs_final, called by the chaotic MODE dispatch below). Placed
|
||||
# AFTER this round's own divergence check (see comment above) so
|
||||
# its slower pace never perturbs that check's timing; which
|
||||
# specific round's kill a tracked job ends up surviving does not
|
||||
# matter, since every tracked job is checked only once, at the end,
|
||||
# after every round's chaos (including this one) has already run.
|
||||
if [ "$WORKER_ENABLED" -eq 1 ]; then
|
||||
restart_scheduler_for_new_primary
|
||||
resync_jid="$(submit_worker_job "$WORKER_RESYNC_SLEEP")" \
|
||||
&& WORKER_RESYNC_JOBIDS+=("$resync_jid") \
|
||||
|| log "round $round: worker-resync job did not reach RUN (transient — not a loss, not tracked)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Restore 3-of-3 and let it settle so the FINAL sweep sees every member's
|
||||
|
|
@ -2447,6 +2745,15 @@ PRIMARY="$(discover_primary)"
|
|||
record_primary "$PRIMARY"
|
||||
log "primary is $PRIMARY"
|
||||
|
||||
# worker-rejoin-resync live coverage: only sequential/chaotic drive it (see
|
||||
# this file's own header comment) -- the other dedicated single-round modes
|
||||
# have their own separate, already-established scenarios and env knobs;
|
||||
# starting an unrelated worker+scheduler alongside them would only add
|
||||
# noise, not coverage.
|
||||
if [ "$MODE" = "sequential" ] || [ "$MODE" = "chaotic" ]; then
|
||||
start_worker_and_scheduler
|
||||
fi
|
||||
|
||||
# ── Step 5: initial submit + verify + waldump ────────────────────────────────
|
||||
submit_jobs "$NJOBS" || fail "initial job submission did not get $NJOBS clean acks"
|
||||
verify_no_loss || fail "initial joblist verification failed"
|
||||
|
|
@ -2640,6 +2947,14 @@ fi
|
|||
if [ "$MODE" = "chaotic" ]; then
|
||||
log "MODE=chaotic — overlapping-kill soak ($ROUNDS rounds)"
|
||||
run_chaotic_rounds
|
||||
# worker-rejoin-resync: hard sweep over every job tracked during the loop
|
||||
# (see run_chaotic_rounds' own submit call and this file's header
|
||||
# comment) -- run AFTER run_chaotic_rounds' own final settle step so the
|
||||
# cluster is stable before judging "stuck". fail()s (exits) on its own if
|
||||
# any job never reconciled; falls through here only on full success.
|
||||
if [ "$WORKER_ENABLED" -eq 1 ]; then
|
||||
verify_worker_resync_jobs_final
|
||||
fi
|
||||
# Final divergence + loss sweep on the settled cluster, then bug-924 tally.
|
||||
walcheck_count "${#ALL_JOBIDS[@]}"
|
||||
verify_classify
|
||||
|
|
@ -2648,7 +2963,7 @@ if [ "$MODE" = "chaotic" ]; then
|
|||
"$V_DIVERGENCE" "$V_JOBLOSS" "$V_BUG854" "$V_BUG924"
|
||||
VERDICT_RC=$?
|
||||
if [ "$VERDICT_RC" -eq 0 ]; then
|
||||
log "PASS (chaotic): $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs, zero committed divergence, zero committed job loss (bug854=$V_BUG854 bug924=$V_BUG924 excluded)"
|
||||
log "PASS (chaotic): $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs, zero committed divergence, zero committed job loss (bug854=$V_BUG854 bug924=$V_BUG924 excluded)$([ "$WORKER_ENABLED" -eq 1 ] && echo ", ${#WORKER_RESYNC_JOBIDS[@]} worker-resync jobs reconciled")"
|
||||
cleanup
|
||||
trap - EXIT INT TERM
|
||||
exit 0
|
||||
|
|
@ -2705,6 +3020,20 @@ for round in $(seq 1 "$ROUNDS"); do
|
|||
[ -n "$PRIMARY" ] || fail "round $round: no serving primary before kill"
|
||||
log "round $round: current primary $PRIMARY"
|
||||
|
||||
# worker-rejoin-resync: dispatch one real job onto the live worker BEFORE
|
||||
# this round's kill, sized (WORKER_RESYNC_SLEEP) so its simulated finish
|
||||
# lands strictly AFTER the kill -- guaranteeing its fire-and-forget
|
||||
# finish notify has nowhere live to land (a genuine loss, not a timing
|
||||
# coincidence) until the worker's own reconnect logic rejoins the new
|
||||
# primary. Skipped entirely when WORKER_ENABLED=0.
|
||||
RESYNC_JID=""
|
||||
if [ "$WORKER_ENABLED" -eq 1 ]; then
|
||||
RESYNC_JID="$(submit_worker_job "$WORKER_RESYNC_SLEEP")" \
|
||||
|| fail "round $round: worker-resync job never reached RUN before the kill"
|
||||
RESYNC_PRE_COUNT="$(worker_log_count "rejoin after transparent reconnect succeeded")"
|
||||
log "round $round: worker-resync job $RESYNC_JID dispatched and RUN, will finish in ${WORKER_RESYNC_SLEEP}s"
|
||||
fi
|
||||
|
||||
# Record the election term BEFORE the kill so we can detect a STRICTLY-newer
|
||||
# CTRL_PRIMARY announcement (a genuine new election) afterwards.
|
||||
PREV_TERM="$(latest_primary_term)"
|
||||
|
|
@ -2722,6 +3051,12 @@ for round in $(seq 1 "$ROUNDS"); do
|
|||
DOWN="$PRIMARY"
|
||||
sleep 2
|
||||
|
||||
# worker-rejoin-resync: let the job's simulated execution finish while
|
||||
# the old primary is confirmed dead, before we even look for a new one.
|
||||
if [ "$WORKER_ENABLED" -eq 1 ] && [ -n "$RESYNC_JID" ]; then
|
||||
sleep "$WORKER_RESYNC_SLEEP"
|
||||
fi
|
||||
|
||||
# Step 7: a NEW primary must be elected (client probe reports a SERVING
|
||||
# primary at a higher term) AND be reachable; then joblist must show no loss.
|
||||
log "round $round: waiting for new primary (term > $PREV_TERM)"
|
||||
|
|
@ -2731,8 +3066,13 @@ for round in $(seq 1 "$ROUNDS"); do
|
|||
record_primary "$NEWP"
|
||||
PRIMARY="$(discover_primary)"
|
||||
[ -n "$PRIMARY" ] || fail "round $round: new primary $NEWP elected but client cannot reach it (bug-854 class)"
|
||||
[ "$WORKER_ENABLED" -eq 1 ] && restart_scheduler_for_new_primary
|
||||
verify_no_loss || fail "round $round: a committed job was LOST after failover"
|
||||
|
||||
if [ "$WORKER_ENABLED" -eq 1 ] && [ -n "$RESYNC_JID" ]; then
|
||||
verify_worker_rejoin_resync "$RESYNC_JID" "$RESYNC_PRE_COUNT" "$round"
|
||||
fi
|
||||
|
||||
# Step 8: submit another batch on the new primary.
|
||||
submit_jobs "$NJOBS" || fail "round $round: post-failover submit did not get $NJOBS acks"
|
||||
|
||||
|
|
@ -2769,7 +3109,7 @@ done
|
|||
[ -z "$MISSING" ] || fail "not all nodes acted as primary — never led:$MISSING"
|
||||
LED="$(printf '%s\n' "${!SEEN_PRIMARY[@]}" | sort | tr '\n' ',' | sed 's/,$//')"
|
||||
|
||||
log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss; all 3 nodes led ($LED)"
|
||||
log "PASS: $ROUNDS rounds, ${#ALL_JOBIDS[@]} jobs submitted, zero job loss; all 3 nodes led ($LED)$([ "$WORKER_ENABLED" -eq 1 ] && echo "; $ROUNDS worker-resync jobs reconciled")"
|
||||
cleanup
|
||||
trap - EXIT INT TERM
|
||||
exit 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue