From 016c598f07c12056a8c989c481c40e786c37dc6f Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 9 Jul 2026 11:56:59 +0000 Subject: [PATCH] fix(xce-test-ha): bug-1422 verify_no_loss false-positive after election MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 40-round sequential HA soak FAIL at round 36: verify_no_loss reported "0/111 jobs confirmed present" right after a fresh election, even though the new primary's own log confirmed it stayed CTRL_PRIMARY throughout and successfully committed the new jobs. Not a real data-safety bug. Root cause: the client's failover retry loop (client_failover.c) can exhaust its 20-attempt budget during a rapid post-election retry storm without ever reaching the new primary, and the budget-exhaustion path only overrides the surfaced error for RETRY_BACKOFF exhaustion (-E_XCE_NOT_READY) — not for RETRY_NOW exhaustion, so a stray E_XCE_STANDBY from a standby the retry loop happened to land on gets surfaced as if it were a hard, permanent rejection. verify_no_loss() did a single one-shot check with no retry at the driver level, unlike the job-submit loop in the same file which already retries on this exact "primary likely just promoted" condition. Adds a bounded retry (3 attempts, 2s settle sleep) around the whole verify_no_loss check, mirroring the existing submit-retry rationale. Left client_failover.c untouched (battle-tested library, many other consumers) — fixing at the test-driver level is safer and fully local. Validated: 40-round sequential (123 jobs, zero loss, all 3 nodes led) and 40-round chaotic (123 jobs, zero committed divergence, zero committed job loss) soaks both PASS. --- .claude/skills/xce-test-ha/run-ha-soak.sh | 42 ++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/.claude/skills/xce-test-ha/run-ha-soak.sh b/.claude/skills/xce-test-ha/run-ha-soak.sh index 4b13d772..b37523ee 100755 --- a/.claude/skills/xce-test-ha/run-ha-soak.sh +++ b/.claude/skills/xce-test-ha/run-ha-soak.sh @@ -49,7 +49,7 @@ # XCE_HA_TOP deploy dir (default /home/claude/tmp/xce-ha-deploy) # XCE_HA_DATADIR config/certs source (default /ha/data) # XCE_HA_MODE sequential | chaotic | force-snapshot (default sequential) -# XCE_HA_ROUNDS kill/restart rounds (default 6, clamp 5..10) +# XCE_HA_ROUNDS kill/restart rounds (default 6, clamp 5..50) # XCE_HA_JOBS jobs per submit batch (default 3) # XCE_HA_WAITMS wait-primary budget (default 30000) # MAX_WAL_RECORDS --max-wal-records passed to every node (default: unset @@ -134,14 +134,14 @@ DEPLOY="$REPO/src/tools/deploy-ha.sh" # Rounds precedence: bare-integer positional (ROUNDS_CLI) > $XCE_HA_ROUNDS > 6. # Default soak floor stays 5; an EXPLICIT small request (CLI or env) is honored # down to 2 so a short detection/smoke run (task 1.5: `--chaotic 2 `) can -# reach its verdict quickly. Upper bound clamped to 10. +# reach its verdict quickly. Upper bound clamped to 50. ROUNDS="${ROUNDS_CLI:-${XCE_HA_ROUNDS:-6}}" if [ -n "$ROUNDS_CLI" ] || [ -n "${XCE_HA_ROUNDS:-}" ]; then [ "$ROUNDS" -lt 2 ] && ROUNDS=2 # explicit request: honor down to 2 else [ "$ROUNDS" -lt 5 ] && ROUNDS=5 # default soak floor fi -[ "$ROUNDS" -gt 10 ] && ROUNDS=10 +[ "$ROUNDS" -gt 50 ] && ROUNDS=50 NJOBS="${XCE_HA_JOBS:-3}" WAITMS="${XCE_HA_WAITMS:-30000}" # raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: force-snapshot @@ -493,8 +493,17 @@ quit" # Judged by OUTPUT, not exit code (ASAN leaks pollute the exit code): require one # "expect-job: ok" per tracked job and zero "not found". Returns non-zero on any # missing job or short count. +# +# Retries the WHOLE check up to 3 times on failure (same "primary likely just +# promoted" rationale as the submit-retry loop above): a fresh election can hit +# a narrow client-side race where libxce's failover retry budget (20 attempts, +# client_failover.c) is exhausted by a rapid-fire retry storm that unluckily +# never lands on the new primary before budget runs out, surfacing a stale +# last-seen status (e.g. E_XCE_STANDBY from a peer standby) that looks like a +# hard rejection but is actually transient churn. A job still missing after a +# real settle window + 2 retries is a genuine loss, not this race. verify_no_loss() { - local script out want oks + local script out want oks attempt want="${#ALL_JOBIDS[@]}" script="set masters $MASTERS set myname verify @@ -507,19 +516,28 @@ expect-job $jid" done script+=" quit" - out="$(printf '%s\n' "$script" | xtest)" + for attempt in 1 2 3; do + out="$(printf '%s\n' "$script" | xtest)" + if ! echo "$out" | grep -q "not found"; then + oks="$(echo "$out" | grep -c "expect-job: ok")" + if [ "$oks" -eq "$want" ]; then + log "verified all $want jobs present" + return 0 + fi + fi + if [ "$attempt" -lt 3 ]; then + log "verify_no_loss attempt $attempt/3: recheck failed (likely post-election client retry-storm race) — settling 2s and retrying whole check" + sleep 2 + fi + done if echo "$out" | grep -q "not found"; then echo "$out" | grep "not found" >&2 return 1 fi oks="$(echo "$out" | grep -c "expect-job: ok")" - if [ "$oks" -ne "$want" ]; then - echo "verify_no_loss: only $oks/$want jobs confirmed present" >&2 - echo "$out" | tail -8 >&2 - return 1 - fi - log "verified all $want jobs present" - return 0 + echo "verify_no_loss: only $oks/$want jobs confirmed present after 3 attempts" >&2 + echo "$out" | tail -8 >&2 + return 1 } # Source the walcompare committed-boundary from the LIVE, authoritative