diff --git a/src/controller/testsuite/clean_failover_preservation.sh b/src/controller/testsuite/clean_failover_preservation.sh index ea0cb527..a46d0953 100755 --- a/src/controller/testsuite/clean_failover_preservation.sh +++ b/src/controller/testsuite/clean_failover_preservation.sh @@ -76,7 +76,6 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$HERE/../../.." && pwd)" BINARY="${XCE_CONTROLLER_BIN:-$REPO_ROOT/src/xce.build/controller/xce_controller}" XCE_TEST="${XCE_TEST_BIN:-$REPO_ROOT/src/xce.build/harness/xce_test}" -DATADIR="${XCE_HA_DATADIR:-$REPO_ROOT/ha/data}" DEPLOY_SH="${XCE_DEPLOY_SH:-$REPO_ROOT/src/tools/deploy-ha.sh}" # A real-fs working dir the harness OWNS (creates + tears down). Override on an @@ -138,8 +137,6 @@ xtest() { # ── Preconditions ──────────────────────────────────────────────────────────── [ -x "$BINARY" ] || fail "controller binary not found/executable: $BINARY (run /xce-build controller)" [ -x "$XCE_TEST" ] || fail "xce_test not found/executable: $XCE_TEST (run /xce-build harness)" -[ -r "$DATADIR/cluster.bin" ] || fail "cluster.bin missing under $DATADIR (deploy prerequisite)" -[ -r "$DATADIR/xce.crt" ] && [ -r "$DATADIR/xce.key" ] || fail "TLS certs missing under $DATADIR" [ -x "$DEPLOY_SH" ] || fail "deploy script missing: $DEPLOY_SH" trap cleanup EXIT INT TERM @@ -147,7 +144,10 @@ trap cleanup EXIT INT TERM # ── 1. Deploy + start ──────────────────────────────────────────────────────── log "deploying 3-node cluster to $TOPDIR (peers: $PEERS)" pkill -9 -f -- "--workdir $TOPDIR/controller" 2>/dev/null; sleep 0.3 -bash "$DEPLOY_SH" "$BINARY" "$DATADIR" "$TOPDIR" "$PEERS" >"$TOPDIR/deploy.log" 2>&1 \ +# bug-1721: deploy-ha.sh generates the TLS certs and cluster.bin into TOPDIR +# itself (the pre-generated ha/data/ directory it once copied is gone, +# ee8473bd) and takes . +bash "$DEPLOY_SH" "$BINARY" "$TOPDIR" "$PEERS" >"$TOPDIR/deploy.log" 2>&1 \ || fail "deploy-ha.sh failed (see $TOPDIR/deploy.log)" log "starting ctrl-1/ctrl-2/ctrl-3" @@ -219,8 +219,10 @@ log "post-failover client masters (excludes killed $PRIMARY): $MASTERS_LIVE" # (Do NOT grep the ha_process_standby_state `committed_lsn=N` heartbeat line: that # is the standby's *learned* committed_lsn, which lags and only logs on the # one-time converged transition — a misleading signal.) Boundary = own-term noop -# LSN 1 + the committed JOB_SUBMIT entries; bug-845 doubles each submit => 1 + 2*N_JOBS. -COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + 2 * N_JOBS))}" +# LSN 1 + one JOB_SUBMIT entry per job => 1 + N_JOBS. (bug-1721: this was +# 1 + 2*N_JOBS while bug-845 wrote every submit twice; since that fix the +# boundary was never reached and the run always failed here.) +COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + N_JOBS))}" # Catch-up of N entries with per-entry fdatasync can be slow on a loaded box; # allow generous headroom (committed advances once the catch-up streamer + ACKs land). COMMIT_WAIT_S="${XCE_FAILOVER_COMMIT_WAIT_S:-90}" @@ -293,9 +295,9 @@ grep -q 'persist_recover done' "$TOPDIR/logs/${PRIMARY}.log" \ # ── 6a. walcompare: committed prefix identical (term, pcrc) ─────────────────── # Derive the committed boundary explicitly = the number of jobs' committed -# entries we KNOW were acked before the kill. Each clean jobsub produces a -# JOB_SUBMIT (the duplicate-submit bug-845 doubles entries, but the FIRST N_JOBS -# distinct submits are committed). Use the minimum recovered tip as a floor and +# entries we KNOW were acked before the kill. Each clean jobsub produces one +# JOB_SUBMIT entry (bug-845's duplicate submits are fixed). Use the minimum +# recovered tip as a floor and # the known acked count; pick the conservative explicit boundary (5.3 advisory: # never let a CRC-broken short tail lower the shared boundary and mask a # divergence). Here we assert the WHOLE pre-failover committed prefix. @@ -303,10 +305,11 @@ D1="$TOPDIR/controller/ctrl-1/data" D2="$TOPDIR/controller/ctrl-2/data" D3="$TOPDIR/controller/ctrl-3/data" -# Committed boundary: the own-term noop (LSN 1) + the committed JOB_SUBMIT -# entries. With bug-845 doubling each submit, N_JOBS submits => 1 + 2*N_JOBS -# committed entries pre-failover. Compute and pass it EXPLICITLY. -COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + 2 * N_JOBS))}" +# Committed boundary: the own-term noop (LSN 1) + one committed JOB_SUBMIT +# entry per job => 1 + N_JOBS pre-failover (bug-1721: was 1 + 2*N_JOBS from +# bug-845's doubled submits, which pointed past every member's WAL). Same value +# as the commit wait above; compute and pass it EXPLICITLY. +COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + N_JOBS))}" log "walcompare --committed $COMMITTED_BOUNDARY across all members" WALCMP_OUT="$(printf 'walcompare --committed %s %s %s %s\nquit\n' \