diff --git a/src/controller/testsuite/clean_failover_preservation.sh b/src/controller/testsuite/clean_failover_preservation.sh index e717ff64..2e81c3c0 100755 --- a/src/controller/testsuite/clean_failover_preservation.sh +++ b/src/controller/testsuite/clean_failover_preservation.sh @@ -195,6 +195,18 @@ done [ -n "$PRIMARY" ] || fail "could not identify the elected primary from logs" log "primary is $PRIMARY" +# Post-failover client probes must NOT lead the masters list with the node we are +# about to kill: libxce positional primary-discovery (bug-854) cannot fail over to +# the next master when the FIRST one is dead, so a client pointed at a dead-first +# list hangs in wait-primary regardless of server-side health. Build a masters +# list that excludes the (to-be-killed) primary so the post-failover client checks +# reach a live node. (This is a CLIENT-side test concern, orthogonal to the +# server-side committed-prefix safety that walcompare verifies.) +name_to_addr() { case "$1" in ctrl-1) echo "$P1";; ctrl-2) echo "$P2";; ctrl-3) echo "$P3";; esac; } +DEAD_ADDR="$(name_to_addr "$PRIMARY")" +MASTERS_LIVE="$(printf '%s\n' "$P1" "$P2" "$P3" | grep -vxF "$DEAD_ADDR" | paste -sd,)" +log "post-failover client masters (excludes killed $PRIMARY): $MASTERS_LIVE" + # ── 2b. Wait for the jobs to COMMIT on a majority BEFORE killing ────────────── # A clean-failover test must kill the primary AFTER the submitted jobs are # committed (durable on a majority). `jobsub` returns the JobId on acceptance, @@ -263,7 +275,7 @@ log "new primary is $NEW_PRIMARY" # hard failure — the authoritative liveness signal is the log-confirmed # promotion + the job-presence check at step 6b. SERVE_OUT="$(printf 'set masters %s\nset myname failover-test\nclient\nwait-primary %s\nquit\n' \ - "$MASTERS" "$WAIT_PRIMARY_MS" | xtest)" + "$MASTERS_LIVE" "$WAIT_PRIMARY_MS" | xtest)" grep -q 'serving' <<<"$SERVE_OUT" \ || log "WARN: post-failover wait-primary probe did not observe 'serving' within ${WAIT_PRIMARY_MS}ms (timing-sensitive; the authoritative checks are the log-confirmed promotion + expect-job at step 6b)" @@ -328,7 +340,7 @@ log "walcompare: committed prefix identical" # ── 6b. every acked-committed job still present (R5.1 / R6.4) ───────────────── log "verifying all ${#JOBIDS[@]} submitted jobs survive the failover" -EXPECT_SCRIPT="set masters $MASTERS +EXPECT_SCRIPT="set masters $MASTERS_LIVE set myname failover-test client wait-primary $WAIT_PRIMARY_MS"