From 88823c59a7fec4a9f10821ce2814a3ab1e12f1df Mon Sep 17 00:00:00 2001
From: Rongsong Shen <rshen@shenrs.eu>
Date: Tue, 23 Jun 2026 04:12:32 +0000
Subject: [PATCH] test(ha-term-safety): harness post-failover client probes
 target a live node (bug-854 workaround)

The post-failover wait-primary/expect-job probes used the full masters list led
by the just-killed node, triggering libxce bug-854 (positional discovery can't
fail over from a dead-first master). Now they use a live-first list excluding the
killed primary.

NOTE: this exposes a deeper, separate post-failover AVAILABILITY issue (not data
loss): the new primary stays stuck in a catch-up loop with a lagging standby
(standby[0] lsn=5 < committed_lsn=6) and does not serve clients, so wait-primary
still times out. That is a catch-up/serving-readiness liveness concern orthogonal
to bug-828's committed-job DATA SAFETY, which is fixed and verified (walcompare
committed-prefix identical, jobs preserved on disk).
---
 .../testsuite/clean_failover_preservation.sh     | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/src/controller/testsuite/clean_failover_preservation.sh b/src/controller/testsuite/clean_failover_preservation.sh
index e717ff64..2e81c3c0 100755
--- a/src/controller/testsuite/clean_failover_preservation.sh
+++ b/src/controller/testsuite/clean_failover_preservation.sh
@@ -195,6 +195,18 @@ done
 [ -n "$PRIMARY" ] || fail "could not identify the elected primary from logs"
 log "primary is $PRIMARY"
 
+# Post-failover client probes must NOT lead the masters list with the node we are
+# about to kill: libxce positional primary-discovery (bug-854) cannot fail over to
+# the next master when the FIRST one is dead, so a client pointed at a dead-first
+# list hangs in wait-primary regardless of server-side health. Build a masters
+# list that excludes the (to-be-killed) primary so the post-failover client checks
+# reach a live node. (This is a CLIENT-side test concern, orthogonal to the
+# server-side committed-prefix safety that walcompare verifies.)
+name_to_addr() { case "$1" in ctrl-1) echo "$P1";; ctrl-2) echo "$P2";; ctrl-3) echo "$P3";; esac; }
+DEAD_ADDR="$(name_to_addr "$PRIMARY")"
+MASTERS_LIVE="$(printf '%s\n' "$P1" "$P2" "$P3" | grep -vxF "$DEAD_ADDR" | paste -sd,)"
+log "post-failover client masters (excludes killed $PRIMARY): $MASTERS_LIVE"
+
 # ── 2b. Wait for the jobs to COMMIT on a majority BEFORE killing ──────────────
 # A clean-failover test must kill the primary AFTER the submitted jobs are
 # committed (durable on a majority). `jobsub` returns the JobId on acceptance,
@@ -263,7 +275,7 @@ log "new primary is $NEW_PRIMARY"
 # hard failure — the authoritative liveness signal is the log-confirmed
 # promotion + the job-presence check at step 6b.
 SERVE_OUT="$(printf 'set masters %s\nset myname failover-test\nclient\nwait-primary %s\nquit\n' \
-            "$MASTERS" "$WAIT_PRIMARY_MS" | xtest)"
+            "$MASTERS_LIVE" "$WAIT_PRIMARY_MS" | xtest)"
 grep -q 'serving' <<<"$SERVE_OUT" \
     || log "WARN: post-failover wait-primary probe did not observe 'serving' within ${WAIT_PRIMARY_MS}ms (timing-sensitive; the authoritative checks are the log-confirmed promotion + expect-job at step 6b)"
 
@@ -328,7 +340,7 @@ log "walcompare: committed prefix identical"
 
 # ── 6b. every acked-committed job still present (R5.1 / R6.4) ─────────────────
 log "verifying all ${#JOBIDS[@]} submitted jobs survive the failover"
-EXPECT_SCRIPT="set masters $MASTERS
+EXPECT_SCRIPT="set masters $MASTERS_LIVE
 set myname failover-test
 client
 wait-primary $WAIT_PRIMARY_MS"