fix(controller-test): clean_failover_preservation.sh runs again (bug-1721)

The script required a pre-generated <repo>/ha/data (cluster.bin + TLS
certs) that ee8473bd removed, and called deploy-ha.sh with that dir as an
extra argument; deploy-ha.sh now generates both into the deploy dir and
takes <binary> <top-dir> <peers>. It failed at its first precondition in
every checkout.

Behind that, both committed-LSN boundaries still assumed bug-845's doubled
submits (1 + 2*N_JOBS): the commit wait never reached it and walcompare
compared an LSN past every member's WAL. Since that fix each job is one
WAL entry, so the boundary is 1 + N_JOBS.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/controller/testsuite/clean_failover_preservation.sh

@@ -76,7 +76,6 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$HERE/../../.." && pwd)"
 BINARY="${XCE_CONTROLLER_BIN:-$REPO_ROOT/src/xce.build/controller/xce_controller}"
 XCE_TEST="${XCE_TEST_BIN:-$REPO_ROOT/src/xce.build/harness/xce_test}"
-DATADIR="${XCE_HA_DATADIR:-$REPO_ROOT/ha/data}"
 DEPLOY_SH="${XCE_DEPLOY_SH:-$REPO_ROOT/src/tools/deploy-ha.sh}"
 
 # A real-fs working dir the harness OWNS (creates + tears down). Override on an
@@ -138,8 +137,6 @@ xtest() {
 # ── Preconditions ────────────────────────────────────────────────────────────
 [ -x "$BINARY" ]    || fail "controller binary not found/executable: $BINARY (run /xce-build controller)"
 [ -x "$XCE_TEST" ]  || fail "xce_test not found/executable: $XCE_TEST (run /xce-build harness)"
-[ -r "$DATADIR/cluster.bin" ] || fail "cluster.bin missing under $DATADIR (deploy prerequisite)"
-[ -r "$DATADIR/xce.crt" ] && [ -r "$DATADIR/xce.key" ] || fail "TLS certs missing under $DATADIR"
 [ -x "$DEPLOY_SH" ] || fail "deploy script missing: $DEPLOY_SH"
 
 trap cleanup EXIT INT TERM
@@ -147,7 +144,10 @@ trap cleanup EXIT INT TERM
 # ── 1. Deploy + start ────────────────────────────────────────────────────────
 log "deploying 3-node cluster to $TOPDIR (peers: $PEERS)"
 pkill -9 -f -- "--workdir $TOPDIR/controller" 2>/dev/null; sleep 0.3
-bash "$DEPLOY_SH" "$BINARY" "$DATADIR" "$TOPDIR" "$PEERS" >"$TOPDIR/deploy.log" 2>&1 \
+# bug-1721: deploy-ha.sh generates the TLS certs and cluster.bin into TOPDIR
+# itself (the pre-generated ha/data/ directory it once copied is gone,
+# ee8473bd) and takes <binary> <top-dir> <peers>.
+bash "$DEPLOY_SH" "$BINARY" "$TOPDIR" "$PEERS" >"$TOPDIR/deploy.log" 2>&1 \
     || fail "deploy-ha.sh failed (see $TOPDIR/deploy.log)"
 
 log "starting ctrl-1/ctrl-2/ctrl-3"
@@ -219,8 +219,10 @@ log "post-failover client masters (excludes killed $PRIMARY): $MASTERS_LIVE"
 # (Do NOT grep the ha_process_standby_state `committed_lsn=N` heartbeat line: that
 # is the standby's *learned* committed_lsn, which lags and only logs on the
 # one-time converged transition — a misleading signal.) Boundary = own-term noop
-# LSN 1 + the committed JOB_SUBMIT entries; bug-845 doubles each submit => 1 + 2*N_JOBS.
+# LSN 1 + one JOB_SUBMIT entry per job => 1 + N_JOBS. (bug-1721: this was
-COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + 2 * N_JOBS))}"
+# 1 + 2*N_JOBS while bug-845 wrote every submit twice; since that fix the
+# boundary was never reached and the run always failed here.)
+COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + N_JOBS))}"
 # Catch-up of N entries with per-entry fdatasync can be slow on a loaded box;
 # allow generous headroom (committed advances once the catch-up streamer + ACKs land).
 COMMIT_WAIT_S="${XCE_FAILOVER_COMMIT_WAIT_S:-90}"
@@ -293,9 +295,9 @@ grep -q 'persist_recover done' "$TOPDIR/logs/${PRIMARY}.log" \
 
 # ── 6a. walcompare: committed prefix identical (term, pcrc) ───────────────────
 # Derive the committed boundary explicitly = the number of jobs' committed
-# entries we KNOW were acked before the kill. Each clean jobsub produces a
+# entries we KNOW were acked before the kill. Each clean jobsub produces one
-# JOB_SUBMIT (the duplicate-submit bug-845 doubles entries, but the FIRST N_JOBS
+# JOB_SUBMIT entry (bug-845's duplicate submits are fixed). Use the minimum
-# distinct submits are committed). Use the minimum recovered tip as a floor and
+# recovered tip as a floor and
 # the known acked count; pick the conservative explicit boundary (5.3 advisory:
 # never let a CRC-broken short tail lower the shared boundary and mask a
 # divergence). Here we assert the WHOLE pre-failover committed prefix.
@@ -303,10 +305,11 @@ D1="$TOPDIR/controller/ctrl-1/data"
 D2="$TOPDIR/controller/ctrl-2/data"
 D3="$TOPDIR/controller/ctrl-3/data"
 
-# Committed boundary: the own-term noop (LSN 1) + the committed JOB_SUBMIT
+# Committed boundary: the own-term noop (LSN 1) + one committed JOB_SUBMIT
-# entries. With bug-845 doubling each submit, N_JOBS submits => 1 + 2*N_JOBS
+# entry per job => 1 + N_JOBS pre-failover (bug-1721: was 1 + 2*N_JOBS from
-# committed entries pre-failover. Compute and pass it EXPLICITLY.
+# bug-845's doubled submits, which pointed past every member's WAL). Same value
-COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + 2 * N_JOBS))}"
+# as the commit wait above; compute and pass it EXPLICITLY.
+COMMITTED_BOUNDARY="${XCE_FAILOVER_COMMITTED:-$((1 + N_JOBS))}"
 log "walcompare --committed $COMMITTED_BOUNDARY across all members"
 
 WALCMP_OUT="$(printf 'walcompare --committed %s %s %s %s\nquit\n' \