fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU

Bundles the fixes validated this session across ha.c/persist.c/worker.c:

- raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression
  fixed via a segment-read cache.
- bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness()
  closed with a recovery-complete gate that closes at process-lifecycle
  start, not function entry.
- bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm
  instead of silently falling through the default job-state case.
- bug-1426: trim_wal_segments() also caps its trim floor at the local
  apply pump's own progress, not just remote standby-acked positions.
- bug-1427: persist_verify_apply_correctness() extended with a worker-count
  check mirroring the existing job-count check (count_jobmgr_entries and
  count_workermgr_entries now share one count_manager_entries(var) helper).
- bug-1428: persist_register_pending_apply's critical section widened to
  close the actual race instead of patching a symptom.
- bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now
  applies mark_worker_unavail()'s mutations via a shared
  apply_worker_dead_mutation() helper instead of duplicating them.
- bug-1474 (angle-a): on_standby_connected's stale-duplicate detection
  fixed for promoted vote endpoints via a shared votes_idx_for_peer()
  index-scale converter.

Each fix independently verified via TDD (RED before fix, GREEN after).
Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live
single-node HA deployment validated Task 7.4 and bug-1416 across 5
restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT
fences).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/tools/deploy-ha.sh

@@ -21,20 +21,59 @@ BINARY=$(realpath "$BINARY")
 # xceadmin is meson-built now (src/xceadmin/meson.build), not a plain
 # `pip install -e .` venv checked out under src/xceadmin/ -- its build
 # output lives under whichever meson build dir xce-build.sh's profile
-# picked, or under a nix-installed prefix's bin/. Try each in turn.
+# picked, or under a nix-installed prefix's bin/, and each candidate is a
+# STATIC SNAPSHOT (meson-build.sh copies src/xceadmin out-of-tree and does
+# a plain, non-editable `pip install`) taken whenever that profile's
+# `xceadmin` target was last built -- editing src/xceadmin/ source does
+# NOT retroactively update an already-built snapshot.
+#
+# Correctness, not just freshness, matters here: xceadmin builds the
+# cluster.bin that $BINARY (the controller daemon being deployed) will
+# decode, so the two must come from the SAME build generation -- e.g. if
+# the capnp wire schema changed between builds, an xceadmin from one
+# generation and a controller from another can silently mis-encode/
+# mis-decode a field (this is exactly how a stale-xceadmin deploy bug
+# went undiagnosed: a fresher-but-DIFFERENT-build xceadmin could just as
+# easily mismatch $BINARY as a staler one). Prefer the xceadmin living in
+# $BINARY's OWN build dir (<builddir>/controller/xce_controller ->
+# <builddir>/xceadmin/.venv/bin/xceadmin) -- guaranteed same-generation.
+# Only if that specific one doesn't exist (e.g. xceadmin was never built
+# in that profile) fall back to whichever other candidate is newest, with
+# a loud warning that same-generation consistency isn't guaranteed.
+BINARY_BUILD_DIR=$(dirname "$(dirname "$BINARY")")
+BINARY_DIR=$(dirname "$BINARY")
 XCEADMIN=""
-for candidate in \
-    "$SCRIPT_DIR/../xce.build/xceadmin/.venv/bin/xceadmin" \
-    "$SCRIPT_DIR/../build.release/xceadmin/.venv/bin/xceadmin" \
-    "$SCRIPT_DIR/../build.dbg/xceadmin/.venv/bin/xceadmin" \
-    "$SCRIPT_DIR/../result/bin/xceadmin" \
-    "$(command -v xceadmin 2>/dev/null || true)"
-do
-    if [ -n "$candidate" ] && [ -x "$candidate" ]; then
-        XCEADMIN="$candidate"
-        break
+if [ -x "$BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin" ]; then
+    # meson layout: <builddir>/controller/xce_controller + <builddir>/xceadmin/.venv/...
+    XCEADMIN="$BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin"
+    echo "Using xceadmin: $XCEADMIN (same build dir as $BINARY)"
+elif [ -x "$BINARY_DIR/xceadmin" ]; then
+    # nix result layout: result/bin/xce_controller + result/bin/xceadmin (flat, no
+    # per-component subdir) -- BINARY_BUILD_DIR's two-levels-up guess doesn't apply.
+    XCEADMIN="$BINARY_DIR/xceadmin"
+    echo "Using xceadmin: $XCEADMIN (same directory as $BINARY)"
+else
+    echo "Warning: no xceadmin found alongside $BINARY (checked $BINARY_BUILD_DIR/xceadmin/.venv/bin/xceadmin and $BINARY_DIR/xceadmin) -- falling back to the newest xceadmin found elsewhere, which is NOT guaranteed to be the same build generation as $BINARY."
+    XCEADMIN_MTIME=0
+    for candidate in \
+        "$SCRIPT_DIR/../xce.build/xceadmin/.venv/bin/xceadmin" \
+        "$SCRIPT_DIR/../build.release/xceadmin/.venv/bin/xceadmin" \
+        "$SCRIPT_DIR/../build.dbg/xceadmin/.venv/bin/xceadmin" \
+        "$SCRIPT_DIR/../result/bin/xceadmin" \
+        "$(command -v xceadmin 2>/dev/null || true)"
+    do
+        if [ -n "$candidate" ] && [ -x "$candidate" ]; then
+            candidate_mtime=$(stat -c %Y "$candidate" 2>/dev/null || stat -f %m "$candidate" 2>/dev/null || echo 0)
+            if [ "$candidate_mtime" -gt "$XCEADMIN_MTIME" ]; then
+                XCEADMIN="$candidate"
+                XCEADMIN_MTIME="$candidate_mtime"
+            fi
+        fi
+    done
+    if [ -n "$XCEADMIN" ]; then
+        echo "Using xceadmin: $XCEADMIN (built $(date -d "@$XCEADMIN_MTIME" 2>/dev/null || date -r "$XCEADMIN_MTIME" 2>/dev/null))"
     fi
-done
+fi
 if [ -z "$XCEADMIN" ]; then
     echo "Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first"
     exit 1

src/tools/xce-build.sh

@@ -85,7 +85,12 @@ case "$COMPONENT" in
         meson compile -C "$BUILD_DIR" xce_scheduler sched_test
         if [ "$RUN_TESTS" = 1 ]; then
             echo "=== Running scheduler tests ==="
-            "./$BUILD_DIR/scheduler/testsuite/sched_test"
+            # bug-1559: run the whole 'scheduler' meson suite (meson test
+            # rebuilds by default), not just the sched_test binary directly
+            # by path -- that bypassed every other scheduler test target
+            # registered under the same suite tag (component-failover-
+            # adoption's 4 task-3.x binaries among them).
+            meson test -C "$BUILD_DIR" --suite scheduler --print-errorlogs
         fi
         ;;
     worker)