fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU

Bundles the fixes validated this session across ha.c/persist.c/worker.c:

- raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression
  fixed via a segment-read cache.
- bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness()
  closed with a recovery-complete gate that closes at process-lifecycle
  start, not function entry.
- bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm
  instead of silently falling through the default job-state case.
- bug-1426: trim_wal_segments() also caps its trim floor at the local
  apply pump's own progress, not just remote standby-acked positions.
- bug-1427: persist_verify_apply_correctness() extended with a worker-count
  check mirroring the existing job-count check (count_jobmgr_entries and
  count_workermgr_entries now share one count_manager_entries(var) helper).
- bug-1428: persist_register_pending_apply's critical section widened to
  close the actual race instead of patching a symptom.
- bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now
  applies mark_worker_unavail()'s mutations via a shared
  apply_worker_dead_mutation() helper instead of duplicating them.
- bug-1474 (angle-a): on_standby_connected's stale-duplicate detection
  fixed for promoted vote endpoints via a shared votes_idx_for_peer()
  index-scale converter.

Each fix independently verified via TDD (RED before fix, GREEN after).
Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live
single-node HA deployment validated Task 7.4 and bug-1416 across 5
restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT
fences).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/controller/testsuite/test_apply_pump.c

@@ -68,6 +68,7 @@
 #include <errno.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdatomic.h>
 #include <stdbool.h>
 #include <stdio.h>
 #include <string.h>
@@ -1406,6 +1407,116 @@ static void test_apply_pump_late_pending_apply_registration_self_heals(void)
           "PASS\n");
 }
 
+/* bug-1428: a narrower TOCTOU residual of the SAME self-heal mechanism the
+ * test above proves. There, the register call arrives after the walk has
+ * ALREADY finished applying `lsn` -- the well-known bug-1415 case, closed
+ * by the fast-path self-heal check. This test targets the tighter,
+ * previously-unclosed window: the register call's check runs BEFORE the
+ * walk applies `lsn` (sees "not yet applied") but the walk's own
+ * advance-then-take sequence completes entirely BEFORE the register call
+ * reaches its insert -- so the walk's persist_take_pending_apply() finds
+ * nothing (register hasn't inserted yet), and the register call then
+ * inserts a node the walk will never revisit. Permanently orphaned,
+ * identical failure mode to bug-1415, different trigger window.
+ *
+ * Reproduced deterministically (no real-time race dependent on timing)
+ * by driving the exact bad ordering through controlled state, mirroring
+ * how the register thread's old code observed state before its lock and
+ * how the walk observed/mutated it independently:
+ *   1. Main thread holds pending_applies_lock (standing in for "the walk's
+ *      advance-then-take sequence is in flight, about to complete").
+ *   2. A second thread calls the real persist_register_pending_apply() for
+ *      an lsn NOT YET applied (last_applied_lsn is still below it) -- under
+ *      the pre-fix code this thread's unlocked check would already have
+ *      run and concluded "not yet applied" by this point; under the fixed
+ *      code it blocks entirely on the lock, check included.
+ *   3. Main thread (still holding the lock) advances last_applied_lsn past
+ *      lsn -- simulating the walk finishing its apply for this lsn while
+ *      the register call is blocked/in-flight.
+ *   4. Main thread releases the lock.
+ *   5. The second thread's persist_register_pending_apply() call proceeds.
+ *
+ * Post-fix: the register call's self-heal check runs AFTER acquiring the
+ * lock in step 5, observes the now-current last_applied_lsn from step 3,
+ * and fires on_applied directly -- no insert, no orphan. Pre-fix, the
+ * check would have already run (unlocked) back in step 2 against the
+ * STALE last_applied_lsn, concluded "not yet applied", and inserted
+ * unconditionally once the lock freed in step 5 -- orphaning it forever
+ * (on_applied never fires; the node stays linked on pending_applies).
+ */
+typedef struct {
+    persist_t* p;
+    uint64_t   lsn;
+    int        fired;
+    uint64_t   fired_lsn;
+} trpa_ctx_t;
+
+static void trpa_on_applied(void* ctx, uint64_t lsn)
+{
+    trpa_ctx_t* c = (trpa_ctx_t*)ctx;
+    c->fired      = 1;
+    c->fired_lsn  = lsn;
+}
+
+static void* trpa_register_thread(void* arg)
+{
+    trpa_ctx_t* c = (trpa_ctx_t*)arg;
+    persist_register_pending_apply(c->p, c->lsn, trpa_on_applied, c);
+    return NULL;
+}
+
+static void test_register_pending_apply_race_against_concurrent_apply_self_heals(void)
+{
+    persist_t p;
+    memset(&p, 0, sizeof(p));
+    TEST_ASSERT_EQUAL_INT(0, pthread_mutex_init(&p.pending_applies_lock, NULL));
+    atomic_store_explicit(&p.last_applied_lsn, 40, memory_order_relaxed);
+
+    const uint64_t target_lsn = 42; /* not yet applied at the moment the
+                                      * register thread's call begins */
+
+    trpa_ctx_t rctx = { .p = &p, .lsn = target_lsn, .fired = 0, .fired_lsn = 0 };
+
+    pthread_mutex_lock(&p.pending_applies_lock);
+
+    pthread_t th;
+    TEST_ASSERT_EQUAL_INT(0,
+        pthread_create(&th, NULL, trpa_register_thread, &rctx));
+
+    /* Give the register thread every opportunity to run past an unlocked
+     * check (pre-fix) before the walk's simulated advance below — a bounded
+     * sleep is fine here since the correctness assertion is deterministic
+     * regardless of exactly how far it got; this only maximizes pre-fix
+     * failure detection sensitivity. */
+    usleep(20000);
+
+    /* Simulate the walk finishing its apply of target_lsn while the
+     * register call is blocked on (or, pre-fix, already past) the check. */
+    atomic_store_explicit(&p.last_applied_lsn, target_lsn, memory_order_relaxed);
+
+    pthread_mutex_unlock(&p.pending_applies_lock);
+
+    struct timespec ts = { .tv_sec = 0, .tv_nsec = 200000000L };
+    nanosleep(&ts, NULL);
+    TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL));
+
+    int has_pending = persist_has_pending_apply(&p, target_lsn);
+
+    pthread_mutex_destroy(&p.pending_applies_lock);
+
+    TEST_ASSERT_EQUAL_INT_MESSAGE(1, rctx.fired,
+        "bug-1428: a registration racing a concurrent apply of the same lsn "
+        "must still fire its hand-off (self-heal) exactly once -- 0 means "
+        "the TOCTOU orphaned it permanently");
+    TEST_ASSERT_EQUAL_UINT64_MESSAGE(target_lsn, rctx.fired_lsn,
+        "the hand-off must fire for the correct lsn");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(0, has_pending,
+        "no orphaned registration may remain linked on pending_applies");
+
+    printf("test_register_pending_apply_race_against_concurrent_apply_self_heals: "
+          "PASS\n");
+}
+
 /* ── 6. Standby-ingest commit-then-apply (2026-07-10 fix) ────────────────
  *
  * Regression coverage for ha.c's standby WAL-replication ingest path.
@@ -1648,6 +1759,7 @@ void test_apply_pump(void)
     RUN_TEST(test_apply_pump_guard_does_not_break_job_ctrl_live_append);
     RUN_TEST(test_apply_pump_fail_racing_finish_merges_against_current_state);
     RUN_TEST(test_apply_pump_late_pending_apply_registration_self_heals);
+    RUN_TEST(test_register_pending_apply_race_against_concurrent_apply_self_heals);
     RUN_TEST(test_standby_apply_deferred_until_committed_then_materializes);
     RUN_TEST(test_standby_truncate_before_commit_leaves_no_phantom_job);
 }