fix(controller): close 6 data-safety races + apply-pump O(N^2) regression + boot TOCTOU

Bundles the fixes validated this session across ha.c/persist.c/worker.c:

- raft-apply-decoupling Task 7.4: apply-pump O(N^2) throughput regression
  fixed via a segment-read cache.
- bug-1416: boot-time TOCTOU race in persist_verify_apply_correctness()
  closed with a recovery-complete gate that closes at process-lifecycle
  start, not function entry.
- bug-451: HOOK_DETAIL_JOB_RUSAGE_UPDATE now has its own persist hook arm
  instead of silently falling through the default job-state case.
- bug-1426: trim_wal_segments() also caps its trim floor at the local
  apply pump's own progress, not just remote standby-acked positions.
- bug-1427: persist_verify_apply_correctness() extended with a worker-count
  check mirroring the existing job-count check (count_jobmgr_entries and
  count_workermgr_entries now share one count_manager_entries(var) helper).
- bug-1428: persist_register_pending_apply's critical section widened to
  close the actual race instead of patching a symptom.
- bug-1472 (remainder): WORKER_DEAD replay in worker_from_capnp() now
  applies mark_worker_unavail()'s mutations via a shared
  apply_worker_dead_mutation() helper instead of duplicating them.
- bug-1474 (angle-a): on_standby_connected's stale-duplicate detection
  fixed for promoted vote endpoints via a shared votes_idx_for_peer()
  index-scale converter.

Each fix independently verified via TDD (RED before fix, GREEN after).
Full ASAN rebuild + controller_unit_test: 1101 tests, 0 failures. Live
single-node HA deployment validated Task 7.4 and bug-1416 across 5
restart/recovery cycles (45/45 job submissions, zero APPLY_DIVERGENT
fences).

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

src/controller/testsuite/test_e2e_checkpoint.c

@@ -158,14 +158,23 @@ static void test_checkpoint_wal_trim_after_checkpoint(void)
     persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
     TEST_ASSERT_NOT_NULL(p);
 
-    /* Write enough entries and checkpoint once */
+    /* Write enough entries and checkpoint once. No live apply pump runs in
+     * this cluster-less fixture, so last_applied_lsn stays 0 unless
+     * simulated -- bug-1426's trim floor now also caps at last_applied_lsn
+     * (the local apply cursor), so it must be advanced here to reflect the
+     * pump having caught up, or trim_wal_segments would correctly refuse
+     * to delete anything this test expects trimmed. */
     for (int i = 0; i < 5; i++)
         persist_wal_async(p, WAL_JOB_UPDATE, "trim", 4);
+    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
+                          memory_order_relaxed);
     TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
 
     /* Write a few more entries then checkpoint again */
     for (int i = 0; i < 3; i++)
         persist_wal_async(p, WAL_JOB_UPDATE, "trim2", 5);
+    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
+                          memory_order_relaxed);
     TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
 
     /* After two checkpoints the wal directory must still be functional
@@ -259,6 +268,14 @@ static void test_trim_guard_preserves_standby_segments(void)
     TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
     TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
 
+    /* bug-1426: simulate the local apply pump having caught up to the
+     * active segment tip -- trim_wal_segments now also caps its floor at
+     * last_applied_lsn (this test targets the standby-floor guard
+     * specifically, not the apply-cursor one, so the pump must not be the
+     * limiting factor here). */
+    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
+                          memory_order_relaxed);
+
     /* Trigger checkpoint → trim_wal_segments runs */
     TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
 
@@ -307,6 +324,10 @@ static void test_trim_without_standbys_proceeds_normally(void)
     TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
     TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
     TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
+    /* bug-1426: simulate the local apply pump having caught up (see
+     * test_trim_guard_preserves_standby_segments's identical note above). */
+    atomic_store_explicit(&p->last_applied_lsn, p->current_lsn - 1,
+                          memory_order_relaxed);
     TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
 
     char wal_path[512];
@@ -331,6 +352,71 @@ static void test_trim_without_standbys_proceeds_normally(void)
     printf("test_trim_without_standbys_proceeds_normally: PASS\n");
 }
 
+/* ------------------------------------------------------------------ */
+/* Test 6: local apply-cursor trim guard — bug-1426. With no standbys   */
+/* registered (so the standby-floor guard alone would allow trimming   */
+/* everything below the active segment), a LAGGING local apply pump     */
+/* (last_applied_lsn below the active segment) must still block         */
+/* trimming of segments it hasn't caught up to yet.                     */
+/*                                                                      */
+/* RED proof: revert the last_applied_lsn floor cap in                  */
+/* trim_wal_segments -> seg-at-2 (which the lagging pump hasn't read     */
+/* yet) is deleted anyway, because the standby-only floor (= active_seg  */
+/* _lsn=3, no standbys) never considers local apply progress -> a        */
+/* concurrent live apply-pump reader relying on an already-open cached   */
+/* fd into that segment (POSIX unlink-keeps-fd-alive) would silently     */
+/* keep succeeding, while any FRESH read of the same range (a restart's  */
+/* persist_recover() replay) would find it already gone -- the apply-    */
+/* correctness gap this bug describes.                                  */
+/* ------------------------------------------------------------------ */
+
+static void test_trim_guard_preserves_unapplied_local_segments(void)
+{
+    char tmpdir[256];
+    persist_t* p = make_ckpt_e2e(tmpdir, sizeof(tmpdir), 2);
+    TEST_ASSERT_NOT_NULL(p);
+
+    /* No standbys registered — the pre-bug-1426 floor would be
+     * active_seg_lsn=3 alone, same as test_trim_without_standbys_proceeds_
+     * normally above. This time, deliberately advance last_applied_lsn only
+     * through the entry in seg-at-1 (LSN 1), NOT the one in seg-at-2 (LSN
+     * 2) — simulating a live apply pump that is still lagging (e.g. mid
+     * catch-up burst) when the checkpoint's trim fires. seg-at-1 is fully
+     * consumed and safe to delete; seg-at-2 is not — the pump hasn't
+     * reached its entry yet. */
+    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
+    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
+    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
+    TEST_ASSERT_EQUAL_INT(0, persist_test_force_new_segment(p));
+    atomic_store_explicit(&p->last_applied_lsn, 1, memory_order_relaxed);
+
+    TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
+
+    char wal_path[512];
+    snprintf(wal_path, sizeof(wal_path), "%s/e2e-ckpt/data/wal", tmpdir);
+
+    /* bug-1426 floor = min(active_seg_lsn=3, last_applied_lsn+1=2) = 2.
+     * seg_at_1 (start=1) < 2 → deleted: its one entry (LSN 1) is fully
+     * applied, safe to reclaim. seg_at_2 (start=2) < 2 is FALSE →
+     * preserved: its entry (LSN 2) has NOT been applied yet — deleting it
+     * would strand data the local apply pump still needs. */
+    TEST_ASSERT_FALSE_MESSAGE(
+        seg_file_exists(wal_path, 1),
+        "seg-at-1: fully applied (last_applied_lsn=1), safe to trim even "
+        "though no standby needs it either");
+    TEST_ASSERT_TRUE_MESSAGE(
+        seg_file_exists(wal_path, 2),
+        "seg-at-2: NOT yet applied by the lagging local pump "
+        "(last_applied_lsn=1 < 2), MUST be preserved (bug-1426 guard) even "
+        "though no standby needs it");
+    TEST_ASSERT_TRUE_MESSAGE(
+        seg_file_exists(wal_path, 3),
+        "seg-at-3: active segment, MUST always be preserved");
+
+    teardown_ckpt_e2e(p);
+    printf("test_trim_guard_preserves_unapplied_local_segments: PASS\n");
+}
+
 /* ------------------------------------------------------------------ */
 /* Suite entry point                                                    */
 /* ------------------------------------------------------------------ */
@@ -345,4 +431,5 @@ void test_e2e_checkpoint(void)
     RUN_TEST(test_snapshot_rotation_keeps_snap_keep_n);
     RUN_TEST(test_trim_guard_preserves_standby_segments);
     RUN_TEST(test_trim_without_standbys_proceeds_normally);
+    RUN_TEST(test_trim_guard_preserves_unapplied_local_segments);
 }