fix(controller): release the promotion fence once a truncated-boot node has caught up (bug-1919)

The fence re-check required !persist_recovery_truncated(), a boot-time
verdict nothing cleared, so a node that booted RECOVER_TRUNCATED stayed
fenced (fenced: true, non-promotable) until restart however far it
caught up. And the "progress is possible" branch did not re-arm, so the
fence was never re-evaluated.

persist_t gains a runtime recovery_truncation_healed latch, which
persist_recovery_truncated() honours (the raw boot verdict stays
available). The re-check marks the truncation healed once the durable
floor has advanced past the fence-time baseline (entries replicated from
the live primary) and reached the HWM, and the progress branch re-arms
with the same baseline and HWM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/controller/ha.c

@@ -1534,10 +1534,12 @@ static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
  *   - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
  *     in_catch_up fence + clear the sticky unrecoverable flag; eligible again
  *     (Req 2.6).
+ *   - booted truncated, but catch-up advanced the durable floor past the
+ *     fence-time baseline up to the HWM → mark the truncation healed
+ *     (bug-1919) and fall through to the release check.
  *   - a healthy peer is reachable now, OR catch-up is making progress → leave
- *     the member fenced under the existing catch-up machinery (it will release
+ *     the member fenced, no loud alert, and RE-ARM with the same baseline/HWM
- *     via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not
+ *     so the next window can release it (bug-1919).
- *     re-arm the re-alert while progress is happening.
  *   - STILL unrecoverable (truncated/behind AND no peer AND no progress) →
  *     RE-EMIT the loud operator alert via persist_recover_loud_fence_if_-
  *     unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the
@@ -1605,6 +1607,23 @@ static int promotion_fence_timeout_fn(void *arg)
      * inside persist_apply_pump_run itself). */
     bool     apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
 
+    /* bug-1919: recovery_truncated is the BOOT-time verdict and nothing else
+     * clears it, so without this a member that booted truncated could never
+     * pass the release check below however far it caught up.  A fenced member
+     * is not primary, so a durable floor that advanced past the fence-time
+     * baseline was advanced by contiguous entries replicated from the live
+     * primary on top of the truncated log; once that reaches the HWM the lost
+     * tail is refilled and the truncation is healed.  No progress means
+     * nothing proves the log matches the primary's, so the fence stays. */
+    if (truncated && !below_hwm && made_progress) {
+        persist_mark_recovery_truncation_healed(&c->persist);
+        truncated = false;
+        INFO("HA: recovery-truncated log healed by catch-up — durable_floor=%"
+             PRIu64 " (baseline=%" PRIu64 ") >= cluster HWM=%" PRIu64
+             " (bug-1919)",
+             durable_floor, t->baseline_durable_floor, t->target_hwm);
+    }
+
     /* Caught up: not truncated AND durable position reached the boundary AND
      * the current apply verdict is not DIVERGENT AND no self-restart is
      * pending AND no live apply resource fence is engaged → release the
@@ -1623,8 +1642,10 @@ static int promotion_fence_timeout_fn(void *arg)
 
     /* Still hollow, but a peer is reachable or catch-up advanced the durable
      * floor: progress is possible.  Leave the member fenced (in_catch_up stays
-     * set) under the existing catch-up machinery; do not re-arm the re-alert
+     * set) without the loud alert, and re-arm with the SAME baseline and HWM so
-     * while progress is happening. */
+     * the next window re-evaluates and releases the fence once catch-up
+     * completes.  Nothing else releases this fence: before bug-1919 this branch
+     * did not re-arm and the member stayed fenced until restart. */
     if (peer_reachable || made_progress) {
         INFO("HA: promotion fence catch-up window elapsed but progress is "
              "possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
@@ -1632,7 +1653,11 @@ static int promotion_fence_timeout_fn(void *arg)
              "(ha-commit-durability Req 2.2)",
              peer_reachable, durable_floor, t->baseline_durable_floor,
              t->target_hwm);
+        bool     was_truncated = t->was_truncated;
+        uint64_t baseline      = t->baseline_durable_floor;
+        uint64_t hwm           = t->target_hwm;
         free(t);
+        ha_arm_promotion_fence_timer(ha, c, was_truncated, baseline, hwm);
         return 0;
     }
 

src/controller/persist.c

@@ -1280,6 +1280,7 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
      * a fresh store has no corruption. persist_recover overwrites this with the
      * replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */
     p->recovery_verdict = RECOVER_CLEAN;
+    atomic_init(&p->recovery_truncation_healed, false); /* bug-1919 */
 
     /* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh
      * store is promotable. Set by persist_recover_loud_fence_if_unrecoverable
@@ -6854,6 +6855,8 @@ int persist_recover(persist_t* p)
         return replayed;
 
     p->recovery_verdict = verdict;
+    atomic_store_explicit(&p->recovery_truncation_healed, false,
+                          memory_order_release); /* bug-1919 */
     if (verdict == RECOVER_TRUNCATED)
         WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally "
              "corrupt (CRC break / mismatch / short read); member marked "
@@ -7004,7 +7007,16 @@ recover_verdict_t persist_recovery_verdict(persist_t* p)
 
 bool persist_recovery_truncated(persist_t* p)
 {
-    return p && p->recovery_verdict == RECOVER_TRUNCATED;
+    return p && p->recovery_verdict == RECOVER_TRUNCATED
+           && !atomic_load_explicit(&p->recovery_truncation_healed,
+                                    memory_order_acquire);
+}
+
+void persist_mark_recovery_truncation_healed(persist_t* p)
+{
+    if (p)
+        atomic_store_explicit(&p->recovery_truncation_healed, true,
+                              memory_order_release);
 }
 
 /* ------------------------------------------------------------------ *

src/controller/testsuite/test_commit_durability.c

@@ -2388,6 +2388,142 @@ static void test_promotion_fence_released_on_catchup(void)
     printf("test_promotion_fence_released_on_catchup: PASS\n");
 }
 
+
+/* Spin until the in_catch_up fence reads `want` or a timeout elapses. */
+static bool dur_wait_in_catch_up(cluster_t* c, int want, int timeout_ms)
+{
+    int waited = 0;
+    while (atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire) != want
+           && waited < timeout_ms) {
+        usleep(2000);
+        waited += 2;
+    }
+    return atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire) == want;
+}
+
+/* bug-1919: a member that BOOTED recovery-truncated, was fenced at a refused
+ * promotion, and then caught up as a standby (durable floor advanced past the
+ * fence-time baseline by entries replicated from the live primary, reaching the
+ * HWM) must have the fence RELEASED by the re-check timer.  The boot-time
+ * RECOVER_TRUNCATED verdict never changes at runtime, so before the fix the
+ * release branch (`!truncated && ...`) was unreachable and the node stayed
+ * `fenced: true` until restart.  Mirrors the soak trace: floor 360 == HWM 360
+ * at fence time, floor 375 after catch-up.  The raw boot verdict is kept for
+ * diagnostics; only persist_recovery_truncated() reports the healed state. */
+static void test_promotion_fence_truncated_boot_released_after_catchup(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_fence_trunc_heal_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
+                                          /*durable_floor=*/360);
+    TEST_ASSERT_NOT_NULL(c);
+
+    ha_set_promotion_fence_window_ms(120);
+
+    extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
+                                                      uint64_t);
+    ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
+                                          /*durable_floor=*/360, /*hwm=*/360);
+    TEST_ASSERT_EQUAL_INT(
+        1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
+
+    /* Catch-up as a standby: replicated entries advance the durable floor. The
+     * recovery verdict is deliberately NOT touched -- production never does. */
+    atomic_store_explicit(&c->persist.durable_lsn, 375, memory_order_release);
+
+    TEST_ASSERT_TRUE_MESSAGE(
+        dur_wait_in_catch_up(c, 0, 2000),
+        "bug-1919: a truncated-boot member that caught up past the HWM must "
+        "have its promotion fence released (Req 2.6)");
+    TEST_ASSERT_FALSE_MESSAGE(
+        persist_recovery_truncated(&c->persist),
+        "bug-1919: catch-up from the primary heals the truncated tail");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        RECOVER_TRUNCATED, persist_recovery_verdict(&c->persist),
+        "bug-1919: the raw boot verdict is kept for diagnostics");
+    TEST_ASSERT_FALSE(persist_recovery_unrecoverable(&c->persist));
+
+    ha_set_promotion_fence_window_ms(0);
+    dur_destroy_fence_cluster(c);
+    printf("test_promotion_fence_truncated_boot_released_after_catchup: PASS\n");
+}
+
+/* bug-1919: when the window elapses with catch-up progressing but not yet
+ * complete, the re-check must keep running (re-arm) so the fence is released
+ * once catch-up reaches the HWM.  Before the fix the "progress is possible"
+ * branch freed its ctx WITHOUT re-arming, so nothing ever re-evaluated. */
+static void test_promotion_fence_rechecks_while_catching_up(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_fence_recheck_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
+                                          /*durable_floor=*/4);
+    TEST_ASSERT_NOT_NULL(c);
+
+    ha_set_promotion_fence_window_ms(60);
+
+    extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
+                                                      uint64_t);
+    ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
+                                          /*durable_floor=*/4, /*hwm=*/10);
+
+    /* Partial catch-up: progress, still below the HWM -> stays fenced. */
+    atomic_store_explicit(&c->persist.durable_lsn, 7, memory_order_release);
+    usleep(200 * 1000); /* >= 3 windows */
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
+        "bug-1919: a member still below the HWM stays fenced");
+    TEST_ASSERT_TRUE_MESSAGE(
+        persist_recovery_truncated(&c->persist),
+        "bug-1919: below the HWM the truncated tail is not yet healed");
+
+    /* Catch-up completes: a later re-check must release the fence. */
+    atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
+    TEST_ASSERT_TRUE_MESSAGE(
+        dur_wait_in_catch_up(c, 0, 2000),
+        "bug-1919: the fence re-check must keep running while catching up and "
+        "release the fence once the HWM is reached");
+
+    ha_set_promotion_fence_window_ms(0);
+    dur_destroy_fence_cluster(c);
+    printf("test_promotion_fence_rechecks_while_catching_up: PASS\n");
+}
+
+/* bug-1919 (conservative heal rule): a truncated-boot member whose durable
+ * floor merely equals the HWM but has received NOTHING from a primary since the
+ * fence is not healed -- reaching the HWM is only proven by entries replicated
+ * on top of the truncated log, so it stays fenced. */
+static void test_promotion_fence_truncated_no_progress_stays_fenced(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_fence_trunc_noprog_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
+                                          /*durable_floor=*/10);
+    TEST_ASSERT_NOT_NULL(c);
+
+    ha_set_promotion_fence_window_ms(40);
+
+    extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
+                                                      uint64_t);
+    ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
+                                          /*durable_floor=*/10, /*hwm=*/10);
+
+    usleep(200 * 1000);
+    TEST_ASSERT_EQUAL_INT_MESSAGE(
+        1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
+        "bug-1919: no replicated progress -> the truncated member stays fenced");
+    TEST_ASSERT_TRUE(persist_recovery_truncated(&c->persist));
+
+    ha_set_promotion_fence_window_ms(0);
+    dur_destroy_fence_cluster(c);
+    printf("test_promotion_fence_truncated_no_progress_stays_fenced: PASS\n");
+}
 /* 4.4: cancelling the timer (shutdown / recovery) before it fires → the
  * re-alert never fires, no use-after-free of the timer ctx, and the process
  * stays alive (it never self-exited even before the cancel). */
@@ -2664,6 +2800,9 @@ void test_commit_durability(void)
     RUN_TEST(test_promotion_fence_unrecoverable_stays_alive_loud);
     RUN_TEST(test_promotion_fence_released_on_catchup);
     RUN_TEST(test_promotion_fence_cancel_stops_realert);
+    RUN_TEST(test_promotion_fence_truncated_boot_released_after_catchup);
+    RUN_TEST(test_promotion_fence_rechecks_while_catching_up);
+    RUN_TEST(test_promotion_fence_truncated_no_progress_stays_fenced);
     /* task 5.3 round-2 remediation (Req 8.4): cross-contamination regression —
      * an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
      * alone. */

src/include/persist.h

@@ -340,6 +340,15 @@ struct persist_s {
      * before the node participates; read by the promotion gate thereafter. */
     recover_verdict_t recovery_verdict;
 
+    /* Set once a RECOVER_TRUNCATED member has caught up from the live primary
+     * past the cluster HWM its promotion fence was armed with (bug-1919):
+     * entries replicated on top of the truncated log prove the lost tail was
+     * refilled. recovery_verdict keeps the raw boot verdict for diagnostics;
+     * persist_recovery_truncated() reports false once this is set. Reset by
+     * persist_recover. Set by the fence re-check timer, read by the
+     * promotion gate. */
+    _Atomic bool recovery_truncation_healed;
+
     /* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4,
      * NEVER-SELF-EXIT rework 2026-06-18). Set by
      * persist_recover_loud_fence_if_unrecoverable when a member is
@@ -789,11 +798,14 @@ uint64_t persist_durable_floor(persist_t* p);
  * persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED
  * verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh
  * store before recovery runs). persist_recovery_truncated is the convenience
- * flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate
+ * flag form (true iff the verdict is RECOVER_TRUNCATED and the truncation has
- * (task 4.3) consumes to mark the member non-promotable. Both are pure reads
+ * not since been healed by catch-up) the promotion gate (task 4.3) consumes
- * of recovery state established single-threaded at startup. */
+ * to mark the member non-promotable. persist_recovery_verdict is a pure read
+ * of the boot-time verdict. persist_mark_recovery_truncation_healed records
+ * that catch-up from the primary refilled the truncated tail (bug-1919). */
 recover_verdict_t persist_recovery_verdict(persist_t* p);
 bool persist_recovery_truncated(persist_t* p);
+void persist_mark_recovery_truncation_healed(persist_t* p);
 
 /* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4;
  * never-self-exit rework 2026-06-18).