diff --git a/src/controller/ha.c b/src/controller/ha.c index e8141af4..a6dd1f13 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -1534,10 +1534,12 @@ static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c, * - caught up (NOT truncated AND durable_floor >= target_hwm) → release the * in_catch_up fence + clear the sticky unrecoverable flag; eligible again * (Req 2.6). + * - booted truncated, but catch-up advanced the durable floor past the + * fence-time baseline up to the HWM → mark the truncation healed + * (bug-1919) and fall through to the release check. * - a healthy peer is reachable now, OR catch-up is making progress → leave - * the member fenced under the existing catch-up machinery (it will release - * via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not - * re-arm the re-alert while progress is happening. + * the member fenced, no loud alert, and RE-ARM with the same baseline/HWM + * so the next window can release it (bug-1919). * - STILL unrecoverable (truncated/behind AND no peer AND no progress) → * RE-EMIT the loud operator alert via persist_recover_loud_fence_if_- * unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the @@ -1605,6 +1607,23 @@ static int promotion_fence_timeout_fn(void *arg) * inside persist_apply_pump_run itself). */ bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist); + /* bug-1919: recovery_truncated is the BOOT-time verdict and nothing else + * clears it, so without this a member that booted truncated could never + * pass the release check below however far it caught up. A fenced member + * is not primary, so a durable floor that advanced past the fence-time + * baseline was advanced by contiguous entries replicated from the live + * primary on top of the truncated log; once that reaches the HWM the lost + * tail is refilled and the truncation is healed. No progress means + * nothing proves the log matches the primary's, so the fence stays. */ + if (truncated && !below_hwm && made_progress) { + persist_mark_recovery_truncation_healed(&c->persist); + truncated = false; + INFO("HA: recovery-truncated log healed by catch-up — durable_floor=%" + PRIu64 " (baseline=%" PRIu64 ") >= cluster HWM=%" PRIu64 + " (bug-1919)", + durable_floor, t->baseline_durable_floor, t->target_hwm); + } + /* Caught up: not truncated AND durable position reached the boundary AND * the current apply verdict is not DIVERGENT AND no self-restart is * pending AND no live apply resource fence is engaged → release the @@ -1623,8 +1642,10 @@ static int promotion_fence_timeout_fn(void *arg) /* Still hollow, but a peer is reachable or catch-up advanced the durable * floor: progress is possible. Leave the member fenced (in_catch_up stays - * set) under the existing catch-up machinery; do not re-arm the re-alert - * while progress is happening. */ + * set) without the loud alert, and re-arm with the SAME baseline and HWM so + * the next window re-evaluates and releases the fence once catch-up + * completes. Nothing else releases this fence: before bug-1919 this branch + * did not re-arm and the member stayed fenced until restart. */ if (peer_reachable || made_progress) { INFO("HA: promotion fence catch-up window elapsed but progress is " "possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%" @@ -1632,7 +1653,11 @@ static int promotion_fence_timeout_fn(void *arg) "(ha-commit-durability Req 2.2)", peer_reachable, durable_floor, t->baseline_durable_floor, t->target_hwm); + bool was_truncated = t->was_truncated; + uint64_t baseline = t->baseline_durable_floor; + uint64_t hwm = t->target_hwm; free(t); + ha_arm_promotion_fence_timer(ha, c, was_truncated, baseline, hwm); return 0; } diff --git a/src/controller/persist.c b/src/controller/persist.c index da59d97f..6cc6785e 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -1280,6 +1280,7 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c) * a fresh store has no corruption. persist_recover overwrites this with the * replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */ p->recovery_verdict = RECOVER_CLEAN; + atomic_init(&p->recovery_truncation_healed, false); /* bug-1919 */ /* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh * store is promotable. Set by persist_recover_loud_fence_if_unrecoverable @@ -6854,6 +6855,8 @@ int persist_recover(persist_t* p) return replayed; p->recovery_verdict = verdict; + atomic_store_explicit(&p->recovery_truncation_healed, false, + memory_order_release); /* bug-1919 */ if (verdict == RECOVER_TRUNCATED) WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally " "corrupt (CRC break / mismatch / short read); member marked " @@ -7004,7 +7007,16 @@ recover_verdict_t persist_recovery_verdict(persist_t* p) bool persist_recovery_truncated(persist_t* p) { - return p && p->recovery_verdict == RECOVER_TRUNCATED; + return p && p->recovery_verdict == RECOVER_TRUNCATED + && !atomic_load_explicit(&p->recovery_truncation_healed, + memory_order_acquire); +} + +void persist_mark_recovery_truncation_healed(persist_t* p) +{ + if (p) + atomic_store_explicit(&p->recovery_truncation_healed, true, + memory_order_release); } /* ------------------------------------------------------------------ * diff --git a/src/controller/testsuite/test_commit_durability.c b/src/controller/testsuite/test_commit_durability.c index 9dc52f7d..8640147e 100644 --- a/src/controller/testsuite/test_commit_durability.c +++ b/src/controller/testsuite/test_commit_durability.c @@ -2388,6 +2388,142 @@ static void test_promotion_fence_released_on_catchup(void) printf("test_promotion_fence_released_on_catchup: PASS\n"); } + +/* Spin until the in_catch_up fence reads `want` or a timeout elapses. */ +static bool dur_wait_in_catch_up(cluster_t* c, int want, int timeout_ms) +{ + int waited = 0; + while (atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire) != want + && waited < timeout_ms) { + usleep(2000); + waited += 2; + } + return atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire) == want; +} + +/* bug-1919: a member that BOOTED recovery-truncated, was fenced at a refused + * promotion, and then caught up as a standby (durable floor advanced past the + * fence-time baseline by entries replicated from the live primary, reaching the + * HWM) must have the fence RELEASED by the re-check timer. The boot-time + * RECOVER_TRUNCATED verdict never changes at runtime, so before the fix the + * release branch (`!truncated && ...`) was unreachable and the node stayed + * `fenced: true` until restart. Mirrors the soak trace: floor 360 == HWM 360 + * at fence time, floor 375 after catch-up. The raw boot verdict is kept for + * diagnostics; only persist_recovery_truncated() reports the healed state. */ +static void test_promotion_fence_truncated_boot_released_after_catchup(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_fence_trunc_heal_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, + /*durable_floor=*/360); + TEST_ASSERT_NOT_NULL(c); + + ha_set_promotion_fence_window_ms(120); + + extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, + uint64_t); + ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, + /*durable_floor=*/360, /*hwm=*/360); + TEST_ASSERT_EQUAL_INT( + 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire)); + + /* Catch-up as a standby: replicated entries advance the durable floor. The + * recovery verdict is deliberately NOT touched -- production never does. */ + atomic_store_explicit(&c->persist.durable_lsn, 375, memory_order_release); + + TEST_ASSERT_TRUE_MESSAGE( + dur_wait_in_catch_up(c, 0, 2000), + "bug-1919: a truncated-boot member that caught up past the HWM must " + "have its promotion fence released (Req 2.6)"); + TEST_ASSERT_FALSE_MESSAGE( + persist_recovery_truncated(&c->persist), + "bug-1919: catch-up from the primary heals the truncated tail"); + TEST_ASSERT_EQUAL_INT_MESSAGE( + RECOVER_TRUNCATED, persist_recovery_verdict(&c->persist), + "bug-1919: the raw boot verdict is kept for diagnostics"); + TEST_ASSERT_FALSE(persist_recovery_unrecoverable(&c->persist)); + + ha_set_promotion_fence_window_ms(0); + dur_destroy_fence_cluster(c); + printf("test_promotion_fence_truncated_boot_released_after_catchup: PASS\n"); +} + +/* bug-1919: when the window elapses with catch-up progressing but not yet + * complete, the re-check must keep running (re-arm) so the fence is released + * once catch-up reaches the HWM. Before the fix the "progress is possible" + * branch freed its ctx WITHOUT re-arming, so nothing ever re-evaluated. */ +static void test_promotion_fence_rechecks_while_catching_up(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_fence_recheck_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, + /*durable_floor=*/4); + TEST_ASSERT_NOT_NULL(c); + + ha_set_promotion_fence_window_ms(60); + + extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, + uint64_t); + ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, + /*durable_floor=*/4, /*hwm=*/10); + + /* Partial catch-up: progress, still below the HWM -> stays fenced. */ + atomic_store_explicit(&c->persist.durable_lsn, 7, memory_order_release); + usleep(200 * 1000); /* >= 3 windows */ + TEST_ASSERT_EQUAL_INT_MESSAGE( + 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), + "bug-1919: a member still below the HWM stays fenced"); + TEST_ASSERT_TRUE_MESSAGE( + persist_recovery_truncated(&c->persist), + "bug-1919: below the HWM the truncated tail is not yet healed"); + + /* Catch-up completes: a later re-check must release the fence. */ + atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release); + TEST_ASSERT_TRUE_MESSAGE( + dur_wait_in_catch_up(c, 0, 2000), + "bug-1919: the fence re-check must keep running while catching up and " + "release the fence once the HWM is reached"); + + ha_set_promotion_fence_window_ms(0); + dur_destroy_fence_cluster(c); + printf("test_promotion_fence_rechecks_while_catching_up: PASS\n"); +} + +/* bug-1919 (conservative heal rule): a truncated-boot member whose durable + * floor merely equals the HWM but has received NOTHING from a primary since the + * fence is not healed -- reaching the HWM is only proven by entries replicated + * on top of the truncated log, so it stays fenced. */ +static void test_promotion_fence_truncated_no_progress_stays_fenced(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_fence_trunc_noprog_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, + /*durable_floor=*/10); + TEST_ASSERT_NOT_NULL(c); + + ha_set_promotion_fence_window_ms(40); + + extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, + uint64_t); + ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, + /*durable_floor=*/10, /*hwm=*/10); + + usleep(200 * 1000); + TEST_ASSERT_EQUAL_INT_MESSAGE( + 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), + "bug-1919: no replicated progress -> the truncated member stays fenced"); + TEST_ASSERT_TRUE(persist_recovery_truncated(&c->persist)); + + ha_set_promotion_fence_window_ms(0); + dur_destroy_fence_cluster(c); + printf("test_promotion_fence_truncated_no_progress_stays_fenced: PASS\n"); +} /* 4.4: cancelling the timer (shutdown / recovery) before it fires → the * re-alert never fires, no use-after-free of the timer ctx, and the process * stays alive (it never self-exited even before the cancel). */ @@ -2664,6 +2800,9 @@ void test_commit_durability(void) RUN_TEST(test_promotion_fence_unrecoverable_stays_alive_loud); RUN_TEST(test_promotion_fence_released_on_catchup); RUN_TEST(test_promotion_fence_cancel_stops_realert); + RUN_TEST(test_promotion_fence_truncated_boot_released_after_catchup); + RUN_TEST(test_promotion_fence_rechecks_while_catching_up); + RUN_TEST(test_promotion_fence_truncated_no_progress_stays_fenced); /* task 5.3 round-2 remediation (Req 8.4): cross-contamination regression — * an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up * alone. */ diff --git a/src/include/persist.h b/src/include/persist.h index c37cb62d..9a86830b 100644 --- a/src/include/persist.h +++ b/src/include/persist.h @@ -340,6 +340,15 @@ struct persist_s { * before the node participates; read by the promotion gate thereafter. */ recover_verdict_t recovery_verdict; + /* Set once a RECOVER_TRUNCATED member has caught up from the live primary + * past the cluster HWM its promotion fence was armed with (bug-1919): + * entries replicated on top of the truncated log prove the lost tail was + * refilled. recovery_verdict keeps the raw boot verdict for diagnostics; + * persist_recovery_truncated() reports false once this is set. Reset by + * persist_recover. Set by the fence re-check timer, read by the + * promotion gate. */ + _Atomic bool recovery_truncation_healed; + /* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4, * NEVER-SELF-EXIT rework 2026-06-18). Set by * persist_recover_loud_fence_if_unrecoverable when a member is @@ -789,11 +798,14 @@ uint64_t persist_durable_floor(persist_t* p); * persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED * verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh * store before recovery runs). persist_recovery_truncated is the convenience - * flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate - * (task 4.3) consumes to mark the member non-promotable. Both are pure reads - * of recovery state established single-threaded at startup. */ + * flag form (true iff the verdict is RECOVER_TRUNCATED and the truncation has + * not since been healed by catch-up) the promotion gate (task 4.3) consumes + * to mark the member non-promotable. persist_recovery_verdict is a pure read + * of the boot-time verdict. persist_mark_recovery_truncation_healed records + * that catch-up from the primary refilled the truncated tail (bug-1919). */ recover_verdict_t persist_recovery_verdict(persist_t* p); bool persist_recovery_truncated(persist_t* p); +void persist_mark_recovery_truncation_healed(persist_t* p); /* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4; * never-self-exit rework 2026-06-18).