lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): release the promotion fence once a truncated-boot node has caught up (bug-1919)

Browse source 
The fence re-check required !persist_recovery_truncated(), a boot-time
verdict nothing cleared, so a node that booted RECOVER_TRUNCATED stayed
fenced (fenced: true, non-promotable) until restart however far it
caught up. And the "progress is possible" branch did not re-arm, so the
fence was never re-evaluated.

persist_t gains a runtime recovery_truncation_healed latch, which
persist_recovery_truncated() honours (the raw boot verdict stays
available). The re-check marks the truncation healed once the durable
floor has advanced past the fence-time baseline (entries replicated from
the live primary) and reached the HWM, and the progress branch re-arms
with the same baseline and HWM.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-04 09:14:48 +00:00
parent 88a8bf8a47
commit 0ab12074f5
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
4 changed files with 197 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

139
src/controller/testsuite/test_commit_durability.c
View file

@ -2388,6 +2388,142 @@ static void test_promotion_fence_released_on_catchup(void)
printf("test_promotion_fence_released_on_catchup: PASS\n");
}
/* Spin until the in_catch_up fence reads `want` or a timeout elapses. */
static bool dur_wait_in_catch_up(cluster_t* c, int want, int timeout_ms)
{
int waited = 0;
while (atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire) != want
&& waited < timeout_ms) {
usleep(2000);
waited += 2;
}
return atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire) == want;
}
/* bug-1919: a member that BOOTED recovery-truncated, was fenced at a refused
* promotion, and then caught up as a standby (durable floor advanced past the
* fence-time baseline by entries replicated from the live primary, reaching the
* HWM) must have the fence RELEASED by the re-check timer. The boot-time
* RECOVER_TRUNCATED verdict never changes at runtime, so before the fix the
* release branch (`!truncated && ...`) was unreachable and the node stayed
* `fenced: true` until restart. Mirrors the soak trace: floor 360 == HWM 360
* at fence time, floor 375 after catch-up. The raw boot verdict is kept for
* diagnostics; only persist_recovery_truncated() reports the healed state. */
static void test_promotion_fence_truncated_boot_released_after_catchup(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_trunc_heal_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/360);
TEST_ASSERT_NOT_NULL(c);
ha_set_promotion_fence_window_ms(120);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/360, /*hwm=*/360);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
/* Catch-up as a standby: replicated entries advance the durable floor. The
* recovery verdict is deliberately NOT touched -- production never does. */
atomic_store_explicit(&c->persist.durable_lsn, 375, memory_order_release);
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_in_catch_up(c, 0, 2000),
"bug-1919: a truncated-boot member that caught up past the HWM must "
"have its promotion fence released (Req 2.6)");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_truncated(&c->persist),
"bug-1919: catch-up from the primary heals the truncated tail");
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_TRUNCATED, persist_recovery_verdict(&c->persist),
"bug-1919: the raw boot verdict is kept for diagnostics");
TEST_ASSERT_FALSE(persist_recovery_unrecoverable(&c->persist));
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_truncated_boot_released_after_catchup: PASS\n");
}
/* bug-1919: when the window elapses with catch-up progressing but not yet
* complete, the re-check must keep running (re-arm) so the fence is released
* once catch-up reaches the HWM. Before the fix the "progress is possible"
* branch freed its ctx WITHOUT re-arming, so nothing ever re-evaluated. */
static void test_promotion_fence_rechecks_while_catching_up(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_recheck_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
ha_set_promotion_fence_window_ms(60);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
/* Partial catch-up: progress, still below the HWM -> stays fenced. */
atomic_store_explicit(&c->persist.durable_lsn, 7, memory_order_release);
usleep(200 * 1000); /* >= 3 windows */
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"bug-1919: a member still below the HWM stays fenced");
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_truncated(&c->persist),
"bug-1919: below the HWM the truncated tail is not yet healed");
/* Catch-up completes: a later re-check must release the fence. */
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_in_catch_up(c, 0, 2000),
"bug-1919: the fence re-check must keep running while catching up and "
"release the fence once the HWM is reached");
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_rechecks_while_catching_up: PASS\n");
}
/* bug-1919 (conservative heal rule): a truncated-boot member whose durable
* floor merely equals the HWM but has received NOTHING from a primary since the
* fence is not healed -- reaching the HWM is only proven by entries replicated
* on top of the truncated log, so it stays fenced. */
static void test_promotion_fence_truncated_no_progress_stays_fenced(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_trunc_noprog_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/10);
TEST_ASSERT_NOT_NULL(c);
ha_set_promotion_fence_window_ms(40);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/10, /*hwm=*/10);
usleep(200 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"bug-1919: no replicated progress -> the truncated member stays fenced");
TEST_ASSERT_TRUE(persist_recovery_truncated(&c->persist));
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_truncated_no_progress_stays_fenced: PASS\n");
}
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → the
* re-alert never fires, no use-after-free of the timer ctx, and the process
* stays alive (it never self-exited even before the cancel). */
@ -2664,6 +2800,9 @@ void test_commit_durability(void)
RUN_TEST(test_promotion_fence_unrecoverable_stays_alive_loud);
RUN_TEST(test_promotion_fence_released_on_catchup);
RUN_TEST(test_promotion_fence_cancel_stops_realert);
RUN_TEST(test_promotion_fence_truncated_boot_released_after_catchup);
RUN_TEST(test_promotion_fence_rechecks_while_catching_up);
RUN_TEST(test_promotion_fence_truncated_no_progress_stays_fenced);
/* task 5.3 round-2 remediation (Req 8.4): cross-contamination regression —
* an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
* alone. */