wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha-commit-durability): pre-promotion eligibility gate + catch-up fence/release (task 4.3)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 09:40:32 +00:00
parent 3049b620f3
commit e9e2a6d703
4 changed files with 409 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

228
src/controller/testsuite/test_commit_durability.c
View file

@ -12,6 +12,7 @@
#include "commit.h"
#include "config.h"
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
#include "persist.h"
#include "wal_types.h"
@ -1302,12 +1303,42 @@ static void test_boot_policy_peer_reachable_never_fatal(void)
* a normal promotable member: the promotion gate (4.3) must NOT fence
* it. WILL ASSERT (4.3): the eligibility predicate returns "eligible"
* for a CLEAN, at/above member; promotion proceeds. (Req 2.3) */
/* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role_lifecycle.c). */
static ha_config_t dur_ha_cfg(void)
{
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
return cfg;
}
static void test_boot_policy_row1_clean_at_or_above_eligible(void)
{
TEST_IGNORE_MESSAGE(
"row 1 (CLEAN + at/above HWM -> ELIGIBLE): pending task 4.3 "
"(promotion-eligibility gate). Will assert the gate admits a clean, "
"not-behind member.");
/* CLEAN + durable at/above HWM -> ELIGIBLE. The live gate must admit a
* structurally-intact, not-behind member and must NOT set the fence. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Pre-set the fence to prove ELIGIBLE clears it (Req 2.6 release path). */
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/8, /*safe_hwm=*/8,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d,
"row 1: a CLEAN, at/above-HWM member must be ELIGIBLE to promote");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 1: ELIGIBLE must clear the in_catch_up fence");
ha_fini(&ha);
printf("test_boot_policy_row1_clean_at_or_above_eligible: PASS\n");
}
/* Row 2: CLEAN + durable below HWM + healthy peer + catch-up succeeds
@ -1318,10 +1349,39 @@ static void test_boot_policy_row1_clean_at_or_above_eligible(void)
* committed boundary, then regains eligibility. (Req 2.2, 2.3, 2.6) */
static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void)
{
TEST_IGNORE_MESSAGE(
"row 2 (CLEAN + below HWM + peer + catch-up ok -> FENCE+CATCH-UP -> "
"ELIGIBLE): pending task 4.3 (fence + release). Will assert denied "
"until caught up, then re-eligible.");
/* CLEAN + below HWM + healthy peer: FENCE first (denied promotion), then
* once catch-up passes the committed boundary, ELIGIBLE (fence released). */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Behind: durable_floor 4 < HWM 10, healthy peer reachable, not yet caught
* up -> FENCE; the in_catch_up fence is SET (refused promotion). */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/4, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 2: a CLEAN-but-behind member with a peer must FENCE (catch up)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 2: FENCE must SET the in_catch_up fence (promotion refused)");
/* Catch-up brings the durable position to/above the committed boundary
* (durable_floor 10 >= HWM 10) -> ELIGIBLE; the fence is released (Req 2.6). */
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 2: after catch-up past the committed boundary the member regains "
"eligibility (Req 2.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 2: regained eligibility must RELEASE the in_catch_up fence");
ha_fini(&ha);
printf("test_boot_policy_row2_clean_below_peer_catchup_eligible: PASS\n");
}
/* Row 3: TRUNCATED + healthy peer + catch-up succeeds
@ -1333,10 +1393,46 @@ static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void)
* triggered. (Req 2.2, 2.6) */
static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
{
TEST_IGNORE_MESSAGE(
"row 3 (TRUNCATED + peer + catch-up ok -> FENCE+CATCH-UP -> "
"ELIGIBLE): pending tasks 3.3/4.3. Will assert truncated member "
"fences, catches up, re-eligible; no FATAL when a peer is reachable.");
/* TRUNCATED + healthy peer: a recovery-truncated member is DENIED promotion
* (FENCE), catches up from the peer, then regains eligibility. FATAL must
* NOT fire while a healthy peer is reachable. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Truncated boundary at 3, cluster HWM 10, peer reachable, not caught up. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 3: a recovery-truncated member with a healthy peer must FENCE "
"(denied promotion), never FATAL");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: a truncated member must be fenced (in_catch_up set)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 3: FATAL must NOT fire while a healthy peer is reachable");
/* After catch-up to/above the committed boundary -> ELIGIBLE, fence released. */
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 3: after catch-up the (previously truncated) member is eligible");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: regained eligibility releases the fence (Req 2.6)");
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row3_truncated_peer_catchup_eligible: PASS\n");
}
/* Row 4: TRUNCATED + below HWM + NO healthy peer + catch-up fails
@ -1435,10 +1531,39 @@ static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
* downstream promotion-fence outcome.) */
static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
{
TEST_IGNORE_MESSAGE(
"row 5 (snapshot-sentinel CLEAN + below + peer -> FENCE+CATCH-UP -> "
"ELIGIBLE, NEVER FATAL): pending task 4.3. Will assert a legitimate "
"sentinel snapshot is treated as behind-must-catch-up, not FATAL.");
/* Snapshot-sentinel CLEAN-but-behind + healthy peer: treated as behind,
* must catch up (FENCE), NEVER FATAL on a legitimate sentinel; then
* ELIGIBLE after catch-up. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Sentinel at LSN 5, cluster has advanced to HWM 12, peer reachable. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/12,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 5: a snapshot-sentinel CLEAN-but-behind member must FENCE, never "
"FATAL while a peer is reachable");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 5: a legitimate sentinel must NEVER trigger FATAL");
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/12, /*safe_hwm=*/12,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 5: after catch-up the sentinel member is eligible (Req 2.6)");
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal: PASS\n");
}
/* Row 6: CLEAN empty first-boot WAL (durable=0, HWM=0, fresh cold start)
@ -1448,10 +1573,71 @@ static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
* fresh cold-start member (no fence, no FATAL). (Req 2.1, 2.3) */
static void test_boot_policy_row6_empty_first_boot_eligible(void)
{
TEST_IGNORE_MESSAGE(
"row 6 (empty first-boot WAL CLEAN, durable=0/HWM=0 -> ELIGIBLE): "
"pending task 4.3. Will assert a fresh cold-start member is "
"promotable (no fence, no FATAL).");
/* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fresh cold-start
* member is not behind anything and must be promotable (no fence, no FATAL),
* even with no peer reachable (single-node / first boot). */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/0, /*safe_hwm=*/0,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d,
"row 6: an empty first-boot WAL (durable=0/HWM=0) must be ELIGIBLE");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 6: a cold-start member must not be fenced");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 6: a cold-start member must never FATAL");
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row6_empty_first_boot_eligible: PASS\n");
}
/* ── task 4.3: hollow node refused even at minimal quorum (Req 2.5) ──────────
* A hollow node (recovery-truncated OR durable below the HWM) must be refused
* promotion regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote. At minimal quorum with NO
* healthy peer, a hollow node FATALs (bounded-unrecoverable) rather than
* silently leading; with a peer it FENCEs. Either way it never becomes
* ELIGIBLE while hollow. */
static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
{
/* Truncated + below HWM + no peer (minimal/degraded quorum) -> NEVER
* ELIGIBLE (FATAL — fail loud rather than promote a hollow node). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
/*safe_hwm=*/10, /*peer_reachable=*/false,
/*catch_up_succeeded=*/false),
"a recovery-truncated node must never be ELIGIBLE at minimal quorum "
"(Req 2.5)");
/* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (FATAL). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/false, /*durable_floor=*/5,
/*safe_hwm=*/10, /*peer_reachable=*/false,
/*catch_up_succeeded=*/false),
"a below-HWM node must never be ELIGIBLE at minimal quorum (Req 2.5)");
/* Hollow node WITH a peer is still refused (FENCE, not ELIGIBLE) until it
* catches up — minimal-quorum size does not relax the gate. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
/*safe_hwm=*/10, /*peer_reachable=*/true,
/*catch_up_succeeded=*/false),
"a hollow node with a peer is FENCED (refused) until caught up");
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
}
#endif /* XCE_TESTING */
@ -1499,6 +1685,8 @@ void test_commit_durability(void)
RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_bounded_fatal);
RUN_TEST(test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal);
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif