lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): loud bounded FATAL on unrecoverable recovery (task 3.3)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 08:15:22 +00:00
parent 6483ee8afd
commit e3fc42c11e
4 changed files with 324 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

149
src/controller/testsuite/test_commit_durability.c
View file

@ -1017,6 +1017,80 @@ static void test_recover_verdict_snapshot_sentinel_clean(void)
* failure. (Req 2.2, 2.4) *
* ------------------------------------------------------------------ */
/* task 3.3 — FATAL-path capture seam. The production FATAL terminates via
* abort(); a test installs persist_fatal_hook so the FATAL decision + boundary
* log can be asserted without the process dying. */
static int g_fatal_hook_calls = 0;
static char g_fatal_hook_msg[256];
static void dur_fatal_hook(const char* msg)
{
g_fatal_hook_calls++;
if (msg != NULL) {
strncpy(g_fatal_hook_msg, msg, sizeof(g_fatal_hook_msg) - 1);
g_fatal_hook_msg[sizeof(g_fatal_hook_msg) - 1] = '\0';
}
}
/* task 3.3 — peer-reachable NEVER FATALs (the never-FATAL aspect of rows
* 3 & 5). This task (3.3) owns only the FATAL terminal; the full FENCE+
* CATCH-UP -> ELIGIBLE eligibility outcome of these rows is task 4.3 (so the
* row3/row5 skeletons below stay IGNORED for 4.3). Here we pin ONLY the
* invariant 3.3 is responsible for: while a healthy peer is reachable, the
* decision is NEVER FATAL — neither for a TRUNCATED log (row 3) nor for a
* legitimate snapshot-sentinel CLEAN-but-behind member (row 5). It is FENCE
* (catch-up pending) and the FATAL terminate path is never fired. (Req 2.2,
* 2.4, design.md "never FATAL when a peer is reachable".) */
static void test_boot_policy_peer_reachable_never_fatal(void)
{
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE, no FATAL. */
recover_policy_inputs_t truncated_peer = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 3,
.cluster_hwm_lsn = 10,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&truncated_peer),
"row 3: a truncated member with a reachable healthy peer must FENCE "
"(catch up), NEVER FATAL");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&truncated_peer));
/* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE, no
* FATAL. A legitimate sentinel is CLEAN-but-behind, not corruption. */
recover_policy_inputs_t sentinel_peer = {
.verdict = RECOVER_CLEAN,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 5,
.cluster_hwm_lsn = 5,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&sentinel_peer),
"row 5: a snapshot-sentinel CLEAN-but-behind member with a reachable "
"peer must FENCE, NEVER FATAL");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&sentinel_peer));
/* The FATAL terminate path must NOT have fired for either peer-reachable
* case — a reachable healthy peer never triggers FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"peer-reachable rows must NEVER fire the FATAL terminate path");
persist_fatal_hook = NULL;
printf("test_boot_policy_peer_reachable_never_fatal: PASS\n");
}
/* Row 1: CLEAN + durable at/above HWM -> ELIGIBLE.
* A structurally intact log that is not behind the cluster boundary is
* a normal promotable member: the promotion gate (4.3) must NOT fence
@ -1068,10 +1142,39 @@ static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
* state. (Req 2.4) */
static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
{
TEST_IGNORE_MESSAGE(
"row 4 (TRUNCATED + below + no peer + bounded catch-up fails -> "
"FATAL): pending task 3.3 (FATAL branch). Will assert loud "
"termination with the truncation boundary logged on bounded failure.");
/* Row 4: TRUNCATED + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Cannot recover locally AND cannot catch up within the bounded
* window: fail loud with the truncation boundary logged (Req 2.4). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
recover_policy_inputs_t in = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 7, /* truncation boundary */
.cluster_hwm_lsn = 12, /* cluster committed high-water */
};
/* Pure decision returns FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4: truncated + below + no peer + bounded catch-up fails must "
"decide FATAL");
/* The action helper routes FATAL through the hook (no abort in test) and
* still returns FATAL — proving the loud-terminate path was taken. */
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4: the FATAL terminate path must fire exactly once (the "
"truncation boundary is logged at ERROR just before it)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4_truncated_no_peer_bounded_fatal: PASS\n");
}
/* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails (bounded
@ -1082,10 +1185,36 @@ static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
* once the bounded window elapses. (Req 2.4, 2.2) */
static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
{
TEST_IGNORE_MESSAGE(
"row 4b (CLEAN + below + no peer + bounded catch-up fails -> FATAL): "
"pending task 3.3. Will assert the behind-no-peer member FATALs on "
"bounded failure — never a silent indefinite fence.");
/* Row 4b: CLEAN + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Behind the HWM with no peer to advance from within the bounded
* window — same bounded-failure terminal as row 4 (Req 2.4, 2.2). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
recover_policy_inputs_t in = {
.verdict = RECOVER_CLEAN, /* structurally intact... */
.durable_below_hwm = true, /* ...but behind the HWM */
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 9,
.cluster_hwm_lsn = 20,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4b: clean-but-behind + no peer + bounded catch-up fails must "
"decide FATAL — never a silent indefinite fence");
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4b: a behind member with no reachable peer must FATAL once the "
"bounded window elapses (boundary logged before terminate)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4b_clean_below_no_peer_bounded_fatal: PASS\n");
}
/* Row 5: CLEAN snapshot-sentinel (committed at sentinel LSN, NO
@ -1148,6 +1277,10 @@ void test_commit_durability(void)
/* task 3.2 — Boot Policy Decision Table skeletons (IGNORED, pending
* 3.3/4.3). Registered so each row is reported as IGNORED, not failed;
* the suite stays GREEN. See design.md "Boot Policy Decision Table". */
/* task 3.3 — FATAL rows (4, 4b) now ACTIVE (PASS), plus the never-FATAL
* invariant for the peer-reachable rows (3, 5). The full FENCE->ELIGIBLE
* eligibility outcome of rows 1,2,3,5,6 stays IGNORED pending task 4.3. */
RUN_TEST(test_boot_policy_peer_reachable_never_fatal);
RUN_TEST(test_boot_policy_row1_clean_at_or_above_eligible);
RUN_TEST(test_boot_policy_row2_clean_below_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row3_truncated_peer_catchup_eligible);