lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): loud bounded FATAL on unrecoverable recovery (task 3.3)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 08:15:22 +00:00
parent 6483ee8afd
commit e3fc42c11e
4 changed files with 324 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

92
src/controller/persist.c
View file

@ -1642,6 +1642,98 @@ bool persist_recovery_truncated(persist_t* p)
return p && p->recovery_verdict == RECOVER_TRUNCATED;
}
/* ------------------------------------------------------------------ *
* Boot-policy FATAL decision (ha-commit-durability task 3.3, Req 2.4).
*
* Implements the FATAL terminal of the Boot Policy Decision Table
* (design.md). The decision is a PURE function of its inputs; the FATAL
* action (loud log + terminate) is separated into
* persist_recover_fatal_if_unrecoverable so the decision can be unit-tested
* without aborting. A reachable healthy peer NEVER FATALs (rows 3, 5);
* FATAL is reserved for a member that cannot recover locally AND cannot catch
* up within the bounded window (rows 4, 4b) — never a silent indefinite fence.
* ------------------------------------------------------------------ */
/* FATAL seam: NULL in production (real abort); a test may install a hook to
* capture the FATAL path without dying. Mirrors ha_fatal_hook. */
void (*persist_fatal_hook)(const char* msg) = NULL;
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in)
{
if (in == NULL)
return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */
bool truncated = (in->verdict == RECOVER_TRUNCATED);
/* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster
* high-water mark is a normal promotable member — no fence, no FATAL. */
if (!truncated && !in->durable_below_hwm)
return RECOVER_DECISION_ELIGIBLE;
/* From here the member is truncated OR behind the HWM — it must NOT lead
* as-is. A truncated log is by definition not provably at/above the HWM
* (design.md note), so it is treated as behind for fence purposes. */
/* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain
* eligibility (task 4.3). A reachable peer NEVER yields FATAL — even for a
* truncated log or a legitimate snapshot-sentinel. If catch-up has already
* succeeded the member is eligible; otherwise it stays fenced (4.3 drives
* the catch-up + release; this task only refuses FATAL while a peer
* exists). */
if (in->healthy_peer_reachable) {
if (in->catch_up_succeeded)
return RECOVER_DECISION_ELIGIBLE;
return RECOVER_DECISION_FENCE;
}
/* No healthy peer. The bounded catch-up window is the member's only path to
* a committed-consistent state; if it did not reach one, the member can
* neither recover locally nor catch up — rows 4 & 4b: FATAL (fail loud),
* never a silent indefinite fence (Req 2.4). */
if (!in->catch_up_succeeded)
return RECOVER_DECISION_FATAL;
/* No peer but catch-up is reported succeeded (e.g. already at/above after a
* local catch-up): eligible. Defensive — the boot path reaches the
* no-peer branch with catch_up_succeeded=false in the bounded-failure case
* this task owns. */
return RECOVER_DECISION_ELIGIBLE;
}
recover_decision_t
persist_recover_fatal_if_unrecoverable(const recover_policy_inputs_t* in)
{
recover_decision_t decision = persist_recover_decision(in);
if (decision != RECOVER_DECISION_FATAL)
return decision;
/* Fail loud: log the truncation boundary (durable_floor_lsn) and the
* cluster high-water mark so an operator sees exactly where the local log
* stops short of the cluster's committed state (Req 2.4). */
const char* why = (in->verdict == RECOVER_TRUNCATED)
? "recovery-truncated log (structural corruption)"
: "log behind the cluster committed high-water mark";
ERROR("persist_recover: FATAL — %s and no healthy peer reachable to catch "
"up within the bounded window (%u ms); truncation boundary "
"durable_floor_lsn=%" PRIu64 " < cluster committed high-water "
"hwm=%" PRIu64 " — terminating rather than serving divergent state "
"(ha-commit-durability Req 2.4)",
why, (unsigned)DEFAULT_RECOVERY_CATCHUP_WINDOW_MS,
in->durable_floor_lsn, in->cluster_hwm_lsn);
if (persist_fatal_hook != NULL) {
/* Test seam: capture the FATAL decision + boundary log without dying.
* Production leaves the hook NULL and aborts below. */
persist_fatal_hook("persist_recover: bounded-unrecoverable (FATAL)");
return RECOVER_DECISION_FATAL;
}
tlog_exit(); /* flush the async log buffer before dying */
abort(); /* NEVER returns in production */
return RECOVER_DECISION_FATAL; /* unreachable */
}
/* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
* entry is already durable on local stable storage (ha-commit-durability task
* 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the

149
src/controller/testsuite/test_commit_durability.c
View file

@ -1017,6 +1017,80 @@ static void test_recover_verdict_snapshot_sentinel_clean(void)
* failure. (Req 2.2, 2.4) *
* ------------------------------------------------------------------ */
/* task 3.3 — FATAL-path capture seam. The production FATAL terminates via
* abort(); a test installs persist_fatal_hook so the FATAL decision + boundary
* log can be asserted without the process dying. */
static int g_fatal_hook_calls = 0;
static char g_fatal_hook_msg[256];
static void dur_fatal_hook(const char* msg)
{
g_fatal_hook_calls++;
if (msg != NULL) {
strncpy(g_fatal_hook_msg, msg, sizeof(g_fatal_hook_msg) - 1);
g_fatal_hook_msg[sizeof(g_fatal_hook_msg) - 1] = '\0';
}
}
/* task 3.3 — peer-reachable NEVER FATALs (the never-FATAL aspect of rows
* 3 & 5). This task (3.3) owns only the FATAL terminal; the full FENCE+
* CATCH-UP -> ELIGIBLE eligibility outcome of these rows is task 4.3 (so the
* row3/row5 skeletons below stay IGNORED for 4.3). Here we pin ONLY the
* invariant 3.3 is responsible for: while a healthy peer is reachable, the
* decision is NEVER FATAL — neither for a TRUNCATED log (row 3) nor for a
* legitimate snapshot-sentinel CLEAN-but-behind member (row 5). It is FENCE
* (catch-up pending) and the FATAL terminate path is never fired. (Req 2.2,
* 2.4, design.md "never FATAL when a peer is reachable".) */
static void test_boot_policy_peer_reachable_never_fatal(void)
{
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE, no FATAL. */
recover_policy_inputs_t truncated_peer = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 3,
.cluster_hwm_lsn = 10,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&truncated_peer),
"row 3: a truncated member with a reachable healthy peer must FENCE "
"(catch up), NEVER FATAL");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&truncated_peer));
/* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE, no
* FATAL. A legitimate sentinel is CLEAN-but-behind, not corruption. */
recover_policy_inputs_t sentinel_peer = {
.verdict = RECOVER_CLEAN,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 5,
.cluster_hwm_lsn = 5,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&sentinel_peer),
"row 5: a snapshot-sentinel CLEAN-but-behind member with a reachable "
"peer must FENCE, NEVER FATAL");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&sentinel_peer));
/* The FATAL terminate path must NOT have fired for either peer-reachable
* case — a reachable healthy peer never triggers FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"peer-reachable rows must NEVER fire the FATAL terminate path");
persist_fatal_hook = NULL;
printf("test_boot_policy_peer_reachable_never_fatal: PASS\n");
}
/* Row 1: CLEAN + durable at/above HWM -> ELIGIBLE.
* A structurally intact log that is not behind the cluster boundary is
* a normal promotable member: the promotion gate (4.3) must NOT fence
@ -1068,10 +1142,39 @@ static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
* state. (Req 2.4) */
static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
{
TEST_IGNORE_MESSAGE(
"row 4 (TRUNCATED + below + no peer + bounded catch-up fails -> "
"FATAL): pending task 3.3 (FATAL branch). Will assert loud "
"termination with the truncation boundary logged on bounded failure.");
/* Row 4: TRUNCATED + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Cannot recover locally AND cannot catch up within the bounded
* window: fail loud with the truncation boundary logged (Req 2.4). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
recover_policy_inputs_t in = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 7, /* truncation boundary */
.cluster_hwm_lsn = 12, /* cluster committed high-water */
};
/* Pure decision returns FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4: truncated + below + no peer + bounded catch-up fails must "
"decide FATAL");
/* The action helper routes FATAL through the hook (no abort in test) and
* still returns FATAL — proving the loud-terminate path was taken. */
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4: the FATAL terminate path must fire exactly once (the "
"truncation boundary is logged at ERROR just before it)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4_truncated_no_peer_bounded_fatal: PASS\n");
}
/* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails (bounded
@ -1082,10 +1185,36 @@ static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
* once the bounded window elapses. (Req 2.4, 2.2) */
static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
{
TEST_IGNORE_MESSAGE(
"row 4b (CLEAN + below + no peer + bounded catch-up fails -> FATAL): "
"pending task 3.3. Will assert the behind-no-peer member FATALs on "
"bounded failure — never a silent indefinite fence.");
/* Row 4b: CLEAN + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Behind the HWM with no peer to advance from within the bounded
* window — same bounded-failure terminal as row 4 (Req 2.4, 2.2). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
recover_policy_inputs_t in = {
.verdict = RECOVER_CLEAN, /* structurally intact... */
.durable_below_hwm = true, /* ...but behind the HWM */
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 9,
.cluster_hwm_lsn = 20,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4b: clean-but-behind + no peer + bounded catch-up fails must "
"decide FATAL — never a silent indefinite fence");
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4b: a behind member with no reachable peer must FATAL once the "
"bounded window elapses (boundary logged before terminate)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4b_clean_below_no_peer_bounded_fatal: PASS\n");
}
/* Row 5: CLEAN snapshot-sentinel (committed at sentinel LSN, NO
@ -1148,6 +1277,10 @@ void test_commit_durability(void)
/* task 3.2 — Boot Policy Decision Table skeletons (IGNORED, pending
* 3.3/4.3). Registered so each row is reported as IGNORED, not failed;
* the suite stays GREEN. See design.md "Boot Policy Decision Table". */
/* task 3.3 — FATAL rows (4, 4b) now ACTIVE (PASS), plus the never-FATAL
* invariant for the peer-reachable rows (3, 5). The full FENCE->ELIGIBLE
* eligibility outcome of rows 1,2,3,5,6 stays IGNORED pending task 4.3. */
RUN_TEST(test_boot_policy_peer_reachable_never_fatal);
RUN_TEST(test_boot_policy_row1_clean_at_or_above_eligible);
RUN_TEST(test_boot_policy_row2_clean_below_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row3_truncated_peer_catchup_eligible);