wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

fix(ha-commit-durability)!: controller never self-terminates — loud non-promotable fence replaces recovery FATAL/abort

Browse source 
Per design invariant (a controller exits only on crash or admin kill), replace the
self-abort recovery terminal (tasks 3.3/4.3/4.4) with a loud, permanent, non-promotable
fence that stays alive and keeps retrying catch-up (self-heals on peer return; no manual
admin recovery). Records the never-self-terminate invariant as project-wide guidance in
src/controller/CLAUDE.md + .kiro/steering/ha-async-io.md.
This commit is contained in:
Rongsong Shen 2026-06-18 11:10:19 +00:00
parent addf53e3b4
commit 59d9df246c
10 changed files with 607 additions and 406 deletions
Download patch file Download diff file Expand all files Collapse all files

390
src/controller/testsuite/test_commit_durability.c
View file

@ -1587,19 +1587,36 @@ static void test_durable_floor_snapshot_sentinel_tip(void)
* failure. (Req 2.2, 2.4) *
* ------------------------------------------------------------------ */
/* task 3.3 — FATAL-path capture seam. The production FATAL terminates via
* abort(); a test installs persist_fatal_hook so the FATAL decision + boundary
* log can be asserted without the process dying. */
static int g_fatal_hook_calls = 0;
static char g_fatal_hook_msg[256];
static void dur_fatal_hook(const char* msg)
/* task 3.3 — unrecoverable loud-fence verification (never-self-exit rework
* 2026-06-18). The OLD behavior installed persist_fatal_hook to capture an
* abort()/exit() without dying. That self-exit terminal is GONE: an
* unrecoverable member STAYS ALIVE and is held permanently non-promotable.
* There is no hook to install — the tests assert the OBSERVABLE
* post-conditions instead:
* - the pure decision returns RECOVER_DECISION_FENCE_UNRECOVERABLE,
* - persist_recover_loud_fence_if_unrecoverable RETURNS (no abort — the test
* simply continuing past the call proves the process stayed alive),
* - the sticky persist_recovery_unrecoverable flag is SET (member is
* permanently non-promotable until catch-up),
* - and the flag CLEARS on catch-up (Req 2.6).
* dur_persist_for_flag builds a real persist_t the loud-fence helper can set
* the sticky flag on. */
static persist_t* dur_flag_persist(const char* tmpdir)
{
g_fatal_hook_calls++;
if (msg != NULL) {
strncpy(g_fatal_hook_msg, msg, sizeof(g_fatal_hook_msg) - 1);
g_fatal_hook_msg[sizeof(g_fatal_hook_msg) - 1] = '\0';
}
persist_t* p = calloc(1, sizeof(*p));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(p, tmpdir, "ctrl-flag");
if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
return p;
}
static void dur_free_flag_persist(persist_t* p)
{
if (!p) return;
persist_fini(p);
free(p);
}
/* task 3.3 — peer-reachable NEVER FATALs (the never-FATAL aspect of rows
@ -1613,10 +1630,13 @@ static void dur_fatal_hook(const char* msg)
* 2.4, design.md "never FATAL when a peer is reachable".) */
static void test_boot_policy_peer_reachable_never_fatal(void)
{
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
char tmp[256];
strcpy(tmp, "/tmp/xce_dur_peerfence_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = dur_flag_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE, no FATAL. */
/* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE. */
recover_policy_inputs_t truncated_peer = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
@ -1628,13 +1648,13 @@ static void test_boot_policy_peer_reachable_never_fatal(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&truncated_peer),
"row 3: a truncated member with a reachable healthy peer must FENCE "
"(catch up), NEVER FATAL");
"(catch up), NEVER the unrecoverable terminal");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&truncated_peer));
persist_recover_loud_fence_if_unrecoverable(p, &truncated_peer));
/* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE, no
* FATAL. A legitimate sentinel is CLEAN-but-behind, not corruption. */
/* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE.
* A legitimate sentinel is CLEAN-but-behind, not corruption. */
recover_policy_inputs_t sentinel_peer = {
.verdict = RECOVER_CLEAN,
.durable_below_hwm = true,
@ -1646,18 +1666,19 @@ static void test_boot_policy_peer_reachable_never_fatal(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&sentinel_peer),
"row 5: a snapshot-sentinel CLEAN-but-behind member with a reachable "
"peer must FENCE, NEVER FATAL");
"peer must FENCE, NEVER the unrecoverable terminal");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&sentinel_peer));
persist_recover_loud_fence_if_unrecoverable(p, &sentinel_peer));
/* The FATAL terminate path must NOT have fired for either peer-reachable
* case — a reachable healthy peer never triggers FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"peer-reachable rows must NEVER fire the FATAL terminate path");
/* The unrecoverable sticky flag must NOT have been set for either
* peer-reachable case — a reachable healthy peer never reaches the
* unrecoverable terminal. */
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"peer-reachable rows must NEVER set the unrecoverable sticky flag");
persist_fatal_hook = NULL;
dur_free_flag_persist(p);
printf("test_boot_policy_peer_reachable_never_fatal: PASS\n");
}
@ -1763,9 +1784,6 @@ static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Truncated boundary at 3, cluster HWM 10, peer reachable, not caught up. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10,
@ -1773,13 +1791,10 @@ static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 3: a recovery-truncated member with a healthy peer must FENCE "
"(denied promotion), never FATAL");
"(denied promotion), never the unrecoverable terminal");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: a truncated member must be fenced (in_catch_up set)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 3: FATAL must NOT fire while a healthy peer is reachable");
/* After catch-up to/above the committed boundary -> ELIGIBLE, fence released. */
recover_decision_t d2 = ha_evaluate_promotion_fence(
@ -1791,28 +1806,29 @@ static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: regained eligibility releases the fence (Req 2.6)");
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row3_truncated_peer_catchup_eligible: PASS\n");
}
/* Row 4: TRUNCATED + below HWM + NO healthy peer + catch-up fails
* (bounded window elapses) -> FATAL.
* Cannot recover locally AND cannot catch up within the bounded window:
* fail loud rather than serve divergent state. WILL ASSERT (3.3): the
* member terminates loudly (FATAL) with the truncation boundary logged
* when the bounded catch-up window elapses without a committed-consistent
* state. (Req 2.4) */
static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
* -> LOUD PERMANENT FENCE (non-promotable, STAYS ALIVE, keeps retrying), and
* RELEASES on catch-up (Req 2.4, 2.6).
*
* never-self-exit rework (2026-06-18): the member cannot recover locally AND
* cannot catch up right now. It does NOT self-exit; it logs LOUDLY, sets the
* sticky non-promotable flag, and stays alive. The flag clears once a peer
* returns and it catches up. RED→GREEN: pre-rework
* persist_recover_fatal_if_unrecoverable aborted (or fired the abort-capture
* hook); post-rework persist_recover_loud_fence_if_unrecoverable RETURNS — the
* test continuing past the call IS the proof the process stayed alive. */
static void test_boot_policy_row4_truncated_no_peer_loud_fence(void)
{
/* Row 4: TRUNCATED + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Cannot recover locally AND cannot catch up within the bounded
* window: fail loud with the truncation boundary logged (Req 2.4). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
char tmp[256];
strcpy(tmp, "/tmp/xce_dur_row4_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = dur_flag_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
recover_policy_inputs_t in = {
.verdict = RECOVER_TRUNCATED,
@ -1823,39 +1839,49 @@ static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
.cluster_hwm_lsn = 12, /* cluster committed high-water */
};
/* Pure decision returns FATAL. */
/* Pure decision returns the non-terminating unrecoverable fence. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4: truncated + below + no peer + bounded catch-up fails must "
"decide FATAL");
RECOVER_DECISION_FENCE_UNRECOVERABLE, persist_recover_decision(&in),
"row 4: truncated + below + no peer + no catch-up must decide the LOUD "
"PERMANENT FENCE — NOT process exit");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4: the sticky flag is set by the ACTION helper, not the pure "
"decision");
/* The action helper routes FATAL through the hook (no abort in test) and
* still returns FATAL — proving the loud-terminate path was taken. */
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
/* The action helper logs LOUDLY, sets the sticky flag, and RETURNS (no
* abort/exit — the process stays alive; the test reaching the next line is
* the proof). */
recover_decision_t d = persist_recover_loud_fence_if_unrecoverable(p, &in);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4: the FATAL terminate path must fire exactly once (the "
"truncation boundary is logged at ERROR just before it)");
RECOVER_DECISION_FENCE_UNRECOVERABLE, d,
"row 4: the loud-fence action returns FENCE_UNRECOVERABLE (stays alive)");
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4: the member is held PERMANENTLY non-promotable (sticky flag set)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4_truncated_no_peer_bounded_fatal: PASS\n");
/* Req 2.6: once catch-up reaches a committed-consistent state the fence
* releases automatically — the flag clears, the member is promotable. */
persist_clear_recovery_unrecoverable(p);
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4: catch-up CLEARS the unrecoverable fence (Req 2.6)");
dur_free_flag_persist(p);
printf("test_boot_policy_row4_truncated_no_peer_loud_fence: PASS\n");
}
/* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails (bounded
* window elapses) -> FATAL.
* Behind the HWM with no peer to advance from within the bounded window:
* same bounded-failure terminal as row 4 — never a silent indefinite
* fence. WILL ASSERT (3.3): a behind member with no reachable peer FATALs
* once the bounded window elapses. (Req 2.4, 2.2) */
static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
/* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails
* -> LOUD PERMANENT FENCE (non-promotable, STAYS ALIVE, keeps retrying).
* Behind the HWM with no peer to advance from — same non-terminating loud-fence
* terminal as row 4. NEVER self-exits. (Req 2.4, 2.2, 2.6) */
static void test_boot_policy_row4b_clean_below_no_peer_loud_fence(void)
{
/* Row 4b: CLEAN + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Behind the HWM with no peer to advance from within the bounded
* window — same bounded-failure terminal as row 4 (Req 2.4, 2.2). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
char tmp[256];
strcpy(tmp, "/tmp/xce_dur_row4b_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = dur_flag_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
recover_policy_inputs_t in = {
.verdict = RECOVER_CLEAN, /* structurally intact... */
@ -1867,19 +1893,24 @@ static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4b: clean-but-behind + no peer + bounded catch-up fails must "
"decide FATAL — never a silent indefinite fence");
RECOVER_DECISION_FENCE_UNRECOVERABLE, persist_recover_decision(&in),
"row 4b: clean-but-behind + no peer + no catch-up must decide the LOUD "
"PERMANENT FENCE — never process exit, never a silent fence");
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4b: a behind member with no reachable peer must FATAL once the "
"bounded window elapses (boundary logged before terminate)");
recover_decision_t d = persist_recover_loud_fence_if_unrecoverable(p, &in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FENCE_UNRECOVERABLE, d);
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4b: a behind member with no peer is held non-promotable + ALIVE "
"(sticky flag set; boundary logged loudly)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4b_clean_below_no_peer_bounded_fatal: PASS\n");
persist_clear_recovery_unrecoverable(p);
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4b: catch-up CLEARS the unrecoverable fence (Req 2.6)");
dur_free_flag_persist(p);
printf("test_boot_policy_row4b_clean_below_no_peer_loud_fence: PASS\n");
}
/* Row 5: CLEAN snapshot-sentinel (committed at sentinel LSN, NO
@ -1901,9 +1932,6 @@ static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Sentinel at LSN 5, cluster has advanced to HWM 12, peer reachable. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/12,
@ -1911,10 +1939,7 @@ static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 5: a snapshot-sentinel CLEAN-but-behind member must FENCE, never "
"FATAL while a peer is reachable");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 5: a legitimate sentinel must NEVER trigger FATAL");
"the unrecoverable terminal while a peer is reachable");
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/12, /*safe_hwm=*/12,
@ -1922,9 +1947,7 @@ static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 5: after catch-up the sentinel member is eligible (Req 2.6)");
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal: PASS\n");
}
@ -1943,9 +1966,6 @@ static void test_boot_policy_row6_empty_first_boot_eligible(void)
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/0, /*safe_hwm=*/0,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
@ -1955,11 +1975,7 @@ static void test_boot_policy_row6_empty_first_boot_eligible(void)
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 6: a cold-start member must not be fenced");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 6: a cold-start member must never FATAL");
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row6_empty_first_boot_eligible: PASS\n");
}
@ -1968,13 +1984,14 @@ static void test_boot_policy_row6_empty_first_boot_eligible(void)
* A hollow node (recovery-truncated OR durable below the HWM) must be refused
* promotion regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote. At minimal quorum with NO
* healthy peer, a hollow node FATALs (bounded-unrecoverable) rather than
* silently leading; with a peer it FENCEs. Either way it never becomes
* ELIGIBLE while hollow. */
* healthy peer, a hollow node is held in the LOUD PERMANENT FENCE
* (FENCE_UNRECOVERABLE, stays alive, non-promotable) rather than silently
* leading; with a peer it FENCEs. Either way it never becomes ELIGIBLE while
* hollow. */
static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
{
/* Truncated + below HWM + no peer (minimal/degraded quorum) -> NEVER
* ELIGIBLE (FATAL — fail loud rather than promote a hollow node). */
* ELIGIBLE (loud permanent fence — never promote a hollow node). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
@ -1983,7 +2000,7 @@ static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
"a recovery-truncated node must never be ELIGIBLE at minimal quorum "
"(Req 2.5)");
/* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (FATAL). */
/* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (loud permanent fence). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/false, /*durable_floor=*/5,
@ -2003,19 +2020,22 @@ static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
}
/* ── task 4.4: bounded-window catch-up before FATAL ──────────────────────────
/* ── task 4.4: catch-up re-alert window (never-self-exit rework 2026-06-18) ───
*
* The 4.3 gate decided FATAL synchronously at the promotion instant when a
* recovery-truncated/behind member had no reachable peer right then. Task 4.4
* defers that FATAL to a one-shot bounded-window timer: the member is fenced
* (in_catch_up) and given DEFAULT_RECOVERY_CATCHUP_WINDOW_MS to find a peer /
* make progress; the 3.3 FATAL is reached ONLY from the timer after a real
* bounded failure (still hollow + no peer + no progress).
* A recovery-truncated/behind member with no reachable peer at the promotion
* instant is FENCED (in_catch_up) and STAYS ALIVE, retrying catch-up. A
* delaymgr re-alert timer fires after DEFAULT_RECOVERY_CATCHUP_WINDOW_MS: if the
* member is still unrecoverable it RE-EMITS the loud operator alert (sets the
* sticky persist_recovery_unrecoverable flag) and the process STAYS ALIVE —
* there is NO abort/exit anywhere. Catch-up releases the fence + clears the
* flag (Req 2.6); shutdown/cancel stops the re-alert.
*
* These tests drive ha_arm_promotion_fence_timer (the production deferral wired
* into raft_cb_become_leader) through a REAL delaymgr with a short window seam,
* so the timer fires fast and deterministically. A minimal cluster_t carries
* the delaymgr + persist + ha exactly as production does. */
* the delaymgr + persist + ha exactly as production does. They assert the
* member STAYS ALIVE + non-promotable (sticky flag) instead of asserting an
* abort/hook — the self-exit terminal is GONE. */
/* The delaymgr runner used by production (run_delay_tasks in cluster.c) is not
* linked into the unit-test target; provide the same trivial pass-through. */
@ -2084,28 +2104,31 @@ static void dur_destroy_fence_cluster(cluster_t* c)
free(c);
}
/* Spin until the fatal hook fires or a timeout elapses (the delaymgr fires the
* one-shot timer on its own worker thread). */
static bool dur_wait_fatal(int expect_calls, int timeout_ms)
/* Spin until the sticky unrecoverable flag is set on c->persist or a timeout
* elapses (the delaymgr fires the re-alert timer on its own worker thread). */
static bool dur_wait_unrecoverable(cluster_t* c, int timeout_ms)
{
int waited = 0;
while (g_fatal_hook_calls < expect_calls && waited < timeout_ms) {
while (!persist_recovery_unrecoverable(&c->persist) && waited < timeout_ms) {
usleep(2000);
waited += 2;
}
return g_fatal_hook_calls >= expect_calls;
return persist_recovery_unrecoverable(&c->persist);
}
/* 4.4-RED carrier + core deferral: a recovery-truncated/behind member with NO
* reachable peer must NOT FATAL synchronously at the promotion instant — it
* fences (in_catch_up set, process alive) and arms a bounded-window timer; the
* 3.3 FATAL fires ONLY from that timer after the window with no peer/progress.
/* 4.4 (never-self-exit): a recovery-truncated/behind member with NO reachable
* peer must NOT self-exit — it fences (in_catch_up set, process ALIVE) and arms
* the re-alert timer; when the window elapses with no peer/progress the timer
* RE-EMITS the loud alert (sets the sticky non-promotable flag) and the process
* STAYS ALIVE.
*
* RED (pre-4.4): the gate routed a no-peer truncated member straight through
* ha_evaluate_promotion_fence → persist_recover_fatal_if_unrecoverable, so the
* fatal hook was called SYNCHRONOUSLY (count==1 with zero delay) — there was no
* bounded window, which is exactly the task-5.2 soak collapse. (Req 2.2, 2.4) */
static void test_promotion_fence_defers_fatal_to_bounded_timer(void)
* RED→GREEN: pre-rework the no-peer truncated member was routed (synchronously
* or via the timer) to persist_recover_fatal_if_unrecoverable, which abort()ed
* (the old fatal-capture hook fired) — that self-exit was the task-5.2 soak
* quorum collapse. Post-rework there is NO abort: the member stays alive,
* fenced, and loudly non-promotable. Reaching the end of this test (the
* process did not die) IS part of the proof. (Req 2.2, 2.4) */
static void test_promotion_fence_unrecoverable_stays_alive_loud(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_defer_XXXXXX");
@ -2115,46 +2138,41 @@ static void test_promotion_fence_defers_fatal_to_bounded_timer(void)
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(40); /* short bounded window for the test */
ha_set_promotion_fence_window_ms(40); /* short re-alert window for the test */
/* Arm the fence exactly as raft_cb_become_leader does for a truncated,
* below-HWM, no-peer winner. This must NOT FATAL synchronously. */
* below-HWM, no-peer winner. This must NOT self-exit and must fence. */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: the promotion gate must NOT FATAL synchronously — FATAL is "
"deferred to the bounded-window timer (Req 2.2/2.4)");
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: a fenced member is held in_catch_up (refused promotion), ALIVE");
/* Let the re-alert window elapse: still truncated + no peer + no progress →
* the timer RE-EMITS the loud alert and SETS the sticky non-promotable flag.
* The process STAYS ALIVE (no abort/exit anywhere on this path). */
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_unrecoverable(c, 1000),
"4.4: the re-alert timer must set the sticky unrecoverable flag after "
"the window with no peer/progress — staying ALIVE, not exiting (Req 2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: a fenced member is held in_catch_up (refused promotion), alive");
"4.4: an unrecoverable member stays fenced (non-promotable) and ALIVE");
/* Now let the bounded window elapse: still truncated + no peer + no
* progress → the 3.3 FATAL fires from the timer (exactly once). */
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_fatal(1, 1000),
"4.4: the deferred FATAL must fire from the timer after the bounded "
"window with no peer and no progress (Req 2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"4.4: the deferred FATAL fires exactly once from the timer");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0); /* restore default */
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_defers_fatal_to_bounded_timer: PASS\n");
printf("test_promotion_fence_unrecoverable_stays_alive_loud: PASS\n");
}
/* 4.4 Req 2.6: catch-up within the window → fence released, FATAL NOT called.
* Arm the fence, then simulate catch-up (clear truncated + advance durable_floor
* past the HWM) before the timer fires; when the timer fires it observes a
* caught-up member, releases in_catch_up, and does NOT FATAL. */
static void test_promotion_fence_released_on_catchup_no_fatal(void)
/* 4.4 Req 2.6: catch-up within the window → fence + sticky flag released, and
* the member NEVER self-exits. Arm the fence, then simulate catch-up (clear
* truncated + advance durable_floor past the HWM) before the timer fires; when
* the timer fires it observes a caught-up member, releases in_catch_up, clears
* the sticky unrecoverable flag, and does NOT set the unrecoverable flag. */
static void test_promotion_fence_released_on_catchup(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_catchup_XXXXXX");
@ -2164,15 +2182,12 @@ static void test_promotion_fence_released_on_catchup_no_fatal(void)
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
@ -2180,26 +2195,26 @@ static void test_promotion_fence_released_on_catchup_no_fatal(void)
c->persist.recovery_verdict = RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
/* Wait long enough for the window to fire; assert it did NOT FATAL and the
* fence was released (Req 2.6). */
/* Wait long enough for the window to fire; assert the fence was released and
* the sticky flag is NOT set (Req 2.6) — the member self-healed, alive. */
usleep(300 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: a member that caught up within the window must NOT FATAL (Req 2.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: catch-up within the window RELEASES the in_catch_up fence (Req 2.6)");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(&c->persist),
"4.4: a member that caught up must NOT be left in the unrecoverable "
"fence (Req 2.6)");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_released_on_catchup_no_fatal: PASS\n");
printf("test_promotion_fence_released_on_catchup: PASS\n");
}
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → NO FATAL.
* Proves the deferred FATAL never fires after a clean cancel — a stopped member
* must not self-terminate, and there is no use-after-free of the timer ctx. */
static void test_promotion_fence_cancel_prevents_fatal(void)
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → the
* re-alert never fires, no use-after-free of the timer ctx, and the process
* stays alive (it never self-exited even before the cancel). */
static void test_promotion_fence_cancel_stops_realert(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_cancel_XXXXXX");
@ -2209,29 +2224,27 @@ static void test_promotion_fence_cancel_prevents_fatal(void)
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(60);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
/* Cancel before the window elapses. */
ha_cancel_promotion_fence_timer(&c->ha);
/* Wait past the window — the cancelled timer must be a no-op, NO FATAL. */
/* Wait past the window — the cancelled timer must be a no-op (no re-alert,
* no UAF). The member stayed alive throughout (no abort anywhere). */
usleep(200 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: a cancelled fence timer must NEVER FATAL (shutdown/recovery)");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(&c->persist),
"4.4: a cancelled fence timer must not fire the re-alert "
"(shutdown/recovery)");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_cancel_prevents_fatal: PASS\n");
printf("test_promotion_fence_cancel_stops_realert: PASS\n");
}
#endif /* XCE_TESTING */
@ -2272,25 +2285,26 @@ void test_commit_durability(void)
/* task 3.2 — Boot Policy Decision Table skeletons (IGNORED, pending
* 3.3/4.3). Registered so each row is reported as IGNORED, not failed;
* the suite stays GREEN. See design.md "Boot Policy Decision Table". */
/* task 3.3 — FATAL rows (4, 4b) now ACTIVE (PASS), plus the never-FATAL
* invariant for the peer-reachable rows (3, 5). The full FENCE->ELIGIBLE
* eligibility outcome of rows 1,2,3,5,6 stays IGNORED pending task 4.3. */
/* task 3.3 — unrecoverable rows (4, 4b) = LOUD PERMANENT FENCE (stays alive,
* non-promotable, clears on catch-up), plus the never-unrecoverable
* invariant for the peer-reachable rows (3, 5) and the FENCE->ELIGIBLE
* eligibility outcome of rows 1,2,3,5,6. (never-self-exit rework) */
RUN_TEST(test_boot_policy_peer_reachable_never_fatal);
RUN_TEST(test_boot_policy_row1_clean_at_or_above_eligible);
RUN_TEST(test_boot_policy_row2_clean_below_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row3_truncated_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row4_truncated_no_peer_bounded_fatal);
RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_bounded_fatal);
RUN_TEST(test_boot_policy_row4_truncated_no_peer_loud_fence);
RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_loud_fence);
RUN_TEST(test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal);
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
/* task 4.4 — bounded-window catch-up before FATAL (no synchronous FATAL;
* deferred-timer FATAL only after a real bounded failure; fence released on
* catch-up; cancelled timer never FATALs). (Req 2.2, 2.4, 2.6) */
RUN_TEST(test_promotion_fence_defers_fatal_to_bounded_timer);
RUN_TEST(test_promotion_fence_released_on_catchup_no_fatal);
RUN_TEST(test_promotion_fence_cancel_prevents_fatal);
/* task 4.4 — catch-up re-alert window (NEVER self-exits; stays alive +
* loudly non-promotable on the unrecoverable path; fence + sticky flag
* released on catch-up; cancelled timer stops the re-alert). (Req 2.2/2.4/2.6) */
RUN_TEST(test_promotion_fence_unrecoverable_stays_alive_loud);
RUN_TEST(test_promotion_fence_released_on_catchup);
RUN_TEST(test_promotion_fence_cancel_stops_realert);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif