feat(ha-commit-durability): pre-promotion eligibility gate + catch-up fence/release (task 4.3)
This commit is contained in:
parent
3049b620f3
commit
e9e2a6d703
4 changed files with 409 additions and 21 deletions
|
|
@ -538,6 +538,91 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
|||
" (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
|
||||
}
|
||||
|
||||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
|
||||
*
|
||||
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
|
||||
* Policy Decision Table inputs from the promoting member's recovery/durability
|
||||
* state and delegate to persist_recover_decision so the 3.2/3.3 table is the
|
||||
* single source of truth (DRY). "Behind the HWM" is durable_floor < safe_hwm;
|
||||
* a truncated log is treated as behind regardless of its volatile position
|
||||
* (design.md note). A hollow node (truncated OR behind) can never be ELIGIBLE
|
||||
* until it catches up — independent of quorum size (Req 2.5).
|
||||
*/
|
||||
recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
|
||||
uint64_t durable_floor,
|
||||
uint64_t safe_hwm,
|
||||
bool peer_reachable,
|
||||
bool catch_up_succeeded)
|
||||
{
|
||||
recover_policy_inputs_t in = {
|
||||
.verdict = recovery_truncated ? RECOVER_TRUNCATED
|
||||
: RECOVER_CLEAN,
|
||||
.durable_below_hwm = (durable_floor < safe_hwm),
|
||||
.healthy_peer_reachable = peer_reachable,
|
||||
.catch_up_succeeded = catch_up_succeeded,
|
||||
.durable_floor_lsn = durable_floor,
|
||||
.cluster_hwm_lsn = safe_hwm,
|
||||
};
|
||||
return persist_recover_decision(&in);
|
||||
}
|
||||
|
||||
/* ha_evaluate_promotion_fence: live gate over the existing in_catch_up fence
|
||||
* (Req 2.2, 2.6). Reuses ha->in_catch_up — does NOT invent a parallel fence.
|
||||
* - ELIGIBLE → clear the fence, return ELIGIBLE (a member that caught up past
|
||||
* the committed boundary regains eligibility, Req 2.6).
|
||||
* - FENCE → set the fence, return FENCE (refuse promotion; catch up from a
|
||||
* healthy peer).
|
||||
* - FATAL → run the 3.3 loud-terminate path (logs the truncation boundary,
|
||||
* then terminates — or fires persist_fatal_hook in tests).
|
||||
*/
|
||||
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
|
||||
bool recovery_truncated,
|
||||
uint64_t durable_floor,
|
||||
uint64_t safe_hwm,
|
||||
bool peer_reachable,
|
||||
bool catch_up_succeeded)
|
||||
{
|
||||
recover_decision_t d = ha_promotion_gate_decision(
|
||||
recovery_truncated, durable_floor, safe_hwm, peer_reachable,
|
||||
catch_up_succeeded);
|
||||
|
||||
switch (d) {
|
||||
case RECOVER_DECISION_ELIGIBLE:
|
||||
/* Release the fence: catch-up brought the member to a
|
||||
* committed-consistent state at/above the cluster boundary (Req 2.6). */
|
||||
if (ha != NULL)
|
||||
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
|
||||
break;
|
||||
case RECOVER_DECISION_FENCE:
|
||||
/* Refuse promotion: extend the catch-up fence so the member catches up
|
||||
* from a healthy peer before it can lead (Req 2.2). */
|
||||
if (ha != NULL)
|
||||
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
|
||||
INFO("HA: promotion FENCED — recovery_truncated=%d durable_floor=%"
|
||||
PRIu64 " < cluster HWM=%" PRIu64 " (catch up before leading; "
|
||||
"ha-commit-durability Req 2.2)",
|
||||
recovery_truncated, durable_floor, safe_hwm);
|
||||
break;
|
||||
case RECOVER_DECISION_FATAL: {
|
||||
/* Bounded-unrecoverable: route through the 3.3 loud-terminate action,
|
||||
* which logs the truncation boundary at ERROR then terminates (or fires
|
||||
* persist_fatal_hook under test). Req 2.4. */
|
||||
recover_policy_inputs_t in = {
|
||||
.verdict = recovery_truncated ? RECOVER_TRUNCATED
|
||||
: RECOVER_CLEAN,
|
||||
.durable_below_hwm = (durable_floor < safe_hwm),
|
||||
.healthy_peer_reachable = peer_reachable,
|
||||
.catch_up_succeeded = catch_up_succeeded,
|
||||
.durable_floor_lsn = durable_floor,
|
||||
.cluster_hwm_lsn = safe_hwm,
|
||||
};
|
||||
persist_recover_fatal_if_unrecoverable(&in);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
/*
|
||||
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
|
||||
*
|
||||
|
|
@ -559,6 +644,13 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
|||
/* Sync ha->term from raft term (raft owns term increments) */
|
||||
atomic_store_explicit(&ha->term, term, memory_order_release);
|
||||
|
||||
/* The cluster committed high-water mark for this promotion — derived from
|
||||
* the voter replies (NOT a stale ctrlhello). The task-4.3 pre-promotion
|
||||
* eligibility gate (below) compares the local CLAMPED durable position
|
||||
* against it. Lifted to function scope so the gate can read it after the
|
||||
* truncate-clamp. */
|
||||
uint64_t promotion_safe_lsn = 0;
|
||||
|
||||
/* Task 4.1 (ha-repl-hardening): compute safe_committed_lsn from vote replies.
|
||||
* Uses voter LSNs collected during election (ha->elect_ctx->votes[]) to find
|
||||
* the highest LSN held by a majority without an extra round-trip.
|
||||
|
|
@ -570,6 +662,7 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
|||
uint64_t safe_lsn = 0;
|
||||
uint64_t safe_tag = 0;
|
||||
ha_compute_safe_committed_lsn(ectx, &safe_lsn, &safe_tag);
|
||||
promotion_safe_lsn = safe_lsn;
|
||||
/* Store results for use in WAL truncation and post-election commit.
|
||||
* safe_lsn is the highest LSN on a majority; entries above it
|
||||
* may not have been committed on all nodes and must be truncated. */
|
||||
|
|
@ -593,6 +686,68 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
|
|||
}
|
||||
}
|
||||
|
||||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────
|
||||
* Refuse promotion BEFORE the node becomes authoritative CTRL_PRIMARY if it
|
||||
* carries the recovery-truncated flag (task 3.1 — a structurally corrupt log
|
||||
* must never lead) OR its CLAMPED durable position is below the cluster
|
||||
* committed high-water mark (promotion_safe_lsn, from the voter replies).
|
||||
* The durable position is read AFTER persist_truncate_wal_above above, so it
|
||||
* reflects the 2.2 truncate-clamp (a node whose durable tip was clamped down
|
||||
* is correctly seen as behind). A hollow node (truncated OR behind) is
|
||||
* refused regardless of quorum size — combined with the 4.2 freshness floor
|
||||
* it can neither win the vote nor self-promote (Req 2.5). On refusal we set
|
||||
* the in_catch_up fence and return WITHOUT becoming primary; the member
|
||||
* catches up from a healthy peer and a later promotion attempt passes the
|
||||
* gate (Req 2.2/2.6). Only the production path (real persist) is gated; the
|
||||
* c==NULL unit shim has no persist and skips it. */
|
||||
if (c != NULL) {
|
||||
bool truncated = persist_recovery_truncated(&c->persist);
|
||||
uint64_t durable_floor = persist_durable_floor(&c->persist);
|
||||
|
||||
/* A healthy peer is reachable for catch-up iff some voter granted with a
|
||||
* live endpoint survives this election (these become the standby
|
||||
* channels promoted below). */
|
||||
bool peer_reachable = false;
|
||||
{
|
||||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||||
&ha->elect_ctx, memory_order_acquire);
|
||||
if (ectx) {
|
||||
for (int i = 0; i < ectx->n_peers; i++) {
|
||||
if (ectx->votes[i].vote_granted && ectx->votes[i].ep != NULL) {
|
||||
peer_reachable = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
|
||||
* the live bounded-window catch-up loop is wired by task 5.1; here the
|
||||
* gate refuses (FENCE) or fails loud (FATAL when no peer) immediately,
|
||||
* never silently proceeding while hollow. */
|
||||
recover_decision_t gate = ha_evaluate_promotion_fence(
|
||||
ha, truncated, durable_floor, promotion_safe_lsn,
|
||||
peer_reachable, /*catch_up_succeeded=*/false);
|
||||
|
||||
if (gate != RECOVER_DECISION_ELIGIBLE) {
|
||||
/* FENCE: refuse promotion — do NOT become CTRL_PRIMARY. The
|
||||
* in_catch_up fence is set; the member catches up from a healthy
|
||||
* peer and re-attempts promotion later (Req 2.2, 2.6).
|
||||
* FATAL already terminated in production (or fired the test hook);
|
||||
* still bail out here so a test-hooked FATAL does not fall through
|
||||
* to authoritative leadership. */
|
||||
WARN("HA: promotion REFUSED (decision=%d) — recovery_truncated=%d "
|
||||
"durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64
|
||||
" peer_reachable=%d; remaining non-primary until caught up "
|
||||
"(ha-commit-durability Req 2.2/2.4)",
|
||||
(int)gate, truncated, durable_floor, promotion_safe_lsn,
|
||||
peer_reachable);
|
||||
return;
|
||||
}
|
||||
/* ELIGIBLE: clear of the fence (release on a caught-up member, Req 2.6)
|
||||
* — proceed to authoritative promotion below. */
|
||||
}
|
||||
|
||||
int is_first_boot = !ha->phase2_done;
|
||||
|
||||
/* First boot: call cluster_init_phase2 */
|
||||
|
|
|
|||
|
|
@ -497,6 +497,50 @@ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
|||
uint64_t *safe_lsn_out,
|
||||
uint64_t *safe_tag_out);
|
||||
|
||||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ──────────
|
||||
*
|
||||
* ha_promotion_gate_decision: PURE pre-promotion eligibility decision (Req 2.2,
|
||||
* 2.5, 2.6). Maps the promoting member's recovery/durability state to the Boot
|
||||
* Policy Decision Table outcome by building recover_policy_inputs_t and
|
||||
* delegating to persist_recover_decision (DRY — the 3.2/3.3 decision table is
|
||||
* the single source of truth; this gate does not re-derive it):
|
||||
* - recovery_truncated → the task-3.1 structural-corruption flag.
|
||||
* - durable_floor → the local CLAMPED durable position (read AFTER the
|
||||
* promotion-path persist_truncate_wal_above so the
|
||||
* 2.2 truncate-clamp is reflected).
|
||||
* - safe_hwm → the cluster committed high-water mark from the voter
|
||||
* replies (ha_compute_safe_committed_lsn), NOT a stale
|
||||
* ctrlhello.
|
||||
* - peer_reachable → a healthy peer is reachable to catch up from.
|
||||
* - catch_up_succeeded → catch-up reached a committed-consistent state.
|
||||
* A member is "behind the HWM" iff durable_floor < safe_hwm. Returns
|
||||
* RECOVER_DECISION_{ELIGIBLE,FENCE,FATAL}; a hollow node (truncated OR behind)
|
||||
* is never ELIGIBLE until it catches up, regardless of quorum size (Req 2.5).
|
||||
*/
|
||||
recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
|
||||
uint64_t durable_floor,
|
||||
uint64_t safe_hwm,
|
||||
bool peer_reachable,
|
||||
bool catch_up_succeeded);
|
||||
|
||||
/* ha_evaluate_promotion_fence: live pre-promotion gate over ha->in_catch_up
|
||||
* (Req 2.2, 2.6). Evaluates ha_promotion_gate_decision and applies the fence:
|
||||
* - ELIGIBLE → clears the in_catch_up fence and returns ELIGIBLE (promotion
|
||||
* may proceed; a member that has caught up past the committed
|
||||
* boundary regains eligibility — Req 2.6).
|
||||
* - FENCE → sets the in_catch_up fence (refuse promotion; catch up from a
|
||||
* healthy peer) and returns FENCE.
|
||||
* - FATAL → routes through persist_recover_fatal_if_unrecoverable (logs the
|
||||
* truncation boundary at ERROR then terminates; the test seam
|
||||
* persist_fatal_hook captures it) and returns FATAL.
|
||||
* Reuses the existing in_catch_up fence — does NOT invent a parallel fence. */
|
||||
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
|
||||
bool recovery_truncated,
|
||||
uint64_t durable_floor,
|
||||
uint64_t safe_hwm,
|
||||
bool peer_reachable,
|
||||
bool catch_up_succeeded);
|
||||
|
||||
/* ── Task 3.2: Unified replication decision enum and evaluator ────────────── */
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
|
||||
#include "commit.h"
|
||||
#include "config.h"
|
||||
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
|
||||
#include "persist.h"
|
||||
#include "wal_types.h"
|
||||
|
||||
|
|
@ -1302,12 +1303,42 @@ static void test_boot_policy_peer_reachable_never_fatal(void)
|
|||
* a normal promotable member: the promotion gate (4.3) must NOT fence
|
||||
* it. WILL ASSERT (4.3): the eligibility predicate returns "eligible"
|
||||
* for a CLEAN, at/above member; promotion proceeds. (Req 2.3) */
|
||||
/* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role_lifecycle.c). */
|
||||
static ha_config_t dur_ha_cfg(void)
|
||||
{
|
||||
ha_config_t cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 10000,
|
||||
.election_timeout_ms = 2000,
|
||||
.probe_connect_timeout_ms = 500,
|
||||
.max_promotion_retries = 5,
|
||||
};
|
||||
return cfg;
|
||||
}
|
||||
|
||||
static void test_boot_policy_row1_clean_at_or_above_eligible(void)
|
||||
{
|
||||
TEST_IGNORE_MESSAGE(
|
||||
"row 1 (CLEAN + at/above HWM -> ELIGIBLE): pending task 4.3 "
|
||||
"(promotion-eligibility gate). Will assert the gate admits a clean, "
|
||||
"not-behind member.");
|
||||
/* CLEAN + durable at/above HWM -> ELIGIBLE. The live gate must admit a
|
||||
* structurally-intact, not-behind member and must NOT set the fence. */
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = dur_ha_cfg();
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
/* Pre-set the fence to prove ELIGIBLE clears it (Req 2.6 release path). */
|
||||
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
|
||||
|
||||
recover_decision_t d = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/8, /*safe_hwm=*/8,
|
||||
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE, d,
|
||||
"row 1: a CLEAN, at/above-HWM member must be ELIGIBLE to promote");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
|
||||
"row 1: ELIGIBLE must clear the in_catch_up fence");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_boot_policy_row1_clean_at_or_above_eligible: PASS\n");
|
||||
}
|
||||
|
||||
/* Row 2: CLEAN + durable below HWM + healthy peer + catch-up succeeds
|
||||
|
|
@ -1318,10 +1349,39 @@ static void test_boot_policy_row1_clean_at_or_above_eligible(void)
|
|||
* committed boundary, then regains eligibility. (Req 2.2, 2.3, 2.6) */
|
||||
static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void)
|
||||
{
|
||||
TEST_IGNORE_MESSAGE(
|
||||
"row 2 (CLEAN + below HWM + peer + catch-up ok -> FENCE+CATCH-UP -> "
|
||||
"ELIGIBLE): pending task 4.3 (fence + release). Will assert denied "
|
||||
"until caught up, then re-eligible.");
|
||||
/* CLEAN + below HWM + healthy peer: FENCE first (denied promotion), then
|
||||
* once catch-up passes the committed boundary, ELIGIBLE (fence released). */
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = dur_ha_cfg();
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
|
||||
/* Behind: durable_floor 4 < HWM 10, healthy peer reachable, not yet caught
|
||||
* up -> FENCE; the in_catch_up fence is SET (refused promotion). */
|
||||
recover_decision_t d1 = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/4, /*safe_hwm=*/10,
|
||||
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_FENCE, d1,
|
||||
"row 2: a CLEAN-but-behind member with a peer must FENCE (catch up)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
|
||||
"row 2: FENCE must SET the in_catch_up fence (promotion refused)");
|
||||
|
||||
/* Catch-up brings the durable position to/above the committed boundary
|
||||
* (durable_floor 10 >= HWM 10) -> ELIGIBLE; the fence is released (Req 2.6). */
|
||||
recover_decision_t d2 = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
|
||||
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE, d2,
|
||||
"row 2: after catch-up past the committed boundary the member regains "
|
||||
"eligibility (Req 2.6)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
|
||||
"row 2: regained eligibility must RELEASE the in_catch_up fence");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_boot_policy_row2_clean_below_peer_catchup_eligible: PASS\n");
|
||||
}
|
||||
|
||||
/* Row 3: TRUNCATED + healthy peer + catch-up succeeds
|
||||
|
|
@ -1333,10 +1393,46 @@ static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void)
|
|||
* triggered. (Req 2.2, 2.6) */
|
||||
static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
|
||||
{
|
||||
TEST_IGNORE_MESSAGE(
|
||||
"row 3 (TRUNCATED + peer + catch-up ok -> FENCE+CATCH-UP -> "
|
||||
"ELIGIBLE): pending tasks 3.3/4.3. Will assert truncated member "
|
||||
"fences, catches up, re-eligible; no FATAL when a peer is reachable.");
|
||||
/* TRUNCATED + healthy peer: a recovery-truncated member is DENIED promotion
|
||||
* (FENCE), catches up from the peer, then regains eligibility. FATAL must
|
||||
* NOT fire while a healthy peer is reachable. */
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = dur_ha_cfg();
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
|
||||
g_fatal_hook_calls = 0;
|
||||
persist_fatal_hook = dur_fatal_hook;
|
||||
|
||||
/* Truncated boundary at 3, cluster HWM 10, peer reachable, not caught up. */
|
||||
recover_decision_t d1 = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10,
|
||||
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_FENCE, d1,
|
||||
"row 3: a recovery-truncated member with a healthy peer must FENCE "
|
||||
"(denied promotion), never FATAL");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
|
||||
"row 3: a truncated member must be fenced (in_catch_up set)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_fatal_hook_calls,
|
||||
"row 3: FATAL must NOT fire while a healthy peer is reachable");
|
||||
|
||||
/* After catch-up to/above the committed boundary -> ELIGIBLE, fence released. */
|
||||
recover_decision_t d2 = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
|
||||
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE, d2,
|
||||
"row 3: after catch-up the (previously truncated) member is eligible");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
|
||||
"row 3: regained eligibility releases the fence (Req 2.6)");
|
||||
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
|
||||
|
||||
persist_fatal_hook = NULL;
|
||||
ha_fini(&ha);
|
||||
printf("test_boot_policy_row3_truncated_peer_catchup_eligible: PASS\n");
|
||||
}
|
||||
|
||||
/* Row 4: TRUNCATED + below HWM + NO healthy peer + catch-up fails
|
||||
|
|
@ -1435,10 +1531,39 @@ static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
|
|||
* downstream promotion-fence outcome.) */
|
||||
static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
|
||||
{
|
||||
TEST_IGNORE_MESSAGE(
|
||||
"row 5 (snapshot-sentinel CLEAN + below + peer -> FENCE+CATCH-UP -> "
|
||||
"ELIGIBLE, NEVER FATAL): pending task 4.3. Will assert a legitimate "
|
||||
"sentinel snapshot is treated as behind-must-catch-up, not FATAL.");
|
||||
/* Snapshot-sentinel CLEAN-but-behind + healthy peer: treated as behind,
|
||||
* must catch up (FENCE), NEVER FATAL on a legitimate sentinel; then
|
||||
* ELIGIBLE after catch-up. */
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = dur_ha_cfg();
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
|
||||
g_fatal_hook_calls = 0;
|
||||
persist_fatal_hook = dur_fatal_hook;
|
||||
|
||||
/* Sentinel at LSN 5, cluster has advanced to HWM 12, peer reachable. */
|
||||
recover_decision_t d1 = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/12,
|
||||
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_FENCE, d1,
|
||||
"row 5: a snapshot-sentinel CLEAN-but-behind member must FENCE, never "
|
||||
"FATAL while a peer is reachable");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_fatal_hook_calls,
|
||||
"row 5: a legitimate sentinel must NEVER trigger FATAL");
|
||||
|
||||
recover_decision_t d2 = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/12, /*safe_hwm=*/12,
|
||||
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE, d2,
|
||||
"row 5: after catch-up the sentinel member is eligible (Req 2.6)");
|
||||
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
|
||||
|
||||
persist_fatal_hook = NULL;
|
||||
ha_fini(&ha);
|
||||
printf("test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal: PASS\n");
|
||||
}
|
||||
|
||||
/* Row 6: CLEAN empty first-boot WAL (durable=0, HWM=0, fresh cold start)
|
||||
|
|
@ -1448,10 +1573,71 @@ static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
|
|||
* fresh cold-start member (no fence, no FATAL). (Req 2.1, 2.3) */
|
||||
static void test_boot_policy_row6_empty_first_boot_eligible(void)
|
||||
{
|
||||
TEST_IGNORE_MESSAGE(
|
||||
"row 6 (empty first-boot WAL CLEAN, durable=0/HWM=0 -> ELIGIBLE): "
|
||||
"pending task 4.3. Will assert a fresh cold-start member is "
|
||||
"promotable (no fence, no FATAL).");
|
||||
/* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fresh cold-start
|
||||
* member is not behind anything and must be promotable (no fence, no FATAL),
|
||||
* even with no peer reachable (single-node / first boot). */
|
||||
ha_state_t ha;
|
||||
ha_config_t cfg = dur_ha_cfg();
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
|
||||
|
||||
g_fatal_hook_calls = 0;
|
||||
persist_fatal_hook = dur_fatal_hook;
|
||||
|
||||
recover_decision_t d = ha_evaluate_promotion_fence(
|
||||
&ha, /*truncated=*/false, /*durable_floor=*/0, /*safe_hwm=*/0,
|
||||
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE, d,
|
||||
"row 6: an empty first-boot WAL (durable=0/HWM=0) must be ELIGIBLE");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
|
||||
"row 6: a cold-start member must not be fenced");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_fatal_hook_calls,
|
||||
"row 6: a cold-start member must never FATAL");
|
||||
|
||||
persist_fatal_hook = NULL;
|
||||
ha_fini(&ha);
|
||||
printf("test_boot_policy_row6_empty_first_boot_eligible: PASS\n");
|
||||
}
|
||||
|
||||
/* ── task 4.3: hollow node refused even at minimal quorum (Req 2.5) ──────────
|
||||
* A hollow node (recovery-truncated OR durable below the HWM) must be refused
|
||||
* promotion regardless of quorum size — combined with the 4.2 freshness floor
|
||||
* it can neither win the vote nor self-promote. At minimal quorum with NO
|
||||
* healthy peer, a hollow node FATALs (bounded-unrecoverable) rather than
|
||||
* silently leading; with a peer it FENCEs. Either way it never becomes
|
||||
* ELIGIBLE while hollow. */
|
||||
static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
|
||||
{
|
||||
/* Truncated + below HWM + no peer (minimal/degraded quorum) -> NEVER
|
||||
* ELIGIBLE (FATAL — fail loud rather than promote a hollow node). */
|
||||
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE,
|
||||
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
|
||||
/*safe_hwm=*/10, /*peer_reachable=*/false,
|
||||
/*catch_up_succeeded=*/false),
|
||||
"a recovery-truncated node must never be ELIGIBLE at minimal quorum "
|
||||
"(Req 2.5)");
|
||||
|
||||
/* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (FATAL). */
|
||||
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_ELIGIBLE,
|
||||
ha_promotion_gate_decision(/*truncated=*/false, /*durable_floor=*/5,
|
||||
/*safe_hwm=*/10, /*peer_reachable=*/false,
|
||||
/*catch_up_succeeded=*/false),
|
||||
"a below-HWM node must never be ELIGIBLE at minimal quorum (Req 2.5)");
|
||||
|
||||
/* Hollow node WITH a peer is still refused (FENCE, not ELIGIBLE) until it
|
||||
* catches up — minimal-quorum size does not relax the gate. */
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
RECOVER_DECISION_FENCE,
|
||||
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
|
||||
/*safe_hwm=*/10, /*peer_reachable=*/true,
|
||||
/*catch_up_succeeded=*/false),
|
||||
"a hollow node with a peer is FENCED (refused) until caught up");
|
||||
|
||||
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
|
@ -1499,6 +1685,8 @@ void test_commit_durability(void)
|
|||
RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_bounded_fatal);
|
||||
RUN_TEST(test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal);
|
||||
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
|
||||
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
|
||||
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
|
||||
#else
|
||||
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue