wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha-commit-durability): bounded-window catch-up before FATAL — fence no longer FATALs synchronously (task 4.4)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 10:35:13 +00:00
parent e14bd62115
commit addf53e3b4
4 changed files with 562 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

282
src/controller/ha.c
View file

@ -623,6 +623,226 @@ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
return d;
}
/* ── ha-commit-durability task 4.4: bounded-window catch-up before FATAL ─────
*
* The 4.3 gate above can DECIDE FATAL synchronously, but a member that won an
* election with no peer reachable AT THE PROMOTION INSTANT may simply be seeing
* a transient no-peer snapshot during aggressive failover (task-5.2 soak: a
* recovery-truncated winner self-FATAL'd immediately and repeatedly until
* quorum was lost — an availability collapse, not durable loss). Per the 3.2
* decision table the FATAL terminal must be reached ONLY after a BOUNDED
* failure: the member is fenced (in_catch_up), given DEFAULT_RECOVERY_CATCHUP_-
* WINDOW_MS to find a peer / make catch-up progress, and FATAL'd from a one-shot
* timer only if it is STILL hollow with no peer and no progress when the window
* elapses.
*
* The window is a settable seam so the timer test runs fast. Default =
* DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */
static _Atomic uint32_t s_promotion_fence_window_ms =
DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
uint32_t ha_promotion_fence_window_ms(void)
{
uint32_t w = atomic_load_explicit(&s_promotion_fence_window_ms,
memory_order_acquire);
return w ? w : DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
}
#ifdef XCE_TESTING
void ha_set_promotion_fence_window_ms(uint32_t ms)
{
atomic_store_explicit(&s_promotion_fence_window_ms, ms,
memory_order_release);
}
#endif
/* promotion_fence_ctx_t: delaymgr context for the bounded catch-up window.
* Captures the cluster committed high-water mark (target_hwm) and the durable
* floor AT SCHEDULE TIME (baseline_durable_floor) so the timer can detect
* catch-up progress (durable_floor advanced toward the HWM). `cancelled` is set
* by ha_cancel_promotion_fence_timer (recovery / shutdown) so a fired-but-
* cancelled callback is a safe no-op. */
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
bool was_truncated; /* recovery-truncated at schedule time */
uint64_t target_hwm; /* cluster committed high-water mark */
uint64_t baseline_durable_floor; /* durable floor at schedule time */
} promotion_fence_ctx_t;
/* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer.
* Idempotent; safe to call when no timer is armed. Used when the member catches
* up (fence released) or when the controller is shutting down — so neither a
* caught-up nor a stopped member ever reaches the deferred FATAL. */
void ha_cancel_promotion_fence_timer(ha_state_t *ha)
{
if (!ha) return;
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)atomic_exchange_explicit(
&ha->promotion_fence_ctx, NULL, memory_order_acq_rel);
if (t)
atomic_store_explicit(&t->cancelled, 1, memory_order_release);
}
/* promotion_fence_timeout_fn: the bounded-window timer callback. Owns its ctx —
* frees it on exit. Re-evaluates the LIVE recovery/durability state and reaches
* the 3.3 FATAL ONLY on a real bounded failure (Req 2.4):
* - still recovery-truncated/behind AND no healthy peer reachable AND no
* catch-up progress (durable_floor has not advanced past the baseline)
* → persist_recover_fatal_if_unrecoverable (fail loud).
* - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
* in_catch_up fence; do NOT FATAL (Req 2.6 — eligible again).
* - a healthy peer is reachable now, OR catch-up is making progress but is not
* yet complete → leave the member fenced under the existing catch-up
* machinery and do NOT FATAL while progress is possible. The single bounded
* window has elapsed, so we do not re-arm: a member with a live peer is no
* longer in the "no peer + no progress" bounded-failure state the FATAL
* guards, and the existing catch-up / re-election path will release the
* fence (ha_evaluate_promotion_fence → ELIGIBLE) when it completes. */
static int promotion_fence_timeout_fn(void *arg)
{
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg;
ha_state_t *ha = t->ha;
cluster_t *c = t->c;
/* Clear the stored handle before checking cancelled so a concurrent
* ha_cancel_promotion_fence_timer is a safe no-op (matches the
* raft_election_timeout_fn convention). */
promotion_fence_ctx_t *cur = (promotion_fence_ctx_t *)atomic_load_explicit(
&ha->promotion_fence_ctx, memory_order_acquire);
if (cur == t)
atomic_store_explicit(&ha->promotion_fence_ctx, NULL,
memory_order_release);
if (atomic_load_explicit(&t->cancelled, memory_order_acquire)) {
free(t);
return 0;
}
if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */
/* Re-read the LIVE state after the bounded window. */
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool below_hwm = durable_floor < t->target_hwm;
bool made_progress = durable_floor > t->baseline_durable_floor;
bool peer_reachable = (ha->repl != NULL)
? (repl_active_standby_count(ha->repl) > 0)
: false;
/* Caught up: not truncated AND durable position reached the boundary →
* release the fence; the member is eligible on its next election (Req 2.6). */
if (!truncated && !below_hwm) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)",
durable_floor, t->target_hwm);
free(t);
return 0;
}
/* Still hollow, but a peer is reachable or catch-up advanced the durable
* floor: progress is possible — do NOT FATAL. Leave the member fenced
* (in_catch_up stays set) under the existing catch-up machinery; the bounded
* window has elapsed so we do not re-arm. A member with a live peer is no
* longer in the no-peer bounded-failure state the FATAL guards. */
if (peer_reachable || made_progress) {
INFO("HA: promotion fence catch-up window elapsed but progress is "
"possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, NOT FATAL "
"(ha-commit-durability Req 2.2)",
peer_reachable, durable_floor, t->baseline_durable_floor,
t->target_hwm);
free(t);
return 0;
}
/* Bounded failure: STILL truncated/behind AND no healthy peer AND no
* catch-up progress after the window → fail loud (Req 2.4). This is the
* ONLY path that reaches the 3.3 FATAL — never synchronously at promotion. */
{
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = below_hwm,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = t->target_hwm,
};
ERROR("HA: promotion catch-up window (%u ms) elapsed with no peer and "
"no progress — recovery_truncated=%d durable_floor=%" PRIu64
" < cluster HWM=%" PRIu64 "; failing loud rather than serving "
"divergent state (ha-commit-durability Req 2.4)",
ha_promotion_fence_window_ms(), truncated, durable_floor,
t->target_hwm);
persist_recover_fatal_if_unrecoverable(&in);
}
free(t);
return 0;
}
/* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule
* the one-shot bounded-window FATAL timer. Cancels any previously-armed timer
* first so a re-evaluation never leaks or double-fires. Returns 0 on success.
* On a failure to schedule the timer it FATALs immediately via the 3.3 action so
* the bounded-failure terminal is never silently lost (a member that could not
* arm its own catch-up deadline must not run on indefinitely). */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm)
{
/* Fence first (Req 2.2): refuse promotion while hollow. */
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
/* Cancel a stale timer (e.g. a prior promotion attempt). */
ha_cancel_promotion_fence_timer(ha);
promotion_fence_ctx_t *t = calloc(1, sizeof(*t));
if (!t)
goto fail_loud;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->was_truncated = truncated;
t->target_hwm = target_hwm;
t->baseline_durable_floor = durable_floor;
uint32_t window_ms = ha_promotion_fence_window_ms();
int rc = delay_task_ms(&c->delaymgr, (int)window_ms, 1,
promotion_fence_timeout_fn, t);
if (rc < 0) {
WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc);
free(t);
goto fail_loud;
}
atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release);
INFO("HA: promotion FENCED + bounded catch-up window armed (%u ms) — "
"recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64
"; FATAL deferred to the timer (ha-commit-durability Req 2.2/2.4)",
window_ms, truncated, durable_floor, target_hwm);
return;
fail_loud:
{
/* Could not arm the bounded deadline — reach the FATAL terminal now
* rather than fence indefinitely. */
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < target_hwm),
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
persist_recover_fatal_if_unrecoverable(&in);
}
}
/*
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
*
@ -722,30 +942,41 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
}
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
* the live bounded-window catch-up loop is wired by task 5.1; here the
* gate refuses (FENCE) or fails loud (FATAL when no peer) immediately,
* never silently proceeding while hollow. */
recover_decision_t gate = ha_evaluate_promotion_fence(
ha, truncated, durable_floor, promotion_safe_lsn,
* a hollow member never silently proceeds to leadership. We evaluate
* the PURE 3.2/3.3 decision here and DEFER its terminal — task 4.4 does
* NOT reach the 3.3 FATAL synchronously at the promotion instant. A
* FENCE or a would-be-FATAL (truncated/behind + no peer right now) both
* fence the member (in_catch_up) and arm the bounded catch-up window;
* the FATAL is reached ONLY from that timer after a real bounded failure
* (still hollow + no peer + no progress). This avoids the task-5.2 soak
* collapse where a transient no-peer snapshot during aggressive failover
* self-FATAL'd a recovering winner repeatedly until quorum was lost. */
recover_decision_t gate = ha_promotion_gate_decision(
truncated, durable_floor, promotion_safe_lsn,
peer_reachable, /*catch_up_succeeded=*/false);
if (gate != RECOVER_DECISION_ELIGIBLE) {
/* FENCE: refuse promotion — do NOT become CTRL_PRIMARY. The
* in_catch_up fence is set; the member catches up from a healthy
* peer and re-attempts promotion later (Req 2.2, 2.6).
* FATAL already terminated in production (or fired the test hook);
* still bail out here so a test-hooked FATAL does not fall through
* to authoritative leadership. */
if (gate == RECOVER_DECISION_ELIGIBLE) {
/* ELIGIBLE: release the fence (a caught-up member regains
* eligibility, Req 2.6) and cancel any pending bounded-window FATAL
* timer — a member that is now promotable must never be FATAL'd by a
* stale timer. Proceed to authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE or would-be-FATAL: refuse promotion — do NOT become
* CTRL_PRIMARY. Fence the member and arm the bounded catch-up
* window; the 3.3 FATAL is deferred to the timer (Req 2.2/2.4).
* Never FATAL synchronously, never fence indefinitely. */
WARN("HA: promotion REFUSED (decision=%d) — recovery_truncated=%d "
"durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64
" peer_reachable=%d; remaining non-primary until caught up "
" peer_reachable=%d; fencing + arming bounded catch-up window "
"(ha-commit-durability Req 2.2/2.4)",
(int)gate, truncated, durable_floor, promotion_safe_lsn,
peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;
}
/* ELIGIBLE: clear of the fence (release on a caught-up member, Req 2.6)
* — proceed to authoritative promotion below. */
}
int is_first_boot = !ha->phase2_done;
@ -1230,6 +1461,13 @@ void ha_fini(ha_state_t *ha)
{
if (!ha)
return;
/* ha-commit-durability task 4.4: cancel any pending bounded-window FATAL
* timer so a stopped member never reaches the deferred FATAL. The delaymgr
* task ctx is freed by promotion_fence_timeout_fn when it fires; cancelling
* makes that firing a no-op (it observes cancelled=1 and frees). */
ha_cancel_promotion_fence_timer(ha);
pthread_mutex_destroy(&ha->lock);
sem_destroy(&ha->heartbeat_seen);
@ -3600,6 +3838,20 @@ void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term)
raft_cb_become_leader((void *)ha, term);
}
#ifdef XCE_TESTING
/* ha_arm_promotion_fence_timer_for_test (ha-commit-durability task 4.4):
* expose the file-static bounded-window arm path so the durability suite can
* drive the exact deferral wired into raft_cb_become_leader without staging a
* full election. Production never references this symbol (XCE_TESTING only). */
void ha_arm_promotion_fence_timer_for_test(cluster_t *c, bool truncated,
uint64_t durable_floor,
uint64_t target_hwm)
{
ha_arm_promotion_fence_timer(&c->ha, c, truncated, durable_floor,
target_hwm);
}
#endif
/*
* raft_cb_become_follower_for_test: drive the follower path without a live
* cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production

48
src/controller/ha.h
View file

@ -220,6 +220,23 @@ typedef struct {
* once per edge (no log storms across
* repeated dispatch-gate rejections).
* 0 at init (cleared by ha_init memset). */
/* ── New field added by ha-commit-durability spec (Task 4.4) ──────── */
void * _Atomic promotion_fence_ctx; /* promotion_fence_ctx_t* for the
* bounded-window catch-up timer (Req
* 2.2/2.4): a recovery-truncated/behind
* member that wins an election with no
* reachable peer is NOT FATAL'd at the
* promotion instant; it is fenced
* (in_catch_up) and this one-shot timer
* is armed for DEFAULT_RECOVERY_CATCHUP_-
* WINDOW_MS. The 3.3 FATAL is reached
* ONLY from that timer iff still
* hollow + no peer + no catch-up
* progress when it fires; cleared/
* cancelled on recovery or shutdown so a
* caught-up or stopped member never
* FATALs. NULL at init / when no fence
* timer is pending. */
} ha_state_t;
/* ── Public API ────────────────────────────────────────────── */
@ -541,6 +558,37 @@ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
bool peer_reachable,
bool catch_up_succeeded);
/* ── ha-commit-durability task 4.4: bounded-window catch-up before FATAL ─────
*
* The promotion gate (4.3) no longer reaches the 3.3 FATAL synchronously at the
* promotion instant. When a recovery-truncated / behind member wins an election
* with no reachable peer AT THAT INSTANT (a transient no-peer snapshot during
* aggressive failover), it FENCES (sets in_catch_up, does NOT become primary)
* and arms a one-shot delaymgr timer for DEFAULT_RECOVERY_CATCHUP_WINDOW_MS.
* The 3.3 FATAL is reached ONLY from that timer, and ONLY if the member is STILL
* recovery-truncated/behind AND no healthy peer is reachable AND no catch-up
* progress was made when it fires (Req 2.4). Catch-up that brings durable_floor
* >= HWM releases the fence and cancels the timer (Req 2.6); shutdown cancels it
* too — never a synchronous FATAL, never an indefinite fence.
*
* ha_promotion_fence_window_ms: the bounded catch-up window in milliseconds.
* Defaults to DEFAULT_RECOVERY_CATCHUP_WINDOW_MS; tests override it (via
* ha_set_promotion_fence_window_ms) to a small value so the timer fires fast. */
uint32_t ha_promotion_fence_window_ms(void);
/* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer
* (idempotent; no-op when none is armed). Called when the member catches up
* (fence released) or on shutdown, so neither a caught-up nor a stopped member
* ever reaches the deferred FATAL. */
void ha_cancel_promotion_fence_timer(ha_state_t *ha);
#ifdef XCE_TESTING
/* Test seam: override the bounded catch-up window so the deferred-FATAL timer
* fires deterministically fast. 0 restores the DEFAULT_RECOVERY_CATCHUP_WINDOW_MS
* default. XCE_TESTING only — compiled out of production. */
void ha_set_promotion_fence_window_ms(uint32_t ms);
#endif
/* ── Task 3.2: Unified replication decision enum and evaluator ────────────── */
/*

239
src/controller/testsuite/test_commit_durability.c
View file

@ -12,11 +12,13 @@
#include "commit.h"
#include "config.h"
#include "delay.h" /* task 4.4: delaymgr for the bounded-window timer */
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
#include "persist.h"
#include "repl.h" /* task 5.1: repl_add_standby + standby ACK feed */
#include "typemap.h" /* task 5.1: wal_catchup_entry_t (send hook) */
#include "wal_types.h"
#include "xce/cluster_def.h" /* task 4.4: cluster_t (delaymgr + persist + ha) */
/* ------------------------------------------------------------------ *
* ha-commit-durability — task 1.2: durability-verification test seam *
@ -2001,6 +2003,237 @@ static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
}
/* ── task 4.4: bounded-window catch-up before FATAL ──────────────────────────
*
* The 4.3 gate decided FATAL synchronously at the promotion instant when a
* recovery-truncated/behind member had no reachable peer right then. Task 4.4
* defers that FATAL to a one-shot bounded-window timer: the member is fenced
* (in_catch_up) and given DEFAULT_RECOVERY_CATCHUP_WINDOW_MS to find a peer /
* make progress; the 3.3 FATAL is reached ONLY from the timer after a real
* bounded failure (still hollow + no peer + no progress).
*
* These tests drive ha_arm_promotion_fence_timer (the production deferral wired
* into raft_cb_become_leader) through a REAL delaymgr with a short window seam,
* so the timer fires fast and deterministically. A minimal cluster_t carries
* the delaymgr + persist + ha exactly as production does. */
/* The delaymgr runner used by production (run_delay_tasks in cluster.c) is not
* linked into the unit-test target; provide the same trivial pass-through. */
static int dur_run_delay_task(delay_task_entry_fn_t entry, void* arg)
{
return entry(arg);
}
/* Build a minimal cluster_t (calloc'd) with a started delaymgr and a recovered
* persist whose recovery verdict + durable floor are set directly, so the
* promotion fence can be evaluated without a full election. truncated → the
* recovery_truncated flag; durable_floor → the durable watermark the gate reads.
* ha.repl stays NULL so "no healthy peer reachable" holds in the timer. */
static cluster_t* dur_make_fence_cluster(const char* tmpdir, bool truncated,
uint64_t durable_floor)
{
cluster_t* c = calloc(1, sizeof(*c));
if (!c) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = 2;
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
if (persist_init(&c->persist, &cfg, NULL) != 0) { free(c); return NULL; }
if (persist_start(&c->persist) != 0) {
persist_fini(&c->persist); free(c); return NULL;
}
/* Set the recovered state the gate reads (mirrors a real persist_recover). */
c->persist.recovery_verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, durable_floor,
memory_order_release);
ha_config_t hcfg = dur_ha_cfg();
if (ha_init(&c->ha, &hcfg, 0) != 0) {
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
c->ha.raft_c = c;
c->ha.repl = NULL; /* no standby registry → no healthy peer reachable */
if (delaymgr_init(&c->delaymgr, 5 /* precision_ms */, dur_run_delay_task) != 0) {
ha_fini(&c->ha);
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
if (delaymgr_start(&c->delaymgr) != 0) {
delaymgr_fini(&c->delaymgr);
ha_fini(&c->ha);
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
return c;
}
static void dur_destroy_fence_cluster(cluster_t* c)
{
if (!c) return;
/* Cancel any pending fence timer BEFORE tearing down the delaymgr so a
* still-armed callback cannot fire against freed state. */
ha_cancel_promotion_fence_timer(&c->ha);
delaymgr_stop(&c->delaymgr);
delaymgr_fini(&c->delaymgr);
ha_fini(&c->ha);
persist_stop(&c->persist);
persist_fini(&c->persist);
free(c);
}
/* Spin until the fatal hook fires or a timeout elapses (the delaymgr fires the
* one-shot timer on its own worker thread). */
static bool dur_wait_fatal(int expect_calls, int timeout_ms)
{
int waited = 0;
while (g_fatal_hook_calls < expect_calls && waited < timeout_ms) {
usleep(2000);
waited += 2;
}
return g_fatal_hook_calls >= expect_calls;
}
/* 4.4-RED carrier + core deferral: a recovery-truncated/behind member with NO
* reachable peer must NOT FATAL synchronously at the promotion instant — it
* fences (in_catch_up set, process alive) and arms a bounded-window timer; the
* 3.3 FATAL fires ONLY from that timer after the window with no peer/progress.
*
* RED (pre-4.4): the gate routed a no-peer truncated member straight through
* ha_evaluate_promotion_fence → persist_recover_fatal_if_unrecoverable, so the
* fatal hook was called SYNCHRONOUSLY (count==1 with zero delay) — there was no
* bounded window, which is exactly the task-5.2 soak collapse. (Req 2.2, 2.4) */
static void test_promotion_fence_defers_fatal_to_bounded_timer(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_defer_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(40); /* short bounded window for the test */
/* Arm the fence exactly as raft_cb_become_leader does for a truncated,
* below-HWM, no-peer winner. This must NOT FATAL synchronously. */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: the promotion gate must NOT FATAL synchronously — FATAL is "
"deferred to the bounded-window timer (Req 2.2/2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: a fenced member is held in_catch_up (refused promotion), alive");
/* Now let the bounded window elapse: still truncated + no peer + no
* progress → the 3.3 FATAL fires from the timer (exactly once). */
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_fatal(1, 1000),
"4.4: the deferred FATAL must fire from the timer after the bounded "
"window with no peer and no progress (Req 2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"4.4: the deferred FATAL fires exactly once from the timer");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0); /* restore default */
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_defers_fatal_to_bounded_timer: PASS\n");
}
/* 4.4 Req 2.6: catch-up within the window → fence released, FATAL NOT called.
* Arm the fence, then simulate catch-up (clear truncated + advance durable_floor
* past the HWM) before the timer fires; when the timer fires it observes a
* caught-up member, releases in_catch_up, and does NOT FATAL. */
static void test_promotion_fence_released_on_catchup_no_fatal(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_catchup_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
/* Simulate catch-up completing: log now intact and durable past the HWM. */
c->persist.recovery_verdict = RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
/* Wait long enough for the window to fire; assert it did NOT FATAL and the
* fence was released (Req 2.6). */
usleep(300 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: a member that caught up within the window must NOT FATAL (Req 2.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: catch-up within the window RELEASES the in_catch_up fence (Req 2.6)");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_released_on_catchup_no_fatal: PASS\n");
}
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → NO FATAL.
* Proves the deferred FATAL never fires after a clean cancel — a stopped member
* must not self-terminate, and there is no use-after-free of the timer ctx. */
static void test_promotion_fence_cancel_prevents_fatal(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_cancel_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(60);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
/* Cancel before the window elapses. */
ha_cancel_promotion_fence_timer(&c->ha);
/* Wait past the window — the cancelled timer must be a no-op, NO FATAL. */
usleep(200 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: a cancelled fence timer must NEVER FATAL (shutdown/recovery)");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_cancel_prevents_fatal: PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
@ -2052,6 +2285,12 @@ void test_commit_durability(void)
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
/* task 4.4 — bounded-window catch-up before FATAL (no synchronous FATAL;
* deferred-timer FATAL only after a real bounded failure; fence released on
* catch-up; cancelled timer never FATALs). (Req 2.2, 2.4, 2.6) */
RUN_TEST(test_promotion_fence_defers_fatal_to_bounded_timer);
RUN_TEST(test_promotion_fence_released_on_catchup_no_fatal);
RUN_TEST(test_promotion_fence_cancel_prevents_fatal);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif