feat(ha-commit-durability): bounded-window catch-up before FATAL — fence no longer FATALs synchronously (task 4.4)
This commit is contained in:
parent
e14bd62115
commit
addf53e3b4
4 changed files with 562 additions and 15 deletions
|
|
@ -12,11 +12,13 @@
|
|||
|
||||
#include "commit.h"
|
||||
#include "config.h"
|
||||
#include "delay.h" /* task 4.4: delaymgr for the bounded-window timer */
|
||||
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
|
||||
#include "persist.h"
|
||||
#include "repl.h" /* task 5.1: repl_add_standby + standby ACK feed */
|
||||
#include "typemap.h" /* task 5.1: wal_catchup_entry_t (send hook) */
|
||||
#include "wal_types.h"
|
||||
#include "xce/cluster_def.h" /* task 4.4: cluster_t (delaymgr + persist + ha) */
|
||||
|
||||
/* ------------------------------------------------------------------ *
|
||||
* ha-commit-durability — task 1.2: durability-verification test seam *
|
||||
|
|
@ -2001,6 +2003,237 @@ static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
|
|||
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
|
||||
}
|
||||
|
||||
/* ── task 4.4: bounded-window catch-up before FATAL ──────────────────────────
|
||||
*
|
||||
* The 4.3 gate decided FATAL synchronously at the promotion instant when a
|
||||
* recovery-truncated/behind member had no reachable peer right then. Task 4.4
|
||||
* defers that FATAL to a one-shot bounded-window timer: the member is fenced
|
||||
* (in_catch_up) and given DEFAULT_RECOVERY_CATCHUP_WINDOW_MS to find a peer /
|
||||
* make progress; the 3.3 FATAL is reached ONLY from the timer after a real
|
||||
* bounded failure (still hollow + no peer + no progress).
|
||||
*
|
||||
* These tests drive ha_arm_promotion_fence_timer (the production deferral wired
|
||||
* into raft_cb_become_leader) through a REAL delaymgr with a short window seam,
|
||||
* so the timer fires fast and deterministically. A minimal cluster_t carries
|
||||
* the delaymgr + persist + ha exactly as production does. */
|
||||
|
||||
/* The delaymgr runner used by production (run_delay_tasks in cluster.c) is not
|
||||
* linked into the unit-test target; provide the same trivial pass-through. */
|
||||
static int dur_run_delay_task(delay_task_entry_fn_t entry, void* arg)
|
||||
{
|
||||
return entry(arg);
|
||||
}
|
||||
|
||||
/* Build a minimal cluster_t (calloc'd) with a started delaymgr and a recovered
|
||||
* persist whose recovery verdict + durable floor are set directly, so the
|
||||
* promotion fence can be evaluated without a full election. truncated → the
|
||||
* recovery_truncated flag; durable_floor → the durable watermark the gate reads.
|
||||
* ha.repl stays NULL so "no healthy peer reachable" holds in the timer. */
|
||||
static cluster_t* dur_make_fence_cluster(const char* tmpdir, bool truncated,
|
||||
uint64_t durable_floor)
|
||||
{
|
||||
cluster_t* c = calloc(1, sizeof(*c));
|
||||
if (!c) return NULL;
|
||||
|
||||
persist_config_t cfg = persist_config_defaults();
|
||||
cfg.wal_sync_ms = 5000;
|
||||
cfg.snap_keep_n = 2;
|
||||
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
|
||||
if (persist_init(&c->persist, &cfg, NULL) != 0) { free(c); return NULL; }
|
||||
if (persist_start(&c->persist) != 0) {
|
||||
persist_fini(&c->persist); free(c); return NULL;
|
||||
}
|
||||
/* Set the recovered state the gate reads (mirrors a real persist_recover). */
|
||||
c->persist.recovery_verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN;
|
||||
atomic_store_explicit(&c->persist.durable_lsn, durable_floor,
|
||||
memory_order_release);
|
||||
|
||||
ha_config_t hcfg = dur_ha_cfg();
|
||||
if (ha_init(&c->ha, &hcfg, 0) != 0) {
|
||||
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
|
||||
return NULL;
|
||||
}
|
||||
c->ha.raft_c = c;
|
||||
c->ha.repl = NULL; /* no standby registry → no healthy peer reachable */
|
||||
|
||||
if (delaymgr_init(&c->delaymgr, 5 /* precision_ms */, dur_run_delay_task) != 0) {
|
||||
ha_fini(&c->ha);
|
||||
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
|
||||
return NULL;
|
||||
}
|
||||
if (delaymgr_start(&c->delaymgr) != 0) {
|
||||
delaymgr_fini(&c->delaymgr);
|
||||
ha_fini(&c->ha);
|
||||
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
|
||||
return NULL;
|
||||
}
|
||||
return c;
|
||||
}
|
||||
|
||||
static void dur_destroy_fence_cluster(cluster_t* c)
|
||||
{
|
||||
if (!c) return;
|
||||
/* Cancel any pending fence timer BEFORE tearing down the delaymgr so a
|
||||
* still-armed callback cannot fire against freed state. */
|
||||
ha_cancel_promotion_fence_timer(&c->ha);
|
||||
delaymgr_stop(&c->delaymgr);
|
||||
delaymgr_fini(&c->delaymgr);
|
||||
ha_fini(&c->ha);
|
||||
persist_stop(&c->persist);
|
||||
persist_fini(&c->persist);
|
||||
free(c);
|
||||
}
|
||||
|
||||
/* Spin until the fatal hook fires or a timeout elapses (the delaymgr fires the
|
||||
* one-shot timer on its own worker thread). */
|
||||
static bool dur_wait_fatal(int expect_calls, int timeout_ms)
|
||||
{
|
||||
int waited = 0;
|
||||
while (g_fatal_hook_calls < expect_calls && waited < timeout_ms) {
|
||||
usleep(2000);
|
||||
waited += 2;
|
||||
}
|
||||
return g_fatal_hook_calls >= expect_calls;
|
||||
}
|
||||
|
||||
/* 4.4-RED carrier + core deferral: a recovery-truncated/behind member with NO
|
||||
* reachable peer must NOT FATAL synchronously at the promotion instant — it
|
||||
* fences (in_catch_up set, process alive) and arms a bounded-window timer; the
|
||||
* 3.3 FATAL fires ONLY from that timer after the window with no peer/progress.
|
||||
*
|
||||
* RED (pre-4.4): the gate routed a no-peer truncated member straight through
|
||||
* ha_evaluate_promotion_fence → persist_recover_fatal_if_unrecoverable, so the
|
||||
* fatal hook was called SYNCHRONOUSLY (count==1 with zero delay) — there was no
|
||||
* bounded window, which is exactly the task-5.2 soak collapse. (Req 2.2, 2.4) */
|
||||
static void test_promotion_fence_defers_fatal_to_bounded_timer(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_fence_defer_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
|
||||
/*durable_floor=*/4);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
|
||||
g_fatal_hook_calls = 0;
|
||||
persist_fatal_hook = dur_fatal_hook;
|
||||
ha_set_promotion_fence_window_ms(40); /* short bounded window for the test */
|
||||
|
||||
/* Arm the fence exactly as raft_cb_become_leader does for a truncated,
|
||||
* below-HWM, no-peer winner. This must NOT FATAL synchronously. */
|
||||
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
|
||||
uint64_t);
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
|
||||
/*durable_floor=*/4, /*hwm=*/10);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_fatal_hook_calls,
|
||||
"4.4: the promotion gate must NOT FATAL synchronously — FATAL is "
|
||||
"deferred to the bounded-window timer (Req 2.2/2.4)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
|
||||
"4.4: a fenced member is held in_catch_up (refused promotion), alive");
|
||||
|
||||
/* Now let the bounded window elapse: still truncated + no peer + no
|
||||
* progress → the 3.3 FATAL fires from the timer (exactly once). */
|
||||
TEST_ASSERT_TRUE_MESSAGE(
|
||||
dur_wait_fatal(1, 1000),
|
||||
"4.4: the deferred FATAL must fire from the timer after the bounded "
|
||||
"window with no peer and no progress (Req 2.4)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
1, g_fatal_hook_calls,
|
||||
"4.4: the deferred FATAL fires exactly once from the timer");
|
||||
|
||||
persist_fatal_hook = NULL;
|
||||
ha_set_promotion_fence_window_ms(0); /* restore default */
|
||||
dur_destroy_fence_cluster(c);
|
||||
printf("test_promotion_fence_defers_fatal_to_bounded_timer: PASS\n");
|
||||
}
|
||||
|
||||
/* 4.4 Req 2.6: catch-up within the window → fence released, FATAL NOT called.
|
||||
* Arm the fence, then simulate catch-up (clear truncated + advance durable_floor
|
||||
* past the HWM) before the timer fires; when the timer fires it observes a
|
||||
* caught-up member, releases in_catch_up, and does NOT FATAL. */
|
||||
static void test_promotion_fence_released_on_catchup_no_fatal(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_fence_catchup_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
|
||||
/*durable_floor=*/4);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
|
||||
g_fatal_hook_calls = 0;
|
||||
persist_fatal_hook = dur_fatal_hook;
|
||||
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
|
||||
|
||||
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
|
||||
uint64_t);
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
|
||||
/*durable_floor=*/4, /*hwm=*/10);
|
||||
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
|
||||
TEST_ASSERT_EQUAL_INT(
|
||||
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
|
||||
|
||||
/* Simulate catch-up completing: log now intact and durable past the HWM. */
|
||||
c->persist.recovery_verdict = RECOVER_CLEAN;
|
||||
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
|
||||
|
||||
/* Wait long enough for the window to fire; assert it did NOT FATAL and the
|
||||
* fence was released (Req 2.6). */
|
||||
usleep(300 * 1000);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_fatal_hook_calls,
|
||||
"4.4: a member that caught up within the window must NOT FATAL (Req 2.6)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
|
||||
"4.4: catch-up within the window RELEASES the in_catch_up fence (Req 2.6)");
|
||||
|
||||
persist_fatal_hook = NULL;
|
||||
ha_set_promotion_fence_window_ms(0);
|
||||
dur_destroy_fence_cluster(c);
|
||||
printf("test_promotion_fence_released_on_catchup_no_fatal: PASS\n");
|
||||
}
|
||||
|
||||
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → NO FATAL.
|
||||
* Proves the deferred FATAL never fires after a clean cancel — a stopped member
|
||||
* must not self-terminate, and there is no use-after-free of the timer ctx. */
|
||||
static void test_promotion_fence_cancel_prevents_fatal(void)
|
||||
{
|
||||
char tmp[256];
|
||||
strcpy(tmp, "/tmp/xce_fence_cancel_XXXXXX");
|
||||
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
|
||||
|
||||
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
|
||||
/*durable_floor=*/4);
|
||||
TEST_ASSERT_NOT_NULL(c);
|
||||
|
||||
g_fatal_hook_calls = 0;
|
||||
persist_fatal_hook = dur_fatal_hook;
|
||||
ha_set_promotion_fence_window_ms(60);
|
||||
|
||||
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
|
||||
uint64_t);
|
||||
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
|
||||
/*durable_floor=*/4, /*hwm=*/10);
|
||||
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
|
||||
|
||||
/* Cancel before the window elapses. */
|
||||
ha_cancel_promotion_fence_timer(&c->ha);
|
||||
|
||||
/* Wait past the window — the cancelled timer must be a no-op, NO FATAL. */
|
||||
usleep(200 * 1000);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(
|
||||
0, g_fatal_hook_calls,
|
||||
"4.4: a cancelled fence timer must NEVER FATAL (shutdown/recovery)");
|
||||
|
||||
persist_fatal_hook = NULL;
|
||||
ha_set_promotion_fence_window_ms(0);
|
||||
dur_destroy_fence_cluster(c);
|
||||
printf("test_promotion_fence_cancel_prevents_fatal: PASS\n");
|
||||
}
|
||||
|
||||
#endif /* XCE_TESTING */
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
|
@ -2052,6 +2285,12 @@ void test_commit_durability(void)
|
|||
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
|
||||
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
|
||||
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
|
||||
/* task 4.4 — bounded-window catch-up before FATAL (no synchronous FATAL;
|
||||
* deferred-timer FATAL only after a real bounded failure; fence released on
|
||||
* catch-up; cancelled timer never FATALs). (Req 2.2, 2.4, 2.6) */
|
||||
RUN_TEST(test_promotion_fence_defers_fatal_to_bounded_timer);
|
||||
RUN_TEST(test_promotion_fence_released_on_catchup_no_fatal);
|
||||
RUN_TEST(test_promotion_fence_cancel_prevents_fatal);
|
||||
#else
|
||||
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
|
||||
#endif
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue