wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/controller/testsuite/test_commit_durability.c

2297 lines
106 KiB
C
Raw Blame History

#define _GNU_SOURCE
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
#include "commit.h"
#include "config.h"
#include "delay.h" /* task 4.4: delaymgr for the bounded-window timer */
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
#include "persist.h"
#include "repl.h" /* task 5.1: repl_add_standby + standby ACK feed */
#include "typemap.h" /* task 5.1: wal_catchup_entry_t (send hook) */
#include "wal_types.h"
#include "xce/cluster_def.h" /* task 4.4: cluster_t (delaymgr + persist + ha) */
/* ------------------------------------------------------------------ *
* ha-commit-durability — task 1.2: durability-verification test seam *
* *
* These tests exercise persist_test_durable_lsn_on_disk(), the *
* XCE_TESTING-only seam that re-reads + CRC-validates the WAL from *
* stable storage and reports the highest fully-persisted LSN — *
* independent of the in-memory current_lsn watermark. This lets a *
* test assert an acknowledged operation is recoverable from disk after *
* an immediate ungraceful termination, without relying on the *
* background flush timer (Req 3.2). The seam is purely inspective: *
* it changes nothing observable about the acknowledgment contract *
* (Req 3.3). *
* *
* The seam is compiled ONLY into the controller_unit_test target *
* (-DXCE_TESTING). The production xce_controller target links without *
* it — see #ifdef XCE_TESTING in persist.c / declaration in persist.h. *
* ------------------------------------------------------------------ */
#ifdef XCE_TESTING
/* Build a persist instance with a deliberately slow background flush
* cadence, so the only durability point is an explicit persist_flush()
* (mirrors make_fresh_persist in test_persist_recovery.c). */
static persist_t* make_durability_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
if (!p)
return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* slow background fsync — durability comes from
* the explicit flush, not the timer */
cfg.snap_keep_n = 2;
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void teardown_durability_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* ------------------------------------------------------------------ *
* Test 1: an empty WAL has no durable entries (floor == 0). *
* An empty first-boot WAL is not corruption — it simply has nothing *
* persisted, so the seam reports 0. *
* ------------------------------------------------------------------ */
static void test_durable_floor_empty(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durfloor0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
uint64_t floor = persist_test_durable_lsn_on_disk(p);
TEST_ASSERT_EQUAL_UINT64(0, floor);
teardown_durability_persist(p);
printf("test_durable_floor_empty: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 2 (core Req 3.2): after append + explicit fdatasync, the seam *
* sees the entries on disk — the durable floor equals the highest *
* appended LSN, recoverable independent of in-memory state. *
* ------------------------------------------------------------------ */
static void test_durable_floor_after_flush(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durfloor1_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Append three entries. LSN 0 is reserved, so these take LSN 1,2,3.
* current_lsn advances to 4 (one past the last appended). */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
/* Durability point: fdatasync the segment so the bytes are on stable
* storage (this is what an acknowledged-before-durable contract must
* have done before replying ok). */
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* The seam re-reads from disk and reports the highest intact LSN. */
uint64_t floor = persist_test_durable_lsn_on_disk(p);
TEST_ASSERT_EQUAL_UINT64(p->current_lsn - 1, floor);
TEST_ASSERT_EQUAL_UINT64(3, floor);
teardown_durability_persist(p);
printf("test_durable_floor_after_flush: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3 (recoverable-after-ungraceful-kill, Req 3.2): a SECOND *
* persist instance opened over the same WAL dir — with NO in-memory *
* state from the writer — recovers exactly the durable floor the seam *
* reported. This is the "survives an immediate ungraceful *
* termination" assertion: durability lives on disk, not in the *
* volatile buffers of the process that wrote it. *
* ------------------------------------------------------------------ */
static void test_durable_floor_survives_process_loss(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durfloor2_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
uint64_t durable_floor = 0;
/* Writer process: append, fdatasync, then "die" (we drop the instance
* without any further flush — the bytes are already durable). */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
durable_floor = persist_test_durable_lsn_on_disk(p);
TEST_ASSERT_EQUAL_UINT64(2, durable_floor);
teardown_durability_persist(p);
}
/* "Restarted" process: a fresh persist_t with zero in-memory state,
* opened over the same on-disk WAL. Its durable floor — read straight
* from disk before any recover/replay — matches what the writer
* persisted. Acknowledged-before-durable ops therefore survive the
* loss of the acking process. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
/* current_lsn is still the fresh-boot value (1, LSN 0 reserved):
* the durable floor does NOT come from in-memory state. */
uint64_t floor2 = persist_test_durable_lsn_on_disk(p2);
TEST_ASSERT_EQUAL_UINT64(durable_floor, floor2);
TEST_ASSERT_EQUAL_UINT64(2, floor2);
teardown_durability_persist(p2);
}
printf("test_durable_floor_survives_process_loss: PASS\n");
}
/* ================================================================== *
* task 2.1 — durable-LSN tracking via a group-commit flusher *
* *
* These tests pin the invariants of persist_t.durable_lsn, the *
* prefix watermark advanced by the demand-driven flusher (the *
* repurposed fsync_worker): *
* - durable_lsn never exceeds the synced tip (<= current_lsn - 1); *
* - durable_lsn is monotonic non-decreasing; *
* - one fdatasync advances durable_lsn past MULTIPLE appended LSNs *
* (batching / group-commit); *
* - durable_lsn advances ONLY after a successful fdatasync; *
* - under concurrent append load the invariants always hold. *
* ------------------------------------------------------------------ */
/* Spin until the flusher catches durable_lsn up to the appended tip,
* or a generous deadline elapses. Returns the observed durable_lsn. */
static uint64_t wait_durable_reaches(persist_t* p, uint64_t target_lsn,
int max_ms)
{
for (int i = 0; i < max_ms; i++) {
uint64_t d = persist_durable_lsn(p);
if (d >= target_lsn)
return d;
usleep(1000); /* 1 ms */
}
return persist_durable_lsn(p);
}
/* ------------------------------------------------------------------ *
* Test 4: a fresh store has durable_lsn == 0 (nothing synced yet). *
* ------------------------------------------------------------------ */
static void test_durable_lsn_initial_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_durable_lsn_initial_zero: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5 (core / batching, Req 1.1): several appends, then the *
* demand-driven flusher advances durable_lsn past ALL of them with a *
* single fdatasync — and never past the appended tip. Uses the fast *
* cadence so the append-signal (not the timer) drives the flush, but *
* the slow-cadence helper would converge too via the bounded fallback.*
* ------------------------------------------------------------------ */
static void test_durable_lsn_batches_multiple_appends(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn1_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* LSN 0 reserved → these take 1,2,3,4,5; current_lsn ends at 6. */
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
uint64_t tip = p->current_lsn - 1; /* 5 */
TEST_ASSERT_EQUAL_UINT64(5, tip);
uint64_t d = wait_durable_reaches(p, tip, 5000);
/* The flusher advanced durable_lsn past several appended LSNs with
* one fdatasync (batching), reaching the synced tip. */
TEST_ASSERT_EQUAL_UINT64(tip, d);
/* Invariant: never exceeds the synced tip. */
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
/* The on-disk seam agrees: those LSNs are genuinely fdatasync'd. */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_batches_multiple_appends: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 6 (advances-only-after-fdatasync + monotonic): the durable *
* floor reported by the on-disk seam must always be >= durable_lsn — *
* i.e. durable_lsn never claims an LSN whose bytes are not actually *
* on stable storage. Also asserts monotonic non-decreasing across *
* successive append+flush rounds. *
* ------------------------------------------------------------------ */
static void test_durable_lsn_never_exceeds_synced_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn2_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
uint64_t prev = persist_durable_lsn(p);
for (int round = 0; round < 4; round++) {
for (int i = 0; i < 3; i++)
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
uint64_t tip = p->current_lsn - 1;
uint64_t d = wait_durable_reaches(p, tip, 5000);
/* Monotonic non-decreasing. */
TEST_ASSERT_TRUE(d >= prev);
/* Never exceeds the synced tip. */
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
/* Advances only after a real fdatasync: the independent on-disk
* scan must confirm every claimed-durable LSN is on disk. */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
prev = d;
}
teardown_durability_persist(p);
printf("test_durable_lsn_never_exceeds_synced_tip: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 7 (UNDER LOAD, Req 1.6 / lost-wakeup): many appends from *
* multiple threads racing the flusher. Asserts the invariants hold *
* throughout (durable_lsn <= synced tip, never past disk) AND that *
* the flusher eventually catches durable_lsn up to the final tip — *
* i.e. no append is left permanently un-synced (no lost wakeup). *
* ------------------------------------------------------------------ */
#define LOAD_THREADS 4
#define LOAD_PER_THREAD 200
typedef struct {
persist_t* p;
int n;
} load_arg_t;
static void* load_appender(void* arg)
{
load_arg_t* a = arg;
for (int i = 0; i < a->n; i++) {
/* Ignore rc: a full-segment rotate or transient EIO is not the
* subject under test; the invariant checks below tolerate it. */
(void)persist_wal_async(a->p, WAL_JOB_UPDATE, "z", 1);
/* Sample the invariant mid-flight: durable_lsn must NEVER exceed
* the appended tip, even while a concurrent flush is in progress. */
uint64_t d = persist_durable_lsn(a->p);
uint64_t cur = atomic_load_explicit(
(const _Atomic uint64_t*)&a->p->current_lsn, memory_order_acquire);
/* current_lsn is read without the lock here purely as a coarse
* upper bound; a torn read can only make `cur` smaller, so the
* assertion stays conservative. durable_lsn is a prefix watermark
* that can never outrun the last assigned LSN. */
TEST_ASSERT_TRUE(d == 0 || d <= cur);
}
return NULL;
}
static void test_durable_lsn_invariants_under_load(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn3_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
pthread_t th[LOAD_THREADS];
load_arg_t args[LOAD_THREADS];
for (int i = 0; i < LOAD_THREADS; i++) {
args[i].p = p;
args[i].n = LOAD_PER_THREAD;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th[i], NULL,
load_appender, &args[i]));
}
for (int i = 0; i < LOAD_THREADS; i++)
pthread_join(th[i], NULL);
/* Final tip after all appends settled. */
uint64_t tip = p->current_lsn - 1;
/* No lost wakeup: the flusher must eventually advance durable_lsn up
* to the final appended tip (every entry gets synced). */
uint64_t d = wait_durable_reaches(p, tip, 10000);
TEST_ASSERT_EQUAL_UINT64(tip, d);
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_invariants_under_load: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 8 (ERROR PATH, Req 1.1/1.2 — fail-loud): when fdatasync FAILS, *
* durable_lsn must NOT advance. Bytes that did not reach stable *
* storage must never be counted durable — otherwise the member would *
* contribute a non-durable LSN toward the commit majority. *
* *
* Mechanism: the XCE_TESTING fault-injection toggle forces every *
* fdatasync site to treat the sync as failed. We append entries while *
* the toggle is ON and assert durable_lsn stays put across the full *
* cadence window (the flusher runs but its advance is gated on rc==0). *
* Then we clear the toggle and confirm the flusher recovers — durable_ *
* lsn advances to the tip once syncs succeed again. *
* ------------------------------------------------------------------ */
static void test_durable_lsn_holds_on_fsync_failure(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn4_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* First, append one entry and let it become durable normally, so the
* watermark has a known non-zero baseline that must HOLD during the
* failure window. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t baseline = wait_durable_reaches(p, 1, 5000);
TEST_ASSERT_EQUAL_UINT64(1, baseline);
/* Now inject fdatasync failure and append more entries. */
persist_test_set_force_fsync_fail(p, 1);
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
uint64_t tip = p->current_lsn - 1; /* 6 */
TEST_ASSERT_EQUAL_UINT64(6, tip);
/* Give the flusher ample time to run several passes (cadence is 5s but
* the append signal fires it immediately) AND an explicit flush attempt.
* Under injected failure, durable_lsn MUST remain at the baseline — the
* new bytes are not on stable storage, so they are not durable. */
(void)persist_flush(p); /* also gated on rc==0 — must not advance either */
for (int i = 0; i < 200; i++) {
TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p));
usleep(1000); /* 1 ms; 200 ms total — many flusher signal cycles */
}
/* Still parked at the baseline after the window. */
TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) <= p->current_lsn - 1);
/* Recovery: clear the fault and confirm the flusher catches up — proving
* the failure path did not corrupt the watermark, only held it. */
persist_test_set_force_fsync_fail(p, 0);
/* Nudge the flusher with one more append so the level predicate signals. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
uint64_t new_tip = p->current_lsn - 1; /* 7 */
uint64_t d = wait_durable_reaches(p, new_tip, 5000);
TEST_ASSERT_EQUAL_UINT64(new_tip, d);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_holds_on_fsync_failure: PASS\n");
}
/* ================================================================== *
* task 2.2 — durable-advance wake drives commit (integration) *
* *
* End-to-end over a REAL persist_t + commit_engine_t (single-member, *
* NULL cluster): the group-commit flusher advances durable_lsn and *
* fires the registered durable-advance callback, which the commit *
* engine uses to advance committed_lsn and wake WAIT callers. Proves *
* the persist↔commit wiring works against the real flusher thread — *
* not just the directly-driven callback (covered in *
* test_commit_engine.c). n_peers=1 means self alone is the majority, *
* so committed_lsn tracks durable_lsn with no standby ACKs needed. *
* ------------------------------------------------------------------ */
static void test_durable_advance_drives_commit_single_member(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durcommit_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Single-member commit engine over the real persist. n_vcpus large enough
* to admit a wait. commit_engine_init registers the durable-advance cb. */
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, 1, 8));
/* committed starts at 0; nothing appended/durable. */
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Append three entries (LSN 1,2,3). They are in page cache only — the slow
* 5s cadence means the append-SIGNAL (not the timer) must drive the flush;
* the real flusher fdatasyncs, advances durable_lsn, and fires the cb,
* which advances committed_lsn for this single-member quorum. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
uint64_t tip = p->current_lsn - 1; /* 3 */
TEST_ASSERT_EQUAL_UINT64(3, tip);
/* Wait for the flusher to make the tip durable AND for the wake-driven
* commit to catch up. committed_lsn must reach the tip — driven by the
* durable-advance callback, not by any ACK (there are no standbys). */
uint64_t committed = 0;
for (int i = 0; i < 5000; i++) {
committed = atomic_load(&p->committed_lsn);
if (committed >= tip)
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
tip, committed,
"durable-advance callback must drive committed_lsn to the durable tip");
/* committed never exceeds durable (the invariant the wake preserves). */
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
/* Tear the engine down BEFORE the persist (unhooks the callback so the
* flusher cannot fire against a destroyed engine). */
commit_engine_fini(&ce);
teardown_durability_persist(p);
printf("test_durable_advance_drives_commit_single_member: PASS\n");
}
/* ================================================================== *
* task 2.3 — standby persist-before-ACK regression (Req 1.3) *
* *
* Req 1.3: a standby durably persists a replicated entry to local *
* stable storage BEFORE sending its acknowledgment. The ACK is sent by *
* the CALLER (ha_apply_single_entry) only after *
* persist_wal_write_replicated returns success — so to pin "persist *
* before ACK" we prove a durable fdatasync ran INSIDE the persist call *
* before it returned. *
* *
* Mechanism: a disk re-read (persist_test_durable_lsn_on_disk) cannot *
* distinguish an fsync'd page from a page-cache-only page, so it CANNOT *
* prove the sync ran. Instead we read the XCE_TESTING fdatasync counter *
* incremented at the inline sync site: snapshot it before the call, *
* assert it incremented after — the sync provably ran within the call, *
* before the caller could ACK. The test FAILS if someone removed or *
* moved the fdatasync out of persist_wal_write_replicated. *
* ------------------------------------------------------------------ */
static void test_replicated_persist_syncs_before_return(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replsync_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Snapshot the sync counter, then write a replicated entry. The standby
* path must fdatasync inline BEFORE returning — so the counter must
* advance during the single call. */
unsigned before = persist_test_replicated_fdatasync_count(p);
int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_EQUAL_INT(0, rc);
unsigned after = persist_test_replicated_fdatasync_count(p);
TEST_ASSERT_TRUE_MESSAGE(
after > before,
"persist_wal_write_replicated must fdatasync inline (persist-before-"
"ACK): the sync counter did not advance during the call");
/* And the entry is genuinely on stable storage by the time the call
* returned (would survive an immediate process loss → recoverable ACK). */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 1);
teardown_durability_persist(p);
printf("test_replicated_persist_syncs_before_return: PASS\n");
}
/* ------------------------------------------------------------------ *
* task 2.3 — forced standby-sync failure ⇒ error, NO success/ACK *
* *
* Closes the Req-1.3 hole: if the inline fdatasync FAILS, the standby *
* must NOT report success — otherwise the caller would ACK an entry *
* that is not durable (durable AFTER, not BEFORE, the ACK). With the *
* force-fail seam ON, persist_wal_write_replicated must return a *
* negative error; the caller (ha_apply_single_entry) already treats any *
* negative return as "do not apply, do not ACK". *
* *
* RED evidence: against the pre-2.3 code (which ignored the fdatasync *
* return at the standby path) this returned 0 (success) under forced *
* failure — a non-durable ACK. The rc-gate makes it return -EIO. *
* ------------------------------------------------------------------ */
static void test_replicated_sync_failure_no_ack(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replsyncfail_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Force the fdatasync to be treated as failed, then attempt a replicated
* write. The durable sync cannot succeed, so the call MUST fail (no ACK).
* A negative return is what ha_apply_single_entry routes to its
* "not applying/acking" path. */
persist_test_set_force_fsync_fail(p, 1);
int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_TRUE_MESSAGE(
rc < 0,
"forced standby fdatasync failure must make persist_wal_write_"
"replicated FAIL (negative rc) so the standby does NOT ACK a "
"non-durable entry");
/* The sync was still attempted (counter advanced) — we fail loud on the
* RESULT, not by skipping the sync. */
TEST_ASSERT_TRUE(persist_test_replicated_fdatasync_count(p) >= 1);
/* Recovery: clear the fault; a fresh replicated write (new lsn, not the
* deduped lsn=1) now succeeds and is durable — proving the failure path
* did not corrupt persist state and the sync runs again on success.
* (lsn=1 already advanced current_lsn to 2, so re-using 1 would dedup;
* lsn=2 is the genuine next entry.) */
persist_test_set_force_fsync_fail(p, 0);
int rc2 = persist_wal_write_replicated(p, /*lsn=*/2, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_EQUAL_INT(0, rc2);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 2);
teardown_durability_persist(p);
printf("test_replicated_sync_failure_no_ack: PASS\n");
}
/* ================================================================== *
* task 2.4 — reconcile the dormant synchronous-append commit path *
* *
* persist_wal_sync_append's LEGACY branch (ce == NULL, or the LSN-0 *
* first entry on the engine path) historically advanced committed_lsn *
* DIRECTLY on a majority ACK (atomic_store committed_lsn = written_lsn) *
* with NO durable precondition. Group-commit removed the premise that *
* the append already fsync'd inline, so a majority ACK no longer *
* implies the local entry is durable. This branch is the structural *
* ACK-without-durability hazard task 2.4 must close: NO path may *
* advance committed_lsn on a majority ACK unless the entry is durable *
* locally (committed_lsn <= durable_lsn). *
* *
* NOTE: persist_wal_sync_append is NOT dormant — it is the LIVE *
* job-submit path (persist_job_hook_fun, WAL_JOB_SUBMIT). So the fix is *
* to ROUTE the legacy direct-store through the durable precondition, *
* not to delete the function. *
* ------------------------------------------------------------------ */
/* Build an ENGINE-LESS persist with standbys registered, so
* persist_wal_sync_append takes the legacy branch (ce == NULL) and its
* has_standbys path: repl_wait_majority_ack returns 0 immediately for
* the LSN-0 first entry, but for a NON-zero LSN it surfaces -EINVAL via
* commit_submit (engine-less). The hazard we guard is the direct
* committed_lsn store that the legacy branch performs on a 0 return. */
/* Drive the legacy branch directly: a single-member commit engine makes
* the ENGINE path durable-safe, so to exercise the legacy direct-store
* we use an engine-less persist and a registered standby that ACKs.
* Rather than stand up the replication wire (out of boundary), we test
* the invariant the fix establishes: committed_lsn must never exceed
* durable_lsn after a commit advance, on EVERY path. */
/* ------------------------------------------------------------------ *
* Test (RED before fix): the legacy direct-store path must NOT advance *
* committed_lsn past durable_lsn. We drive the legacy branch via the *
* LSN-0-style direct store by appending one entry that is NOT yet *
* durable (slow cadence, no flush) and confirming committed_lsn stays *
* at/below durable_lsn. *
* *
* Mechanism: persist_test_sync_append_legacy_store() is an XCE_TESTING *
* seam that invokes the SAME guarded store the legacy branch performs *
* (advance committed_lsn to written_lsn only if written_lsn <= *
* durable_lsn). Pre-fix the store was unconditional, so this over- *
* advanced committed past durable (RED). Post-fix it holds. *
* ------------------------------------------------------------------ */
static void test_legacy_sync_append_holds_committed_at_durable(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_legacycommit_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Append one entry (LSN 1) but do NOT flush — durable_lsn lags at 0,
* while the appended tip is 1. The legacy direct-store would advance
* committed_lsn to the ACK'd written_lsn (1) — over-advancing past
* the durable watermark (0). The durable precondition must hold it. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t written_lsn = p->current_lsn - 1; /* 1 */
TEST_ASSERT_EQUAL_UINT64(1, written_lsn);
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Invoke the guarded legacy store directly (simulates a majority ACK
* for an entry that is not yet durable locally). It must NOT advance
* committed_lsn past durable_lsn. */
persist_test_sync_append_legacy_commit_store(p, written_lsn);
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&p->committed_lsn) <= persist_durable_lsn(p),
"legacy sync-append commit store must NOT advance committed_lsn past "
"durable_lsn (ACK-without-durability hazard)");
/* Concretely: committed stays at 0 because durable_lsn is still 0. */
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Now make the entry durable and re-invoke: the store is now allowed
* (written_lsn <= durable_lsn) and committed advances to it. */
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) >= written_lsn);
persist_test_sync_append_legacy_commit_store(p, written_lsn);
TEST_ASSERT_EQUAL_UINT64(written_lsn, atomic_load(&p->committed_lsn));
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_legacy_sync_append_holds_committed_at_durable: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test: the LSN-0 first-entry edge stays valid. durable_lsn and *
* committed_lsn both start at 0; the legacy store of written_lsn == 0 *
* (0 <= 0) is permitted and is a harmless no-op store of 0. This is the *
* once-per-cluster-lifetime path the engine branch routes here. *
* ------------------------------------------------------------------ */
static void test_legacy_sync_append_lsn0_edge_valid(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_legacylsn0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Fresh store: durable_lsn == 0, committed_lsn == 0. */
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Storing written_lsn == 0 (0 <= durable 0) is allowed — a no-op. */
persist_test_sync_append_legacy_commit_store(p, 0);
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_legacy_sync_append_lsn0_edge_valid: PASS\n");
}
/* ------------------------------------------------------------------ *
* Guard test (the Observable's "test guards against a future caller"): *
* the committed_lsn <= durable_lsn invariant holds across the commit- *
* engine advance path too. Drives the real engine + flusher and asserts *
* that at every observed point committed never exceeds durable — the *
* future-proof guard against reintroducing an ACK-without-durability *
* commit on ANY path. In a debug build the in-code assert would abort *
* if a path ever advanced committed past durable. *
* ------------------------------------------------------------------ */
static void test_commit_never_exceeds_durable_invariant(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cinvariant_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, 1, 8));
/* Append a burst while sampling the invariant: committed must never
* outrun durable, even mid-flight while the flusher advances both. */
for (int round = 0; round < 6; round++) {
for (int i = 0; i < 4; i++)
TEST_ASSERT_EQUAL_INT(
0, persist_wal_async(p, WAL_JOB_UPDATE, "q", 1));
for (int s = 0; s < 50; s++) {
uint64_t committed = atomic_load(&p->committed_lsn);
uint64_t durable = persist_durable_lsn(p);
TEST_ASSERT_TRUE_MESSAGE(
committed <= durable,
"committed_lsn must never exceed durable_lsn on any commit "
"advance path");
usleep(1000);
}
}
/* And eventually committed catches up to the durable tip. */
uint64_t tip = p->current_lsn - 1;
uint64_t committed = 0;
for (int i = 0; i < 5000; i++) {
committed = atomic_load(&p->committed_lsn);
if (committed >= tip)
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64(tip, committed);
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
commit_engine_fini(&ce);
teardown_durability_persist(p);
printf("test_commit_never_exceeds_durable_invariant: PASS\n");
}
/* ================================================================== *
* task 5.1 — wire durable-majority commit end to end *
* *
* These three tests verify the full durable-majority commit path the *
* 2.x/4.x wiring assembled, end to end over a REAL persist_t (real WAL *
* + real group-commit flusher thread) and a REAL commit_engine_t with *
* registered standbys: *
* *
* 1. reply-ok ⇒ durable-majority (Req 1.1, 1.4): committed_lsn *
* advances and a WAIT caller releases ONLY after the entry is *
* durable on a fault-tolerant majority — self durable (durable_lsn *
* covers it, via the real flusher) AND a standby ACK. With *
* durability held off, even a standby ACK does NOT release. *
* 2. sole-durable-contributor loss survives (Req 1.5, 2.5): an op *
* committed on self + a standby is durable on >= floor(N/2)+1 *
* members, so losing one durable contributor leaves it present on *
* a surviving member (modeled at the integration level; the live *
* failover proof is task 5.2). *
* 3. liveness — durability work off the iomgr (Req 1.6): fdatasync *
* runs on the dedicated flusher thread, not the caller/iomgr *
* thread, so a heartbeat-style callback is not stalled while a *
* durable flush is in flight. *
* ------------------------------------------------------------------ */
/* Dummy standby endpoints — the replicator only compares pointers, and
* commit_on_ack reads acked_lsn from the registry slot. */
static int g_dur_eps[MAX_STANDBYS];
/* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps
* repl_broadcast from dereferencing the dummy endpoints as real
* endpoint_t when persist_wal_async fans out to the registered standbys.
* Same pattern as test_commit_engine.c. */
extern int (*repl_broadcast_send_hook)(void* ep, wal_catchup_entry_t** entries,
int n_entries);
static int dur_drop_send_hook(void* ep, wal_catchup_entry_t** entries,
int n_entries)
{
(void)ep;
(void)entries;
(void)n_entries;
return 0; /* pretend success; drop the data (no real wire in a unit test) */
}
/* Feed a standby ACK exactly as the iomgr read path does: store the
* watermark into the registry slot, then notify the commit engine. */
static void durable_standby_ack(persist_t* p, commit_engine_t* ce, int idx,
uint64_t acked_lsn)
{
atomic_store(&p->repl.standbys[idx].acked_lsn, acked_lsn);
commit_on_ack(ce, idx, acked_lsn);
}
/* A WAIT submit that runs on its own thread, so the test can observe that
* it does NOT return while the entry is not yet durable-majority, then DOES
* return once it becomes durable-majority (mirrors the off-iomgr AFTER_EVENT
* caller blocking on the commit barrier). */
typedef struct {
commit_engine_t* ce;
uint64_t lsn;
uint32_t timeout_ms;
_Atomic int returned;
int rc;
} wait_submit_arg_t;
static void* wait_submit_thread(void* a)
{
wait_submit_arg_t* w = (wait_submit_arg_t*)a;
w->rc = commit_submit(w->ce, w->lsn, COMMIT_MODE_WAIT, w->timeout_ms);
atomic_store(&w->returned, 1);
return NULL;
}
/* ------------------------------------------------------------------ *
* Test 5.1a (MUST-HAVE, Req 1.1 / 1.4): reply-ok only after a durable *
* majority. 3-member cluster (self + 2 standbys), so the majority for *
* an entry is self + at least 1 standby. We: *
* - force the local fdatasync to FAIL, so the flusher cannot advance *
* durable_lsn (self is NOT durable); *
* - append an entry and deliver a standby ACK covering it; *
* - assert committed_lsn does NOT advance and a WAIT submit does NOT *
* release — a majority of ACKs is not enough without local *
* durability (this is the durable precondition gate; pre-2.2 this *
* would have committed on the ACK alone — RED); *
* - clear the fault so the real flusher fdatasyncs and advances *
* durable_lsn, firing the durable-advance wake; *
* - assert committed_lsn now advances and the WAIT submit releases *
* promptly (durable self + durable standby ACK = durable majority). *
* ------------------------------------------------------------------ */
static void test_durable_majority_commit_end_to_end(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durmaj_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Intercept the replication fan-out so persist_wal_async does not try to
* send over the dummy endpoints (they are not real endpoint_t). */
repl_broadcast_send_hook = dur_drop_send_hook;
/* Register two standbys against the REAL persist's replicator. n_peers=3
* (self + 2 standbys); majority for entry N = self durable + >= 1 standby
* with acked_lsn >= N. n_vcpus high enough to admit a WAIT. */
TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&p->repl, &g_dur_eps[0], "sb0",
7000));
TEST_ASSERT_EQUAL_INT(1, repl_add_standby(&p->repl, &g_dur_eps[1], "sb1",
7001));
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, /*n_peers=*/3,
/*n_vcpus=*/8));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Hold durability off: every flusher fdatasync is forced to fail, so
* durable_lsn cannot advance. (Fail-loud: the entry is never counted
* durable, so the commit cannot advance regardless of ACKs.) */
persist_test_set_force_fsync_fail(p, 1);
/* Append one entry (LSN 1). It is in page cache only; the forced-fail
* flusher will NOT make it durable. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t lsn = p->current_lsn - 1; /* 1 */
TEST_ASSERT_EQUAL_UINT64(1, lsn);
/* Register a WAIT for LSN 1 on its own thread (the off-iomgr commit-barrier
* caller). It must block: the entry is not durable-majority. */
wait_submit_arg_t w = {.ce = &ce, .lsn = lsn, .timeout_ms = 2000};
pthread_t th;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th, NULL, wait_submit_thread, &w));
/* Deliver a covering standby ACK from BOTH standbys (a clear majority of
* ACKs). Pre-durability this alone advanced committed_lsn (the bug);
* post-2.2 the self-cap holds it at the non-durable self (durable_lsn=0). */
durable_standby_ack(p, &ce, 0, lsn);
durable_standby_ack(p, &ce, 1, lsn);
/* Give the (forced-failing) flusher ample time to run; durable_lsn must
* stay 0, committed must stay 0, and the WAIT must NOT have returned. */
for (int i = 0; i < 300; i++) { /* up to ~300 ms */
if (persist_durable_lsn(p) != 0 ||
atomic_load(&p->committed_lsn) != 0 ||
atomic_load(&w.returned))
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, persist_durable_lsn(p),
"durable_lsn must not advance while fdatasync is failing");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, atomic_load(&p->committed_lsn),
"committed_lsn must NOT advance on a standby-ACK majority while the "
"local entry is not durable (durable precondition gate)");
TEST_ASSERT_FALSE_MESSAGE(
atomic_load(&w.returned),
"the WAIT (client reply) must NOT release before durable-majority");
/* Now allow durability: clear the fault and force a flush. The REAL
* flusher fdatasyncs, advances durable_lsn over LSN 1, and fires the
* durable-advance wake — self is now durable AND both standbys already
* ACKed, so the entry is durable on a majority. committed_lsn advances
* and the WAIT releases promptly. */
persist_test_set_force_fsync_fail(p, 0);
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) >= lsn);
/* The WAIT must now return success (committed) within its timeout. */
TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL));
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&w.returned),
"the WAIT must release once the entry is durable on a majority");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, w.rc,
"commit_submit(WAIT) must return 0 (committed) after durable-majority");
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&p->committed_lsn) >= lsn,
"committed_lsn must advance once self is durable AND a standby ACK "
"covers the entry (durable majority)");
/* The invariant the whole spec rests on: committed never exceeds durable. */
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
commit_engine_fini(&ce);
repl_remove_standby(&p->repl, &g_dur_eps[0]);
repl_remove_standby(&p->repl, &g_dur_eps[1]);
teardown_durability_persist(p);
repl_broadcast_send_hook = NULL;
printf("test_durable_majority_commit_end_to_end: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1b (Req 1.5 / 2.5): losing the sole durable contributor *
* leaves the acknowledged op present on a surviving member. *
* *
* The durable-majority property says: when an op is acknowledged, its *
* entry is fdatasync'd on a fault-tolerant majority (self + >= *
* floor(N/2) others). For N=3 that is 2 durable copies. So losing ONE *
* durable contributor still leaves >= 1 surviving durable copy that *
* holds the op — it is recoverable from the survivor. *
* *
* We model the two durable copies as two independent persist_t over *
* separate WAL dirs (a primary's copy and a standby's copy), both made *
* genuinely durable (fdatasync'd). We then DROP the "sole durable *
* contributor" (the first copy, simulating its loss/restart-with-loss) *
* and prove the op is still durably present and recoverable on the *
* surviving copy via a fresh-process recovery (no in-memory state). *
* *
* This is the integration-level assertion of the property that makes *
* survival true (committed ⇒ >= floor(N/2)+1 durable copies); the live *
* multi-host SIGKILL failover proof is task 5.2. *
* ------------------------------------------------------------------ */
static void test_sole_durable_contributor_loss_survives(void)
{
char tmp_a[256];
char tmp_b[256];
strcpy(tmp_a, "/tmp/xce_durlossA_XXXXXX");
strcpy(tmp_b, "/tmp/xce_durlossB_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp_a));
TEST_ASSERT_NOT_NULL(mkdtemp(tmp_b));
const char* payload = "committed-op";
uint32_t plen = (uint32_t)strlen(payload);
/* Durable copy A (the contributor we will lose) and durable copy B (the
* survivor): write the SAME committed op to both and fdatasync each, so
* the op is durable on a 2-of-3 majority. */
{
persist_t* a = make_durability_persist(tmp_a);
TEST_ASSERT_NOT_NULL(a);
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(a, WAL_JOB_UPDATE, payload, plen));
TEST_ASSERT_EQUAL_INT(0, persist_flush(a));
TEST_ASSERT_TRUE(persist_durable_lsn(a) >= 1);
teardown_durability_persist(a);
}
{
persist_t* b = make_durability_persist(tmp_b);
TEST_ASSERT_NOT_NULL(b);
/* The standby applies the SAME replicated entry durably before ACK
* (persist_wal_write_replicated fdatasyncs inline — Req 1.3). */
TEST_ASSERT_EQUAL_INT(0,
persist_wal_write_replicated(b, /*lsn=*/1, WAL_JOB_UPDATE,
payload, plen));
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(b) >= 1);
teardown_durability_persist(b);
}
/* "Lose" the sole durable contributor A: destroy its on-disk WAL entirely
* (simulates a member that is gone / came back with a truncated log). */
{
char rm[600];
snprintf(rm, sizeof(rm), "rm -rf %s", tmp_a);
TEST_ASSERT_EQUAL_INT(0, system(rm));
}
/* The acknowledged op MUST still be present and recoverable on the
* surviving member B — read straight from disk by a fresh process with
* NO in-memory state (the durable-majority property in action). */
{
persist_t* b2 = make_durability_persist(tmp_b);
TEST_ASSERT_NOT_NULL(b2);
/* On-disk durable floor (CRC-validated re-read) still covers the op. */
TEST_ASSERT_TRUE_MESSAGE(
persist_test_durable_lsn_on_disk(b2) >= 1,
"the acknowledged op must survive on the durable majority after "
"the sole-durable-contributor copy is lost");
/* And a real recovery reconstructs it cleanly (not truncated). */
TEST_ASSERT_EQUAL_INT(0, persist_recover(b2));
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(b2),
"the survivor recovers the committed op as a clean intact log");
TEST_ASSERT_TRUE_MESSAGE(
persist_durable_floor(b2) >= 1,
"the survivor's recovered durable floor covers the committed op");
teardown_durability_persist(b2);
}
{
char rm[600];
snprintf(rm, sizeof(rm), "rm -rf %s", tmp_b);
(void)system(rm);
}
printf("test_sole_durable_contributor_loss_survives: PASS "
"(durable-majority property; live failover proof is task 5.2)\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1c (Req 1.6): durability work does not stall heartbeat / *
* election handling — fdatasync runs on the dedicated flusher thread, *
* NOT on the caller (iomgr/heartbeat) thread. *
* *
* Structural + timing assertion: while a durable flush is in flight on *
* the flusher thread, a heartbeat-style callback executed on the *
* caller/iomgr thread completes within a tight tolerance. We prove the *
* flush is OFF the caller thread by capturing the flusher's thread id at *
* the fdatasync site (via the XCE_TESTING seam) and asserting it is NOT *
* the caller thread. (persist_wal_async only appends + signals; it never *
* fdatasyncs inline, so the append path itself never blocks on disk.) *
* ------------------------------------------------------------------ */
static void test_durability_work_off_iomgr_preserves_liveness(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durliveness_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
pthread_t caller = pthread_self();
/* Drive a steady stream of appends on the CALLER (iomgr-equivalent)
* thread; the real flusher thread does the fdatasync work concurrently.
* After each append, a heartbeat-style callback must complete promptly —
* the append path signals the flusher and returns without blocking on
* fdatasync, so heartbeat handling on this thread is never stalled. */
const long tolerance_us = 50 * 1000; /* 50 ms — generous heartbeat budget */
for (int i = 0; i < 64; i++) {
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "h", 1));
struct timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
/* Heartbeat-style work on the caller thread (a cheap atomic read of
* commit/durable state, exactly what the heartbeat path inspects). */
volatile uint64_t hb = atomic_load(&p->committed_lsn) +
persist_durable_lsn(p);
(void)hb;
clock_gettime(CLOCK_MONOTONIC, &t1);
long elapsed_us = (t1.tv_sec - t0.tv_sec) * 1000000L +
(t1.tv_nsec - t0.tv_nsec) / 1000L;
TEST_ASSERT_TRUE_MESSAGE(
elapsed_us < tolerance_us,
"heartbeat-style work on the caller thread must not be stalled "
"by durability (fdatasync) work in flight on the flusher thread");
}
/* Let the flusher make everything durable, then confirm the fdatasync
* provably ran on a DIFFERENT thread than this caller — durability work
* is off the iomgr/heartbeat thread (Req 1.6). The seam records the
* thread id of the last successful flusher fdatasync. */
uint64_t tip = p->current_lsn - 1;
TEST_ASSERT_TRUE(wait_durable_reaches(p, tip, 5000) >= tip);
pthread_t flusher_tid = persist_test_last_fdatasync_thread(p);
TEST_ASSERT_FALSE_MESSAGE(
pthread_equal(flusher_tid, caller),
"fdatasync must run on the dedicated flusher thread, never on the "
"caller/iomgr/heartbeat thread");
teardown_durability_persist(p);
printf("test_durability_work_off_iomgr_preserves_liveness: PASS\n");
}
/* ================================================================== *
* task 3.1 — recovery integrity verdict (Req 2.1) *
* *
* persist_recover/replay_wal must distinguish a CLEAN log end (EOF, *
* pre-allocated-zero tail, snapshot-sentinel + clean replay, empty *
* first-boot WAL) from a corruption-TRUNCATED end (CRC chain break, *
* CRC mismatch, short/partial entry read). A corruption-truncated *
* recovery is NEVER reported as a clean recovery and sets a *
* member-level recovery_truncated flag the promotion gate (4.3) *
* consumes. 3.1 only SETS the verdict/flag — it does NOT FATAL and *
* does NOT fence (3.3 / 4.3 own those). *
* *
* NB the verdict is about STRUCTURAL integrity of the local log, not *
* about being behind the cluster: a clean log that ends below the *
* cluster high-water is still CLEAN (the freshness/promotion fence *
* catches "behind", not this verdict). *
* ------------------------------------------------------------------ */
/* The persist paths derive as <workdir>/<name>/data/wal; make_durability_persist
* uses name "ctrl-test" (see persist_set_workdir call above). */
#define DUR_WAL_SEG_FMT "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal"
/* Open the first WAL segment (starts at LSN 1; LSN 0 reserved) for the
* given tmpdir and return its fd, or -1. */
static int open_first_seg_rw(const char* tmpdir)
{
char seg_path[512];
/* The first-ever entry is LSN 1, so the first segment is seg-...0001. */
snprintf(seg_path, sizeof(seg_path), DUR_WAL_SEG_FMT, tmpdir,
(uint64_t)1);
return open(seg_path, O_RDWR);
}
/* Open a SECOND persist instance over the same on-disk WAL with NO
* in-memory writer state, run recovery, and return the recovery verdict
* (via persist_recovery_truncated). The writer must already be torn down
* so the segments are closed and on disk. */
/* ------------------------------------------------------------------ *
* Test 3.1a: an intact log yields RECOVER_CLEAN (flag NOT set). *
* ------------------------------------------------------------------ */
static void test_recover_verdict_clean_intact_log(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recclean_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: append + flush three entries, then "die". */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Recover into a fresh instance: an intact CRC chain must be CLEAN. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"an intact CRC chain must recover CLEAN");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_truncated(p2),
"a clean recovery must NOT set the recovery_truncated flag");
/* current_lsn advanced past the three replayed entries. */
TEST_ASSERT_EQUAL_UINT64(4, p2->current_lsn);
teardown_durability_persist(p2);
}
printf("test_recover_verdict_clean_intact_log: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1b: an empty first-boot WAL is CLEAN, not corruption. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_empty_first_boot_clean(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recempty_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* No appends: a first-boot WAL has nothing to replay — that is a clean
* (empty) recovery, never a corruption-truncated one. */
int rc = persist_recover(p);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p),
"an empty first-boot WAL must be CLEAN (no entries != corruption)");
TEST_ASSERT_FALSE(persist_recovery_truncated(p));
teardown_durability_persist(p);
printf("test_recover_verdict_empty_first_boot_clean: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1c (core RED): a CRC break injected at the FIRST entry yields *
* RECOVER_TRUNCATED and SETS the recovery_truncated flag. *
* *
* We append+flush two entries, then corrupt the on-disk `crc` field of *
* the FIRST entry (LSN 1). On recovery the CRC verification at LSN 1 *
* fails, replay stops before any clean boundary, and the verdict must *
* be TRUNCATED — never silently accepted as a complete clean recovery. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_crc_break_first_entry_truncated(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recbreak_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: two durable entries, then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Corrupt the FIRST entry's CRC field on disk: read its header at
* offset 0, flip the crc, write it back. This breaks the chain at
* LSN 1 — replay cannot reach any clean boundary. */
{
int fd = open_first_seg_rw(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
wal_entry_hdr_t hdr;
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), 0));
hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pwrite(fd, &hdr, sizeof(hdr), 0));
fsync(fd);
close(fd);
}
/* Recover: the corruption at LSN 1 must yield TRUNCATED + set the flag. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
/* persist_recover still returns 0 (no I/O error); the VERDICT is the
* signal — 3.1 classifies, it does not FATAL (that is task 3.3). */
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_TRUNCATED, persist_recovery_verdict(p2),
"a CRC break at the first entry must yield RECOVER_TRUNCATED");
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_truncated(p2),
"a corruption-truncated recovery must SET the recovery_truncated "
"flag (consumed by the promotion gate, task 4.3)");
teardown_durability_persist(p2);
}
printf("test_recover_verdict_crc_break_first_entry_truncated: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1d (snapshot-sentinel edge): a member that recovered from a *
* 16-byte LSN-sentinel snapshot has a committed position at the *
* sentinel LSN but NO log entries below it. A clean replay over that *
* sentinel snapshot must classify CLEAN, never TRUNCATED — the absence *
* of sub-sentinel entries is normal, not corruption (design Open *
* Questions / snapshot-recovery edge). *
* *
* We checkpoint (writes the sentinel snapshot + trims the WAL), append *
* + flush one fresh entry above the checkpoint, then recover in a fresh *
* instance: snapshot-sentinel + clean replay ⇒ CLEAN. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_snapshot_sentinel_clean(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recsnap_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: a couple of entries, a checkpoint (sentinel snapshot), then a
* further entry above the checkpoint, all flushed; then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Sentinel snapshot at the current committed/checkpoint position. */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Recover: load_snapshot sets checkpoint_lsn from the sentinel, replay
* picks up entries above it cleanly — verdict CLEAN, flag NOT set. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"snapshot-sentinel + clean replay must be CLEAN, not TRUNCATED");
TEST_ASSERT_FALSE(persist_recovery_truncated(p2));
teardown_durability_persist(p2);
}
printf("test_recover_verdict_snapshot_sentinel_clean: PASS\n");
}
/* ================================================================== *
* task 3.4 — re-derive the durable floor from the recovered WAL tip *
* (Req 2.3) *
* *
* On recovery the member publishes its durable floor as the highest *
* INTACT replayed LSN (the WAL tip / durable last-log index), so the *
* freshness advertisement (task 4.2) is backed by durable on-disk *
* state rather than a volatile in-memory position. The floor is the *
* durable last-log index, NOT the commit position, and NOT a stale *
* higher in-memory value. persist_durable_floor() exposes it (a thin *
* wrapper over the durable_lsn watermark, which recovery sets to the *
* recovered tip). *
* ------------------------------------------------------------------ */
/* Test 3.4a (CRUX, Req 2.3): after a TRUNCATING restart the published
* durable floor equals the highest INTACT LSN (the WAL tip), NOT a stale
* higher in-memory value and NOT the commit position.
*
* Writer lays down 4 durable entries (LSN 1..4). We then corrupt entry 3's
* CRC on disk so the intact prefix is LSN 1..2 — replay stops at the chain
* break and the recovered tip is LSN 2. To make the "not a stale higher
* in-memory value" assertion meaningful we deliberately seed the fresh
* instance's in-memory durable_lsn / committed_lsn to an over-advertised
* value (4) BEFORE recovery; recovery must publish the durable FLOOR at the
* intact tip 2, never the seeded 4. */
static void test_durable_floor_truncating_restart_is_intact_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floortrunc_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: four durable entries (LSN 1..4), then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "d", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* All four are durable on disk before the truncation. */
TEST_ASSERT_EQUAL_UINT64(4, persist_test_durable_lsn_on_disk(p));
teardown_durability_persist(p);
}
/* Corrupt entry 3 (LSN 3): walk the segment to its offset and flip its
* crc. Each entry is sizeof(hdr) + payload_len (=1) bytes; entries 1 and 2
* precede it. This breaks the chain at LSN 3, so the intact tip is LSN 2. */
{
int fd = open_first_seg_rw(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
uint32_t off = 0;
wal_entry_hdr_t hdr;
/* Skip entries 1 and 2 (LSN 1, LSN 2). */
for (int i = 0; i < 2; i++) {
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), (off_t)off));
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
/* Entry 3 (LSN 3) is at `off`: corrupt its CRC. */
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), (off_t)off));
TEST_ASSERT_EQUAL_UINT64(3, hdr.lsn);
hdr.crc ^= 0xDEADBEEFu;
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pwrite(fd, &hdr, sizeof(hdr), (off_t)off));
fsync(fd);
close(fd);
}
/* Recover into a fresh instance that we deliberately poison with a stale,
* over-advertised in-memory position (4) BEFORE recovery. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
/* Stale higher in-memory values: simulate a volatile position that
* over-states what is durable. The published floor must NOT come from
* these — it must come from the recovered intact WAL tip. */
atomic_store(&p2->durable_lsn, 4);
atomic_store(&p2->committed_lsn, 4);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Verdict is TRUNCATED (chain break at LSN 3). */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_TRUNCATED, persist_recovery_verdict(p2),
"a CRC break at LSN 3 must yield RECOVER_TRUNCATED");
/* The published durable floor == the highest INTACT LSN (2): the WAL
* tip, equal to current_lsn - 1 after replay (replay advanced
* current_lsn only to one past the last intact entry). */
uint64_t floor = persist_durable_floor(p2);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
2, floor,
"durable floor after a truncating restart must equal the highest "
"intact LSN (the WAL tip), NOT the stale in-memory value");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
p2->current_lsn - 1, floor,
"durable floor must equal the recovered WAL tip (current_lsn - 1)");
/* Independently confirmed against the on-disk re-read seam. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
2, persist_test_durable_lsn_on_disk(p2),
"on-disk re-read agrees: only LSN 1..2 are intact");
teardown_durability_persist(p2);
}
printf("test_durable_floor_truncating_restart_is_intact_tip: PASS\n");
}
/* Test 3.4b: an INTACT-log recovery publishes the FULL tip (no truncation).
* Three durable entries (LSN 1..3) recover cleanly; the floor is the full
* recovered tip 3 == current_lsn - 1. */
static void test_durable_floor_intact_recovery_full_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floorintact_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
3, persist_durable_floor(p2),
"intact recovery publishes the full recovered WAL tip");
TEST_ASSERT_EQUAL_UINT64(p2->current_lsn - 1, persist_durable_floor(p2));
teardown_durability_persist(p2);
}
printf("test_durable_floor_intact_recovery_full_tip: PASS\n");
}
/* Test 3.4c: an EMPTY first-boot WAL publishes floor 0 (nothing durable). */
static void test_durable_floor_empty_recovery_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floorempty_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
int rc = persist_recover(p);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, persist_durable_floor(p),
"an empty first-boot WAL has nothing durable — floor is 0");
teardown_durability_persist(p);
printf("test_durable_floor_empty_recovery_zero: PASS\n");
}
/* Test 3.4d (snapshot-sentinel edge): a member recovered from a 16-byte
* LSN-sentinel snapshot plus a clean replay above it publishes the recovered
* tip (the highest replayed LSN). The floor is the recovered tip, consistent
* with 3.1's CLEAN classification — "behind-and-must-catch-up", not
* over-advertising. */
static void test_durable_floor_snapshot_sentinel_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floorsnap_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Sentinel snapshot at the checkpoint position; then one entry above. */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
/* The floor is the recovered tip = current_lsn - 1 (whatever the
* highest replayed LSN above the sentinel is). It is the recovered
* tip, not a stale higher value. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
p2->current_lsn - 1, persist_durable_floor(p2),
"snapshot-sentinel recovery publishes the recovered WAL tip");
TEST_ASSERT_TRUE_MESSAGE(
persist_durable_floor(p2) > 0,
"a sentinel + clean replay-above has a non-zero recovered tip");
teardown_durability_persist(p2);
}
printf("test_durable_floor_snapshot_sentinel_tip: PASS\n");
}
/* ================================================================== *
* task 3.2 — Boot Policy Decision Table (DECISION ONLY) *
* *
* Task 3.2 pins the fail-loud-vs-fence boot policy as a DECISION and *
* encodes it as a table of input states -> expected outcome. It makes *
* NO production behavior change: the FATAL branch is implemented by *
* task 3.3 and the promotion fence + release by task 4.3. *
* *
* The decision table lives in design.md ("Boot Policy Decision Table *
* (task 3.2)"). The skeletons below mirror it ONE TEST PER ROW. Each *
* skeleton documents the row's inputs + the expected outcome it WILL *
* assert once 3.3/4.3 land, then TEST_IGNORE_MESSAGE()-skips so the *
* row is REGISTERED and reported as IGNORED (not failed) — the suite *
* stays GREEN. These become the RED tests 3.3/4.3 turn GREEN. *
* *
* Inputs (boot / promotion-eligibility evaluation): *
* verdict : CLEAN | TRUNCATED (from task 3.1) *
* durable-vs-HWM : at/above | below (durable position) *
* healthy peer : yes | no (catch-up source) *
* catch-up : succeeds | fails (bounded window) *
* Outcomes: *
* ELIGIBLE | FENCE+CATCH-UP -> ELIGIBLE | FATAL *
* INVARIANT: never a silent indefinite fence — a truncated/behind *
* member either catches up (-> eligible) or FATALs on a BOUNDED *
* failure. (Req 2.2, 2.4) *
* ------------------------------------------------------------------ */
/* task 3.3 — FATAL-path capture seam. The production FATAL terminates via
* abort(); a test installs persist_fatal_hook so the FATAL decision + boundary
* log can be asserted without the process dying. */
static int g_fatal_hook_calls = 0;
static char g_fatal_hook_msg[256];
static void dur_fatal_hook(const char* msg)
{
g_fatal_hook_calls++;
if (msg != NULL) {
strncpy(g_fatal_hook_msg, msg, sizeof(g_fatal_hook_msg) - 1);
g_fatal_hook_msg[sizeof(g_fatal_hook_msg) - 1] = '\0';
}
}
/* task 3.3 — peer-reachable NEVER FATALs (the never-FATAL aspect of rows
* 3 & 5). This task (3.3) owns only the FATAL terminal; the full FENCE+
* CATCH-UP -> ELIGIBLE eligibility outcome of these rows is task 4.3 (so the
* row3/row5 skeletons below stay IGNORED for 4.3). Here we pin ONLY the
* invariant 3.3 is responsible for: while a healthy peer is reachable, the
* decision is NEVER FATAL — neither for a TRUNCATED log (row 3) nor for a
* legitimate snapshot-sentinel CLEAN-but-behind member (row 5). It is FENCE
* (catch-up pending) and the FATAL terminate path is never fired. (Req 2.2,
* 2.4, design.md "never FATAL when a peer is reachable".) */
static void test_boot_policy_peer_reachable_never_fatal(void)
{
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE, no FATAL. */
recover_policy_inputs_t truncated_peer = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 3,
.cluster_hwm_lsn = 10,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&truncated_peer),
"row 3: a truncated member with a reachable healthy peer must FENCE "
"(catch up), NEVER FATAL");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&truncated_peer));
/* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE, no
* FATAL. A legitimate sentinel is CLEAN-but-behind, not corruption. */
recover_policy_inputs_t sentinel_peer = {
.verdict = RECOVER_CLEAN,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 5,
.cluster_hwm_lsn = 5,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&sentinel_peer),
"row 5: a snapshot-sentinel CLEAN-but-behind member with a reachable "
"peer must FENCE, NEVER FATAL");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_fatal_if_unrecoverable(&sentinel_peer));
/* The FATAL terminate path must NOT have fired for either peer-reachable
* case — a reachable healthy peer never triggers FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"peer-reachable rows must NEVER fire the FATAL terminate path");
persist_fatal_hook = NULL;
printf("test_boot_policy_peer_reachable_never_fatal: PASS\n");
}
/* Row 1: CLEAN + durable at/above HWM -> ELIGIBLE.
* A structurally intact log that is not behind the cluster boundary is
* a normal promotable member: the promotion gate (4.3) must NOT fence
* it. WILL ASSERT (4.3): the eligibility predicate returns "eligible"
* for a CLEAN, at/above member; promotion proceeds. (Req 2.3) */
/* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role_lifecycle.c). */
static ha_config_t dur_ha_cfg(void)
{
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
return cfg;
}
static void test_boot_policy_row1_clean_at_or_above_eligible(void)
{
/* CLEAN + durable at/above HWM -> ELIGIBLE. The live gate must admit a
* structurally-intact, not-behind member and must NOT set the fence. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Pre-set the fence to prove ELIGIBLE clears it (Req 2.6 release path). */
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/8, /*safe_hwm=*/8,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d,
"row 1: a CLEAN, at/above-HWM member must be ELIGIBLE to promote");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 1: ELIGIBLE must clear the in_catch_up fence");
ha_fini(&ha);
printf("test_boot_policy_row1_clean_at_or_above_eligible: PASS\n");
}
/* Row 2: CLEAN + durable below HWM + healthy peer + catch-up succeeds
* -> FENCE+CATCH-UP -> ELIGIBLE.
* Intact but behind: must NOT lead below the boundary; fence, catch up
* from a healthy peer, regain eligibility. WILL ASSERT (4.3): a
* below-HWM member is denied promotion until catch-up passes the last
* committed boundary, then regains eligibility. (Req 2.2, 2.3, 2.6) */
static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void)
{
/* CLEAN + below HWM + healthy peer: FENCE first (denied promotion), then
* once catch-up passes the committed boundary, ELIGIBLE (fence released). */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Behind: durable_floor 4 < HWM 10, healthy peer reachable, not yet caught
* up -> FENCE; the in_catch_up fence is SET (refused promotion). */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/4, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 2: a CLEAN-but-behind member with a peer must FENCE (catch up)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 2: FENCE must SET the in_catch_up fence (promotion refused)");
/* Catch-up brings the durable position to/above the committed boundary
* (durable_floor 10 >= HWM 10) -> ELIGIBLE; the fence is released (Req 2.6). */
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 2: after catch-up past the committed boundary the member regains "
"eligibility (Req 2.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 2: regained eligibility must RELEASE the in_catch_up fence");
ha_fini(&ha);
printf("test_boot_policy_row2_clean_below_peer_catchup_eligible: PASS\n");
}
/* Row 3: TRUNCATED + healthy peer + catch-up succeeds
* -> FENCE+CATCH-UP -> ELIGIBLE.
* Structurally truncated (task-3.1 verdict) but a healthy peer exists:
* fence, catch up, regain eligibility — NEVER FATAL when a peer is
* reachable. WILL ASSERT (4.3): a recovery_truncated member is denied
* promotion, catches up from the peer, and is re-enabled; FATAL is NOT
* triggered. (Req 2.2, 2.6) */
static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
{
/* TRUNCATED + healthy peer: a recovery-truncated member is DENIED promotion
* (FENCE), catches up from the peer, then regains eligibility. FATAL must
* NOT fire while a healthy peer is reachable. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Truncated boundary at 3, cluster HWM 10, peer reachable, not caught up. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 3: a recovery-truncated member with a healthy peer must FENCE "
"(denied promotion), never FATAL");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: a truncated member must be fenced (in_catch_up set)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 3: FATAL must NOT fire while a healthy peer is reachable");
/* After catch-up to/above the committed boundary -> ELIGIBLE, fence released. */
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 3: after catch-up the (previously truncated) member is eligible");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: regained eligibility releases the fence (Req 2.6)");
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row3_truncated_peer_catchup_eligible: PASS\n");
}
/* Row 4: TRUNCATED + below HWM + NO healthy peer + catch-up fails
* (bounded window elapses) -> FATAL.
* Cannot recover locally AND cannot catch up within the bounded window:
* fail loud rather than serve divergent state. WILL ASSERT (3.3): the
* member terminates loudly (FATAL) with the truncation boundary logged
* when the bounded catch-up window elapses without a committed-consistent
* state. (Req 2.4) */
static void test_boot_policy_row4_truncated_no_peer_bounded_fatal(void)
{
/* Row 4: TRUNCATED + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Cannot recover locally AND cannot catch up within the bounded
* window: fail loud with the truncation boundary logged (Req 2.4). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
recover_policy_inputs_t in = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 7, /* truncation boundary */
.cluster_hwm_lsn = 12, /* cluster committed high-water */
};
/* Pure decision returns FATAL. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4: truncated + below + no peer + bounded catch-up fails must "
"decide FATAL");
/* The action helper routes FATAL through the hook (no abort in test) and
* still returns FATAL — proving the loud-terminate path was taken. */
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4: the FATAL terminate path must fire exactly once (the "
"truncation boundary is logged at ERROR just before it)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4_truncated_no_peer_bounded_fatal: PASS\n");
}
/* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails (bounded
* window elapses) -> FATAL.
* Behind the HWM with no peer to advance from within the bounded window:
* same bounded-failure terminal as row 4 — never a silent indefinite
* fence. WILL ASSERT (3.3): a behind member with no reachable peer FATALs
* once the bounded window elapses. (Req 2.4, 2.2) */
static void test_boot_policy_row4b_clean_below_no_peer_bounded_fatal(void)
{
/* Row 4b: CLEAN + below HWM + NO healthy peer + bounded catch-up fails
* -> FATAL. Behind the HWM with no peer to advance from within the bounded
* window — same bounded-failure terminal as row 4 (Req 2.4, 2.2). */
g_fatal_hook_calls = 0;
g_fatal_hook_msg[0] = '\0';
persist_fatal_hook = dur_fatal_hook;
recover_policy_inputs_t in = {
.verdict = RECOVER_CLEAN, /* structurally intact... */
.durable_below_hwm = true, /* ...but behind the HWM */
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 9,
.cluster_hwm_lsn = 20,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FATAL, persist_recover_decision(&in),
"row 4b: clean-but-behind + no peer + bounded catch-up fails must "
"decide FATAL — never a silent indefinite fence");
recover_decision_t d = persist_recover_fatal_if_unrecoverable(&in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FATAL, d);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"row 4b: a behind member with no reachable peer must FATAL once the "
"bounded window elapses (boundary logged before terminate)");
persist_fatal_hook = NULL;
printf("test_boot_policy_row4b_clean_below_no_peer_bounded_fatal: PASS\n");
}
/* Row 5: CLEAN snapshot-sentinel (committed at sentinel LSN, NO
* sub-sentinel entries) + below + healthy peer + catch-up succeeds
* -> FENCE+CATCH-UP -> ELIGIBLE.
* A legitimate sentinel snapshot is CLEAN-but-behind, NOT corruption:
* treat as must-catch-up, NEVER FATAL on a legitimate sentinel. WILL
* ASSERT (4.3): a snapshot-sentinel member is fenced+caught-up like any
* behind member and is NEVER routed to the FATAL branch. (Req 2.1, 2.2)
* (3.1 already verifies the verdict itself is CLEAN — see
* test_recover_verdict_snapshot_sentinel_clean; this row pins the
* downstream promotion-fence outcome.) */
static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
{
/* Snapshot-sentinel CLEAN-but-behind + healthy peer: treated as behind,
* must catch up (FENCE), NEVER FATAL on a legitimate sentinel; then
* ELIGIBLE after catch-up. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
/* Sentinel at LSN 5, cluster has advanced to HWM 12, peer reachable. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/12,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 5: a snapshot-sentinel CLEAN-but-behind member must FENCE, never "
"FATAL while a peer is reachable");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 5: a legitimate sentinel must NEVER trigger FATAL");
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/12, /*safe_hwm=*/12,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 5: after catch-up the sentinel member is eligible (Req 2.6)");
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal: PASS\n");
}
/* Row 6: CLEAN empty first-boot WAL (durable=0, HWM=0, fresh cold start)
* -> ELIGIBLE.
* An empty first-boot WAL is not behind anything — cold-start members
* are promotable. WILL ASSERT (4.3): the eligibility gate admits a
* fresh cold-start member (no fence, no FATAL). (Req 2.1, 2.3) */
static void test_boot_policy_row6_empty_first_boot_eligible(void)
{
/* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fresh cold-start
* member is not behind anything and must be promotable (no fence, no FATAL),
* even with no peer reachable (single-node / first boot). */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/0, /*safe_hwm=*/0,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d,
"row 6: an empty first-boot WAL (durable=0/HWM=0) must be ELIGIBLE");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 6: a cold-start member must not be fenced");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"row 6: a cold-start member must never FATAL");
persist_fatal_hook = NULL;
ha_fini(&ha);
printf("test_boot_policy_row6_empty_first_boot_eligible: PASS\n");
}
/* ── task 4.3: hollow node refused even at minimal quorum (Req 2.5) ──────────
* A hollow node (recovery-truncated OR durable below the HWM) must be refused
* promotion regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote. At minimal quorum with NO
* healthy peer, a hollow node FATALs (bounded-unrecoverable) rather than
* silently leading; with a peer it FENCEs. Either way it never becomes
* ELIGIBLE while hollow. */
static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
{
/* Truncated + below HWM + no peer (minimal/degraded quorum) -> NEVER
* ELIGIBLE (FATAL — fail loud rather than promote a hollow node). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
/*safe_hwm=*/10, /*peer_reachable=*/false,
/*catch_up_succeeded=*/false),
"a recovery-truncated node must never be ELIGIBLE at minimal quorum "
"(Req 2.5)");
/* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (FATAL). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/false, /*durable_floor=*/5,
/*safe_hwm=*/10, /*peer_reachable=*/false,
/*catch_up_succeeded=*/false),
"a below-HWM node must never be ELIGIBLE at minimal quorum (Req 2.5)");
/* Hollow node WITH a peer is still refused (FENCE, not ELIGIBLE) until it
* catches up — minimal-quorum size does not relax the gate. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
/*safe_hwm=*/10, /*peer_reachable=*/true,
/*catch_up_succeeded=*/false),
"a hollow node with a peer is FENCED (refused) until caught up");
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
}
/* ── task 4.4: bounded-window catch-up before FATAL ──────────────────────────
*
* The 4.3 gate decided FATAL synchronously at the promotion instant when a
* recovery-truncated/behind member had no reachable peer right then. Task 4.4
* defers that FATAL to a one-shot bounded-window timer: the member is fenced
* (in_catch_up) and given DEFAULT_RECOVERY_CATCHUP_WINDOW_MS to find a peer /
* make progress; the 3.3 FATAL is reached ONLY from the timer after a real
* bounded failure (still hollow + no peer + no progress).
*
* These tests drive ha_arm_promotion_fence_timer (the production deferral wired
* into raft_cb_become_leader) through a REAL delaymgr with a short window seam,
* so the timer fires fast and deterministically. A minimal cluster_t carries
* the delaymgr + persist + ha exactly as production does. */
/* The delaymgr runner used by production (run_delay_tasks in cluster.c) is not
* linked into the unit-test target; provide the same trivial pass-through. */
static int dur_run_delay_task(delay_task_entry_fn_t entry, void* arg)
{
return entry(arg);
}
/* Build a minimal cluster_t (calloc'd) with a started delaymgr and a recovered
* persist whose recovery verdict + durable floor are set directly, so the
* promotion fence can be evaluated without a full election. truncated → the
* recovery_truncated flag; durable_floor → the durable watermark the gate reads.
* ha.repl stays NULL so "no healthy peer reachable" holds in the timer. */
static cluster_t* dur_make_fence_cluster(const char* tmpdir, bool truncated,
uint64_t durable_floor)
{
cluster_t* c = calloc(1, sizeof(*c));
if (!c) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = 2;
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
if (persist_init(&c->persist, &cfg, NULL) != 0) { free(c); return NULL; }
if (persist_start(&c->persist) != 0) {
persist_fini(&c->persist); free(c); return NULL;
}
/* Set the recovered state the gate reads (mirrors a real persist_recover). */
c->persist.recovery_verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, durable_floor,
memory_order_release);
ha_config_t hcfg = dur_ha_cfg();
if (ha_init(&c->ha, &hcfg, 0) != 0) {
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
c->ha.raft_c = c;
c->ha.repl = NULL; /* no standby registry → no healthy peer reachable */
if (delaymgr_init(&c->delaymgr, 5 /* precision_ms */, dur_run_delay_task) != 0) {
ha_fini(&c->ha);
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
if (delaymgr_start(&c->delaymgr) != 0) {
delaymgr_fini(&c->delaymgr);
ha_fini(&c->ha);
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
return c;
}
static void dur_destroy_fence_cluster(cluster_t* c)
{
if (!c) return;
/* Cancel any pending fence timer BEFORE tearing down the delaymgr so a
* still-armed callback cannot fire against freed state. */
ha_cancel_promotion_fence_timer(&c->ha);
delaymgr_stop(&c->delaymgr);
delaymgr_fini(&c->delaymgr);
ha_fini(&c->ha);
persist_stop(&c->persist);
persist_fini(&c->persist);
free(c);
}
/* Spin until the fatal hook fires or a timeout elapses (the delaymgr fires the
* one-shot timer on its own worker thread). */
static bool dur_wait_fatal(int expect_calls, int timeout_ms)
{
int waited = 0;
while (g_fatal_hook_calls < expect_calls && waited < timeout_ms) {
usleep(2000);
waited += 2;
}
return g_fatal_hook_calls >= expect_calls;
}
/* 4.4-RED carrier + core deferral: a recovery-truncated/behind member with NO
* reachable peer must NOT FATAL synchronously at the promotion instant — it
* fences (in_catch_up set, process alive) and arms a bounded-window timer; the
* 3.3 FATAL fires ONLY from that timer after the window with no peer/progress.
*
* RED (pre-4.4): the gate routed a no-peer truncated member straight through
* ha_evaluate_promotion_fence → persist_recover_fatal_if_unrecoverable, so the
* fatal hook was called SYNCHRONOUSLY (count==1 with zero delay) — there was no
* bounded window, which is exactly the task-5.2 soak collapse. (Req 2.2, 2.4) */
static void test_promotion_fence_defers_fatal_to_bounded_timer(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_defer_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(40); /* short bounded window for the test */
/* Arm the fence exactly as raft_cb_become_leader does for a truncated,
* below-HWM, no-peer winner. This must NOT FATAL synchronously. */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: the promotion gate must NOT FATAL synchronously — FATAL is "
"deferred to the bounded-window timer (Req 2.2/2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: a fenced member is held in_catch_up (refused promotion), alive");
/* Now let the bounded window elapse: still truncated + no peer + no
* progress → the 3.3 FATAL fires from the timer (exactly once). */
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_fatal(1, 1000),
"4.4: the deferred FATAL must fire from the timer after the bounded "
"window with no peer and no progress (Req 2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, g_fatal_hook_calls,
"4.4: the deferred FATAL fires exactly once from the timer");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0); /* restore default */
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_defers_fatal_to_bounded_timer: PASS\n");
}
/* 4.4 Req 2.6: catch-up within the window → fence released, FATAL NOT called.
* Arm the fence, then simulate catch-up (clear truncated + advance durable_floor
* past the HWM) before the timer fires; when the timer fires it observes a
* caught-up member, releases in_catch_up, and does NOT FATAL. */
static void test_promotion_fence_released_on_catchup_no_fatal(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_catchup_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
/* Simulate catch-up completing: log now intact and durable past the HWM. */
c->persist.recovery_verdict = RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
/* Wait long enough for the window to fire; assert it did NOT FATAL and the
* fence was released (Req 2.6). */
usleep(300 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: a member that caught up within the window must NOT FATAL (Req 2.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: catch-up within the window RELEASES the in_catch_up fence (Req 2.6)");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_released_on_catchup_no_fatal: PASS\n");
}
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → NO FATAL.
* Proves the deferred FATAL never fires after a clean cancel — a stopped member
* must not self-terminate, and there is no use-after-free of the timer ctx. */
static void test_promotion_fence_cancel_prevents_fatal(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_cancel_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
g_fatal_hook_calls = 0;
persist_fatal_hook = dur_fatal_hook;
ha_set_promotion_fence_window_ms(60);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(0, g_fatal_hook_calls);
/* Cancel before the window elapses. */
ha_cancel_promotion_fence_timer(&c->ha);
/* Wait past the window — the cancelled timer must be a no-op, NO FATAL. */
usleep(200 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_fatal_hook_calls,
"4.4: a cancelled fence timer must NEVER FATAL (shutdown/recovery)");
persist_fatal_hook = NULL;
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_cancel_prevents_fatal: PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
/* Test suite entry point */
/* ------------------------------------------------------------------ */
void test_commit_durability(void)
{
#ifdef XCE_TESTING
RUN_TEST(test_durable_floor_empty);
RUN_TEST(test_durable_floor_after_flush);
RUN_TEST(test_durable_floor_survives_process_loss);
RUN_TEST(test_durable_lsn_initial_zero);
RUN_TEST(test_durable_lsn_batches_multiple_appends);
RUN_TEST(test_durable_lsn_never_exceeds_synced_tip);
RUN_TEST(test_durable_lsn_invariants_under_load);
RUN_TEST(test_durable_lsn_holds_on_fsync_failure);
RUN_TEST(test_durable_advance_drives_commit_single_member);
RUN_TEST(test_replicated_persist_syncs_before_return);
RUN_TEST(test_replicated_sync_failure_no_ack);
RUN_TEST(test_legacy_sync_append_holds_committed_at_durable);
RUN_TEST(test_legacy_sync_append_lsn0_edge_valid);
RUN_TEST(test_commit_never_exceeds_durable_invariant);
/* task 5.1 — durable-majority commit wired end to end */
RUN_TEST(test_durable_majority_commit_end_to_end);
RUN_TEST(test_sole_durable_contributor_loss_survives);
RUN_TEST(test_durability_work_off_iomgr_preserves_liveness);
RUN_TEST(test_recover_verdict_clean_intact_log);
RUN_TEST(test_recover_verdict_empty_first_boot_clean);
RUN_TEST(test_recover_verdict_crc_break_first_entry_truncated);
RUN_TEST(test_recover_verdict_snapshot_sentinel_clean);
/* task 3.4 — re-derive the durable floor from the recovered WAL tip. */
RUN_TEST(test_durable_floor_truncating_restart_is_intact_tip);
RUN_TEST(test_durable_floor_intact_recovery_full_tip);
RUN_TEST(test_durable_floor_empty_recovery_zero);
RUN_TEST(test_durable_floor_snapshot_sentinel_tip);
/* task 3.2 — Boot Policy Decision Table skeletons (IGNORED, pending
* 3.3/4.3). Registered so each row is reported as IGNORED, not failed;
* the suite stays GREEN. See design.md "Boot Policy Decision Table". */
/* task 3.3 — FATAL rows (4, 4b) now ACTIVE (PASS), plus the never-FATAL
* invariant for the peer-reachable rows (3, 5). The full FENCE->ELIGIBLE
* eligibility outcome of rows 1,2,3,5,6 stays IGNORED pending task 4.3. */
RUN_TEST(test_boot_policy_peer_reachable_never_fatal);
RUN_TEST(test_boot_policy_row1_clean_at_or_above_eligible);
RUN_TEST(test_boot_policy_row2_clean_below_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row3_truncated_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row4_truncated_no_peer_bounded_fatal);
RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_bounded_fatal);
RUN_TEST(test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal);
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
/* task 4.4 — bounded-window catch-up before FATAL (no synchronous FATAL;
* deferred-timer FATAL only after a real bounded failure; fence released on
* catch-up; cancelled timer never FATALs). (Req 2.2, 2.4, 2.6) */
RUN_TEST(test_promotion_fence_defers_fatal_to_bounded_timer);
RUN_TEST(test_promotion_fence_released_on_catchup_no_fatal);
RUN_TEST(test_promotion_fence_cancel_prevents_fatal);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif
}
View git blame Copy permalink