wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/controller/testsuite/test_commit_durability.c
Rongsong Shen d3da3ca8bd fix(raft-apply-decoupling): prevent apply-divergence fence from being live-cleared (task 5.3) 
Add explicit test coverage for the DIVERGENT fence's stay-alive and
restart-only-clear guarantees (task 5.3), which surfaced a critical,
pre-existing safety gap: the sticky recovery_unrecoverable flag is
trigger-agnostic, and two pre-existing live call sites in ha.c
(promotion_fence_timeout_fn's periodic timer, and
raft_cb_become_leader's promotion-eligibility gate) clear it based
purely on structural conditions, never checking the apply verdict.

A node whose local apply dispatch diverges from the committed log,
but whose WAL bytes are otherwise structurally healthy, would have
its fence silently released by the live timer, then be eligible for
promotion to primary on a later election -- serving genuinely
divergent state.

Fix: gate both clear-call-sites on persist_apply_verdict(&c->persist)
!= APPLY_DIVERGENT in addition to their existing structural checks.
An apply-divergence-triggered fence can now only clear via a fresh
process restart's clean persist_recover() replay.

New tests: test_promotion_fence_apply_divergent_survives_structural_catchup
(test_commit_durability.c) covers the timer site; and
test_promotion_refused_when_apply_divergent_even_if_eligible
(test_convergence_verification.c) covers the promotion-gate site,
structurally isolated so it cannot pass for the wrong reason.

Closes out task-group 5 (restart apply-correctness verification).
2026-07-04 13:33:03 +00:00

2464 lines
115 KiB
C
Raw Blame History

#define _GNU_SOURCE
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <unity/unity.h>
#include "commit.h"
#include "config.h"
#include "delay.h" /* task 4.4: delaymgr for the bounded-window timer */
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
#include "persist.h"
#include "repl.h" /* task 5.1: repl_add_standby + standby ACK feed */
#include "typemap.h" /* task 5.1: wal_catchup_entry_t (send hook) */
#include "wal_types.h"
#include "xce/cluster_def.h" /* task 4.4: cluster_t (delaymgr + persist + ha) */
/* ------------------------------------------------------------------ *
* ha-commit-durability — task 1.2: durability-verification test seam *
* *
* These tests exercise persist_test_durable_lsn_on_disk(), the *
* XCE_TESTING-only seam that re-reads + CRC-validates the WAL from *
* stable storage and reports the highest fully-persisted LSN — *
* independent of the in-memory current_lsn watermark. This lets a *
* test assert an acknowledged operation is recoverable from disk after *
* an immediate ungraceful termination, without relying on the *
* background flush timer (Req 3.2). The seam is purely inspective: *
* it changes nothing observable about the acknowledgment contract *
* (Req 3.3). *
* *
* The seam is compiled ONLY into the controller_unit_test target *
* (-DXCE_TESTING). The production xce_controller target links without *
* it — see #ifdef XCE_TESTING in persist.c / declaration in persist.h. *
* ------------------------------------------------------------------ */
#ifdef XCE_TESTING
/* Build a persist instance with a deliberately slow background flush
* cadence, so the only durability point is an explicit persist_flush()
* (mirrors make_fresh_persist in test_persist_recovery.c). */
static persist_t* make_durability_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(persist_t));
if (!p)
return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000; /* slow background fsync — durability comes from
* the explicit flush, not the timer */
cfg.snap_keep_n = 2;
persist_set_workdir(p, tmpdir, "ctrl-test");
if (persist_init(p, &cfg, NULL) != 0) {
free(p);
return NULL;
}
if (persist_start(p) != 0) {
persist_fini(p);
free(p);
return NULL;
}
return p;
}
static void teardown_durability_persist(persist_t* p)
{
persist_stop(p);
persist_fini(p);
free(p);
}
/* ------------------------------------------------------------------ *
* Test 1: an empty WAL has no durable entries (floor == 0). *
* An empty first-boot WAL is not corruption — it simply has nothing *
* persisted, so the seam reports 0. *
* ------------------------------------------------------------------ */
static void test_durable_floor_empty(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durfloor0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
uint64_t floor = persist_test_durable_lsn_on_disk(p);
TEST_ASSERT_EQUAL_UINT64(0, floor);
teardown_durability_persist(p);
printf("test_durable_floor_empty: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 2 (core Req 3.2): after append + explicit fdatasync, the seam *
* sees the entries on disk — the durable floor equals the highest *
* appended LSN, recoverable independent of in-memory state. *
* ------------------------------------------------------------------ */
static void test_durable_floor_after_flush(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durfloor1_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Append three entries. LSN 0 is reserved, so these take LSN 1,2,3.
* current_lsn advances to 4 (one past the last appended). */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
/* Durability point: fdatasync the segment so the bytes are on stable
* storage (this is what an acknowledged-before-durable contract must
* have done before replying ok). */
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* The seam re-reads from disk and reports the highest intact LSN. */
uint64_t floor = persist_test_durable_lsn_on_disk(p);
TEST_ASSERT_EQUAL_UINT64(p->current_lsn - 1, floor);
TEST_ASSERT_EQUAL_UINT64(3, floor);
teardown_durability_persist(p);
printf("test_durable_floor_after_flush: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3 (recoverable-after-ungraceful-kill, Req 3.2): a SECOND *
* persist instance opened over the same WAL dir — with NO in-memory *
* state from the writer — recovers exactly the durable floor the seam *
* reported. This is the "survives an immediate ungraceful *
* termination" assertion: durability lives on disk, not in the *
* volatile buffers of the process that wrote it. *
* ------------------------------------------------------------------ */
static void test_durable_floor_survives_process_loss(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durfloor2_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
uint64_t durable_floor = 0;
/* Writer process: append, fdatasync, then "die" (we drop the instance
* without any further flush — the bytes are already durable). */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
durable_floor = persist_test_durable_lsn_on_disk(p);
TEST_ASSERT_EQUAL_UINT64(2, durable_floor);
teardown_durability_persist(p);
}
/* "Restarted" process: a fresh persist_t with zero in-memory state,
* opened over the same on-disk WAL. Its durable floor — read straight
* from disk before any recover/replay — matches what the writer
* persisted. Acknowledged-before-durable ops therefore survive the
* loss of the acking process. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
/* current_lsn is still the fresh-boot value (1, LSN 0 reserved):
* the durable floor does NOT come from in-memory state. */
uint64_t floor2 = persist_test_durable_lsn_on_disk(p2);
TEST_ASSERT_EQUAL_UINT64(durable_floor, floor2);
TEST_ASSERT_EQUAL_UINT64(2, floor2);
teardown_durability_persist(p2);
}
printf("test_durable_floor_survives_process_loss: PASS\n");
}
/* ================================================================== *
* task 2.1 — durable-LSN tracking via a group-commit flusher *
* *
* These tests pin the invariants of persist_t.durable_lsn, the *
* prefix watermark advanced by the demand-driven flusher (the *
* repurposed fsync_worker): *
* - durable_lsn never exceeds the synced tip (<= current_lsn - 1); *
* - durable_lsn is monotonic non-decreasing; *
* - one fdatasync advances durable_lsn past MULTIPLE appended LSNs *
* (batching / group-commit); *
* - durable_lsn advances ONLY after a successful fdatasync; *
* - under concurrent append load the invariants always hold. *
* ------------------------------------------------------------------ */
/* Spin until the flusher catches durable_lsn up to the appended tip,
* or a generous deadline elapses. Returns the observed durable_lsn. */
static uint64_t wait_durable_reaches(persist_t* p, uint64_t target_lsn,
int max_ms)
{
for (int i = 0; i < max_ms; i++) {
uint64_t d = persist_durable_lsn(p);
if (d >= target_lsn)
return d;
usleep(1000); /* 1 ms */
}
return persist_durable_lsn(p);
}
/* ------------------------------------------------------------------ *
* Test 4: a fresh store has durable_lsn == 0 (nothing synced yet). *
* ------------------------------------------------------------------ */
static void test_durable_lsn_initial_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_durable_lsn_initial_zero: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5 (core / batching, Req 1.1): several appends, then the *
* demand-driven flusher advances durable_lsn past ALL of them with a *
* single fdatasync — and never past the appended tip. Uses the fast *
* cadence so the append-signal (not the timer) drives the flush, but *
* the slow-cadence helper would converge too via the bounded fallback.*
* ------------------------------------------------------------------ */
static void test_durable_lsn_batches_multiple_appends(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn1_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* LSN 0 reserved → these take 1,2,3,4,5; current_lsn ends at 6. */
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
uint64_t tip = p->current_lsn - 1; /* 5 */
TEST_ASSERT_EQUAL_UINT64(5, tip);
uint64_t d = wait_durable_reaches(p, tip, 5000);
/* The flusher advanced durable_lsn past several appended LSNs with
* one fdatasync (batching), reaching the synced tip. */
TEST_ASSERT_EQUAL_UINT64(tip, d);
/* Invariant: never exceeds the synced tip. */
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
/* The on-disk seam agrees: those LSNs are genuinely fdatasync'd. */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_batches_multiple_appends: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 6 (advances-only-after-fdatasync + monotonic): the durable *
* floor reported by the on-disk seam must always be >= durable_lsn — *
* i.e. durable_lsn never claims an LSN whose bytes are not actually *
* on stable storage. Also asserts monotonic non-decreasing across *
* successive append+flush rounds. *
* ------------------------------------------------------------------ */
static void test_durable_lsn_never_exceeds_synced_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn2_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
uint64_t prev = persist_durable_lsn(p);
for (int round = 0; round < 4; round++) {
for (int i = 0; i < 3; i++)
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
uint64_t tip = p->current_lsn - 1;
uint64_t d = wait_durable_reaches(p, tip, 5000);
/* Monotonic non-decreasing. */
TEST_ASSERT_TRUE(d >= prev);
/* Never exceeds the synced tip. */
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
/* Advances only after a real fdatasync: the independent on-disk
* scan must confirm every claimed-durable LSN is on disk. */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
prev = d;
}
teardown_durability_persist(p);
printf("test_durable_lsn_never_exceeds_synced_tip: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 7 (UNDER LOAD, Req 1.6 / lost-wakeup): many appends from *
* multiple threads racing the flusher. Asserts the invariants hold *
* throughout (durable_lsn <= synced tip, never past disk) AND that *
* the flusher eventually catches durable_lsn up to the final tip — *
* i.e. no append is left permanently un-synced (no lost wakeup). *
* ------------------------------------------------------------------ */
#define LOAD_THREADS 4
#define LOAD_PER_THREAD 200
typedef struct {
persist_t* p;
int n;
} load_arg_t;
static void* load_appender(void* arg)
{
load_arg_t* a = arg;
for (int i = 0; i < a->n; i++) {
/* Ignore rc: a full-segment rotate or transient EIO is not the
* subject under test; the invariant checks below tolerate it. */
(void)persist_wal_async(a->p, WAL_JOB_UPDATE, "z", 1);
/* Sample the invariant mid-flight: durable_lsn must NEVER exceed
* the appended tip, even while a concurrent flush is in progress. */
uint64_t d = persist_durable_lsn(a->p);
uint64_t cur = atomic_load_explicit(
(const _Atomic uint64_t*)&a->p->current_lsn, memory_order_acquire);
/* current_lsn is read without the lock here purely as a coarse
* upper bound; a torn read can only make `cur` smaller, so the
* assertion stays conservative. durable_lsn is a prefix watermark
* that can never outrun the last assigned LSN. */
TEST_ASSERT_TRUE(d == 0 || d <= cur);
}
return NULL;
}
static void test_durable_lsn_invariants_under_load(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn3_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
pthread_t th[LOAD_THREADS];
load_arg_t args[LOAD_THREADS];
for (int i = 0; i < LOAD_THREADS; i++) {
args[i].p = p;
args[i].n = LOAD_PER_THREAD;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th[i], NULL,
load_appender, &args[i]));
}
for (int i = 0; i < LOAD_THREADS; i++)
pthread_join(th[i], NULL);
/* Final tip after all appends settled. */
uint64_t tip = p->current_lsn - 1;
/* No lost wakeup: the flusher must eventually advance durable_lsn up
* to the final appended tip (every entry gets synced). */
uint64_t d = wait_durable_reaches(p, tip, 10000);
TEST_ASSERT_EQUAL_UINT64(tip, d);
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_invariants_under_load: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 8 (ERROR PATH, Req 1.1/1.2 — fail-loud): when fdatasync FAILS, *
* durable_lsn must NOT advance. Bytes that did not reach stable *
* storage must never be counted durable — otherwise the member would *
* contribute a non-durable LSN toward the commit majority. *
* *
* Mechanism: the XCE_TESTING fault-injection toggle forces every *
* fdatasync site to treat the sync as failed. We append entries while *
* the toggle is ON and assert durable_lsn stays put across the full *
* cadence window (the flusher runs but its advance is gated on rc==0). *
* Then we clear the toggle and confirm the flusher recovers — durable_ *
* lsn advances to the tip once syncs succeed again. *
* ------------------------------------------------------------------ */
static void test_durable_lsn_holds_on_fsync_failure(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn4_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* First, append one entry and let it become durable normally, so the
* watermark has a known non-zero baseline that must HOLD during the
* failure window. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t baseline = wait_durable_reaches(p, 1, 5000);
TEST_ASSERT_EQUAL_UINT64(1, baseline);
/* Now inject fdatasync failure and append more entries. */
persist_test_set_force_fsync_fail(p, 1);
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
uint64_t tip = p->current_lsn - 1; /* 6 */
TEST_ASSERT_EQUAL_UINT64(6, tip);
/* Give the flusher ample time to run several passes (cadence is 5s but
* the append signal fires it immediately) AND an explicit flush attempt.
* Under injected failure, durable_lsn MUST remain at the baseline — the
* new bytes are not on stable storage, so they are not durable. */
(void)persist_flush(p); /* also gated on rc==0 — must not advance either */
for (int i = 0; i < 200; i++) {
TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p));
usleep(1000); /* 1 ms; 200 ms total — many flusher signal cycles */
}
/* Still parked at the baseline after the window. */
TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) <= p->current_lsn - 1);
/* Recovery: clear the fault and confirm the flusher catches up — proving
* the failure path did not corrupt the watermark, only held it. */
persist_test_set_force_fsync_fail(p, 0);
/* Nudge the flusher with one more append so the level predicate signals. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
uint64_t new_tip = p->current_lsn - 1; /* 7 */
uint64_t d = wait_durable_reaches(p, new_tip, 5000);
TEST_ASSERT_EQUAL_UINT64(new_tip, d);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_holds_on_fsync_failure: PASS\n");
}
/* ================================================================== *
* task 2.2 — durable-advance wake drives commit (integration) *
* *
* End-to-end over a REAL persist_t + commit_engine_t (single-member, *
* NULL cluster): the group-commit flusher advances durable_lsn and *
* fires the registered durable-advance callback, which the commit *
* engine uses to advance committed_lsn and wake WAIT callers. Proves *
* the persist↔commit wiring works against the real flusher thread — *
* not just the directly-driven callback (covered in *
* test_commit_engine.c). n_peers=1 means self alone is the majority, *
* so committed_lsn tracks durable_lsn with no standby ACKs needed. *
* ------------------------------------------------------------------ */
static void test_durable_advance_drives_commit_single_member(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durcommit_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Single-member commit engine over the real persist. n_vcpus large enough
* to admit a wait. commit_engine_init registers the durable-advance cb. */
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, 1, 8));
/* committed starts at 0; nothing appended/durable. */
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Append three entries (LSN 1,2,3). They are in page cache only — the slow
* 5s cadence means the append-SIGNAL (not the timer) must drive the flush;
* the real flusher fdatasyncs, advances durable_lsn, and fires the cb,
* which advances committed_lsn for this single-member quorum. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
uint64_t tip = p->current_lsn - 1; /* 3 */
TEST_ASSERT_EQUAL_UINT64(3, tip);
/* Wait for the flusher to make the tip durable AND for the wake-driven
* commit to catch up. committed_lsn must reach the tip — driven by the
* durable-advance callback, not by any ACK (there are no standbys). */
uint64_t committed = 0;
for (int i = 0; i < 5000; i++) {
committed = atomic_load(&p->committed_lsn);
if (committed >= tip)
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
tip, committed,
"durable-advance callback must drive committed_lsn to the durable tip");
/* committed never exceeds durable (the invariant the wake preserves). */
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
/* Tear the engine down BEFORE the persist (unhooks the callback so the
* flusher cannot fire against a destroyed engine). */
commit_engine_fini(&ce);
teardown_durability_persist(p);
printf("test_durable_advance_drives_commit_single_member: PASS\n");
}
/* ================================================================== *
* task 2.3 — standby persist-before-ACK regression (Req 1.3) *
* *
* Req 1.3: a standby durably persists a replicated entry to local *
* stable storage BEFORE sending its acknowledgment. The ACK is sent by *
* the CALLER (ha_apply_single_entry) only after *
* persist_wal_write_replicated returns success — so to pin "persist *
* before ACK" we prove a durable fdatasync ran INSIDE the persist call *
* before it returned. *
* *
* Mechanism: a disk re-read (persist_test_durable_lsn_on_disk) cannot *
* distinguish an fsync'd page from a page-cache-only page, so it CANNOT *
* prove the sync ran. Instead we read the XCE_TESTING fdatasync counter *
* incremented at the inline sync site: snapshot it before the call, *
* assert it incremented after — the sync provably ran within the call, *
* before the caller could ACK. The test FAILS if someone removed or *
* moved the fdatasync out of persist_wal_write_replicated. *
* ------------------------------------------------------------------ */
static void test_replicated_persist_syncs_before_return(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replsync_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Snapshot the sync counter, then write a replicated entry. The standby
* path must fdatasync inline BEFORE returning — so the counter must
* advance during the single call. */
unsigned before = persist_test_replicated_fdatasync_count(p);
int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_EQUAL_INT(0, rc);
unsigned after = persist_test_replicated_fdatasync_count(p);
TEST_ASSERT_TRUE_MESSAGE(
after > before,
"persist_wal_write_replicated must fdatasync inline (persist-before-"
"ACK): the sync counter did not advance during the call");
/* And the entry is genuinely on stable storage by the time the call
* returned (would survive an immediate process loss → recoverable ACK). */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 1);
teardown_durability_persist(p);
printf("test_replicated_persist_syncs_before_return: PASS\n");
}
/* ------------------------------------------------------------------ *
* task 2.3 — forced standby-sync failure ⇒ error, NO success/ACK *
* *
* Closes the Req-1.3 hole: if the inline fdatasync FAILS, the standby *
* must NOT report success — otherwise the caller would ACK an entry *
* that is not durable (durable AFTER, not BEFORE, the ACK). With the *
* force-fail seam ON, persist_wal_write_replicated must return a *
* negative error; the caller (ha_apply_single_entry) already treats any *
* negative return as "do not apply, do not ACK". *
* *
* RED evidence: against the pre-2.3 code (which ignored the fdatasync *
* return at the standby path) this returned 0 (success) under forced *
* failure — a non-durable ACK. The rc-gate makes it return -EIO. *
* ------------------------------------------------------------------ */
static void test_replicated_sync_failure_no_ack(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replsyncfail_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Force the fdatasync to be treated as failed, then attempt a replicated
* write. The durable sync cannot succeed, so the call MUST fail (no ACK).
* A negative return is what ha_apply_single_entry routes to its
* "not applying/acking" path. */
persist_test_set_force_fsync_fail(p, 1);
int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_TRUE_MESSAGE(
rc < 0,
"forced standby fdatasync failure must make persist_wal_write_"
"replicated FAIL (negative rc) so the standby does NOT ACK a "
"non-durable entry");
/* The sync was still attempted (counter advanced) — we fail loud on the
* RESULT, not by skipping the sync. */
TEST_ASSERT_TRUE(persist_test_replicated_fdatasync_count(p) >= 1);
/* Recovery: clear the fault; a fresh replicated write (new lsn, not the
* deduped lsn=1) now succeeds and is durable — proving the failure path
* did not corrupt persist state and the sync runs again on success.
* (lsn=1 already advanced current_lsn to 2, so re-using 1 would dedup;
* lsn=2 is the genuine next entry.) */
persist_test_set_force_fsync_fail(p, 0);
int rc2 = persist_wal_write_replicated(p, /*lsn=*/2, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_EQUAL_INT(0, rc2);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 2);
teardown_durability_persist(p);
printf("test_replicated_sync_failure_no_ack: PASS\n");
}
/* ================================================================== *
* task 2.4 — reconcile the dormant synchronous-append commit path *
* *
* persist_wal_sync_append's LEGACY branch (ce == NULL, or the LSN-0 *
* first entry on the engine path) historically advanced committed_lsn *
* DIRECTLY on a majority ACK (atomic_store committed_lsn = written_lsn) *
* with NO durable precondition. Group-commit removed the premise that *
* the append already fsync'd inline, so a majority ACK no longer *
* implies the local entry is durable. This branch is the structural *
* ACK-without-durability hazard task 2.4 must close: NO path may *
* advance committed_lsn on a majority ACK unless the entry is durable *
* locally (committed_lsn <= durable_lsn). *
* *
* NOTE: persist_wal_sync_append is NOT dormant — it is the LIVE *
* job-submit path (persist_job_hook_fun, WAL_JOB_SUBMIT). So the fix is *
* to ROUTE the legacy direct-store through the durable precondition, *
* not to delete the function. *
* ------------------------------------------------------------------ */
/* Build an ENGINE-LESS persist with standbys registered, so
* persist_wal_sync_append takes the legacy branch (ce == NULL) and its
* has_standbys path: repl_wait_majority_ack returns 0 immediately for
* the LSN-0 first entry, but for a NON-zero LSN it surfaces -EINVAL via
* commit_submit (engine-less). The hazard we guard is the direct
* committed_lsn store that the legacy branch performs on a 0 return. */
/* Drive the legacy branch directly: a single-member commit engine makes
* the ENGINE path durable-safe, so to exercise the legacy direct-store
* we use an engine-less persist and a registered standby that ACKs.
* Rather than stand up the replication wire (out of boundary), we test
* the invariant the fix establishes: committed_lsn must never exceed
* durable_lsn after a commit advance, on EVERY path. */
/* ------------------------------------------------------------------ *
* Test (RED before fix): the legacy direct-store path must NOT advance *
* committed_lsn past durable_lsn. We drive the legacy branch via the *
* LSN-0-style direct store by appending one entry that is NOT yet *
* durable (slow cadence, no flush) and confirming committed_lsn stays *
* at/below durable_lsn. *
* *
* Mechanism: persist_test_sync_append_legacy_store() is an XCE_TESTING *
* seam that invokes the SAME guarded store the legacy branch performs *
* (advance committed_lsn to written_lsn only if written_lsn <= *
* durable_lsn). Pre-fix the store was unconditional, so this over- *
* advanced committed past durable (RED). Post-fix it holds. *
* ------------------------------------------------------------------ */
static void test_legacy_sync_append_holds_committed_at_durable(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_legacycommit_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Append one entry (LSN 1) but do NOT flush — durable_lsn lags at 0,
* while the appended tip is 1. The legacy direct-store would advance
* committed_lsn to the ACK'd written_lsn (1) — over-advancing past
* the durable watermark (0). The durable precondition must hold it. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t written_lsn = p->current_lsn - 1; /* 1 */
TEST_ASSERT_EQUAL_UINT64(1, written_lsn);
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Invoke the guarded legacy store directly (simulates a majority ACK
* for an entry that is not yet durable locally). It must NOT advance
* committed_lsn past durable_lsn. */
persist_test_sync_append_legacy_commit_store(p, written_lsn);
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&p->committed_lsn) <= persist_durable_lsn(p),
"legacy sync-append commit store must NOT advance committed_lsn past "
"durable_lsn (ACK-without-durability hazard)");
/* Concretely: committed stays at 0 because durable_lsn is still 0. */
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Now make the entry durable and re-invoke: the store is now allowed
* (written_lsn <= durable_lsn) and committed advances to it. */
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) >= written_lsn);
persist_test_sync_append_legacy_commit_store(p, written_lsn);
TEST_ASSERT_EQUAL_UINT64(written_lsn, atomic_load(&p->committed_lsn));
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_legacy_sync_append_holds_committed_at_durable: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test: the LSN-0 first-entry edge stays valid. durable_lsn and *
* committed_lsn both start at 0; the legacy store of written_lsn == 0 *
* (0 <= 0) is permitted and is a harmless no-op store of 0. This is the *
* once-per-cluster-lifetime path the engine branch routes here. *
* ------------------------------------------------------------------ */
static void test_legacy_sync_append_lsn0_edge_valid(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_legacylsn0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Fresh store: durable_lsn == 0, committed_lsn == 0. */
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Storing written_lsn == 0 (0 <= durable 0) is allowed — a no-op. */
persist_test_sync_append_legacy_commit_store(p, 0);
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_legacy_sync_append_lsn0_edge_valid: PASS\n");
}
/* ------------------------------------------------------------------ *
* Guard test (the Observable's "test guards against a future caller"): *
* the committed_lsn <= durable_lsn invariant holds across the commit- *
* engine advance path too. Drives the real engine + flusher and asserts *
* that at every observed point committed never exceeds durable — the *
* future-proof guard against reintroducing an ACK-without-durability *
* commit on ANY path. In a debug build the in-code assert would abort *
* if a path ever advanced committed past durable. *
* ------------------------------------------------------------------ */
static void test_commit_never_exceeds_durable_invariant(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_cinvariant_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, 1, 8));
/* Append a burst while sampling the invariant: committed must never
* outrun durable, even mid-flight while the flusher advances both. */
for (int round = 0; round < 6; round++) {
for (int i = 0; i < 4; i++)
TEST_ASSERT_EQUAL_INT(
0, persist_wal_async(p, WAL_JOB_UPDATE, "q", 1));
for (int s = 0; s < 50; s++) {
uint64_t committed = atomic_load(&p->committed_lsn);
uint64_t durable = persist_durable_lsn(p);
TEST_ASSERT_TRUE_MESSAGE(
committed <= durable,
"committed_lsn must never exceed durable_lsn on any commit "
"advance path");
usleep(1000);
}
}
/* And eventually committed catches up to the durable tip. */
uint64_t tip = p->current_lsn - 1;
uint64_t committed = 0;
for (int i = 0; i < 5000; i++) {
committed = atomic_load(&p->committed_lsn);
if (committed >= tip)
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64(tip, committed);
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
commit_engine_fini(&ce);
teardown_durability_persist(p);
printf("test_commit_never_exceeds_durable_invariant: PASS\n");
}
/* ================================================================== *
* task 5.1 — wire durable-majority commit end to end *
* *
* These three tests verify the full durable-majority commit path the *
* 2.x/4.x wiring assembled, end to end over a REAL persist_t (real WAL *
* + real group-commit flusher thread) and a REAL commit_engine_t with *
* registered standbys: *
* *
* 1. reply-ok ⇒ durable-majority (Req 1.1, 1.4): committed_lsn *
* advances and a WAIT caller releases ONLY after the entry is *
* durable on a fault-tolerant majority — self durable (durable_lsn *
* covers it, via the real flusher) AND a standby ACK. With *
* durability held off, even a standby ACK does NOT release. *
* 2. sole-durable-contributor loss survives (Req 1.5, 2.5): an op *
* committed on self + a standby is durable on >= floor(N/2)+1 *
* members, so losing one durable contributor leaves it present on *
* a surviving member (modeled at the integration level; the live *
* failover proof is task 5.2). *
* 3. liveness — durability work off the iomgr (Req 1.6): fdatasync *
* runs on the dedicated flusher thread, not the caller/iomgr *
* thread, so a heartbeat-style callback is not stalled while a *
* durable flush is in flight. *
* ------------------------------------------------------------------ */
/* Dummy standby endpoints — the replicator only compares pointers, and
* commit_on_ack reads acked_lsn from the registry slot. */
static int g_dur_eps[MAX_STANDBYS];
/* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps
* repl_broadcast from dereferencing the dummy endpoints as real
* endpoint_t when persist_wal_async fans out to the registered standbys.
* Same pattern as test_commit_engine.c. */
extern int (*repl_broadcast_send_hook)(void* ep, wal_catchup_entry_t** entries,
int n_entries);
static int dur_drop_send_hook(void* ep, wal_catchup_entry_t** entries,
int n_entries)
{
(void)ep;
(void)entries;
(void)n_entries;
return 0; /* pretend success; drop the data (no real wire in a unit test) */
}
/* Feed a standby ACK exactly as the iomgr read path does: store the
* watermark into the registry slot, then notify the commit engine. */
static void durable_standby_ack(persist_t* p, commit_engine_t* ce, int idx,
uint64_t acked_lsn)
{
atomic_store(&p->repl.standbys[idx].acked_lsn, acked_lsn);
commit_on_ack(ce, idx, acked_lsn);
}
/* A WAIT submit that runs on its own thread, so the test can observe that
* it does NOT return while the entry is not yet durable-majority, then DOES
* return once it becomes durable-majority (mirrors the off-iomgr AFTER_EVENT
* caller blocking on the commit barrier). */
typedef struct {
commit_engine_t* ce;
uint64_t lsn;
uint32_t timeout_ms;
_Atomic int returned;
int rc;
} wait_submit_arg_t;
static void* wait_submit_thread(void* a)
{
wait_submit_arg_t* w = (wait_submit_arg_t*)a;
w->rc = commit_submit(w->ce, w->lsn, COMMIT_MODE_WAIT, w->timeout_ms);
atomic_store(&w->returned, 1);
return NULL;
}
/* ------------------------------------------------------------------ *
* Test 5.1a (MUST-HAVE, Req 1.1 / 1.4): reply-ok only after a durable *
* majority. 3-member cluster (self + 2 standbys), so the majority for *
* an entry is self + at least 1 standby. We: *
* - force the local fdatasync to FAIL, so the flusher cannot advance *
* durable_lsn (self is NOT durable); *
* - append an entry and deliver a standby ACK covering it; *
* - assert committed_lsn does NOT advance and a WAIT submit does NOT *
* release — a majority of ACKs is not enough without local *
* durability (this is the durable precondition gate; pre-2.2 this *
* would have committed on the ACK alone — RED); *
* - clear the fault so the real flusher fdatasyncs and advances *
* durable_lsn, firing the durable-advance wake; *
* - assert committed_lsn now advances and the WAIT submit releases *
* promptly (durable self + durable standby ACK = durable majority). *
* ------------------------------------------------------------------ */
static void test_durable_majority_commit_end_to_end(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durmaj_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Intercept the replication fan-out so persist_wal_async does not try to
* send over the dummy endpoints (they are not real endpoint_t). */
repl_broadcast_send_hook = dur_drop_send_hook;
/* Register two standbys against the REAL persist's replicator. n_peers=3
* (self + 2 standbys); majority for entry N = self durable + >= 1 standby
* with acked_lsn >= N. n_vcpus high enough to admit a WAIT. */
TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&p->repl, &g_dur_eps[0], "sb0",
7000));
TEST_ASSERT_EQUAL_INT(1, repl_add_standby(&p->repl, &g_dur_eps[1], "sb1",
7001));
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, /*n_peers=*/3,
/*n_vcpus=*/8));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Hold durability off: every flusher fdatasync is forced to fail, so
* durable_lsn cannot advance. (Fail-loud: the entry is never counted
* durable, so the commit cannot advance regardless of ACKs.) */
persist_test_set_force_fsync_fail(p, 1);
/* Append one entry (LSN 1). It is in page cache only; the forced-fail
* flusher will NOT make it durable. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t lsn = p->current_lsn - 1; /* 1 */
TEST_ASSERT_EQUAL_UINT64(1, lsn);
/* Register a WAIT for LSN 1 on its own thread (the off-iomgr commit-barrier
* caller). It must block: the entry is not durable-majority. */
wait_submit_arg_t w = {.ce = &ce, .lsn = lsn, .timeout_ms = 2000};
pthread_t th;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th, NULL, wait_submit_thread, &w));
/* Deliver a covering standby ACK from BOTH standbys (a clear majority of
* ACKs). Pre-durability this alone advanced committed_lsn (the bug);
* post-2.2 the self-cap holds it at the non-durable self (durable_lsn=0). */
durable_standby_ack(p, &ce, 0, lsn);
durable_standby_ack(p, &ce, 1, lsn);
/* Give the (forced-failing) flusher ample time to run; durable_lsn must
* stay 0, committed must stay 0, and the WAIT must NOT have returned. */
for (int i = 0; i < 300; i++) { /* up to ~300 ms */
if (persist_durable_lsn(p) != 0 ||
atomic_load(&p->committed_lsn) != 0 ||
atomic_load(&w.returned))
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, persist_durable_lsn(p),
"durable_lsn must not advance while fdatasync is failing");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, atomic_load(&p->committed_lsn),
"committed_lsn must NOT advance on a standby-ACK majority while the "
"local entry is not durable (durable precondition gate)");
TEST_ASSERT_FALSE_MESSAGE(
atomic_load(&w.returned),
"the WAIT (client reply) must NOT release before durable-majority");
/* Now allow durability: clear the fault and force a flush. The REAL
* flusher fdatasyncs, advances durable_lsn over LSN 1, and fires the
* durable-advance wake — self is now durable AND both standbys already
* ACKed, so the entry is durable on a majority. committed_lsn advances
* and the WAIT releases promptly. */
persist_test_set_force_fsync_fail(p, 0);
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) >= lsn);
/* The WAIT must now return success (committed) within its timeout. */
TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL));
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&w.returned),
"the WAIT must release once the entry is durable on a majority");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, w.rc,
"commit_submit(WAIT) must return 0 (committed) after durable-majority");
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&p->committed_lsn) >= lsn,
"committed_lsn must advance once self is durable AND a standby ACK "
"covers the entry (durable majority)");
/* The invariant the whole spec rests on: committed never exceeds durable. */
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
commit_engine_fini(&ce);
repl_remove_standby(&p->repl, &g_dur_eps[0]);
repl_remove_standby(&p->repl, &g_dur_eps[1]);
teardown_durability_persist(p);
repl_broadcast_send_hook = NULL;
printf("test_durable_majority_commit_end_to_end: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1b (Req 1.5 / 2.5): losing the sole durable contributor *
* leaves the acknowledged op present on a surviving member. *
* *
* The durable-majority property says: when an op is acknowledged, its *
* entry is fdatasync'd on a fault-tolerant majority (self + >= *
* floor(N/2) others). For N=3 that is 2 durable copies. So losing ONE *
* durable contributor still leaves >= 1 surviving durable copy that *
* holds the op — it is recoverable from the survivor. *
* *
* We model the two durable copies as two independent persist_t over *
* separate WAL dirs (a primary's copy and a standby's copy), both made *
* genuinely durable (fdatasync'd). We then DROP the "sole durable *
* contributor" (the first copy, simulating its loss/restart-with-loss) *
* and prove the op is still durably present and recoverable on the *
* surviving copy via a fresh-process recovery (no in-memory state). *
* *
* This is the integration-level assertion of the property that makes *
* survival true (committed ⇒ >= floor(N/2)+1 durable copies); the live *
* multi-host SIGKILL failover proof is task 5.2. *
* ------------------------------------------------------------------ */
static void test_sole_durable_contributor_loss_survives(void)
{
char tmp_a[256];
char tmp_b[256];
strcpy(tmp_a, "/tmp/xce_durlossA_XXXXXX");
strcpy(tmp_b, "/tmp/xce_durlossB_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp_a));
TEST_ASSERT_NOT_NULL(mkdtemp(tmp_b));
const char* payload = "committed-op";
uint32_t plen = (uint32_t)strlen(payload);
/* Durable copy A (the contributor we will lose) and durable copy B (the
* survivor): write the SAME committed op to both and fdatasync each, so
* the op is durable on a 2-of-3 majority. */
{
persist_t* a = make_durability_persist(tmp_a);
TEST_ASSERT_NOT_NULL(a);
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(a, WAL_JOB_UPDATE, payload, plen));
TEST_ASSERT_EQUAL_INT(0, persist_flush(a));
TEST_ASSERT_TRUE(persist_durable_lsn(a) >= 1);
teardown_durability_persist(a);
}
{
persist_t* b = make_durability_persist(tmp_b);
TEST_ASSERT_NOT_NULL(b);
/* The standby applies the SAME replicated entry durably before ACK
* (persist_wal_write_replicated fdatasyncs inline — Req 1.3). */
TEST_ASSERT_EQUAL_INT(0,
persist_wal_write_replicated(b, /*lsn=*/1, WAL_JOB_UPDATE,
payload, plen));
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(b) >= 1);
teardown_durability_persist(b);
}
/* "Lose" the sole durable contributor A: destroy its on-disk WAL entirely
* (simulates a member that is gone / came back with a truncated log). */
{
char rm[600];
snprintf(rm, sizeof(rm), "rm -rf %s", tmp_a);
TEST_ASSERT_EQUAL_INT(0, system(rm));
}
/* The acknowledged op MUST still be present and recoverable on the
* surviving member B — read straight from disk by a fresh process with
* NO in-memory state (the durable-majority property in action). */
{
persist_t* b2 = make_durability_persist(tmp_b);
TEST_ASSERT_NOT_NULL(b2);
/* On-disk durable floor (CRC-validated re-read) still covers the op. */
TEST_ASSERT_TRUE_MESSAGE(
persist_test_durable_lsn_on_disk(b2) >= 1,
"the acknowledged op must survive on the durable majority after "
"the sole-durable-contributor copy is lost");
/* And a real recovery reconstructs it cleanly (not truncated). */
TEST_ASSERT_EQUAL_INT(0, persist_recover(b2));
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(b2),
"the survivor recovers the committed op as a clean intact log");
TEST_ASSERT_TRUE_MESSAGE(
persist_durable_floor(b2) >= 1,
"the survivor's recovered durable floor covers the committed op");
teardown_durability_persist(b2);
}
{
char rm[600];
snprintf(rm, sizeof(rm), "rm -rf %s", tmp_b);
(void)system(rm);
}
printf("test_sole_durable_contributor_loss_survives: PASS "
"(durable-majority property; live failover proof is task 5.2)\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1c (Req 1.6): durability work does not stall heartbeat / *
* election handling — fdatasync runs on the dedicated flusher thread, *
* NOT on the caller (iomgr/heartbeat) thread. *
* *
* Structural + timing assertion: while a durable flush is in flight on *
* the flusher thread, a heartbeat-style callback executed on the *
* caller/iomgr thread completes within a tight tolerance. We prove the *
* flush is OFF the caller thread by capturing the flusher's thread id at *
* the fdatasync site (via the XCE_TESTING seam) and asserting it is NOT *
* the caller thread. (persist_wal_async only appends + signals; it never *
* fdatasyncs inline, so the append path itself never blocks on disk.) *
* ------------------------------------------------------------------ */
static void test_durability_work_off_iomgr_preserves_liveness(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durliveness_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
pthread_t caller = pthread_self();
/* Drive a steady stream of appends on the CALLER (iomgr-equivalent)
* thread; the real flusher thread does the fdatasync work concurrently.
* After each append, a heartbeat-style callback must complete promptly —
* the append path signals the flusher and returns without blocking on
* fdatasync, so heartbeat handling on this thread is never stalled. */
const long tolerance_us = 50 * 1000; /* 50 ms — generous heartbeat budget */
for (int i = 0; i < 64; i++) {
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "h", 1));
struct timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
/* Heartbeat-style work on the caller thread (a cheap atomic read of
* commit/durable state, exactly what the heartbeat path inspects). */
volatile uint64_t hb = atomic_load(&p->committed_lsn) +
persist_durable_lsn(p);
(void)hb;
clock_gettime(CLOCK_MONOTONIC, &t1);
long elapsed_us = (t1.tv_sec - t0.tv_sec) * 1000000L +
(t1.tv_nsec - t0.tv_nsec) / 1000L;
TEST_ASSERT_TRUE_MESSAGE(
elapsed_us < tolerance_us,
"heartbeat-style work on the caller thread must not be stalled "
"by durability (fdatasync) work in flight on the flusher thread");
}
/* Let the flusher make everything durable, then confirm the fdatasync
* provably ran on a DIFFERENT thread than this caller — durability work
* is off the iomgr/heartbeat thread (Req 1.6). The seam records the
* thread id of the last successful flusher fdatasync. */
uint64_t tip = p->current_lsn - 1;
TEST_ASSERT_TRUE(wait_durable_reaches(p, tip, 5000) >= tip);
pthread_t flusher_tid = persist_test_last_fdatasync_thread(p);
TEST_ASSERT_FALSE_MESSAGE(
pthread_equal(flusher_tid, caller),
"fdatasync must run on the dedicated flusher thread, never on the "
"caller/iomgr/heartbeat thread");
teardown_durability_persist(p);
printf("test_durability_work_off_iomgr_preserves_liveness: PASS\n");
}
/* ================================================================== *
* task 3.1 — recovery integrity verdict (Req 2.1) *
* *
* persist_recover/replay_wal must distinguish a CLEAN log end (EOF, *
* pre-allocated-zero tail, snapshot-sentinel + clean replay, empty *
* first-boot WAL) from a corruption-TRUNCATED end (CRC chain break, *
* CRC mismatch, short/partial entry read). A corruption-truncated *
* recovery is NEVER reported as a clean recovery and sets a *
* member-level recovery_truncated flag the promotion gate (4.3) *
* consumes. 3.1 only SETS the verdict/flag — it does NOT FATAL and *
* does NOT fence (3.3 / 4.3 own those). *
* *
* NB the verdict is about STRUCTURAL integrity of the local log, not *
* about being behind the cluster: a clean log that ends below the *
* cluster high-water is still CLEAN (the freshness/promotion fence *
* catches "behind", not this verdict). *
* ------------------------------------------------------------------ */
/* The persist paths derive as <workdir>/<name>/data/wal; make_durability_persist
* uses name "ctrl-test" (see persist_set_workdir call above). */
#define DUR_WAL_SEG_FMT "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal"
/* Open the first WAL segment (starts at LSN 1; LSN 0 reserved) for the
* given tmpdir and return its fd, or -1. */
static int open_first_seg_rw(const char* tmpdir)
{
char seg_path[512];
/* The first-ever entry is LSN 1, so the first segment is seg-...0001. */
snprintf(seg_path, sizeof(seg_path), DUR_WAL_SEG_FMT, tmpdir,
(uint64_t)1);
return open(seg_path, O_RDWR);
}
/* Open a SECOND persist instance over the same on-disk WAL with NO
* in-memory writer state, run recovery, and return the recovery verdict
* (via persist_recovery_truncated). The writer must already be torn down
* so the segments are closed and on disk. */
/* ------------------------------------------------------------------ *
* Test 3.1a: an intact log yields RECOVER_CLEAN (flag NOT set). *
* ------------------------------------------------------------------ */
static void test_recover_verdict_clean_intact_log(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recclean_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: append + flush three entries, then "die". */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Recover into a fresh instance: an intact CRC chain must be CLEAN. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"an intact CRC chain must recover CLEAN");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_truncated(p2),
"a clean recovery must NOT set the recovery_truncated flag");
/* current_lsn advanced past the three replayed entries. */
TEST_ASSERT_EQUAL_UINT64(4, p2->current_lsn);
teardown_durability_persist(p2);
}
printf("test_recover_verdict_clean_intact_log: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1b: an empty first-boot WAL is CLEAN, not corruption. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_empty_first_boot_clean(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recempty_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* No appends: a first-boot WAL has nothing to replay — that is a clean
* (empty) recovery, never a corruption-truncated one. */
int rc = persist_recover(p);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p),
"an empty first-boot WAL must be CLEAN (no entries != corruption)");
TEST_ASSERT_FALSE(persist_recovery_truncated(p));
teardown_durability_persist(p);
printf("test_recover_verdict_empty_first_boot_clean: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1c (core RED): a CRC break injected at the FIRST entry yields *
* RECOVER_TRUNCATED and SETS the recovery_truncated flag. *
* *
* We append+flush two entries, then corrupt the on-disk `crc` field of *
* the FIRST entry (LSN 1). On recovery the CRC verification at LSN 1 *
* fails, replay stops before any clean boundary, and the verdict must *
* be TRUNCATED — never silently accepted as a complete clean recovery. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_crc_break_first_entry_truncated(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recbreak_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: two durable entries, then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Corrupt the FIRST entry's CRC field on disk: read its header at
* offset 0, flip the crc, write it back. This breaks the chain at
* LSN 1 — replay cannot reach any clean boundary. */
{
int fd = open_first_seg_rw(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
wal_entry_hdr_t hdr;
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), 0));
hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pwrite(fd, &hdr, sizeof(hdr), 0));
fsync(fd);
close(fd);
}
/* Recover: the corruption at LSN 1 must yield TRUNCATED + set the flag. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
/* persist_recover still returns 0 (no I/O error); the VERDICT is the
* signal — 3.1 classifies, it does not FATAL (that is task 3.3). */
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_TRUNCATED, persist_recovery_verdict(p2),
"a CRC break at the first entry must yield RECOVER_TRUNCATED");
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_truncated(p2),
"a corruption-truncated recovery must SET the recovery_truncated "
"flag (consumed by the promotion gate, task 4.3)");
teardown_durability_persist(p2);
}
printf("test_recover_verdict_crc_break_first_entry_truncated: PASS\n");
}
/* ------------------------------------------------------------------ *
* bug-778 regression: the standby replicated-apply path must leave an *
* intact on-disk CRC chain that cold-restart replay can recover. *
* *
* Drives the REAL standby path (ha_apply_wal_catchup_entry -> *
* ha_apply_single_entry -> persist_wal_write_replicated + apply_wal_ *
* entry) for N entries, then cold-recovers. Before the fix, *
* apply_wal_entry clobbered p->last_crc with the synthetic header's *
* crc=0, so every entry after the first wrote prev_crc=0 -> CRC chain *
* break -> RECOVER_TRUNCATED and the committed entries were lost on *
* cold restart. cluster==NULL here (entity restore is skipped); the *
* physical WAL write + CRC chain is what is under test. *
* ------------------------------------------------------------------ */
static void test_replicated_apply_chain_survives_cold_restart(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replchain_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
const int N = 6;
/* Writer: apply N replicated entries via the real standby path, then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
ha_state_t ha;
memset(&ha, 0, sizeof(ha));
for (int i = 1; i <= N; i++) {
wal_jobstate_t js = { .jobid = (uint64_t)i, .state = 0x10 };
wal_catchup_entry_t e;
memset(&e, 0, sizeof(e));
e.lsn = (uint64_t)i;
e.entry_type = WAL_JOB_UPDATE;
e.n_entries = 0; /* single-entry mode */
e.raw_payload = (uint8_t*)&js;
e.raw_payload_len = sizeof(js);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, ha_apply_wal_catchup_entry(&ha, p, &e, NULL, NULL),
"replicated apply of a streamed entry must succeed");
}
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Cold restart: the standby's own WAL must form an intact CRC chain. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
TEST_ASSERT_EQUAL_INT(0, persist_recover(p2));
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"replicated-apply WAL must form an intact CRC chain on cold restart "
"(bug-778: apply_wal_entry must not clobber the chain link)");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_truncated(p2),
"no truncation: every replicated entry must be recoverable");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
(uint64_t)N + 1, p2->current_lsn,
"all N replicated entries must replay (current_lsn == N+1)");
teardown_durability_persist(p2);
}
printf("test_replicated_apply_chain_survives_cold_restart: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1d (snapshot-sentinel edge): a member that recovered from a *
* 16-byte LSN-sentinel snapshot has a committed position at the *
* 16-byte LSN-sentinel snapshot has a committed position at the *
* sentinel LSN but NO log entries below it. A clean replay over that *
* sentinel snapshot must classify CLEAN, never TRUNCATED — the absence *
* of sub-sentinel entries is normal, not corruption (design Open *
* Questions / snapshot-recovery edge). *
* *
* We checkpoint (writes the sentinel snapshot + trims the WAL), append *
* + flush one fresh entry above the checkpoint, then recover in a fresh *
* instance: snapshot-sentinel + clean replay ⇒ CLEAN. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_snapshot_sentinel_clean(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recsnap_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: a couple of entries, a checkpoint (sentinel snapshot), then a
* further entry above the checkpoint, all flushed; then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Sentinel snapshot at the current committed/checkpoint position. */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Recover: load_snapshot sets checkpoint_lsn from the sentinel, replay
* picks up entries above it cleanly — verdict CLEAN, flag NOT set. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"snapshot-sentinel + clean replay must be CLEAN, not TRUNCATED");
TEST_ASSERT_FALSE(persist_recovery_truncated(p2));
teardown_durability_persist(p2);
}
printf("test_recover_verdict_snapshot_sentinel_clean: PASS\n");
}
/* ================================================================== *
* task 3.4 — re-derive the durable floor from the recovered WAL tip *
* (Req 2.3) *
* *
* On recovery the member publishes its durable floor as the highest *
* INTACT replayed LSN (the WAL tip / durable last-log index), so the *
* freshness advertisement (task 4.2) is backed by durable on-disk *
* state rather than a volatile in-memory position. The floor is the *
* durable last-log index, NOT the commit position, and NOT a stale *
* higher in-memory value. persist_durable_floor() exposes it (a thin *
* wrapper over the durable_lsn watermark, which recovery sets to the *
* recovered tip). *
* ------------------------------------------------------------------ */
/* Test 3.4a (CRUX, Req 2.3): after a TRUNCATING restart the published
* durable floor equals the highest INTACT LSN (the WAL tip), NOT a stale
* higher in-memory value and NOT the commit position.
*
* Writer lays down 4 durable entries (LSN 1..4). We then corrupt entry 3's
* CRC on disk so the intact prefix is LSN 1..2 — replay stops at the chain
* break and the recovered tip is LSN 2. To make the "not a stale higher
* in-memory value" assertion meaningful we deliberately seed the fresh
* instance's in-memory durable_lsn / committed_lsn to an over-advertised
* value (4) BEFORE recovery; recovery must publish the durable FLOOR at the
* intact tip 2, never the seeded 4. */
static void test_durable_floor_truncating_restart_is_intact_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floortrunc_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: four durable entries (LSN 1..4), then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "d", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* All four are durable on disk before the truncation. */
TEST_ASSERT_EQUAL_UINT64(4, persist_test_durable_lsn_on_disk(p));
teardown_durability_persist(p);
}
/* Corrupt entry 3 (LSN 3): walk the segment to its offset and flip its
* crc. Each entry is sizeof(hdr) + payload_len (=1) bytes; entries 1 and 2
* precede it. This breaks the chain at LSN 3, so the intact tip is LSN 2. */
{
int fd = open_first_seg_rw(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
uint32_t off = 0;
wal_entry_hdr_t hdr;
/* Skip entries 1 and 2 (LSN 1, LSN 2). */
for (int i = 0; i < 2; i++) {
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), (off_t)off));
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
/* Entry 3 (LSN 3) is at `off`: corrupt its CRC. */
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), (off_t)off));
TEST_ASSERT_EQUAL_UINT64(3, hdr.lsn);
hdr.crc ^= 0xDEADBEEFu;
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pwrite(fd, &hdr, sizeof(hdr), (off_t)off));
fsync(fd);
close(fd);
}
/* Recover into a fresh instance that we deliberately poison with a stale,
* over-advertised in-memory position (4) BEFORE recovery. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
/* Stale higher in-memory values: simulate a volatile position that
* over-states what is durable. The published floor must NOT come from
* these — it must come from the recovered intact WAL tip. */
atomic_store(&p2->durable_lsn, 4);
atomic_store(&p2->committed_lsn, 4);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Verdict is TRUNCATED (chain break at LSN 3). */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_TRUNCATED, persist_recovery_verdict(p2),
"a CRC break at LSN 3 must yield RECOVER_TRUNCATED");
/* The published durable floor == the highest INTACT LSN (2): the WAL
* tip, equal to current_lsn - 1 after replay (replay advanced
* current_lsn only to one past the last intact entry). */
uint64_t floor = persist_durable_floor(p2);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
2, floor,
"durable floor after a truncating restart must equal the highest "
"intact LSN (the WAL tip), NOT the stale in-memory value");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
p2->current_lsn - 1, floor,
"durable floor must equal the recovered WAL tip (current_lsn - 1)");
/* Independently confirmed against the on-disk re-read seam. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
2, persist_test_durable_lsn_on_disk(p2),
"on-disk re-read agrees: only LSN 1..2 are intact");
teardown_durability_persist(p2);
}
printf("test_durable_floor_truncating_restart_is_intact_tip: PASS\n");
}
/* Test 3.4b: an INTACT-log recovery publishes the FULL tip (no truncation).
* Three durable entries (LSN 1..3) recover cleanly; the floor is the full
* recovered tip 3 == current_lsn - 1. */
static void test_durable_floor_intact_recovery_full_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floorintact_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
3, persist_durable_floor(p2),
"intact recovery publishes the full recovered WAL tip");
TEST_ASSERT_EQUAL_UINT64(p2->current_lsn - 1, persist_durable_floor(p2));
teardown_durability_persist(p2);
}
printf("test_durable_floor_intact_recovery_full_tip: PASS\n");
}
/* Test 3.4c: an EMPTY first-boot WAL publishes floor 0 (nothing durable). */
static void test_durable_floor_empty_recovery_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floorempty_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
int rc = persist_recover(p);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, persist_durable_floor(p),
"an empty first-boot WAL has nothing durable — floor is 0");
teardown_durability_persist(p);
printf("test_durable_floor_empty_recovery_zero: PASS\n");
}
/* Test 3.4d (snapshot-sentinel edge): a member recovered from a 16-byte
* LSN-sentinel snapshot plus a clean replay above it publishes the recovered
* tip (the highest replayed LSN). The floor is the recovered tip, consistent
* with 3.1's CLEAN classification — "behind-and-must-catch-up", not
* over-advertising. */
static void test_durable_floor_snapshot_sentinel_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_floorsnap_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Sentinel snapshot at the checkpoint position; then one entry above. */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2));
/* The floor is the recovered tip = current_lsn - 1 (whatever the
* highest replayed LSN above the sentinel is). It is the recovered
* tip, not a stale higher value. */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
p2->current_lsn - 1, persist_durable_floor(p2),
"snapshot-sentinel recovery publishes the recovered WAL tip");
TEST_ASSERT_TRUE_MESSAGE(
persist_durable_floor(p2) > 0,
"a sentinel + clean replay-above has a non-zero recovered tip");
teardown_durability_persist(p2);
}
printf("test_durable_floor_snapshot_sentinel_tip: PASS\n");
}
/* ================================================================== *
* task 3.2 — Boot Policy Decision Table (DECISION ONLY) *
* *
* Task 3.2 pins the fail-loud-vs-fence boot policy as a DECISION and *
* encodes it as a table of input states -> expected outcome. It makes *
* NO production behavior change: the FATAL branch is implemented by *
* task 3.3 and the promotion fence + release by task 4.3. *
* *
* The decision table lives in design.md ("Boot Policy Decision Table *
* (task 3.2)"). The skeletons below mirror it ONE TEST PER ROW. Each *
* skeleton documents the row's inputs + the expected outcome it WILL *
* assert once 3.3/4.3 land, then TEST_IGNORE_MESSAGE()-skips so the *
* row is REGISTERED and reported as IGNORED (not failed) — the suite *
* stays GREEN. These become the RED tests 3.3/4.3 turn GREEN. *
* *
* Inputs (boot / promotion-eligibility evaluation): *
* verdict : CLEAN | TRUNCATED (from task 3.1) *
* durable-vs-HWM : at/above | below (durable position) *
* healthy peer : yes | no (catch-up source) *
* catch-up : succeeds | fails (bounded window) *
* Outcomes: *
* ELIGIBLE | FENCE+CATCH-UP -> ELIGIBLE | FATAL *
* INVARIANT: never a silent indefinite fence — a truncated/behind *
* member either catches up (-> eligible) or FATALs on a BOUNDED *
* failure. (Req 2.2, 2.4) *
* ------------------------------------------------------------------ */
/* task 3.3 — unrecoverable loud-fence verification (never-self-exit rework
* 2026-06-18). The OLD behavior installed persist_fatal_hook to capture an
* abort()/exit() without dying. That self-exit terminal is GONE: an
* unrecoverable member STAYS ALIVE and is held permanently non-promotable.
* There is no hook to install — the tests assert the OBSERVABLE
* post-conditions instead:
* - the pure decision returns RECOVER_DECISION_FENCE_UNRECOVERABLE,
* - persist_recover_loud_fence_if_unrecoverable RETURNS (no abort — the test
* simply continuing past the call proves the process stayed alive),
* - the sticky persist_recovery_unrecoverable flag is SET (member is
* permanently non-promotable until catch-up),
* - and the flag CLEARS on catch-up (Req 2.6).
* dur_persist_for_flag builds a real persist_t the loud-fence helper can set
* the sticky flag on. */
static persist_t* dur_flag_persist(const char* tmpdir)
{
persist_t* p = calloc(1, sizeof(*p));
if (!p) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
persist_set_workdir(p, tmpdir, "ctrl-flag");
if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; }
return p;
}
static void dur_free_flag_persist(persist_t* p)
{
if (!p) return;
persist_fini(p);
free(p);
}
/* task 3.3 — peer-reachable NEVER FATALs (the never-FATAL aspect of rows
* 3 & 5). This task (3.3) owns only the FATAL terminal; the full FENCE+
* CATCH-UP -> ELIGIBLE eligibility outcome of these rows is task 4.3 (so the
* row3/row5 skeletons below stay IGNORED for 4.3). Here we pin ONLY the
* invariant 3.3 is responsible for: while a healthy peer is reachable, the
* decision is NEVER FATAL — neither for a TRUNCATED log (row 3) nor for a
* legitimate snapshot-sentinel CLEAN-but-behind member (row 5). It is FENCE
* (catch-up pending) and the FATAL terminate path is never fired. (Req 2.2,
* 2.4, design.md "never FATAL when a peer is reachable".) */
static void test_boot_policy_peer_reachable_never_fatal(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_dur_peerfence_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = dur_flag_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE. */
recover_policy_inputs_t truncated_peer = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 3,
.cluster_hwm_lsn = 10,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&truncated_peer),
"row 3: a truncated member with a reachable healthy peer must FENCE "
"(catch up), NEVER the unrecoverable terminal");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_loud_fence_if_unrecoverable(p, &truncated_peer));
/* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE.
* A legitimate sentinel is CLEAN-but-behind, not corruption. */
recover_policy_inputs_t sentinel_peer = {
.verdict = RECOVER_CLEAN,
.durable_below_hwm = true,
.healthy_peer_reachable = true,
.catch_up_succeeded = false,
.durable_floor_lsn = 5,
.cluster_hwm_lsn = 5,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, persist_recover_decision(&sentinel_peer),
"row 5: a snapshot-sentinel CLEAN-but-behind member with a reachable "
"peer must FENCE, NEVER the unrecoverable terminal");
TEST_ASSERT_EQUAL_INT(
RECOVER_DECISION_FENCE,
persist_recover_loud_fence_if_unrecoverable(p, &sentinel_peer));
/* The unrecoverable sticky flag must NOT have been set for either
* peer-reachable case — a reachable healthy peer never reaches the
* unrecoverable terminal. */
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"peer-reachable rows must NEVER set the unrecoverable sticky flag");
dur_free_flag_persist(p);
printf("test_boot_policy_peer_reachable_never_fatal: PASS\n");
}
/* Row 1: CLEAN + durable at/above HWM -> ELIGIBLE.
* A structurally intact log that is not behind the cluster boundary is
* a normal promotable member: the promotion gate (4.3) must NOT fence
* it. WILL ASSERT (4.3): the eligibility predicate returns "eligible"
* for a CLEAN, at/above member; promotion proceeds. (Req 2.3) */
/* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role_lifecycle.c). */
static ha_config_t dur_ha_cfg(void)
{
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
return cfg;
}
static void test_boot_policy_row1_clean_at_or_above_eligible(void)
{
/* CLEAN + durable at/above HWM -> ELIGIBLE. The live gate must admit a
* structurally-intact, not-behind member and must NOT set the fence. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Pre-set the fence to prove ELIGIBLE clears it (Req 2.6 release path). */
atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release);
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/8, /*safe_hwm=*/8,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d,
"row 1: a CLEAN, at/above-HWM member must be ELIGIBLE to promote");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 1: ELIGIBLE must clear the in_catch_up fence");
ha_fini(&ha);
printf("test_boot_policy_row1_clean_at_or_above_eligible: PASS\n");
}
/* Row 2: CLEAN + durable below HWM + healthy peer + catch-up succeeds
* -> FENCE+CATCH-UP -> ELIGIBLE.
* Intact but behind: must NOT lead below the boundary; fence, catch up
* from a healthy peer, regain eligibility. WILL ASSERT (4.3): a
* below-HWM member is denied promotion until catch-up passes the last
* committed boundary, then regains eligibility. (Req 2.2, 2.3, 2.6) */
static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void)
{
/* CLEAN + below HWM + healthy peer: FENCE first (denied promotion), then
* once catch-up passes the committed boundary, ELIGIBLE (fence released). */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Behind: durable_floor 4 < HWM 10, healthy peer reachable, not yet caught
* up -> FENCE; the in_catch_up fence is SET (refused promotion). */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/4, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 2: a CLEAN-but-behind member with a peer must FENCE (catch up)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 2: FENCE must SET the in_catch_up fence (promotion refused)");
/* Catch-up brings the durable position to/above the committed boundary
* (durable_floor 10 >= HWM 10) -> ELIGIBLE; the fence is released (Req 2.6). */
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 2: after catch-up past the committed boundary the member regains "
"eligibility (Req 2.6)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 2: regained eligibility must RELEASE the in_catch_up fence");
ha_fini(&ha);
printf("test_boot_policy_row2_clean_below_peer_catchup_eligible: PASS\n");
}
/* Row 3: TRUNCATED + healthy peer + catch-up succeeds
* -> FENCE+CATCH-UP -> ELIGIBLE.
* Structurally truncated (task-3.1 verdict) but a healthy peer exists:
* fence, catch up, regain eligibility — NEVER FATAL when a peer is
* reachable. WILL ASSERT (4.3): a recovery_truncated member is denied
* promotion, catches up from the peer, and is re-enabled; FATAL is NOT
* triggered. (Req 2.2, 2.6) */
static void test_boot_policy_row3_truncated_peer_catchup_eligible(void)
{
/* TRUNCATED + healthy peer: a recovery-truncated member is DENIED promotion
* (FENCE), catches up from the peer, then regains eligibility. FATAL must
* NOT fire while a healthy peer is reachable. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Truncated boundary at 3, cluster HWM 10, peer reachable, not caught up. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 3: a recovery-truncated member with a healthy peer must FENCE "
"(denied promotion), never the unrecoverable terminal");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: a truncated member must be fenced (in_catch_up set)");
/* After catch-up to/above the committed boundary -> ELIGIBLE, fence released. */
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 3: after catch-up the (previously truncated) member is eligible");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 3: regained eligibility releases the fence (Req 2.6)");
ha_fini(&ha);
printf("test_boot_policy_row3_truncated_peer_catchup_eligible: PASS\n");
}
/* Row 4: TRUNCATED + below HWM + NO healthy peer + catch-up fails
* -> LOUD PERMANENT FENCE (non-promotable, STAYS ALIVE, keeps retrying), and
* RELEASES on catch-up (Req 2.4, 2.6).
*
* never-self-exit rework (2026-06-18): the member cannot recover locally AND
* cannot catch up right now. It does NOT self-exit; it logs LOUDLY, sets the
* sticky non-promotable flag, and stays alive. The flag clears once a peer
* returns and it catches up. RED→GREEN: pre-rework
* persist_recover_fatal_if_unrecoverable aborted (or fired the abort-capture
* hook); post-rework persist_recover_loud_fence_if_unrecoverable RETURNS — the
* test continuing past the call IS the proof the process stayed alive. */
static void test_boot_policy_row4_truncated_no_peer_loud_fence(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_dur_row4_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = dur_flag_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
recover_policy_inputs_t in = {
.verdict = RECOVER_TRUNCATED,
.durable_below_hwm = true,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 7, /* truncation boundary */
.cluster_hwm_lsn = 12, /* cluster committed high-water */
};
/* Pure decision returns the non-terminating unrecoverable fence. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE_UNRECOVERABLE, persist_recover_decision(&in),
"row 4: truncated + below + no peer + no catch-up must decide the LOUD "
"PERMANENT FENCE — NOT process exit");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4: the sticky flag is set by the ACTION helper, not the pure "
"decision");
/* The action helper logs LOUDLY, sets the sticky flag, and RETURNS (no
* abort/exit — the process stays alive; the test reaching the next line is
* the proof). */
recover_decision_t d = persist_recover_loud_fence_if_unrecoverable(p, &in);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE_UNRECOVERABLE, d,
"row 4: the loud-fence action returns FENCE_UNRECOVERABLE (stays alive)");
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4: the member is held PERMANENTLY non-promotable (sticky flag set)");
/* Req 2.6: once catch-up reaches a committed-consistent state the fence
* releases automatically — the flag clears, the member is promotable. */
persist_clear_recovery_unrecoverable(p);
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4: catch-up CLEARS the unrecoverable fence (Req 2.6)");
dur_free_flag_persist(p);
printf("test_boot_policy_row4_truncated_no_peer_loud_fence: PASS\n");
}
/* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails
* -> LOUD PERMANENT FENCE (non-promotable, STAYS ALIVE, keeps retrying).
* Behind the HWM with no peer to advance from — same non-terminating loud-fence
* terminal as row 4. NEVER self-exits. (Req 2.4, 2.2, 2.6) */
static void test_boot_policy_row4b_clean_below_no_peer_loud_fence(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_dur_row4b_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = dur_flag_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
recover_policy_inputs_t in = {
.verdict = RECOVER_CLEAN, /* structurally intact... */
.durable_below_hwm = true, /* ...but behind the HWM */
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = 9,
.cluster_hwm_lsn = 20,
};
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE_UNRECOVERABLE, persist_recover_decision(&in),
"row 4b: clean-but-behind + no peer + no catch-up must decide the LOUD "
"PERMANENT FENCE — never process exit, never a silent fence");
recover_decision_t d = persist_recover_loud_fence_if_unrecoverable(p, &in);
TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FENCE_UNRECOVERABLE, d);
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4b: a behind member with no peer is held non-promotable + ALIVE "
"(sticky flag set; boundary logged loudly)");
persist_clear_recovery_unrecoverable(p);
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(p),
"row 4b: catch-up CLEARS the unrecoverable fence (Req 2.6)");
dur_free_flag_persist(p);
printf("test_boot_policy_row4b_clean_below_no_peer_loud_fence: PASS\n");
}
/* Row 5: CLEAN snapshot-sentinel (committed at sentinel LSN, NO
* sub-sentinel entries) + below + healthy peer + catch-up succeeds
* -> FENCE+CATCH-UP -> ELIGIBLE.
* A legitimate sentinel snapshot is CLEAN-but-behind, NOT corruption:
* treat as must-catch-up, NEVER FATAL on a legitimate sentinel. WILL
* ASSERT (4.3): a snapshot-sentinel member is fenced+caught-up like any
* behind member and is NEVER routed to the FATAL branch. (Req 2.1, 2.2)
* (3.1 already verifies the verdict itself is CLEAN — see
* test_recover_verdict_snapshot_sentinel_clean; this row pins the
* downstream promotion-fence outcome.) */
static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void)
{
/* Snapshot-sentinel CLEAN-but-behind + healthy peer: treated as behind,
* must catch up (FENCE), NEVER FATAL on a legitimate sentinel; then
* ELIGIBLE after catch-up. */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
/* Sentinel at LSN 5, cluster has advanced to HWM 12, peer reachable. */
recover_decision_t d1 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/12,
/*peer_reachable=*/true, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE, d1,
"row 5: a snapshot-sentinel CLEAN-but-behind member must FENCE, never "
"the unrecoverable terminal while a peer is reachable");
recover_decision_t d2 = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/12, /*safe_hwm=*/12,
/*peer_reachable=*/true, /*catch_up_succeeded=*/true);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d2,
"row 5: after catch-up the sentinel member is eligible (Req 2.6)");
ha_fini(&ha);
printf("test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal: PASS\n");
}
/* Row 6: CLEAN empty first-boot WAL (durable=0, HWM=0, fresh cold start)
* -> ELIGIBLE.
* An empty first-boot WAL is not behind anything — cold-start members
* are promotable. WILL ASSERT (4.3): the eligibility gate admits a
* fresh cold-start member (no fence, no FATAL). (Req 2.1, 2.3) */
static void test_boot_policy_row6_empty_first_boot_eligible(void)
{
/* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fresh cold-start
* member is not behind anything and must be promotable (no fence, no FATAL),
* even with no peer reachable (single-node / first boot). */
ha_state_t ha;
ha_config_t cfg = dur_ha_cfg();
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
recover_decision_t d = ha_evaluate_promotion_fence(
&ha, /*truncated=*/false, /*durable_floor=*/0, /*safe_hwm=*/0,
/*peer_reachable=*/false, /*catch_up_succeeded=*/false);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE, d,
"row 6: an empty first-boot WAL (durable=0/HWM=0) must be ELIGIBLE");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire),
"row 6: a cold-start member must not be fenced");
ha_fini(&ha);
printf("test_boot_policy_row6_empty_first_boot_eligible: PASS\n");
}
/* ── task 4.3: hollow node refused even at minimal quorum (Req 2.5) ──────────
* A hollow node (recovery-truncated OR durable below the HWM) must be refused
* promotion regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote. At minimal quorum with NO
* healthy peer, a hollow node is held in the LOUD PERMANENT FENCE
* (FENCE_UNRECOVERABLE, stays alive, non-promotable) rather than silently
* leading; with a peer it FENCEs. Either way it never becomes ELIGIBLE while
* hollow. */
static void test_promotion_gate_hollow_refused_at_minimal_quorum(void)
{
/* Truncated + below HWM + no peer (minimal/degraded quorum) -> NEVER
* ELIGIBLE (loud permanent fence — never promote a hollow node). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
/*safe_hwm=*/10, /*peer_reachable=*/false,
/*catch_up_succeeded=*/false),
"a recovery-truncated node must never be ELIGIBLE at minimal quorum "
"(Req 2.5)");
/* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (loud permanent fence). */
TEST_ASSERT_NOT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_ELIGIBLE,
ha_promotion_gate_decision(/*truncated=*/false, /*durable_floor=*/5,
/*safe_hwm=*/10, /*peer_reachable=*/false,
/*catch_up_succeeded=*/false),
"a below-HWM node must never be ELIGIBLE at minimal quorum (Req 2.5)");
/* Hollow node WITH a peer is still refused (FENCE, not ELIGIBLE) until it
* catches up — minimal-quorum size does not relax the gate. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_DECISION_FENCE,
ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3,
/*safe_hwm=*/10, /*peer_reachable=*/true,
/*catch_up_succeeded=*/false),
"a hollow node with a peer is FENCED (refused) until caught up");
printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n");
}
/* ── task 4.4: catch-up re-alert window (never-self-exit rework 2026-06-18) ───
*
* A recovery-truncated/behind member with no reachable peer at the promotion
* instant is FENCED (in_catch_up) and STAYS ALIVE, retrying catch-up. A
* delaymgr re-alert timer fires after DEFAULT_RECOVERY_CATCHUP_WINDOW_MS: if the
* member is still unrecoverable it RE-EMITS the loud operator alert (sets the
* sticky persist_recovery_unrecoverable flag) and the process STAYS ALIVE —
* there is NO abort/exit anywhere. Catch-up releases the fence + clears the
* flag (Req 2.6); shutdown/cancel stops the re-alert.
*
* These tests drive ha_arm_promotion_fence_timer (the production deferral wired
* into raft_cb_become_leader) through a REAL delaymgr with a short window seam,
* so the timer fires fast and deterministically. A minimal cluster_t carries
* the delaymgr + persist + ha exactly as production does. They assert the
* member STAYS ALIVE + non-promotable (sticky flag) instead of asserting an
* abort/hook — the self-exit terminal is GONE. */
/* The delaymgr runner used by production (run_delay_tasks in cluster.c) is not
* linked into the unit-test target; provide the same trivial pass-through. */
static int dur_run_delay_task(delay_task_entry_fn_t entry, void* arg)
{
return entry(arg);
}
/* Build a minimal cluster_t (calloc'd) with a started delaymgr and a recovered
* persist whose recovery verdict + durable floor are set directly, so the
* promotion fence can be evaluated without a full election. truncated → the
* recovery_truncated flag; durable_floor → the durable watermark the gate reads.
* ha.repl stays NULL so "no healthy peer reachable" holds in the timer. */
static cluster_t* dur_make_fence_cluster(const char* tmpdir, bool truncated,
uint64_t durable_floor)
{
cluster_t* c = calloc(1, sizeof(*c));
if (!c) return NULL;
persist_config_t cfg = persist_config_defaults();
cfg.wal_sync_ms = 5000;
cfg.snap_keep_n = 2;
persist_set_workdir(&c->persist, tmpdir, "ctrl-test");
if (persist_init(&c->persist, &cfg, NULL) != 0) { free(c); return NULL; }
if (persist_start(&c->persist) != 0) {
persist_fini(&c->persist); free(c); return NULL;
}
/* Set the recovered state the gate reads (mirrors a real persist_recover). */
c->persist.recovery_verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, durable_floor,
memory_order_release);
ha_config_t hcfg = dur_ha_cfg();
if (ha_init(&c->ha, &hcfg, 0) != 0) {
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
c->ha.raft_c = c;
c->ha.repl = NULL; /* no standby registry → no healthy peer reachable */
if (delaymgr_init(&c->delaymgr, 5 /* precision_ms */, dur_run_delay_task) != 0) {
ha_fini(&c->ha);
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
if (delaymgr_start(&c->delaymgr) != 0) {
delaymgr_fini(&c->delaymgr);
ha_fini(&c->ha);
persist_stop(&c->persist); persist_fini(&c->persist); free(c);
return NULL;
}
return c;
}
static void dur_destroy_fence_cluster(cluster_t* c)
{
if (!c) return;
/* Cancel any pending fence timer BEFORE tearing down the delaymgr so a
* still-armed callback cannot fire against freed state. */
ha_cancel_promotion_fence_timer(&c->ha);
delaymgr_stop(&c->delaymgr);
delaymgr_fini(&c->delaymgr);
ha_fini(&c->ha);
persist_stop(&c->persist);
persist_fini(&c->persist);
free(c);
}
/* Spin until the sticky unrecoverable flag is set on c->persist or a timeout
* elapses (the delaymgr fires the re-alert timer on its own worker thread). */
static bool dur_wait_unrecoverable(cluster_t* c, int timeout_ms)
{
int waited = 0;
while (!persist_recovery_unrecoverable(&c->persist) && waited < timeout_ms) {
usleep(2000);
waited += 2;
}
return persist_recovery_unrecoverable(&c->persist);
}
/* 4.4 (never-self-exit): a recovery-truncated/behind member with NO reachable
* peer must NOT self-exit — it fences (in_catch_up set, process ALIVE) and arms
* the re-alert timer; when the window elapses with no peer/progress the timer
* RE-EMITS the loud alert (sets the sticky non-promotable flag) and the process
* STAYS ALIVE.
*
* RED→GREEN: pre-rework the no-peer truncated member was routed (synchronously
* or via the timer) to persist_recover_fatal_if_unrecoverable, which abort()ed
* (the old fatal-capture hook fired) — that self-exit was the task-5.2 soak
* quorum collapse. Post-rework there is NO abort: the member stays alive,
* fenced, and loudly non-promotable. Reaching the end of this test (the
* process did not die) IS part of the proof. (Req 2.2, 2.4) */
static void test_promotion_fence_unrecoverable_stays_alive_loud(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_defer_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
ha_set_promotion_fence_window_ms(40); /* short re-alert window for the test */
/* Arm the fence exactly as raft_cb_become_leader does for a truncated,
* below-HWM, no-peer winner. This must NOT self-exit and must fence. */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: a fenced member is held in_catch_up (refused promotion), ALIVE");
/* Let the re-alert window elapse: still truncated + no peer + no progress →
* the timer RE-EMITS the loud alert and SETS the sticky non-promotable flag.
* The process STAYS ALIVE (no abort/exit anywhere on this path). */
TEST_ASSERT_TRUE_MESSAGE(
dur_wait_unrecoverable(c, 1000),
"4.4: the re-alert timer must set the sticky unrecoverable flag after "
"the window with no peer/progress — staying ALIVE, not exiting (Req 2.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: an unrecoverable member stays fenced (non-promotable) and ALIVE");
ha_set_promotion_fence_window_ms(0); /* restore default */
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_unrecoverable_stays_alive_loud: PASS\n");
}
/* 4.4 Req 2.6: catch-up within the window → fence + sticky flag released, and
* the member NEVER self-exits. Arm the fence, then simulate catch-up (clear
* truncated + advance durable_floor past the HWM) before the timer fires; when
* the timer fires it observes a caught-up member, releases in_catch_up, clears
* the sticky unrecoverable flag, and does NOT set the unrecoverable flag. */
static void test_promotion_fence_released_on_catchup(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_catchup_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
/* Simulate catch-up completing: log now intact and durable past the HWM. */
c->persist.recovery_verdict = RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
/* Wait long enough for the window to fire; assert the fence was released and
* the sticky flag is NOT set (Req 2.6) — the member self-healed, alive. */
usleep(300 * 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"4.4: catch-up within the window RELEASES the in_catch_up fence (Req 2.6)");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(&c->persist),
"4.4: a member that caught up must NOT be left in the unrecoverable "
"fence (Req 2.6)");
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_released_on_catchup: PASS\n");
}
/* 4.4: cancelling the timer (shutdown / recovery) before it fires → the
* re-alert never fires, no use-after-free of the timer ctx, and the process
* stays alive (it never self-exited even before the cancel). */
static void test_promotion_fence_cancel_stops_realert(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_cancel_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
ha_set_promotion_fence_window_ms(60);
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
/* Cancel before the window elapses. */
ha_cancel_promotion_fence_timer(&c->ha);
/* Wait past the window — the cancelled timer must be a no-op (no re-alert,
* no UAF). The member stayed alive throughout (no abort anywhere). */
usleep(200 * 1000);
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_unrecoverable(&c->persist),
"4.4: a cancelled fence timer must not fire the re-alert "
"(shutdown/recovery)");
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_cancel_stops_realert: PASS\n");
}
/* ── raft-apply-decoupling task 5.3 (round-2 remediation, Req 8.4) ───────────
* CROSS-CONTAMINATION REGRESSION: an APPLY_DIVERGENT-triggered fence must
* NEVER be live-cleared by this purely-structural re-check timer, even when
* the structural conditions it actually inspects (truncated / durable_floor
* vs HWM) become healthy. Round-1 review found BOTH ha.c live-clear call
* sites (this timer, ha.c ~867, and the promotion-eligibility gate in
* raft_cb_become_leader, ha.c ~1115) blind to WHY the sticky flag was set —
* they only ever inspected structural getters. A node whose local apply
* dispatch is defective (materialized state diverges from the committed log
* — task 5.1's APPLY_DIVERGENT verdict) can have a perfectly healthy WAL
* (not truncated, durable_floor >= HWM) by construction, since the two
* properties are orthogonal. Before the fix, this test drives exactly that
* cross-contamination and asserts the fence does NOT clear.
*
* This mirrors test_promotion_fence_released_on_catchup's harness and
* catch-up simulation EXACTLY, differing only in also marking the fence as
* apply_divergent-triggered up front — proving the fix discriminates on the
* apply verdict, not merely on the structural fields. */
static void test_promotion_fence_apply_divergent_survives_structural_catchup(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_fence_apply_div_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true,
/*durable_floor=*/4);
TEST_ASSERT_NOT_NULL(c);
/* Mark the fence as APPLY_DIVERGENT-triggered, exactly as persist_recover()
* does at boot (task 5.1/5.2) — the WAL-structural fields above
* (truncated/durable_floor) were already set independently above, proving
* the two conditions are orthogonal by construction. */
c->persist.apply_verdict = APPLY_DIVERGENT;
atomic_store_explicit(&c->persist.recovery_unrecoverable, true,
memory_order_release);
ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */
extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t,
uint64_t);
ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true,
/*durable_floor=*/4, /*hwm=*/10);
TEST_ASSERT_EQUAL_INT(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire));
/* Simulate the EXACT SAME structural catch-up as
* test_promotion_fence_released_on_catchup: log now intact and durable
* past the HWM. The apply_verdict / apply-divergent fence trigger is
* UNTOUCHED by this — it can only ever be recomputed by a fresh
* persist_recover() boot-time replay (task 5.1), never by a live
* structural mutation like this one. */
c->persist.recovery_verdict = RECOVER_CLEAN;
atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release);
/* Wait long enough for the window to fire. */
usleep(300 * 1000);
/* THE REGRESSION ASSERTION (Req 8.4): even though the structural
* conditions the timer inspects are now fully healthy, an
* APPLY_DIVERGENT-triggered fence must NOT be released — a local apply
* defect cannot be proven fixed by more WAL bytes arriving. Before the
* fix this assertion FAILS (the timer clears the flag purely on the
* structural check, oblivious to apply_verdict). */
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_unrecoverable(&c->persist),
"5.3 round-2: an APPLY_DIVERGENT-triggered fence must survive a "
"purely-structural catch-up re-check — only a fresh, clean "
"persist_recover() restart may clear it (Req 8.4)");
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire),
"5.3 round-2: the member must remain fenced (non-promotable) while "
"apply_divergent, regardless of structural catch-up");
ha_set_promotion_fence_window_ms(0);
dur_destroy_fence_cluster(c);
printf("test_promotion_fence_apply_divergent_survives_structural_catchup: "
"PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
/* Test suite entry point */
/* ------------------------------------------------------------------ */
void test_commit_durability(void)
{
#ifdef XCE_TESTING
RUN_TEST(test_durable_floor_empty);
RUN_TEST(test_durable_floor_after_flush);
RUN_TEST(test_durable_floor_survives_process_loss);
RUN_TEST(test_durable_lsn_initial_zero);
RUN_TEST(test_durable_lsn_batches_multiple_appends);
RUN_TEST(test_durable_lsn_never_exceeds_synced_tip);
RUN_TEST(test_durable_lsn_invariants_under_load);
RUN_TEST(test_durable_lsn_holds_on_fsync_failure);
RUN_TEST(test_durable_advance_drives_commit_single_member);
RUN_TEST(test_replicated_persist_syncs_before_return);
RUN_TEST(test_replicated_sync_failure_no_ack);
RUN_TEST(test_legacy_sync_append_holds_committed_at_durable);
RUN_TEST(test_legacy_sync_append_lsn0_edge_valid);
RUN_TEST(test_commit_never_exceeds_durable_invariant);
/* task 5.1 — durable-majority commit wired end to end */
RUN_TEST(test_durable_majority_commit_end_to_end);
RUN_TEST(test_sole_durable_contributor_loss_survives);
RUN_TEST(test_durability_work_off_iomgr_preserves_liveness);
RUN_TEST(test_recover_verdict_clean_intact_log);
RUN_TEST(test_recover_verdict_empty_first_boot_clean);
RUN_TEST(test_recover_verdict_crc_break_first_entry_truncated);
RUN_TEST(test_recover_verdict_snapshot_sentinel_clean);
/* bug-778 — standby replicated-apply must leave an intact CRC chain. */
RUN_TEST(test_replicated_apply_chain_survives_cold_restart);
/* task 3.4 — re-derive the durable floor from the recovered WAL tip. */
RUN_TEST(test_durable_floor_truncating_restart_is_intact_tip);
RUN_TEST(test_durable_floor_intact_recovery_full_tip);
RUN_TEST(test_durable_floor_empty_recovery_zero);
RUN_TEST(test_durable_floor_snapshot_sentinel_tip);
/* task 3.2 — Boot Policy Decision Table skeletons (IGNORED, pending
* 3.3/4.3). Registered so each row is reported as IGNORED, not failed;
* the suite stays GREEN. See design.md "Boot Policy Decision Table". */
/* task 3.3 — unrecoverable rows (4, 4b) = LOUD PERMANENT FENCE (stays alive,
* non-promotable, clears on catch-up), plus the never-unrecoverable
* invariant for the peer-reachable rows (3, 5) and the FENCE->ELIGIBLE
* eligibility outcome of rows 1,2,3,5,6. (never-self-exit rework) */
RUN_TEST(test_boot_policy_peer_reachable_never_fatal);
RUN_TEST(test_boot_policy_row1_clean_at_or_above_eligible);
RUN_TEST(test_boot_policy_row2_clean_below_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row3_truncated_peer_catchup_eligible);
RUN_TEST(test_boot_policy_row4_truncated_no_peer_loud_fence);
RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_loud_fence);
RUN_TEST(test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal);
RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible);
/* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */
RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum);
/* task 4.4 — catch-up re-alert window (NEVER self-exits; stays alive +
* loudly non-promotable on the unrecoverable path; fence + sticky flag
* released on catch-up; cancelled timer stops the re-alert). (Req 2.2/2.4/2.6) */
RUN_TEST(test_promotion_fence_unrecoverable_stays_alive_loud);
RUN_TEST(test_promotion_fence_released_on_catchup);
RUN_TEST(test_promotion_fence_cancel_stops_realert);
/* task 5.3 round-2 remediation (Req 8.4): cross-contamination regression —
* an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up
* alone. */
RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif
}
View git blame Copy permalink