#define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include "commit.h" #include "config.h" #include "delay.h" /* task 4.4: delaymgr for the bounded-window timer */ #include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */ #include "persist.h" #include "repl.h" /* task 5.1: repl_add_standby + standby ACK feed */ #include "typemap.h" /* task 5.1: wal_catchup_entry_t (send hook) */ #include "wal_types.h" #include "xce/cluster_def.h" /* task 4.4: cluster_t (delaymgr + persist + ha) */ /* ------------------------------------------------------------------ * * ha-commit-durability — task 1.2: durability-verification test seam * * * * These tests exercise persist_test_durable_lsn_on_disk(), the * * XCE_TESTING-only seam that re-reads + CRC-validates the WAL from * * stable storage and reports the highest fully-persisted LSN — * * independent of the in-memory current_lsn watermark. This lets a * * test assert an acknowledged operation is recoverable from disk after * * an immediate ungraceful termination, without relying on the * * background flush timer (Req 3.2). The seam is purely inspective: * * it changes nothing observable about the acknowledgment contract * * (Req 3.3). * * * * The seam is compiled ONLY into the controller_unit_test target * * (-DXCE_TESTING). The production xce_controller target links without * * it — see #ifdef XCE_TESTING in persist.c / declaration in persist.h. * * ------------------------------------------------------------------ */ #ifdef XCE_TESTING /* Build a persist instance with a deliberately slow background flush * cadence, so the only durability point is an explicit persist_flush() * (mirrors make_fresh_persist in test_persist_recovery.c). */ static persist_t* make_durability_persist(const char* tmpdir) { persist_t* p = calloc(1, sizeof(persist_t)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; /* slow background fsync — durability comes from * the explicit flush, not the timer */ cfg.snap_keep_n = 2; persist_set_workdir(p, tmpdir, "ctrl-test"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } if (persist_start(p) != 0) { persist_fini(p); free(p); return NULL; } return p; } static void teardown_durability_persist(persist_t* p) { persist_stop(p); persist_fini(p); free(p); } /* ------------------------------------------------------------------ * * Test 1: an empty WAL has no durable entries (floor == 0). * * An empty first-boot WAL is not corruption — it simply has nothing * * persisted, so the seam reports 0. * * ------------------------------------------------------------------ */ static void test_durable_floor_empty(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durfloor0_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); uint64_t floor = persist_test_durable_lsn_on_disk(p); TEST_ASSERT_EQUAL_UINT64(0, floor); teardown_durability_persist(p); printf("test_durable_floor_empty: PASS\n"); } /* ------------------------------------------------------------------ * * Test 2 (core Req 3.2): after append + explicit fdatasync, the seam * * sees the entries on disk — the durable floor equals the highest * * appended LSN, recoverable independent of in-memory state. * * ------------------------------------------------------------------ */ static void test_durable_floor_after_flush(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durfloor1_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Append three entries. LSN 0 is reserved, so these take LSN 1,2,3. * current_lsn advances to 4 (one past the last appended). */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); /* Durability point: fdatasync the segment so the bytes are on stable * storage (this is what an acknowledged-before-durable contract must * have done before replying ok). */ TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); /* The seam re-reads from disk and reports the highest intact LSN. */ uint64_t floor = persist_test_durable_lsn_on_disk(p); TEST_ASSERT_EQUAL_UINT64(p->current_lsn - 1, floor); TEST_ASSERT_EQUAL_UINT64(3, floor); teardown_durability_persist(p); printf("test_durable_floor_after_flush: PASS\n"); } /* ------------------------------------------------------------------ * * Test 3 (recoverable-after-ungraceful-kill, Req 3.2): a SECOND * * persist instance opened over the same WAL dir — with NO in-memory * * state from the writer — recovers exactly the durable floor the seam * * reported. This is the "survives an immediate ungraceful * * termination" assertion: durability lives on disk, not in the * * volatile buffers of the process that wrote it. * * ------------------------------------------------------------------ */ static void test_durable_floor_survives_process_loss(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durfloor2_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); uint64_t durable_floor = 0; /* Writer process: append, fdatasync, then "die" (we drop the instance * without any further flush — the bytes are already durable). */ { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); durable_floor = persist_test_durable_lsn_on_disk(p); TEST_ASSERT_EQUAL_UINT64(2, durable_floor); teardown_durability_persist(p); } /* "Restarted" process: a fresh persist_t with zero in-memory state, * opened over the same on-disk WAL. Its durable floor — read straight * from disk before any recover/replay — matches what the writer * persisted. Acknowledged-before-durable ops therefore survive the * loss of the acking process. */ { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); /* current_lsn is still the fresh-boot value (1, LSN 0 reserved): * the durable floor does NOT come from in-memory state. */ uint64_t floor2 = persist_test_durable_lsn_on_disk(p2); TEST_ASSERT_EQUAL_UINT64(durable_floor, floor2); TEST_ASSERT_EQUAL_UINT64(2, floor2); teardown_durability_persist(p2); } printf("test_durable_floor_survives_process_loss: PASS\n"); } /* ================================================================== * * task 2.1 — durable-LSN tracking via a group-commit flusher * * * * These tests pin the invariants of persist_t.durable_lsn, the * * prefix watermark advanced by the demand-driven flusher (the * * repurposed fsync_worker): * * - durable_lsn never exceeds the synced tip (<= current_lsn - 1); * * - durable_lsn is monotonic non-decreasing; * * - one fdatasync advances durable_lsn past MULTIPLE appended LSNs * * (batching / group-commit); * * - durable_lsn advances ONLY after a successful fdatasync; * * - under concurrent append load the invariants always hold. * * ------------------------------------------------------------------ */ /* Spin until the flusher catches durable_lsn up to the appended tip, * or a generous deadline elapses. Returns the observed durable_lsn. */ static uint64_t wait_durable_reaches(persist_t* p, uint64_t target_lsn, int max_ms) { for (int i = 0; i < max_ms; i++) { uint64_t d = persist_durable_lsn(p); if (d >= target_lsn) return d; usleep(1000); /* 1 ms */ } return persist_durable_lsn(p); } /* ------------------------------------------------------------------ * * Test 4: a fresh store has durable_lsn == 0 (nothing synced yet). * * ------------------------------------------------------------------ */ static void test_durable_lsn_initial_zero(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durlsn0_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p)); teardown_durability_persist(p); printf("test_durable_lsn_initial_zero: PASS\n"); } /* ------------------------------------------------------------------ * * Test 5 (core / batching, Req 1.1): several appends, then the * * demand-driven flusher advances durable_lsn past ALL of them with a * * single fdatasync — and never past the appended tip. Uses the fast * * cadence so the append-signal (not the timer) drives the flush, but * * the slow-cadence helper would converge too via the bounded fallback.* * ------------------------------------------------------------------ */ static void test_durable_lsn_batches_multiple_appends(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durlsn1_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* LSN 0 reserved → these take 1,2,3,4,5; current_lsn ends at 6. */ for (int i = 0; i < 5; i++) TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1)); uint64_t tip = p->current_lsn - 1; /* 5 */ TEST_ASSERT_EQUAL_UINT64(5, tip); uint64_t d = wait_durable_reaches(p, tip, 5000); /* The flusher advanced durable_lsn past several appended LSNs with * one fdatasync (batching), reaching the synced tip. */ TEST_ASSERT_EQUAL_UINT64(tip, d); /* Invariant: never exceeds the synced tip. */ TEST_ASSERT_TRUE(d <= p->current_lsn - 1); /* The on-disk seam agrees: those LSNs are genuinely fdatasync'd. */ TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d); teardown_durability_persist(p); printf("test_durable_lsn_batches_multiple_appends: PASS\n"); } /* ------------------------------------------------------------------ * * Test 6 (advances-only-after-fdatasync + monotonic): the durable * * floor reported by the on-disk seam must always be >= durable_lsn — * * i.e. durable_lsn never claims an LSN whose bytes are not actually * * on stable storage. Also asserts monotonic non-decreasing across * * successive append+flush rounds. * * ------------------------------------------------------------------ */ static void test_durable_lsn_never_exceeds_synced_tip(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durlsn2_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); uint64_t prev = persist_durable_lsn(p); for (int round = 0; round < 4; round++) { for (int i = 0; i < 3; i++) TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1)); uint64_t tip = p->current_lsn - 1; uint64_t d = wait_durable_reaches(p, tip, 5000); /* Monotonic non-decreasing. */ TEST_ASSERT_TRUE(d >= prev); /* Never exceeds the synced tip. */ TEST_ASSERT_TRUE(d <= p->current_lsn - 1); /* Advances only after a real fdatasync: the independent on-disk * scan must confirm every claimed-durable LSN is on disk. */ TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d); prev = d; } teardown_durability_persist(p); printf("test_durable_lsn_never_exceeds_synced_tip: PASS\n"); } /* ------------------------------------------------------------------ * * Test 7 (UNDER LOAD, Req 1.6 / lost-wakeup): many appends from * * multiple threads racing the flusher. Asserts the invariants hold * * throughout (durable_lsn <= synced tip, never past disk) AND that * * the flusher eventually catches durable_lsn up to the final tip — * * i.e. no append is left permanently un-synced (no lost wakeup). * * ------------------------------------------------------------------ */ #define LOAD_THREADS 4 #define LOAD_PER_THREAD 200 typedef struct { persist_t* p; int n; } load_arg_t; static void* load_appender(void* arg) { load_arg_t* a = arg; for (int i = 0; i < a->n; i++) { /* Ignore rc: a full-segment rotate or transient EIO is not the * subject under test; the invariant checks below tolerate it. */ (void)persist_wal_async(a->p, WAL_JOB_UPDATE, "z", 1); /* Sample the invariant mid-flight: durable_lsn must NEVER exceed * the appended tip, even while a concurrent flush is in progress. */ uint64_t d = persist_durable_lsn(a->p); uint64_t cur = atomic_load_explicit( (const _Atomic uint64_t*)&a->p->current_lsn, memory_order_acquire); /* current_lsn is read without the lock here purely as a coarse * upper bound; a torn read can only make `cur` smaller, so the * assertion stays conservative. durable_lsn is a prefix watermark * that can never outrun the last assigned LSN. */ TEST_ASSERT_TRUE(d == 0 || d <= cur); } return NULL; } static void test_durable_lsn_invariants_under_load(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durlsn3_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); pthread_t th[LOAD_THREADS]; load_arg_t args[LOAD_THREADS]; for (int i = 0; i < LOAD_THREADS; i++) { args[i].p = p; args[i].n = LOAD_PER_THREAD; TEST_ASSERT_EQUAL_INT(0, pthread_create(&th[i], NULL, load_appender, &args[i])); } for (int i = 0; i < LOAD_THREADS; i++) pthread_join(th[i], NULL); /* Final tip after all appends settled. */ uint64_t tip = p->current_lsn - 1; /* No lost wakeup: the flusher must eventually advance durable_lsn up * to the final appended tip (every entry gets synced). */ uint64_t d = wait_durable_reaches(p, tip, 10000); TEST_ASSERT_EQUAL_UINT64(tip, d); TEST_ASSERT_TRUE(d <= p->current_lsn - 1); TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d); teardown_durability_persist(p); printf("test_durable_lsn_invariants_under_load: PASS\n"); } /* ------------------------------------------------------------------ * * Test 8 (ERROR PATH, Req 1.1/1.2 — fail-loud): when fdatasync FAILS, * * durable_lsn must NOT advance. Bytes that did not reach stable * * storage must never be counted durable — otherwise the member would * * contribute a non-durable LSN toward the commit majority. * * * * Mechanism: the XCE_TESTING fault-injection toggle forces every * * fdatasync site to treat the sync as failed. We append entries while * * the toggle is ON and assert durable_lsn stays put across the full * * cadence window (the flusher runs but its advance is gated on rc==0). * * Then we clear the toggle and confirm the flusher recovers — durable_ * * lsn advances to the tip once syncs succeed again. * * ------------------------------------------------------------------ */ static void test_durable_lsn_holds_on_fsync_failure(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durlsn4_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* First, append one entry and let it become durable normally, so the * watermark has a known non-zero baseline that must HOLD during the * failure window. */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); uint64_t baseline = wait_durable_reaches(p, 1, 5000); TEST_ASSERT_EQUAL_UINT64(1, baseline); /* Now inject fdatasync failure and append more entries. */ persist_test_set_force_fsync_fail(p, 1); for (int i = 0; i < 5; i++) TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); uint64_t tip = p->current_lsn - 1; /* 6 */ TEST_ASSERT_EQUAL_UINT64(6, tip); /* Give the flusher ample time to run several passes (cadence is 5s but * the append signal fires it immediately) AND an explicit flush attempt. * Under injected failure, durable_lsn MUST remain at the baseline — the * new bytes are not on stable storage, so they are not durable. */ (void)persist_flush(p); /* also gated on rc==0 — must not advance either */ for (int i = 0; i < 200; i++) { TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p)); usleep(1000); /* 1 ms; 200 ms total — many flusher signal cycles */ } /* Still parked at the baseline after the window. */ TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p)); TEST_ASSERT_TRUE(persist_durable_lsn(p) <= p->current_lsn - 1); /* Recovery: clear the fault and confirm the flusher catches up — proving * the failure path did not corrupt the watermark, only held it. */ persist_test_set_force_fsync_fail(p, 0); /* Nudge the flusher with one more append so the level predicate signals. */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); uint64_t new_tip = p->current_lsn - 1; /* 7 */ uint64_t d = wait_durable_reaches(p, new_tip, 5000); TEST_ASSERT_EQUAL_UINT64(new_tip, d); TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d); teardown_durability_persist(p); printf("test_durable_lsn_holds_on_fsync_failure: PASS\n"); } /* ================================================================== * * task 2.2 — durable-advance wake drives commit (integration) * * * * End-to-end over a REAL persist_t + commit_engine_t (single-member, * * NULL cluster): the group-commit flusher advances durable_lsn and * * fires the registered durable-advance callback, which the commit * * engine uses to advance committed_lsn and wake WAIT callers. Proves * * the persist↔commit wiring works against the real flusher thread — * * not just the directly-driven callback (covered in * * test_commit_engine.c). n_peers=1 means self alone is the majority, * * so committed_lsn tracks durable_lsn with no standby ACKs needed. * * ------------------------------------------------------------------ */ static void test_durable_advance_drives_commit_single_member(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durcommit_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Single-member commit engine over the real persist. n_vcpus large enough * to admit a wait. commit_engine_init registers the durable-advance cb. */ commit_engine_t ce; TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, 1, 8)); /* committed starts at 0; nothing appended/durable. */ TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn)); /* Append three entries (LSN 1,2,3). They are in page cache only — the slow * 5s cadence means the append-SIGNAL (not the timer) must drive the flush; * the real flusher fdatasyncs, advances durable_lsn, and fires the cb, * which advances committed_lsn for this single-member quorum. */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); uint64_t tip = p->current_lsn - 1; /* 3 */ TEST_ASSERT_EQUAL_UINT64(3, tip); /* Wait for the flusher to make the tip durable AND for the wake-driven * commit to catch up. committed_lsn must reach the tip — driven by the * durable-advance callback, not by any ACK (there are no standbys). */ uint64_t committed = 0; for (int i = 0; i < 5000; i++) { committed = atomic_load(&p->committed_lsn); if (committed >= tip) break; usleep(1000); } TEST_ASSERT_EQUAL_UINT64_MESSAGE( tip, committed, "durable-advance callback must drive committed_lsn to the durable tip"); /* committed never exceeds durable (the invariant the wake preserves). */ TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p)); /* Tear the engine down BEFORE the persist (unhooks the callback so the * flusher cannot fire against a destroyed engine). */ commit_engine_fini(&ce); teardown_durability_persist(p); printf("test_durable_advance_drives_commit_single_member: PASS\n"); } /* ================================================================== * * task 2.3 — standby persist-before-ACK regression (Req 1.3) * * * * Req 1.3: a standby durably persists a replicated entry to local * * stable storage BEFORE sending its acknowledgment. The ACK is sent by * * the CALLER (ha_apply_single_entry) only after * * persist_wal_write_replicated returns success — so to pin "persist * * before ACK" we prove a durable fdatasync ran INSIDE the persist call * * before it returned. * * * * Mechanism: a disk re-read (persist_test_durable_lsn_on_disk) cannot * * distinguish an fsync'd page from a page-cache-only page, so it CANNOT * * prove the sync ran. Instead we read the XCE_TESTING fdatasync counter * * incremented at the inline sync site: snapshot it before the call, * * assert it incremented after — the sync provably ran within the call, * * before the caller could ACK. The test FAILS if someone removed or * * moved the fdatasync out of persist_wal_write_replicated. * * ------------------------------------------------------------------ */ static void test_replicated_persist_syncs_before_return(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_replsync_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Snapshot the sync counter, then write a replicated entry. The standby * path must fdatasync inline BEFORE returning — so the counter must * advance during the single call. */ unsigned before = persist_test_replicated_fdatasync_count(p); int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, rc); unsigned after = persist_test_replicated_fdatasync_count(p); TEST_ASSERT_TRUE_MESSAGE( after > before, "persist_wal_write_replicated must fdatasync inline (persist-before-" "ACK): the sync counter did not advance during the call"); /* And the entry is genuinely on stable storage by the time the call * returned (would survive an immediate process loss → recoverable ACK). */ TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 1); teardown_durability_persist(p); printf("test_replicated_persist_syncs_before_return: PASS\n"); } /* ------------------------------------------------------------------ * * task 2.3 — forced standby-sync failure ⇒ error, NO success/ACK * * * * Closes the Req-1.3 hole: if the inline fdatasync FAILS, the standby * * must NOT report success — otherwise the caller would ACK an entry * * that is not durable (durable AFTER, not BEFORE, the ACK). With the * * force-fail seam ON, persist_wal_write_replicated must return a * * negative error; the caller (ha_apply_single_entry) already treats any * * negative return as "do not apply, do not ACK". * * * * RED evidence: against the pre-2.3 code (which ignored the fdatasync * * return at the standby path) this returned 0 (success) under forced * * failure — a non-durable ACK. The rc-gate makes it return -EIO. * * ------------------------------------------------------------------ */ static void test_replicated_sync_failure_no_ack(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_replsyncfail_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Force the fdatasync to be treated as failed, then attempt a replicated * write. The durable sync cannot succeed, so the call MUST fail (no ACK). * A negative return is what ha_apply_single_entry routes to its * "not applying/acking" path. */ persist_test_set_force_fsync_fail(p, 1); int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_TRUE_MESSAGE( rc < 0, "forced standby fdatasync failure must make persist_wal_write_" "replicated FAIL (negative rc) so the standby does NOT ACK a " "non-durable entry"); /* The sync was still attempted (counter advanced) — we fail loud on the * RESULT, not by skipping the sync. */ TEST_ASSERT_TRUE(persist_test_replicated_fdatasync_count(p) >= 1); /* Recovery: clear the fault; a fresh replicated write (new lsn, not the * deduped lsn=1) now succeeds and is durable — proving the failure path * did not corrupt persist state and the sync runs again on success. * (lsn=1 already advanced current_lsn to 2, so re-using 1 would dedup; * lsn=2 is the genuine next entry.) */ persist_test_set_force_fsync_fail(p, 0); int rc2 = persist_wal_write_replicated(p, /*lsn=*/2, WAL_JOB_UPDATE, "x", 1); TEST_ASSERT_EQUAL_INT(0, rc2); TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 2); teardown_durability_persist(p); printf("test_replicated_sync_failure_no_ack: PASS\n"); } /* ================================================================== * * task 2.4 — reconcile the dormant synchronous-append commit path * * * * persist_wal_sync_append's LEGACY branch (ce == NULL, or the LSN-0 * * first entry on the engine path) historically advanced committed_lsn * * DIRECTLY on a majority ACK (atomic_store committed_lsn = written_lsn) * * with NO durable precondition. Group-commit removed the premise that * * the append already fsync'd inline, so a majority ACK no longer * * implies the local entry is durable. This branch is the structural * * ACK-without-durability hazard task 2.4 must close: NO path may * * advance committed_lsn on a majority ACK unless the entry is durable * * locally (committed_lsn <= durable_lsn). * * * * NOTE: persist_wal_sync_append is NOT dormant — it is the LIVE * * job-submit path (persist_job_hook_fun, WAL_JOB_SUBMIT). So the fix is * * to ROUTE the legacy direct-store through the durable precondition, * * not to delete the function. * * ------------------------------------------------------------------ */ /* Build an ENGINE-LESS persist with standbys registered, so * persist_wal_sync_append takes the legacy branch (ce == NULL) and its * has_standbys path: repl_wait_majority_ack returns 0 immediately for * the LSN-0 first entry, but for a NON-zero LSN it surfaces -EINVAL via * commit_submit (engine-less). The hazard we guard is the direct * committed_lsn store that the legacy branch performs on a 0 return. */ /* Drive the legacy branch directly: a single-member commit engine makes * the ENGINE path durable-safe, so to exercise the legacy direct-store * we use an engine-less persist and a registered standby that ACKs. * Rather than stand up the replication wire (out of boundary), we test * the invariant the fix establishes: committed_lsn must never exceed * durable_lsn after a commit advance, on EVERY path. */ /* ------------------------------------------------------------------ * * Test (RED before fix): the legacy direct-store path must NOT advance * * committed_lsn past durable_lsn. We drive the legacy branch via the * * LSN-0-style direct store by appending one entry that is NOT yet * * durable (slow cadence, no flush) and confirming committed_lsn stays * * at/below durable_lsn. * * * * Mechanism: persist_test_sync_append_legacy_store() is an XCE_TESTING * * seam that invokes the SAME guarded store the legacy branch performs * * (advance committed_lsn to written_lsn only if written_lsn <= * * durable_lsn). Pre-fix the store was unconditional, so this over- * * advanced committed past durable (RED). Post-fix it holds. * * ------------------------------------------------------------------ */ static void test_legacy_sync_append_holds_committed_at_durable(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_legacycommit_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Append one entry (LSN 1) but do NOT flush — durable_lsn lags at 0, * while the appended tip is 1. The legacy direct-store would advance * committed_lsn to the ACK'd written_lsn (1) — over-advancing past * the durable watermark (0). The durable precondition must hold it. */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); uint64_t written_lsn = p->current_lsn - 1; /* 1 */ TEST_ASSERT_EQUAL_UINT64(1, written_lsn); TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p)); TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn)); /* Invoke the guarded legacy store directly (simulates a majority ACK * for an entry that is not yet durable locally). It must NOT advance * committed_lsn past durable_lsn. */ persist_test_sync_append_legacy_commit_store(p, written_lsn); TEST_ASSERT_TRUE_MESSAGE( atomic_load(&p->committed_lsn) <= persist_durable_lsn(p), "legacy sync-append commit store must NOT advance committed_lsn past " "durable_lsn (ACK-without-durability hazard)"); /* Concretely: committed stays at 0 because durable_lsn is still 0. */ TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn)); /* Now make the entry durable and re-invoke: the store is now allowed * (written_lsn <= durable_lsn) and committed advances to it. */ TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); TEST_ASSERT_TRUE(persist_durable_lsn(p) >= written_lsn); persist_test_sync_append_legacy_commit_store(p, written_lsn); TEST_ASSERT_EQUAL_UINT64(written_lsn, atomic_load(&p->committed_lsn)); TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p)); teardown_durability_persist(p); printf("test_legacy_sync_append_holds_committed_at_durable: PASS\n"); } /* ------------------------------------------------------------------ * * Test: the LSN-0 first-entry edge stays valid. durable_lsn and * * committed_lsn both start at 0; the legacy store of written_lsn == 0 * * (0 <= 0) is permitted and is a harmless no-op store of 0. This is the * * once-per-cluster-lifetime path the engine branch routes here. * * ------------------------------------------------------------------ */ static void test_legacy_sync_append_lsn0_edge_valid(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_legacylsn0_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Fresh store: durable_lsn == 0, committed_lsn == 0. */ TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p)); TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn)); /* Storing written_lsn == 0 (0 <= durable 0) is allowed — a no-op. */ persist_test_sync_append_legacy_commit_store(p, 0); TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn)); TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p)); teardown_durability_persist(p); printf("test_legacy_sync_append_lsn0_edge_valid: PASS\n"); } /* ------------------------------------------------------------------ * * Guard test (the Observable's "test guards against a future caller"): * * the committed_lsn <= durable_lsn invariant holds across the commit- * * engine advance path too. Drives the real engine + flusher and asserts * * that at every observed point committed never exceeds durable — the * * future-proof guard against reintroducing an ACK-without-durability * * commit on ANY path. In a debug build the in-code assert would abort * * if a path ever advanced committed past durable. * * ------------------------------------------------------------------ */ static void test_commit_never_exceeds_durable_invariant(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_cinvariant_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); commit_engine_t ce; TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, 1, 8)); /* Append a burst while sampling the invariant: committed must never * outrun durable, even mid-flight while the flusher advances both. */ for (int round = 0; round < 6; round++) { for (int i = 0; i < 4; i++) TEST_ASSERT_EQUAL_INT( 0, persist_wal_async(p, WAL_JOB_UPDATE, "q", 1)); for (int s = 0; s < 50; s++) { uint64_t committed = atomic_load(&p->committed_lsn); uint64_t durable = persist_durable_lsn(p); TEST_ASSERT_TRUE_MESSAGE( committed <= durable, "committed_lsn must never exceed durable_lsn on any commit " "advance path"); usleep(1000); } } /* And eventually committed catches up to the durable tip. */ uint64_t tip = p->current_lsn - 1; uint64_t committed = 0; for (int i = 0; i < 5000; i++) { committed = atomic_load(&p->committed_lsn); if (committed >= tip) break; usleep(1000); } TEST_ASSERT_EQUAL_UINT64(tip, committed); TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p)); commit_engine_fini(&ce); teardown_durability_persist(p); printf("test_commit_never_exceeds_durable_invariant: PASS\n"); } /* ================================================================== * * task 5.1 — wire durable-majority commit end to end * * * * These three tests verify the full durable-majority commit path the * * 2.x/4.x wiring assembled, end to end over a REAL persist_t (real WAL * * + real group-commit flusher thread) and a REAL commit_engine_t with * * registered standbys: * * * * 1. reply-ok ⇒ durable-majority (Req 1.1, 1.4): committed_lsn * * advances and a WAIT caller releases ONLY after the entry is * * durable on a fault-tolerant majority — self durable (durable_lsn * * covers it, via the real flusher) AND a standby ACK. With * * durability held off, even a standby ACK does NOT release. * * 2. sole-durable-contributor loss survives (Req 1.5, 2.5): an op * * committed on self + a standby is durable on >= floor(N/2)+1 * * members, so losing one durable contributor leaves it present on * * a surviving member (modeled at the integration level; the live * * failover proof is task 5.2). * * 3. liveness — durability work off the iomgr (Req 1.6): fdatasync * * runs on the dedicated flusher thread, not the caller/iomgr * * thread, so a heartbeat-style callback is not stalled while a * * durable flush is in flight. * * ------------------------------------------------------------------ */ /* Dummy standby endpoints — the replicator only compares pointers, and * commit_on_ack reads acked_lsn from the registry slot. */ static int g_dur_eps[MAX_STANDBYS]; /* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps * repl_broadcast from dereferencing the dummy endpoints as real * endpoint_t when persist_wal_async fans out to the registered standbys. * Same pattern as test_commit_engine.c. */ extern int (*repl_broadcast_send_hook)(void* ep, wal_catchup_entry_t** entries, int n_entries); static int dur_drop_send_hook(void* ep, wal_catchup_entry_t** entries, int n_entries) { (void)ep; (void)entries; (void)n_entries; return 0; /* pretend success; drop the data (no real wire in a unit test) */ } /* Feed a standby ACK exactly as the iomgr read path does: store the * watermark into the registry slot, then notify the commit engine. */ static void durable_standby_ack(persist_t* p, commit_engine_t* ce, int idx, uint64_t acked_lsn) { atomic_store(&p->repl.standbys[idx].acked_lsn, acked_lsn); commit_on_ack(ce, idx, acked_lsn); } /* A WAIT submit that runs on its own thread, so the test can observe that * it does NOT return while the entry is not yet durable-majority, then DOES * return once it becomes durable-majority (mirrors the off-iomgr AFTER_EVENT * caller blocking on the commit barrier). */ typedef struct { commit_engine_t* ce; uint64_t lsn; uint32_t timeout_ms; _Atomic int returned; int rc; } wait_submit_arg_t; static void* wait_submit_thread(void* a) { wait_submit_arg_t* w = (wait_submit_arg_t*)a; w->rc = commit_submit(w->ce, w->lsn, COMMIT_MODE_WAIT, w->timeout_ms); atomic_store(&w->returned, 1); return NULL; } /* ------------------------------------------------------------------ * * Test 5.1a (MUST-HAVE, Req 1.1 / 1.4): reply-ok only after a durable * * majority. 3-member cluster (self + 2 standbys), so the majority for * * an entry is self + at least 1 standby. We: * * - force the local fdatasync to FAIL, so the flusher cannot advance * * durable_lsn (self is NOT durable); * * - append an entry and deliver a standby ACK covering it; * * - assert committed_lsn does NOT advance and a WAIT submit does NOT * * release — a majority of ACKs is not enough without local * * durability (this is the durable precondition gate; pre-2.2 this * * would have committed on the ACK alone — RED); * * - clear the fault so the real flusher fdatasyncs and advances * * durable_lsn, firing the durable-advance wake; * * - assert committed_lsn now advances and the WAIT submit releases * * promptly (durable self + durable standby ACK = durable majority). * * ------------------------------------------------------------------ */ static void test_durable_majority_commit_end_to_end(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durmaj_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Intercept the replication fan-out so persist_wal_async does not try to * send over the dummy endpoints (they are not real endpoint_t). */ repl_broadcast_send_hook = dur_drop_send_hook; /* Register two standbys against the REAL persist's replicator. n_peers=3 * (self + 2 standbys); majority for entry N = self durable + >= 1 standby * with acked_lsn >= N. n_vcpus high enough to admit a WAIT. */ TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&p->repl, &g_dur_eps[0], "sb0", 7000)); TEST_ASSERT_EQUAL_INT(1, repl_add_standby(&p->repl, &g_dur_eps[1], "sb1", 7001)); commit_engine_t ce; TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, /*n_peers=*/3, /*n_vcpus=*/8)); TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn)); /* Hold durability off: every flusher fdatasync is forced to fail, so * durable_lsn cannot advance. (Fail-loud: the entry is never counted * durable, so the commit cannot advance regardless of ACKs.) */ persist_test_set_force_fsync_fail(p, 1); /* Append one entry (LSN 1). It is in page cache only; the forced-fail * flusher will NOT make it durable. */ TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); uint64_t lsn = p->current_lsn - 1; /* 1 */ TEST_ASSERT_EQUAL_UINT64(1, lsn); /* Register a WAIT for LSN 1 on its own thread (the off-iomgr commit-barrier * caller). It must block: the entry is not durable-majority. */ wait_submit_arg_t w = {.ce = &ce, .lsn = lsn, .timeout_ms = 2000}; pthread_t th; TEST_ASSERT_EQUAL_INT(0, pthread_create(&th, NULL, wait_submit_thread, &w)); /* Deliver a covering standby ACK from BOTH standbys (a clear majority of * ACKs). Pre-durability this alone advanced committed_lsn (the bug); * post-2.2 the self-cap holds it at the non-durable self (durable_lsn=0). */ durable_standby_ack(p, &ce, 0, lsn); durable_standby_ack(p, &ce, 1, lsn); /* Give the (forced-failing) flusher ample time to run; durable_lsn must * stay 0, committed must stay 0, and the WAIT must NOT have returned. */ for (int i = 0; i < 300; i++) { /* up to ~300 ms */ if (persist_durable_lsn(p) != 0 || atomic_load(&p->committed_lsn) != 0 || atomic_load(&w.returned)) break; usleep(1000); } TEST_ASSERT_EQUAL_UINT64_MESSAGE( 0, persist_durable_lsn(p), "durable_lsn must not advance while fdatasync is failing"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 0, atomic_load(&p->committed_lsn), "committed_lsn must NOT advance on a standby-ACK majority while the " "local entry is not durable (durable precondition gate)"); TEST_ASSERT_FALSE_MESSAGE( atomic_load(&w.returned), "the WAIT (client reply) must NOT release before durable-majority"); /* Now allow durability: clear the fault and force a flush. The REAL * flusher fdatasyncs, advances durable_lsn over LSN 1, and fires the * durable-advance wake — self is now durable AND both standbys already * ACKed, so the entry is durable on a majority. committed_lsn advances * and the WAIT releases promptly. */ persist_test_set_force_fsync_fail(p, 0); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); TEST_ASSERT_TRUE(persist_durable_lsn(p) >= lsn); /* The WAIT must now return success (committed) within its timeout. */ TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL)); TEST_ASSERT_TRUE_MESSAGE( atomic_load(&w.returned), "the WAIT must release once the entry is durable on a majority"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, w.rc, "commit_submit(WAIT) must return 0 (committed) after durable-majority"); TEST_ASSERT_TRUE_MESSAGE( atomic_load(&p->committed_lsn) >= lsn, "committed_lsn must advance once self is durable AND a standby ACK " "covers the entry (durable majority)"); /* The invariant the whole spec rests on: committed never exceeds durable. */ TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p)); commit_engine_fini(&ce); repl_remove_standby(&p->repl, &g_dur_eps[0]); repl_remove_standby(&p->repl, &g_dur_eps[1]); teardown_durability_persist(p); repl_broadcast_send_hook = NULL; printf("test_durable_majority_commit_end_to_end: PASS\n"); } /* ------------------------------------------------------------------ * * Test 5.1b (Req 1.5 / 2.5): losing the sole durable contributor * * leaves the acknowledged op present on a surviving member. * * * * The durable-majority property says: when an op is acknowledged, its * * entry is fdatasync'd on a fault-tolerant majority (self + >= * * floor(N/2) others). For N=3 that is 2 durable copies. So losing ONE * * durable contributor still leaves >= 1 surviving durable copy that * * holds the op — it is recoverable from the survivor. * * * * We model the two durable copies as two independent persist_t over * * separate WAL dirs (a primary's copy and a standby's copy), both made * * genuinely durable (fdatasync'd). We then DROP the "sole durable * * contributor" (the first copy, simulating its loss/restart-with-loss) * * and prove the op is still durably present and recoverable on the * * surviving copy via a fresh-process recovery (no in-memory state). * * * * This is the integration-level assertion of the property that makes * * survival true (committed ⇒ >= floor(N/2)+1 durable copies); the live * * multi-host SIGKILL failover proof is task 5.2. * * ------------------------------------------------------------------ */ static void test_sole_durable_contributor_loss_survives(void) { char tmp_a[256]; char tmp_b[256]; strcpy(tmp_a, "/tmp/xce_durlossA_XXXXXX"); strcpy(tmp_b, "/tmp/xce_durlossB_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp_a)); TEST_ASSERT_NOT_NULL(mkdtemp(tmp_b)); const char* payload = "committed-op"; uint32_t plen = (uint32_t)strlen(payload); /* Durable copy A (the contributor we will lose) and durable copy B (the * survivor): write the SAME committed op to both and fdatasync each, so * the op is durable on a 2-of-3 majority. */ { persist_t* a = make_durability_persist(tmp_a); TEST_ASSERT_NOT_NULL(a); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(a, WAL_JOB_UPDATE, payload, plen)); TEST_ASSERT_EQUAL_INT(0, persist_flush(a)); TEST_ASSERT_TRUE(persist_durable_lsn(a) >= 1); teardown_durability_persist(a); } { persist_t* b = make_durability_persist(tmp_b); TEST_ASSERT_NOT_NULL(b); /* The standby applies the SAME replicated entry durably before ACK * (persist_wal_write_replicated fdatasyncs inline — Req 1.3). */ TEST_ASSERT_EQUAL_INT(0, persist_wal_write_replicated(b, /*lsn=*/1, WAL_JOB_UPDATE, payload, plen)); TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(b) >= 1); teardown_durability_persist(b); } /* "Lose" the sole durable contributor A: destroy its on-disk WAL entirely * (simulates a member that is gone / came back with a truncated log). */ { char rm[600]; snprintf(rm, sizeof(rm), "rm -rf %s", tmp_a); TEST_ASSERT_EQUAL_INT(0, system(rm)); } /* The acknowledged op MUST still be present and recoverable on the * surviving member B — read straight from disk by a fresh process with * NO in-memory state (the durable-majority property in action). */ { persist_t* b2 = make_durability_persist(tmp_b); TEST_ASSERT_NOT_NULL(b2); /* On-disk durable floor (CRC-validated re-read) still covers the op. */ TEST_ASSERT_TRUE_MESSAGE( persist_test_durable_lsn_on_disk(b2) >= 1, "the acknowledged op must survive on the durable majority after " "the sole-durable-contributor copy is lost"); /* And a real recovery reconstructs it cleanly (not truncated). */ TEST_ASSERT_EQUAL_INT(0, persist_recover(b2)); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_CLEAN, persist_recovery_verdict(b2), "the survivor recovers the committed op as a clean intact log"); TEST_ASSERT_TRUE_MESSAGE( persist_durable_floor(b2) >= 1, "the survivor's recovered durable floor covers the committed op"); teardown_durability_persist(b2); } { char rm[600]; snprintf(rm, sizeof(rm), "rm -rf %s", tmp_b); (void)system(rm); } printf("test_sole_durable_contributor_loss_survives: PASS " "(durable-majority property; live failover proof is task 5.2)\n"); } /* ------------------------------------------------------------------ * * Test 5.1c (Req 1.6): durability work does not stall heartbeat / * * election handling — fdatasync runs on the dedicated flusher thread, * * NOT on the caller (iomgr/heartbeat) thread. * * * * Structural + timing assertion: while a durable flush is in flight on * * the flusher thread, a heartbeat-style callback executed on the * * caller/iomgr thread completes within a tight tolerance. We prove the * * flush is OFF the caller thread by capturing the flusher's thread id at * * the fdatasync site (via the XCE_TESTING seam) and asserting it is NOT * * the caller thread. (persist_wal_async only appends + signals; it never * * fdatasyncs inline, so the append path itself never blocks on disk.) * * ------------------------------------------------------------------ */ static void test_durability_work_off_iomgr_preserves_liveness(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_durliveness_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); pthread_t caller = pthread_self(); /* Drive a steady stream of appends on the CALLER (iomgr-equivalent) * thread; the real flusher thread does the fdatasync work concurrently. * After each append, a heartbeat-style callback must complete promptly — * the append path signals the flusher and returns without blocking on * fdatasync, so heartbeat handling on this thread is never stalled. */ const long tolerance_us = 50 * 1000; /* 50 ms — generous heartbeat budget */ for (int i = 0; i < 64; i++) { TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "h", 1)); struct timespec t0, t1; clock_gettime(CLOCK_MONOTONIC, &t0); /* Heartbeat-style work on the caller thread (a cheap atomic read of * commit/durable state, exactly what the heartbeat path inspects). */ volatile uint64_t hb = atomic_load(&p->committed_lsn) + persist_durable_lsn(p); (void)hb; clock_gettime(CLOCK_MONOTONIC, &t1); long elapsed_us = (t1.tv_sec - t0.tv_sec) * 1000000L + (t1.tv_nsec - t0.tv_nsec) / 1000L; TEST_ASSERT_TRUE_MESSAGE( elapsed_us < tolerance_us, "heartbeat-style work on the caller thread must not be stalled " "by durability (fdatasync) work in flight on the flusher thread"); } /* Let the flusher make everything durable, then confirm the fdatasync * provably ran on a DIFFERENT thread than this caller — durability work * is off the iomgr/heartbeat thread (Req 1.6). The seam records the * thread id of the last successful flusher fdatasync. */ uint64_t tip = p->current_lsn - 1; TEST_ASSERT_TRUE(wait_durable_reaches(p, tip, 5000) >= tip); pthread_t flusher_tid = persist_test_last_fdatasync_thread(p); TEST_ASSERT_FALSE_MESSAGE( pthread_equal(flusher_tid, caller), "fdatasync must run on the dedicated flusher thread, never on the " "caller/iomgr/heartbeat thread"); teardown_durability_persist(p); printf("test_durability_work_off_iomgr_preserves_liveness: PASS\n"); } /* ================================================================== * * task 3.1 — recovery integrity verdict (Req 2.1) * * * * persist_recover/replay_wal must distinguish a CLEAN log end (EOF, * * pre-allocated-zero tail, snapshot-sentinel + clean replay, empty * * first-boot WAL) from a corruption-TRUNCATED end (CRC chain break, * * CRC mismatch, short/partial entry read). A corruption-truncated * * recovery is NEVER reported as a clean recovery and sets a * * member-level recovery_truncated flag the promotion gate (4.3) * * consumes. 3.1 only SETS the verdict/flag — it does NOT FATAL and * * does NOT fence (3.3 / 4.3 own those). * * * * NB the verdict is about STRUCTURAL integrity of the local log, not * * about being behind the cluster: a clean log that ends below the * * cluster high-water is still CLEAN (the freshness/promotion fence * * catches "behind", not this verdict). * * ------------------------------------------------------------------ */ /* The persist paths derive as //data/wal; make_durability_persist * uses name "ctrl-test" (see persist_set_workdir call above). */ #define DUR_WAL_SEG_FMT "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal" /* Open the first WAL segment (starts at LSN 1; LSN 0 reserved) for the * given tmpdir and return its fd, or -1. */ static int open_first_seg_rw(const char* tmpdir) { char seg_path[512]; /* The first-ever entry is LSN 1, so the first segment is seg-...0001. */ snprintf(seg_path, sizeof(seg_path), DUR_WAL_SEG_FMT, tmpdir, (uint64_t)1); return open(seg_path, O_RDWR); } /* Open a SECOND persist instance over the same on-disk WAL with NO * in-memory writer state, run recovery, and return the recovery verdict * (via persist_recovery_truncated). The writer must already be torn down * so the segments are closed and on disk. */ /* ------------------------------------------------------------------ * * Test 3.1a: an intact log yields RECOVER_CLEAN (flag NOT set). * * ------------------------------------------------------------------ */ static void test_recover_verdict_clean_intact_log(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_recclean_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Writer: append + flush three entries, then "die". */ { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_durability_persist(p); } /* Recover into a fresh instance: an intact CRC chain must be CLEAN. */ { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_CLEAN, persist_recovery_verdict(p2), "an intact CRC chain must recover CLEAN"); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_truncated(p2), "a clean recovery must NOT set the recovery_truncated flag"); /* current_lsn advanced past the three replayed entries. */ TEST_ASSERT_EQUAL_UINT64(4, p2->current_lsn); teardown_durability_persist(p2); } printf("test_recover_verdict_clean_intact_log: PASS\n"); } /* ------------------------------------------------------------------ * * Test 3.1b: an empty first-boot WAL is CLEAN, not corruption. * * ------------------------------------------------------------------ */ static void test_recover_verdict_empty_first_boot_clean(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_recempty_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* No appends: a first-boot WAL has nothing to replay — that is a clean * (empty) recovery, never a corruption-truncated one. */ int rc = persist_recover(p); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_CLEAN, persist_recovery_verdict(p), "an empty first-boot WAL must be CLEAN (no entries != corruption)"); TEST_ASSERT_FALSE(persist_recovery_truncated(p)); teardown_durability_persist(p); printf("test_recover_verdict_empty_first_boot_clean: PASS\n"); } /* ------------------------------------------------------------------ * * Test 3.1c (core RED): a CRC break injected at the FIRST entry yields * * RECOVER_TRUNCATED and SETS the recovery_truncated flag. * * * * We append+flush two entries, then corrupt the on-disk `crc` field of * * the FIRST entry (LSN 1). On recovery the CRC verification at LSN 1 * * fails, replay stops before any clean boundary, and the verdict must * * be TRUNCATED — never silently accepted as a complete clean recovery. * * ------------------------------------------------------------------ */ static void test_recover_verdict_crc_break_first_entry_truncated(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_recbreak_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Writer: two durable entries, then die. */ { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_durability_persist(p); } /* Corrupt the FIRST entry's CRC field on disk: read its header at * offset 0, flip the crc, write it back. This breaks the chain at * LSN 1 — replay cannot reach any clean boundary. */ { int fd = open_first_seg_rw(tmp); TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist"); wal_entry_hdr_t hdr; TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pread(fd, &hdr, sizeof(hdr), 0)); hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */ TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pwrite(fd, &hdr, sizeof(hdr), 0)); fsync(fd); close(fd); } /* Recover: the corruption at LSN 1 must yield TRUNCATED + set the flag. */ { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); int rc = persist_recover(p2); /* persist_recover still returns 0 (no I/O error); the VERDICT is the * signal — 3.1 classifies, it does not FATAL (that is task 3.3). */ TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_TRUNCATED, persist_recovery_verdict(p2), "a CRC break at the first entry must yield RECOVER_TRUNCATED"); TEST_ASSERT_TRUE_MESSAGE( persist_recovery_truncated(p2), "a corruption-truncated recovery must SET the recovery_truncated " "flag (consumed by the promotion gate, task 4.3)"); teardown_durability_persist(p2); } printf("test_recover_verdict_crc_break_first_entry_truncated: PASS\n"); } /* ------------------------------------------------------------------ * * bug-778 regression: the standby replicated-apply path must leave an * * intact on-disk CRC chain that cold-restart replay can recover. * * * * Drives the REAL standby path (ha_apply_wal_catchup_entry -> * * ha_apply_single_entry -> persist_wal_write_replicated + apply_wal_ * * entry) for N entries, then cold-recovers. Before the fix, * * apply_wal_entry clobbered p->last_crc with the synthetic header's * * crc=0, so every entry after the first wrote prev_crc=0 -> CRC chain * * break -> RECOVER_TRUNCATED and the committed entries were lost on * * cold restart. cluster==NULL here (entity restore is skipped); the * * physical WAL write + CRC chain is what is under test. * * ------------------------------------------------------------------ */ static void test_replicated_apply_chain_survives_cold_restart(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_replchain_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); const int N = 6; /* Writer: apply N replicated entries via the real standby path, then die. */ { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); ha_state_t ha; memset(&ha, 0, sizeof(ha)); for (int i = 1; i <= N; i++) { wal_jobstate_t js = { .jobid = (uint64_t)i, .state = 0x10 }; wal_catchup_entry_t e; memset(&e, 0, sizeof(e)); e.lsn = (uint64_t)i; e.entry_type = WAL_JOB_UPDATE; e.n_entries = 0; /* single-entry mode */ e.raw_payload = (uint8_t*)&js; e.raw_payload_len = sizeof(js); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, ha_apply_wal_catchup_entry(&ha, p, &e, NULL, NULL), "replicated apply of a streamed entry must succeed"); } TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_durability_persist(p); } /* Cold restart: the standby's own WAL must form an intact CRC chain. */ { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); TEST_ASSERT_EQUAL_INT(0, persist_recover(p2)); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_CLEAN, persist_recovery_verdict(p2), "replicated-apply WAL must form an intact CRC chain on cold restart " "(bug-778: apply_wal_entry must not clobber the chain link)"); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_truncated(p2), "no truncation: every replicated entry must be recoverable"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( (uint64_t)N + 1, p2->current_lsn, "all N replicated entries must replay (current_lsn == N+1)"); teardown_durability_persist(p2); } printf("test_replicated_apply_chain_survives_cold_restart: PASS\n"); } /* ------------------------------------------------------------------ * * Test 3.1d (snapshot-sentinel edge): a member that recovered from a * * 16-byte LSN-sentinel snapshot has a committed position at the * * 16-byte LSN-sentinel snapshot has a committed position at the * * sentinel LSN but NO log entries below it. A clean replay over that * * sentinel snapshot must classify CLEAN, never TRUNCATED — the absence * * of sub-sentinel entries is normal, not corruption (design Open * * Questions / snapshot-recovery edge). * * * * We checkpoint (writes the sentinel snapshot + trims the WAL), append * * + flush one fresh entry above the checkpoint, then recover in a fresh * * instance: snapshot-sentinel + clean replay ⇒ CLEAN. * * ------------------------------------------------------------------ */ static void test_recover_verdict_snapshot_sentinel_clean(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_recsnap_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Writer: a couple of entries, a checkpoint (sentinel snapshot), then a * further entry above the checkpoint, all flushed; then die. */ { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); /* Sentinel snapshot at the current committed/checkpoint position. */ TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_durability_persist(p); } /* Recover: load_snapshot sets checkpoint_lsn from the sentinel, replay * picks up entries above it cleanly — verdict CLEAN, flag NOT set. */ { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_CLEAN, persist_recovery_verdict(p2), "snapshot-sentinel + clean replay must be CLEAN, not TRUNCATED"); TEST_ASSERT_FALSE(persist_recovery_truncated(p2)); teardown_durability_persist(p2); } printf("test_recover_verdict_snapshot_sentinel_clean: PASS\n"); } /* ================================================================== * * task 3.4 — re-derive the durable floor from the recovered WAL tip * * (Req 2.3) * * * * On recovery the member publishes its durable floor as the highest * * INTACT replayed LSN (the WAL tip / durable last-log index), so the * * freshness advertisement (task 4.2) is backed by durable on-disk * * state rather than a volatile in-memory position. The floor is the * * durable last-log index, NOT the commit position, and NOT a stale * * higher in-memory value. persist_durable_floor() exposes it (a thin * * wrapper over the durable_lsn watermark, which recovery sets to the * * recovered tip). * * ------------------------------------------------------------------ */ /* Test 3.4a (CRUX, Req 2.3): after a TRUNCATING restart the published * durable floor equals the highest INTACT LSN (the WAL tip), NOT a stale * higher in-memory value and NOT the commit position. * * Writer lays down 4 durable entries (LSN 1..4). We then corrupt entry 3's * CRC on disk so the intact prefix is LSN 1..2 — replay stops at the chain * break and the recovered tip is LSN 2. To make the "not a stale higher * in-memory value" assertion meaningful we deliberately seed the fresh * instance's in-memory durable_lsn / committed_lsn to an over-advertised * value (4) BEFORE recovery; recovery must publish the durable FLOOR at the * intact tip 2, never the seeded 4. */ static void test_durable_floor_truncating_restart_is_intact_tip(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_floortrunc_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); /* Writer: four durable entries (LSN 1..4), then die. */ { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "d", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); /* All four are durable on disk before the truncation. */ TEST_ASSERT_EQUAL_UINT64(4, persist_test_durable_lsn_on_disk(p)); teardown_durability_persist(p); } /* Corrupt entry 3 (LSN 3): walk the segment to its offset and flip its * crc. Each entry is sizeof(hdr) + payload_len (=1) bytes; entries 1 and 2 * precede it. This breaks the chain at LSN 3, so the intact tip is LSN 2. */ { int fd = open_first_seg_rw(tmp); TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist"); uint32_t off = 0; wal_entry_hdr_t hdr; /* Skip entries 1 and 2 (LSN 1, LSN 2). */ for (int i = 0; i < 2; i++) { TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pread(fd, &hdr, sizeof(hdr), (off_t)off)); off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } /* Entry 3 (LSN 3) is at `off`: corrupt its CRC. */ TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pread(fd, &hdr, sizeof(hdr), (off_t)off)); TEST_ASSERT_EQUAL_UINT64(3, hdr.lsn); hdr.crc ^= 0xDEADBEEFu; TEST_ASSERT_EQUAL_INT((int)sizeof(hdr), (int)pwrite(fd, &hdr, sizeof(hdr), (off_t)off)); fsync(fd); close(fd); } /* Recover into a fresh instance that we deliberately poison with a stale, * over-advertised in-memory position (4) BEFORE recovery. */ { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); /* Stale higher in-memory values: simulate a volatile position that * over-states what is durable. The published floor must NOT come from * these — it must come from the recovered intact WAL tip. */ atomic_store(&p2->durable_lsn, 4); atomic_store(&p2->committed_lsn, 4); int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); /* Verdict is TRUNCATED (chain break at LSN 3). */ TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_TRUNCATED, persist_recovery_verdict(p2), "a CRC break at LSN 3 must yield RECOVER_TRUNCATED"); /* The published durable floor == the highest INTACT LSN (2): the WAL * tip, equal to current_lsn - 1 after replay (replay advanced * current_lsn only to one past the last intact entry). */ uint64_t floor = persist_durable_floor(p2); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 2, floor, "durable floor after a truncating restart must equal the highest " "intact LSN (the WAL tip), NOT the stale in-memory value"); TEST_ASSERT_EQUAL_UINT64_MESSAGE( p2->current_lsn - 1, floor, "durable floor must equal the recovered WAL tip (current_lsn - 1)"); /* Independently confirmed against the on-disk re-read seam. */ TEST_ASSERT_EQUAL_UINT64_MESSAGE( 2, persist_test_durable_lsn_on_disk(p2), "on-disk re-read agrees: only LSN 1..2 are intact"); teardown_durability_persist(p2); } printf("test_durable_floor_truncating_restart_is_intact_tip: PASS\n"); } /* Test 3.4b: an INTACT-log recovery publishes the FULL tip (no truncation). * Three durable entries (LSN 1..3) recover cleanly; the floor is the full * recovered tip 3 == current_lsn - 1. */ static void test_durable_floor_intact_recovery_full_tip(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_floorintact_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_durability_persist(p); } { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2)); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 3, persist_durable_floor(p2), "intact recovery publishes the full recovered WAL tip"); TEST_ASSERT_EQUAL_UINT64(p2->current_lsn - 1, persist_durable_floor(p2)); teardown_durability_persist(p2); } printf("test_durable_floor_intact_recovery_full_tip: PASS\n"); } /* Test 3.4c: an EMPTY first-boot WAL publishes floor 0 (nothing durable). */ static void test_durable_floor_empty_recovery_zero(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_floorempty_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); int rc = persist_recover(p); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_UINT64_MESSAGE( 0, persist_durable_floor(p), "an empty first-boot WAL has nothing durable — floor is 0"); teardown_durability_persist(p); printf("test_durable_floor_empty_recovery_zero: PASS\n"); } /* Test 3.4d (snapshot-sentinel edge): a member recovered from a 16-byte * LSN-sentinel snapshot plus a clean replay above it publishes the recovered * tip (the highest replayed LSN). The floor is the recovered tip, consistent * with 3.1's CLEAN classification — "behind-and-must-catch-up", not * over-advertising. */ static void test_durable_floor_snapshot_sentinel_tip(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_floorsnap_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); { persist_t* p = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); /* Sentinel snapshot at the checkpoint position; then one entry above. */ TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p)); TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); teardown_durability_persist(p); } { persist_t* p2 = make_durability_persist(tmp); TEST_ASSERT_NOT_NULL(p2); int rc = persist_recover(p2); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_EQUAL_INT(RECOVER_CLEAN, persist_recovery_verdict(p2)); /* The floor is the recovered tip = current_lsn - 1 (whatever the * highest replayed LSN above the sentinel is). It is the recovered * tip, not a stale higher value. */ TEST_ASSERT_EQUAL_UINT64_MESSAGE( p2->current_lsn - 1, persist_durable_floor(p2), "snapshot-sentinel recovery publishes the recovered WAL tip"); TEST_ASSERT_TRUE_MESSAGE( persist_durable_floor(p2) > 0, "a sentinel + clean replay-above has a non-zero recovered tip"); teardown_durability_persist(p2); } printf("test_durable_floor_snapshot_sentinel_tip: PASS\n"); } /* ================================================================== * * task 3.2 — Boot Policy Decision Table (DECISION ONLY) * * * * Task 3.2 pins the fail-loud-vs-fence boot policy as a DECISION and * * encodes it as a table of input states -> expected outcome. It makes * * NO production behavior change: the FATAL branch is implemented by * * task 3.3 and the promotion fence + release by task 4.3. * * * * The decision table lives in design.md ("Boot Policy Decision Table * * (task 3.2)"). The skeletons below mirror it ONE TEST PER ROW. Each * * skeleton documents the row's inputs + the expected outcome it WILL * * assert once 3.3/4.3 land, then TEST_IGNORE_MESSAGE()-skips so the * * row is REGISTERED and reported as IGNORED (not failed) — the suite * * stays GREEN. These become the RED tests 3.3/4.3 turn GREEN. * * * * Inputs (boot / promotion-eligibility evaluation): * * verdict : CLEAN | TRUNCATED (from task 3.1) * * durable-vs-HWM : at/above | below (durable position) * * healthy peer : yes | no (catch-up source) * * catch-up : succeeds | fails (bounded window) * * Outcomes: * * ELIGIBLE | FENCE+CATCH-UP -> ELIGIBLE | FATAL * * INVARIANT: never a silent indefinite fence — a truncated/behind * * member either catches up (-> eligible) or FATALs on a BOUNDED * * failure. (Req 2.2, 2.4) * * ------------------------------------------------------------------ */ /* task 3.3 — unrecoverable loud-fence verification (never-self-exit rework * 2026-06-18). The OLD behavior installed persist_fatal_hook to capture an * abort()/exit() without dying. That self-exit terminal is GONE: an * unrecoverable member STAYS ALIVE and is held permanently non-promotable. * There is no hook to install — the tests assert the OBSERVABLE * post-conditions instead: * - the pure decision returns RECOVER_DECISION_FENCE_UNRECOVERABLE, * - persist_recover_loud_fence_if_unrecoverable RETURNS (no abort — the test * simply continuing past the call proves the process stayed alive), * - the sticky persist_recovery_unrecoverable flag is SET (member is * permanently non-promotable until catch-up), * - and the flag CLEARS on catch-up (Req 2.6). * dur_persist_for_flag builds a real persist_t the loud-fence helper can set * the sticky flag on. */ static persist_t* dur_flag_persist(const char* tmpdir) { persist_t* p = calloc(1, sizeof(*p)); if (!p) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; persist_set_workdir(p, tmpdir, "ctrl-flag"); if (persist_init(p, &cfg, NULL) != 0) { free(p); return NULL; } return p; } static void dur_free_flag_persist(persist_t* p) { if (!p) return; persist_fini(p); free(p); } /* task 3.3 — peer-reachable NEVER FATALs (the never-FATAL aspect of rows * 3 & 5). This task (3.3) owns only the FATAL terminal; the full FENCE+ * CATCH-UP -> ELIGIBLE eligibility outcome of these rows is task 4.3 (so the * row3/row5 skeletons below stay IGNORED for 4.3). Here we pin ONLY the * invariant 3.3 is responsible for: while a healthy peer is reachable, the * decision is NEVER FATAL — neither for a TRUNCATED log (row 3) nor for a * legitimate snapshot-sentinel CLEAN-but-behind member (row 5). It is FENCE * (catch-up pending) and the FATAL terminate path is never fired. (Req 2.2, * 2.4, design.md "never FATAL when a peer is reachable".) */ static void test_boot_policy_peer_reachable_never_fatal(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_dur_peerfence_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = dur_flag_persist(tmp); TEST_ASSERT_NOT_NULL(p); /* Row 3: TRUNCATED + healthy peer (catch-up not yet done) -> FENCE. */ recover_policy_inputs_t truncated_peer = { .verdict = RECOVER_TRUNCATED, .durable_below_hwm = true, .healthy_peer_reachable = true, .catch_up_succeeded = false, .durable_floor_lsn = 3, .cluster_hwm_lsn = 10, }; TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE, persist_recover_decision(&truncated_peer), "row 3: a truncated member with a reachable healthy peer must FENCE " "(catch up), NEVER the unrecoverable terminal"); TEST_ASSERT_EQUAL_INT( RECOVER_DECISION_FENCE, persist_recover_loud_fence_if_unrecoverable(p, &truncated_peer)); /* Row 5: snapshot-sentinel CLEAN-but-behind + healthy peer -> FENCE. * A legitimate sentinel is CLEAN-but-behind, not corruption. */ recover_policy_inputs_t sentinel_peer = { .verdict = RECOVER_CLEAN, .durable_below_hwm = true, .healthy_peer_reachable = true, .catch_up_succeeded = false, .durable_floor_lsn = 5, .cluster_hwm_lsn = 5, }; TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE, persist_recover_decision(&sentinel_peer), "row 5: a snapshot-sentinel CLEAN-but-behind member with a reachable " "peer must FENCE, NEVER the unrecoverable terminal"); TEST_ASSERT_EQUAL_INT( RECOVER_DECISION_FENCE, persist_recover_loud_fence_if_unrecoverable(p, &sentinel_peer)); /* The unrecoverable sticky flag must NOT have been set for either * peer-reachable case — a reachable healthy peer never reaches the * unrecoverable terminal. */ TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(p), "peer-reachable rows must NEVER set the unrecoverable sticky flag"); dur_free_flag_persist(p); printf("test_boot_policy_peer_reachable_never_fatal: PASS\n"); } /* Row 1: CLEAN + durable at/above HWM -> ELIGIBLE. * A structurally intact log that is not behind the cluster boundary is * a normal promotable member: the promotion gate (4.3) must NOT fence * it. WILL ASSERT (4.3): the eligibility predicate returns "eligible" * for a CLEAN, at/above member; promotion proceeds. (Req 2.3) */ /* Minimal ha_config_t accepted by ha_init (mirrors test_ha_role_lifecycle.c). */ static ha_config_t dur_ha_cfg(void) { ha_config_t cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 10000, .election_timeout_ms = 2000, .probe_connect_timeout_ms = 500, .max_promotion_retries = 5, }; return cfg; } static void test_boot_policy_row1_clean_at_or_above_eligible(void) { /* CLEAN + durable at/above HWM -> ELIGIBLE. The live gate must admit a * structurally-intact, not-behind member and must NOT set the fence. */ ha_state_t ha; ha_config_t cfg = dur_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* Pre-set the fence to prove ELIGIBLE clears it (Req 2.6 release path). */ atomic_store_explicit(&ha.in_catch_up, 1, memory_order_release); recover_decision_t d = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/8, /*safe_hwm=*/8, /*peer_reachable=*/false, /*catch_up_succeeded=*/false); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, d, "row 1: a CLEAN, at/above-HWM member must be ELIGIBLE to promote"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire), "row 1: ELIGIBLE must clear the in_catch_up fence"); ha_fini(&ha); printf("test_boot_policy_row1_clean_at_or_above_eligible: PASS\n"); } /* Row 2: CLEAN + durable below HWM + healthy peer + catch-up succeeds * -> FENCE+CATCH-UP -> ELIGIBLE. * Intact but behind: must NOT lead below the boundary; fence, catch up * from a healthy peer, regain eligibility. WILL ASSERT (4.3): a * below-HWM member is denied promotion until catch-up passes the last * committed boundary, then regains eligibility. (Req 2.2, 2.3, 2.6) */ static void test_boot_policy_row2_clean_below_peer_catchup_eligible(void) { /* CLEAN + below HWM + healthy peer: FENCE first (denied promotion), then * once catch-up passes the committed boundary, ELIGIBLE (fence released). */ ha_state_t ha; ha_config_t cfg = dur_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* Behind: durable_floor 4 < HWM 10, healthy peer reachable, not yet caught * up -> FENCE; the in_catch_up fence is SET (refused promotion). */ recover_decision_t d1 = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/4, /*safe_hwm=*/10, /*peer_reachable=*/true, /*catch_up_succeeded=*/false); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE, d1, "row 2: a CLEAN-but-behind member with a peer must FENCE (catch up)"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire), "row 2: FENCE must SET the in_catch_up fence (promotion refused)"); /* Catch-up brings the durable position to/above the committed boundary * (durable_floor 10 >= HWM 10) -> ELIGIBLE; the fence is released (Req 2.6). */ recover_decision_t d2 = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10, /*peer_reachable=*/true, /*catch_up_succeeded=*/true); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, d2, "row 2: after catch-up past the committed boundary the member regains " "eligibility (Req 2.6)"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire), "row 2: regained eligibility must RELEASE the in_catch_up fence"); ha_fini(&ha); printf("test_boot_policy_row2_clean_below_peer_catchup_eligible: PASS\n"); } /* Row 3: TRUNCATED + healthy peer + catch-up succeeds * -> FENCE+CATCH-UP -> ELIGIBLE. * Structurally truncated (task-3.1 verdict) but a healthy peer exists: * fence, catch up, regain eligibility — NEVER FATAL when a peer is * reachable. WILL ASSERT (4.3): a recovery_truncated member is denied * promotion, catches up from the peer, and is re-enabled; FATAL is NOT * triggered. (Req 2.2, 2.6) */ static void test_boot_policy_row3_truncated_peer_catchup_eligible(void) { /* TRUNCATED + healthy peer: a recovery-truncated member is DENIED promotion * (FENCE), catches up from the peer, then regains eligibility. FATAL must * NOT fire while a healthy peer is reachable. */ ha_state_t ha; ha_config_t cfg = dur_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* Truncated boundary at 3, cluster HWM 10, peer reachable, not caught up. */ recover_decision_t d1 = ha_evaluate_promotion_fence( &ha, /*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10, /*peer_reachable=*/true, /*catch_up_succeeded=*/false); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE, d1, "row 3: a recovery-truncated member with a healthy peer must FENCE " "(denied promotion), never the unrecoverable terminal"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire), "row 3: a truncated member must be fenced (in_catch_up set)"); /* After catch-up to/above the committed boundary -> ELIGIBLE, fence released. */ recover_decision_t d2 = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/10, /*safe_hwm=*/10, /*peer_reachable=*/true, /*catch_up_succeeded=*/true); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, d2, "row 3: after catch-up the (previously truncated) member is eligible"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire), "row 3: regained eligibility releases the fence (Req 2.6)"); ha_fini(&ha); printf("test_boot_policy_row3_truncated_peer_catchup_eligible: PASS\n"); } /* Row 4: TRUNCATED + below HWM + NO healthy peer + catch-up fails * -> LOUD PERMANENT FENCE (non-promotable, STAYS ALIVE, keeps retrying), and * RELEASES on catch-up (Req 2.4, 2.6). * * never-self-exit rework (2026-06-18): the member cannot recover locally AND * cannot catch up right now. It does NOT self-exit; it logs LOUDLY, sets the * sticky non-promotable flag, and stays alive. The flag clears once a peer * returns and it catches up. RED→GREEN: pre-rework * persist_recover_fatal_if_unrecoverable aborted (or fired the abort-capture * hook); post-rework persist_recover_loud_fence_if_unrecoverable RETURNS — the * test continuing past the call IS the proof the process stayed alive. */ static void test_boot_policy_row4_truncated_no_peer_loud_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_dur_row4_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = dur_flag_persist(tmp); TEST_ASSERT_NOT_NULL(p); recover_policy_inputs_t in = { .verdict = RECOVER_TRUNCATED, .durable_below_hwm = true, .healthy_peer_reachable = false, .catch_up_succeeded = false, .durable_floor_lsn = 7, /* truncation boundary */ .cluster_hwm_lsn = 12, /* cluster committed high-water */ }; /* Pure decision returns the non-terminating unrecoverable fence. */ TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE_UNRECOVERABLE, persist_recover_decision(&in), "row 4: truncated + below + no peer + no catch-up must decide the LOUD " "PERMANENT FENCE — NOT process exit"); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(p), "row 4: the sticky flag is set by the ACTION helper, not the pure " "decision"); /* The action helper logs LOUDLY, sets the sticky flag, and RETURNS (no * abort/exit — the process stays alive; the test reaching the next line is * the proof). */ recover_decision_t d = persist_recover_loud_fence_if_unrecoverable(p, &in); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE_UNRECOVERABLE, d, "row 4: the loud-fence action returns FENCE_UNRECOVERABLE (stays alive)"); TEST_ASSERT_TRUE_MESSAGE( persist_recovery_unrecoverable(p), "row 4: the member is held PERMANENTLY non-promotable (sticky flag set)"); /* Req 2.6: once catch-up reaches a committed-consistent state the fence * releases automatically — the flag clears, the member is promotable. */ persist_clear_recovery_unrecoverable(p); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(p), "row 4: catch-up CLEARS the unrecoverable fence (Req 2.6)"); dur_free_flag_persist(p); printf("test_boot_policy_row4_truncated_no_peer_loud_fence: PASS\n"); } /* Row 4b: CLEAN + below HWM + NO healthy peer + catch-up fails * -> LOUD PERMANENT FENCE (non-promotable, STAYS ALIVE, keeps retrying). * Behind the HWM with no peer to advance from — same non-terminating loud-fence * terminal as row 4. NEVER self-exits. (Req 2.4, 2.2, 2.6) */ static void test_boot_policy_row4b_clean_below_no_peer_loud_fence(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_dur_row4b_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); persist_t* p = dur_flag_persist(tmp); TEST_ASSERT_NOT_NULL(p); recover_policy_inputs_t in = { .verdict = RECOVER_CLEAN, /* structurally intact... */ .durable_below_hwm = true, /* ...but behind the HWM */ .healthy_peer_reachable = false, .catch_up_succeeded = false, .durable_floor_lsn = 9, .cluster_hwm_lsn = 20, }; TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE_UNRECOVERABLE, persist_recover_decision(&in), "row 4b: clean-but-behind + no peer + no catch-up must decide the LOUD " "PERMANENT FENCE — never process exit, never a silent fence"); recover_decision_t d = persist_recover_loud_fence_if_unrecoverable(p, &in); TEST_ASSERT_EQUAL_INT(RECOVER_DECISION_FENCE_UNRECOVERABLE, d); TEST_ASSERT_TRUE_MESSAGE( persist_recovery_unrecoverable(p), "row 4b: a behind member with no peer is held non-promotable + ALIVE " "(sticky flag set; boundary logged loudly)"); persist_clear_recovery_unrecoverable(p); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(p), "row 4b: catch-up CLEARS the unrecoverable fence (Req 2.6)"); dur_free_flag_persist(p); printf("test_boot_policy_row4b_clean_below_no_peer_loud_fence: PASS\n"); } /* Row 5: CLEAN snapshot-sentinel (committed at sentinel LSN, NO * sub-sentinel entries) + below + healthy peer + catch-up succeeds * -> FENCE+CATCH-UP -> ELIGIBLE. * A legitimate sentinel snapshot is CLEAN-but-behind, NOT corruption: * treat as must-catch-up, NEVER FATAL on a legitimate sentinel. WILL * ASSERT (4.3): a snapshot-sentinel member is fenced+caught-up like any * behind member and is NEVER routed to the FATAL branch. (Req 2.1, 2.2) * (3.1 already verifies the verdict itself is CLEAN — see * test_recover_verdict_snapshot_sentinel_clean; this row pins the * downstream promotion-fence outcome.) */ static void test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal(void) { /* Snapshot-sentinel CLEAN-but-behind + healthy peer: treated as behind, * must catch up (FENCE), NEVER FATAL on a legitimate sentinel; then * ELIGIBLE after catch-up. */ ha_state_t ha; ha_config_t cfg = dur_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); /* Sentinel at LSN 5, cluster has advanced to HWM 12, peer reachable. */ recover_decision_t d1 = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/12, /*peer_reachable=*/true, /*catch_up_succeeded=*/false); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE, d1, "row 5: a snapshot-sentinel CLEAN-but-behind member must FENCE, never " "the unrecoverable terminal while a peer is reachable"); recover_decision_t d2 = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/12, /*safe_hwm=*/12, /*peer_reachable=*/true, /*catch_up_succeeded=*/true); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, d2, "row 5: after catch-up the sentinel member is eligible (Req 2.6)"); ha_fini(&ha); printf("test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal: PASS\n"); } /* Row 6: CLEAN empty first-boot WAL (durable=0, HWM=0, fresh cold start) * -> ELIGIBLE. * An empty first-boot WAL is not behind anything — cold-start members * are promotable. WILL ASSERT (4.3): the eligibility gate admits a * fresh cold-start member (no fence, no FATAL). (Req 2.1, 2.3) */ static void test_boot_policy_row6_empty_first_boot_eligible(void) { /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fresh cold-start * member is not behind anything and must be promotable (no fence, no FATAL), * even with no peer reachable (single-node / first boot). */ ha_state_t ha; ha_config_t cfg = dur_ha_cfg(); TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0)); recover_decision_t d = ha_evaluate_promotion_fence( &ha, /*truncated=*/false, /*durable_floor=*/0, /*safe_hwm=*/0, /*peer_reachable=*/false, /*catch_up_succeeded=*/false); TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, d, "row 6: an empty first-boot WAL (durable=0/HWM=0) must be ELIGIBLE"); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load_explicit(&ha.in_catch_up, memory_order_acquire), "row 6: a cold-start member must not be fenced"); ha_fini(&ha); printf("test_boot_policy_row6_empty_first_boot_eligible: PASS\n"); } /* ── task 4.3: hollow node refused even at minimal quorum (Req 2.5) ────────── * A hollow node (recovery-truncated OR durable below the HWM) must be refused * promotion regardless of quorum size — combined with the 4.2 freshness floor * it can neither win the vote nor self-promote. At minimal quorum with NO * healthy peer, a hollow node is held in the LOUD PERMANENT FENCE * (FENCE_UNRECOVERABLE, stays alive, non-promotable) rather than silently * leading; with a peer it FENCEs. Either way it never becomes ELIGIBLE while * hollow. */ static void test_promotion_gate_hollow_refused_at_minimal_quorum(void) { /* Truncated + below HWM + no peer (minimal/degraded quorum) -> NEVER * ELIGIBLE (loud permanent fence — never promote a hollow node). */ TEST_ASSERT_NOT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10, /*peer_reachable=*/false, /*catch_up_succeeded=*/false), "a recovery-truncated node must never be ELIGIBLE at minimal quorum " "(Req 2.5)"); /* CLEAN but below HWM + no peer -> NEVER ELIGIBLE (loud permanent fence). */ TEST_ASSERT_NOT_EQUAL_INT_MESSAGE( RECOVER_DECISION_ELIGIBLE, ha_promotion_gate_decision(/*truncated=*/false, /*durable_floor=*/5, /*safe_hwm=*/10, /*peer_reachable=*/false, /*catch_up_succeeded=*/false), "a below-HWM node must never be ELIGIBLE at minimal quorum (Req 2.5)"); /* Hollow node WITH a peer is still refused (FENCE, not ELIGIBLE) until it * catches up — minimal-quorum size does not relax the gate. */ TEST_ASSERT_EQUAL_INT_MESSAGE( RECOVER_DECISION_FENCE, ha_promotion_gate_decision(/*truncated=*/true, /*durable_floor=*/3, /*safe_hwm=*/10, /*peer_reachable=*/true, /*catch_up_succeeded=*/false), "a hollow node with a peer is FENCED (refused) until caught up"); printf("test_promotion_gate_hollow_refused_at_minimal_quorum: PASS\n"); } /* ── task 4.4: catch-up re-alert window (never-self-exit rework 2026-06-18) ─── * * A recovery-truncated/behind member with no reachable peer at the promotion * instant is FENCED (in_catch_up) and STAYS ALIVE, retrying catch-up. A * delaymgr re-alert timer fires after DEFAULT_RECOVERY_CATCHUP_WINDOW_MS: if the * member is still unrecoverable it RE-EMITS the loud operator alert (sets the * sticky persist_recovery_unrecoverable flag) and the process STAYS ALIVE — * there is NO abort/exit anywhere. Catch-up releases the fence + clears the * flag (Req 2.6); shutdown/cancel stops the re-alert. * * These tests drive ha_arm_promotion_fence_timer (the production deferral wired * into raft_cb_become_leader) through a REAL delaymgr with a short window seam, * so the timer fires fast and deterministically. A minimal cluster_t carries * the delaymgr + persist + ha exactly as production does. They assert the * member STAYS ALIVE + non-promotable (sticky flag) instead of asserting an * abort/hook — the self-exit terminal is GONE. */ /* The delaymgr runner used by production (run_delay_tasks in cluster.c) is not * linked into the unit-test target; provide the same trivial pass-through. */ static int dur_run_delay_task(delay_task_entry_fn_t entry, void* arg) { return entry(arg); } /* Build a minimal cluster_t (calloc'd) with a started delaymgr and a recovered * persist whose recovery verdict + durable floor are set directly, so the * promotion fence can be evaluated without a full election. truncated → the * recovery_truncated flag; durable_floor → the durable watermark the gate reads. * ha.repl stays NULL so "no healthy peer reachable" holds in the timer. */ static cluster_t* dur_make_fence_cluster(const char* tmpdir, bool truncated, uint64_t durable_floor) { cluster_t* c = calloc(1, sizeof(*c)); if (!c) return NULL; persist_config_t cfg = persist_config_defaults(); cfg.wal_sync_ms = 5000; cfg.snap_keep_n = 2; persist_set_workdir(&c->persist, tmpdir, "ctrl-test"); if (persist_init(&c->persist, &cfg, NULL) != 0) { free(c); return NULL; } if (persist_start(&c->persist) != 0) { persist_fini(&c->persist); free(c); return NULL; } /* Set the recovered state the gate reads (mirrors a real persist_recover). */ c->persist.recovery_verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN; atomic_store_explicit(&c->persist.durable_lsn, durable_floor, memory_order_release); ha_config_t hcfg = dur_ha_cfg(); if (ha_init(&c->ha, &hcfg, 0) != 0) { persist_stop(&c->persist); persist_fini(&c->persist); free(c); return NULL; } c->ha.raft_c = c; c->ha.repl = NULL; /* no standby registry → no healthy peer reachable */ if (delaymgr_init(&c->delaymgr, 5 /* precision_ms */, dur_run_delay_task) != 0) { ha_fini(&c->ha); persist_stop(&c->persist); persist_fini(&c->persist); free(c); return NULL; } if (delaymgr_start(&c->delaymgr) != 0) { delaymgr_fini(&c->delaymgr); ha_fini(&c->ha); persist_stop(&c->persist); persist_fini(&c->persist); free(c); return NULL; } return c; } static void dur_destroy_fence_cluster(cluster_t* c) { if (!c) return; /* Cancel any pending fence timer BEFORE tearing down the delaymgr so a * still-armed callback cannot fire against freed state. */ ha_cancel_promotion_fence_timer(&c->ha); delaymgr_stop(&c->delaymgr); delaymgr_fini(&c->delaymgr); ha_fini(&c->ha); persist_stop(&c->persist); persist_fini(&c->persist); free(c); } /* Spin until the sticky unrecoverable flag is set on c->persist or a timeout * elapses (the delaymgr fires the re-alert timer on its own worker thread). */ static bool dur_wait_unrecoverable(cluster_t* c, int timeout_ms) { int waited = 0; while (!persist_recovery_unrecoverable(&c->persist) && waited < timeout_ms) { usleep(2000); waited += 2; } return persist_recovery_unrecoverable(&c->persist); } /* 4.4 (never-self-exit): a recovery-truncated/behind member with NO reachable * peer must NOT self-exit — it fences (in_catch_up set, process ALIVE) and arms * the re-alert timer; when the window elapses with no peer/progress the timer * RE-EMITS the loud alert (sets the sticky non-promotable flag) and the process * STAYS ALIVE. * * RED→GREEN: pre-rework the no-peer truncated member was routed (synchronously * or via the timer) to persist_recover_fatal_if_unrecoverable, which abort()ed * (the old fatal-capture hook fired) — that self-exit was the task-5.2 soak * quorum collapse. Post-rework there is NO abort: the member stays alive, * fenced, and loudly non-promotable. Reaching the end of this test (the * process did not die) IS part of the proof. (Req 2.2, 2.4) */ static void test_promotion_fence_unrecoverable_stays_alive_loud(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_fence_defer_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, /*durable_floor=*/4); TEST_ASSERT_NOT_NULL(c); ha_set_promotion_fence_window_ms(40); /* short re-alert window for the test */ /* Arm the fence exactly as raft_cb_become_leader does for a truncated, * below-HWM, no-peer winner. This must NOT self-exit and must fence. */ extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, uint64_t); ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, /*durable_floor=*/4, /*hwm=*/10); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), "4.4: a fenced member is held in_catch_up (refused promotion), ALIVE"); /* Let the re-alert window elapse: still truncated + no peer + no progress → * the timer RE-EMITS the loud alert and SETS the sticky non-promotable flag. * The process STAYS ALIVE (no abort/exit anywhere on this path). */ TEST_ASSERT_TRUE_MESSAGE( dur_wait_unrecoverable(c, 1000), "4.4: the re-alert timer must set the sticky unrecoverable flag after " "the window with no peer/progress — staying ALIVE, not exiting (Req 2.4)"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), "4.4: an unrecoverable member stays fenced (non-promotable) and ALIVE"); ha_set_promotion_fence_window_ms(0); /* restore default */ dur_destroy_fence_cluster(c); printf("test_promotion_fence_unrecoverable_stays_alive_loud: PASS\n"); } /* 4.4 Req 2.6: catch-up within the window → fence + sticky flag released, and * the member NEVER self-exits. Arm the fence, then simulate catch-up (clear * truncated + advance durable_floor past the HWM) before the timer fires; when * the timer fires it observes a caught-up member, releases in_catch_up, clears * the sticky unrecoverable flag, and does NOT set the unrecoverable flag. */ static void test_promotion_fence_released_on_catchup(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_fence_catchup_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, /*durable_floor=*/4); TEST_ASSERT_NOT_NULL(c); ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */ extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, uint64_t); ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, /*durable_floor=*/4, /*hwm=*/10); TEST_ASSERT_EQUAL_INT( 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire)); /* Simulate catch-up completing: log now intact and durable past the HWM. */ c->persist.recovery_verdict = RECOVER_CLEAN; atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release); /* Wait long enough for the window to fire; assert the fence was released and * the sticky flag is NOT set (Req 2.6) — the member self-healed, alive. */ usleep(300 * 1000); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), "4.4: catch-up within the window RELEASES the in_catch_up fence (Req 2.6)"); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(&c->persist), "4.4: a member that caught up must NOT be left in the unrecoverable " "fence (Req 2.6)"); ha_set_promotion_fence_window_ms(0); dur_destroy_fence_cluster(c); printf("test_promotion_fence_released_on_catchup: PASS\n"); } /* 4.4: cancelling the timer (shutdown / recovery) before it fires → the * re-alert never fires, no use-after-free of the timer ctx, and the process * stays alive (it never self-exited even before the cancel). */ static void test_promotion_fence_cancel_stops_realert(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_fence_cancel_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, /*durable_floor=*/4); TEST_ASSERT_NOT_NULL(c); ha_set_promotion_fence_window_ms(60); extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, uint64_t); ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, /*durable_floor=*/4, /*hwm=*/10); /* Cancel before the window elapses. */ ha_cancel_promotion_fence_timer(&c->ha); /* Wait past the window — the cancelled timer must be a no-op (no re-alert, * no UAF). The member stayed alive throughout (no abort anywhere). */ usleep(200 * 1000); TEST_ASSERT_FALSE_MESSAGE( persist_recovery_unrecoverable(&c->persist), "4.4: a cancelled fence timer must not fire the re-alert " "(shutdown/recovery)"); ha_set_promotion_fence_window_ms(0); dur_destroy_fence_cluster(c); printf("test_promotion_fence_cancel_stops_realert: PASS\n"); } /* ── raft-apply-decoupling task 5.3 (round-2 remediation, Req 8.4) ─────────── * CROSS-CONTAMINATION REGRESSION: an APPLY_DIVERGENT-triggered fence must * NEVER be live-cleared by this purely-structural re-check timer, even when * the structural conditions it actually inspects (truncated / durable_floor * vs HWM) become healthy. Round-1 review found BOTH ha.c live-clear call * sites (this timer, ha.c ~867, and the promotion-eligibility gate in * raft_cb_become_leader, ha.c ~1115) blind to WHY the sticky flag was set — * they only ever inspected structural getters. A node whose local apply * dispatch is defective (materialized state diverges from the committed log * — task 5.1's APPLY_DIVERGENT verdict) can have a perfectly healthy WAL * (not truncated, durable_floor >= HWM) by construction, since the two * properties are orthogonal. Before the fix, this test drives exactly that * cross-contamination and asserts the fence does NOT clear. * * This mirrors test_promotion_fence_released_on_catchup's harness and * catch-up simulation EXACTLY, differing only in also marking the fence as * apply_divergent-triggered up front — proving the fix discriminates on the * apply verdict, not merely on the structural fields. */ static void test_promotion_fence_apply_divergent_survives_structural_catchup(void) { char tmp[256]; strcpy(tmp, "/tmp/xce_fence_apply_div_XXXXXX"); TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); cluster_t* c = dur_make_fence_cluster(tmp, /*truncated=*/true, /*durable_floor=*/4); TEST_ASSERT_NOT_NULL(c); /* Mark the fence as APPLY_DIVERGENT-triggered, exactly as persist_recover() * does at boot (task 5.1/5.2) — the WAL-structural fields above * (truncated/durable_floor) were already set independently above, proving * the two conditions are orthogonal by construction. */ c->persist.apply_verdict = APPLY_DIVERGENT; atomic_store_explicit(&c->persist.recovery_unrecoverable, true, memory_order_release); ha_set_promotion_fence_window_ms(120); /* generous so we can mutate first */ extern void ha_arm_promotion_fence_timer_for_test(cluster_t*, bool, uint64_t, uint64_t); ha_arm_promotion_fence_timer_for_test(c, /*truncated=*/true, /*durable_floor=*/4, /*hwm=*/10); TEST_ASSERT_EQUAL_INT( 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire)); /* Simulate the EXACT SAME structural catch-up as * test_promotion_fence_released_on_catchup: log now intact and durable * past the HWM. The apply_verdict / apply-divergent fence trigger is * UNTOUCHED by this — it can only ever be recomputed by a fresh * persist_recover() boot-time replay (task 5.1), never by a live * structural mutation like this one. */ c->persist.recovery_verdict = RECOVER_CLEAN; atomic_store_explicit(&c->persist.durable_lsn, 10, memory_order_release); /* Wait long enough for the window to fire. */ usleep(300 * 1000); /* THE REGRESSION ASSERTION (Req 8.4): even though the structural * conditions the timer inspects are now fully healthy, an * APPLY_DIVERGENT-triggered fence must NOT be released — a local apply * defect cannot be proven fixed by more WAL bytes arriving. Before the * fix this assertion FAILS (the timer clears the flag purely on the * structural check, oblivious to apply_verdict). */ TEST_ASSERT_TRUE_MESSAGE( persist_recovery_unrecoverable(&c->persist), "5.3 round-2: an APPLY_DIVERGENT-triggered fence must survive a " "purely-structural catch-up re-check — only a fresh, clean " "persist_recover() restart may clear it (Req 8.4)"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load_explicit(&c->ha.in_catch_up, memory_order_acquire), "5.3 round-2: the member must remain fenced (non-promotable) while " "apply_divergent, regardless of structural catch-up"); ha_set_promotion_fence_window_ms(0); dur_destroy_fence_cluster(c); printf("test_promotion_fence_apply_divergent_survives_structural_catchup: " "PASS\n"); } #endif /* XCE_TESTING */ /* ------------------------------------------------------------------ */ /* Test suite entry point */ /* ------------------------------------------------------------------ */ void test_commit_durability(void) { #ifdef XCE_TESTING RUN_TEST(test_durable_floor_empty); RUN_TEST(test_durable_floor_after_flush); RUN_TEST(test_durable_floor_survives_process_loss); RUN_TEST(test_durable_lsn_initial_zero); RUN_TEST(test_durable_lsn_batches_multiple_appends); RUN_TEST(test_durable_lsn_never_exceeds_synced_tip); RUN_TEST(test_durable_lsn_invariants_under_load); RUN_TEST(test_durable_lsn_holds_on_fsync_failure); RUN_TEST(test_durable_advance_drives_commit_single_member); RUN_TEST(test_replicated_persist_syncs_before_return); RUN_TEST(test_replicated_sync_failure_no_ack); RUN_TEST(test_legacy_sync_append_holds_committed_at_durable); RUN_TEST(test_legacy_sync_append_lsn0_edge_valid); RUN_TEST(test_commit_never_exceeds_durable_invariant); /* task 5.1 — durable-majority commit wired end to end */ RUN_TEST(test_durable_majority_commit_end_to_end); RUN_TEST(test_sole_durable_contributor_loss_survives); RUN_TEST(test_durability_work_off_iomgr_preserves_liveness); RUN_TEST(test_recover_verdict_clean_intact_log); RUN_TEST(test_recover_verdict_empty_first_boot_clean); RUN_TEST(test_recover_verdict_crc_break_first_entry_truncated); RUN_TEST(test_recover_verdict_snapshot_sentinel_clean); /* bug-778 — standby replicated-apply must leave an intact CRC chain. */ RUN_TEST(test_replicated_apply_chain_survives_cold_restart); /* task 3.4 — re-derive the durable floor from the recovered WAL tip. */ RUN_TEST(test_durable_floor_truncating_restart_is_intact_tip); RUN_TEST(test_durable_floor_intact_recovery_full_tip); RUN_TEST(test_durable_floor_empty_recovery_zero); RUN_TEST(test_durable_floor_snapshot_sentinel_tip); /* task 3.2 — Boot Policy Decision Table skeletons (IGNORED, pending * 3.3/4.3). Registered so each row is reported as IGNORED, not failed; * the suite stays GREEN. See design.md "Boot Policy Decision Table". */ /* task 3.3 — unrecoverable rows (4, 4b) = LOUD PERMANENT FENCE (stays alive, * non-promotable, clears on catch-up), plus the never-unrecoverable * invariant for the peer-reachable rows (3, 5) and the FENCE->ELIGIBLE * eligibility outcome of rows 1,2,3,5,6. (never-self-exit rework) */ RUN_TEST(test_boot_policy_peer_reachable_never_fatal); RUN_TEST(test_boot_policy_row1_clean_at_or_above_eligible); RUN_TEST(test_boot_policy_row2_clean_below_peer_catchup_eligible); RUN_TEST(test_boot_policy_row3_truncated_peer_catchup_eligible); RUN_TEST(test_boot_policy_row4_truncated_no_peer_loud_fence); RUN_TEST(test_boot_policy_row4b_clean_below_no_peer_loud_fence); RUN_TEST(test_boot_policy_row5_snapshot_sentinel_catchup_never_fatal); RUN_TEST(test_boot_policy_row6_empty_first_boot_eligible); /* task 4.3 — hollow node refused even at minimal quorum (Req 2.5). */ RUN_TEST(test_promotion_gate_hollow_refused_at_minimal_quorum); /* task 4.4 — catch-up re-alert window (NEVER self-exits; stays alive + * loudly non-promotable on the unrecoverable path; fence + sticky flag * released on catch-up; cancelled timer stops the re-alert). (Req 2.2/2.4/2.6) */ RUN_TEST(test_promotion_fence_unrecoverable_stays_alive_loud); RUN_TEST(test_promotion_fence_released_on_catchup); RUN_TEST(test_promotion_fence_cancel_stops_realert); /* task 5.3 round-2 remediation (Req 8.4): cross-contamination regression — * an APPLY_DIVERGENT-triggered fence must not clear on structural catch-up * alone. */ RUN_TEST(test_promotion_fence_apply_divergent_survives_structural_catchup); #else printf("test_commit_durability: skipped (no XCE_TESTING)\n"); #endif }