wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

test(ha-commit-durability): end-to-end durable-majority commit + liveness integration tests (task 5.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 09:56:26 +00:00
parent e9e2a6d703
commit e14bd62115
4 changed files with 404 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

365
src/controller/testsuite/test_commit_durability.c
View file

@ -14,6 +14,8 @@
#include "config.h"
#include "ha.h" /* task 4.3: ha_promotion_gate_decision / fence */
#include "persist.h"
#include "repl.h" /* task 5.1: repl_add_standby + standby ACK feed */
#include "typemap.h" /* task 5.1: wal_catchup_entry_t (send hook) */
#include "wal_types.h"
/* ------------------------------------------------------------------ *
@ -779,6 +781,365 @@ static void test_commit_never_exceeds_durable_invariant(void)
printf("test_commit_never_exceeds_durable_invariant: PASS\n");
}
/* ================================================================== *
* task 5.1 — wire durable-majority commit end to end *
* *
* These three tests verify the full durable-majority commit path the *
* 2.x/4.x wiring assembled, end to end over a REAL persist_t (real WAL *
* + real group-commit flusher thread) and a REAL commit_engine_t with *
* registered standbys: *
* *
* 1. reply-ok ⇒ durable-majority (Req 1.1, 1.4): committed_lsn *
* advances and a WAIT caller releases ONLY after the entry is *
* durable on a fault-tolerant majority — self durable (durable_lsn *
* covers it, via the real flusher) AND a standby ACK. With *
* durability held off, even a standby ACK does NOT release. *
* 2. sole-durable-contributor loss survives (Req 1.5, 2.5): an op *
* committed on self + a standby is durable on >= floor(N/2)+1 *
* members, so losing one durable contributor leaves it present on *
* a surviving member (modeled at the integration level; the live *
* failover proof is task 5.2). *
* 3. liveness — durability work off the iomgr (Req 1.6): fdatasync *
* runs on the dedicated flusher thread, not the caller/iomgr *
* thread, so a heartbeat-style callback is not stalled while a *
* durable flush is in flight. *
* ------------------------------------------------------------------ */
/* Dummy standby endpoints — the replicator only compares pointers, and
* commit_on_ack reads acked_lsn from the registry slot. */
static int g_dur_eps[MAX_STANDBYS];
/* Test-injectable send hook (defined in repl.c under XCE_TESTING) — keeps
* repl_broadcast from dereferencing the dummy endpoints as real
* endpoint_t when persist_wal_async fans out to the registered standbys.
* Same pattern as test_commit_engine.c. */
extern int (*repl_broadcast_send_hook)(void* ep, wal_catchup_entry_t** entries,
int n_entries);
static int dur_drop_send_hook(void* ep, wal_catchup_entry_t** entries,
int n_entries)
{
(void)ep;
(void)entries;
(void)n_entries;
return 0; /* pretend success; drop the data (no real wire in a unit test) */
}
/* Feed a standby ACK exactly as the iomgr read path does: store the
* watermark into the registry slot, then notify the commit engine. */
static void durable_standby_ack(persist_t* p, commit_engine_t* ce, int idx,
uint64_t acked_lsn)
{
atomic_store(&p->repl.standbys[idx].acked_lsn, acked_lsn);
commit_on_ack(ce, idx, acked_lsn);
}
/* A WAIT submit that runs on its own thread, so the test can observe that
* it does NOT return while the entry is not yet durable-majority, then DOES
* return once it becomes durable-majority (mirrors the off-iomgr AFTER_EVENT
* caller blocking on the commit barrier). */
typedef struct {
commit_engine_t* ce;
uint64_t lsn;
uint32_t timeout_ms;
_Atomic int returned;
int rc;
} wait_submit_arg_t;
static void* wait_submit_thread(void* a)
{
wait_submit_arg_t* w = (wait_submit_arg_t*)a;
w->rc = commit_submit(w->ce, w->lsn, COMMIT_MODE_WAIT, w->timeout_ms);
atomic_store(&w->returned, 1);
return NULL;
}
/* ------------------------------------------------------------------ *
* Test 5.1a (MUST-HAVE, Req 1.1 / 1.4): reply-ok only after a durable *
* majority. 3-member cluster (self + 2 standbys), so the majority for *
* an entry is self + at least 1 standby. We: *
* - force the local fdatasync to FAIL, so the flusher cannot advance *
* durable_lsn (self is NOT durable); *
* - append an entry and deliver a standby ACK covering it; *
* - assert committed_lsn does NOT advance and a WAIT submit does NOT *
* release — a majority of ACKs is not enough without local *
* durability (this is the durable precondition gate; pre-2.2 this *
* would have committed on the ACK alone — RED); *
* - clear the fault so the real flusher fdatasyncs and advances *
* durable_lsn, firing the durable-advance wake; *
* - assert committed_lsn now advances and the WAIT submit releases *
* promptly (durable self + durable standby ACK = durable majority). *
* ------------------------------------------------------------------ */
static void test_durable_majority_commit_end_to_end(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durmaj_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Intercept the replication fan-out so persist_wal_async does not try to
* send over the dummy endpoints (they are not real endpoint_t). */
repl_broadcast_send_hook = dur_drop_send_hook;
/* Register two standbys against the REAL persist's replicator. n_peers=3
* (self + 2 standbys); majority for entry N = self durable + >= 1 standby
* with acked_lsn >= N. n_vcpus high enough to admit a WAIT. */
TEST_ASSERT_EQUAL_INT(0, repl_add_standby(&p->repl, &g_dur_eps[0], "sb0",
7000));
TEST_ASSERT_EQUAL_INT(1, repl_add_standby(&p->repl, &g_dur_eps[1], "sb1",
7001));
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0, commit_engine_init(&ce, p, /*n_peers=*/3,
/*n_vcpus=*/8));
TEST_ASSERT_EQUAL_UINT64(0, atomic_load(&p->committed_lsn));
/* Hold durability off: every flusher fdatasync is forced to fail, so
* durable_lsn cannot advance. (Fail-loud: the entry is never counted
* durable, so the commit cannot advance regardless of ACKs.) */
persist_test_set_force_fsync_fail(p, 1);
/* Append one entry (LSN 1). It is in page cache only; the forced-fail
* flusher will NOT make it durable. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t lsn = p->current_lsn - 1; /* 1 */
TEST_ASSERT_EQUAL_UINT64(1, lsn);
/* Register a WAIT for LSN 1 on its own thread (the off-iomgr commit-barrier
* caller). It must block: the entry is not durable-majority. */
wait_submit_arg_t w = {.ce = &ce, .lsn = lsn, .timeout_ms = 2000};
pthread_t th;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th, NULL, wait_submit_thread, &w));
/* Deliver a covering standby ACK from BOTH standbys (a clear majority of
* ACKs). Pre-durability this alone advanced committed_lsn (the bug);
* post-2.2 the self-cap holds it at the non-durable self (durable_lsn=0). */
durable_standby_ack(p, &ce, 0, lsn);
durable_standby_ack(p, &ce, 1, lsn);
/* Give the (forced-failing) flusher ample time to run; durable_lsn must
* stay 0, committed must stay 0, and the WAIT must NOT have returned. */
for (int i = 0; i < 300; i++) { /* up to ~300 ms */
if (persist_durable_lsn(p) != 0 ||
atomic_load(&p->committed_lsn) != 0 ||
atomic_load(&w.returned))
break;
usleep(1000);
}
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, persist_durable_lsn(p),
"durable_lsn must not advance while fdatasync is failing");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(
0, atomic_load(&p->committed_lsn),
"committed_lsn must NOT advance on a standby-ACK majority while the "
"local entry is not durable (durable precondition gate)");
TEST_ASSERT_FALSE_MESSAGE(
atomic_load(&w.returned),
"the WAIT (client reply) must NOT release before durable-majority");
/* Now allow durability: clear the fault and force a flush. The REAL
* flusher fdatasyncs, advances durable_lsn over LSN 1, and fires the
* durable-advance wake — self is now durable AND both standbys already
* ACKed, so the entry is durable on a majority. committed_lsn advances
* and the WAIT releases promptly. */
persist_test_set_force_fsync_fail(p, 0);
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) >= lsn);
/* The WAIT must now return success (committed) within its timeout. */
TEST_ASSERT_EQUAL_INT(0, pthread_join(th, NULL));
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&w.returned),
"the WAIT must release once the entry is durable on a majority");
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, w.rc,
"commit_submit(WAIT) must return 0 (committed) after durable-majority");
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&p->committed_lsn) >= lsn,
"committed_lsn must advance once self is durable AND a standby ACK "
"covers the entry (durable majority)");
/* The invariant the whole spec rests on: committed never exceeds durable. */
TEST_ASSERT_TRUE(atomic_load(&p->committed_lsn) <= persist_durable_lsn(p));
commit_engine_fini(&ce);
repl_remove_standby(&p->repl, &g_dur_eps[0]);
repl_remove_standby(&p->repl, &g_dur_eps[1]);
teardown_durability_persist(p);
repl_broadcast_send_hook = NULL;
printf("test_durable_majority_commit_end_to_end: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1b (Req 1.5 / 2.5): losing the sole durable contributor *
* leaves the acknowledged op present on a surviving member. *
* *
* The durable-majority property says: when an op is acknowledged, its *
* entry is fdatasync'd on a fault-tolerant majority (self + >= *
* floor(N/2) others). For N=3 that is 2 durable copies. So losing ONE *
* durable contributor still leaves >= 1 surviving durable copy that *
* holds the op — it is recoverable from the survivor. *
* *
* We model the two durable copies as two independent persist_t over *
* separate WAL dirs (a primary's copy and a standby's copy), both made *
* genuinely durable (fdatasync'd). We then DROP the "sole durable *
* contributor" (the first copy, simulating its loss/restart-with-loss) *
* and prove the op is still durably present and recoverable on the *
* surviving copy via a fresh-process recovery (no in-memory state). *
* *
* This is the integration-level assertion of the property that makes *
* survival true (committed ⇒ >= floor(N/2)+1 durable copies); the live *
* multi-host SIGKILL failover proof is task 5.2. *
* ------------------------------------------------------------------ */
static void test_sole_durable_contributor_loss_survives(void)
{
char tmp_a[256];
char tmp_b[256];
strcpy(tmp_a, "/tmp/xce_durlossA_XXXXXX");
strcpy(tmp_b, "/tmp/xce_durlossB_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp_a));
TEST_ASSERT_NOT_NULL(mkdtemp(tmp_b));
const char* payload = "committed-op";
uint32_t plen = (uint32_t)strlen(payload);
/* Durable copy A (the contributor we will lose) and durable copy B (the
* survivor): write the SAME committed op to both and fdatasync each, so
* the op is durable on a 2-of-3 majority. */
{
persist_t* a = make_durability_persist(tmp_a);
TEST_ASSERT_NOT_NULL(a);
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(a, WAL_JOB_UPDATE, payload, plen));
TEST_ASSERT_EQUAL_INT(0, persist_flush(a));
TEST_ASSERT_TRUE(persist_durable_lsn(a) >= 1);
teardown_durability_persist(a);
}
{
persist_t* b = make_durability_persist(tmp_b);
TEST_ASSERT_NOT_NULL(b);
/* The standby applies the SAME replicated entry durably before ACK
* (persist_wal_write_replicated fdatasyncs inline — Req 1.3). */
TEST_ASSERT_EQUAL_INT(0,
persist_wal_write_replicated(b, /*lsn=*/1, WAL_JOB_UPDATE,
payload, plen));
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(b) >= 1);
teardown_durability_persist(b);
}
/* "Lose" the sole durable contributor A: destroy its on-disk WAL entirely
* (simulates a member that is gone / came back with a truncated log). */
{
char rm[600];
snprintf(rm, sizeof(rm), "rm -rf %s", tmp_a);
TEST_ASSERT_EQUAL_INT(0, system(rm));
}
/* The acknowledged op MUST still be present and recoverable on the
* surviving member B — read straight from disk by a fresh process with
* NO in-memory state (the durable-majority property in action). */
{
persist_t* b2 = make_durability_persist(tmp_b);
TEST_ASSERT_NOT_NULL(b2);
/* On-disk durable floor (CRC-validated re-read) still covers the op. */
TEST_ASSERT_TRUE_MESSAGE(
persist_test_durable_lsn_on_disk(b2) >= 1,
"the acknowledged op must survive on the durable majority after "
"the sole-durable-contributor copy is lost");
/* And a real recovery reconstructs it cleanly (not truncated). */
TEST_ASSERT_EQUAL_INT(0, persist_recover(b2));
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(b2),
"the survivor recovers the committed op as a clean intact log");
TEST_ASSERT_TRUE_MESSAGE(
persist_durable_floor(b2) >= 1,
"the survivor's recovered durable floor covers the committed op");
teardown_durability_persist(b2);
}
{
char rm[600];
snprintf(rm, sizeof(rm), "rm -rf %s", tmp_b);
(void)system(rm);
}
printf("test_sole_durable_contributor_loss_survives: PASS "
"(durable-majority property; live failover proof is task 5.2)\n");
}
/* ------------------------------------------------------------------ *
* Test 5.1c (Req 1.6): durability work does not stall heartbeat / *
* election handling — fdatasync runs on the dedicated flusher thread, *
* NOT on the caller (iomgr/heartbeat) thread. *
* *
* Structural + timing assertion: while a durable flush is in flight on *
* the flusher thread, a heartbeat-style callback executed on the *
* caller/iomgr thread completes within a tight tolerance. We prove the *
* flush is OFF the caller thread by capturing the flusher's thread id at *
* the fdatasync site (via the XCE_TESTING seam) and asserting it is NOT *
* the caller thread. (persist_wal_async only appends + signals; it never *
* fdatasyncs inline, so the append path itself never blocks on disk.) *
* ------------------------------------------------------------------ */
static void test_durability_work_off_iomgr_preserves_liveness(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durliveness_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
pthread_t caller = pthread_self();
/* Drive a steady stream of appends on the CALLER (iomgr-equivalent)
* thread; the real flusher thread does the fdatasync work concurrently.
* After each append, a heartbeat-style callback must complete promptly —
* the append path signals the flusher and returns without blocking on
* fdatasync, so heartbeat handling on this thread is never stalled. */
const long tolerance_us = 50 * 1000; /* 50 ms — generous heartbeat budget */
for (int i = 0; i < 64; i++) {
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "h", 1));
struct timespec t0, t1;
clock_gettime(CLOCK_MONOTONIC, &t0);
/* Heartbeat-style work on the caller thread (a cheap atomic read of
* commit/durable state, exactly what the heartbeat path inspects). */
volatile uint64_t hb = atomic_load(&p->committed_lsn) +
persist_durable_lsn(p);
(void)hb;
clock_gettime(CLOCK_MONOTONIC, &t1);
long elapsed_us = (t1.tv_sec - t0.tv_sec) * 1000000L +
(t1.tv_nsec - t0.tv_nsec) / 1000L;
TEST_ASSERT_TRUE_MESSAGE(
elapsed_us < tolerance_us,
"heartbeat-style work on the caller thread must not be stalled "
"by durability (fdatasync) work in flight on the flusher thread");
}
/* Let the flusher make everything durable, then confirm the fdatasync
* provably ran on a DIFFERENT thread than this caller — durability work
* is off the iomgr/heartbeat thread (Req 1.6). The seam records the
* thread id of the last successful flusher fdatasync. */
uint64_t tip = p->current_lsn - 1;
TEST_ASSERT_TRUE(wait_durable_reaches(p, tip, 5000) >= tip);
pthread_t flusher_tid = persist_test_last_fdatasync_thread(p);
TEST_ASSERT_FALSE_MESSAGE(
pthread_equal(flusher_tid, caller),
"fdatasync must run on the dedicated flusher thread, never on the "
"caller/iomgr/heartbeat thread");
teardown_durability_persist(p);
printf("test_durability_work_off_iomgr_preserves_liveness: PASS\n");
}
/* ================================================================== *
* task 3.1 — recovery integrity verdict (Req 2.1) *
* *
@ -1662,6 +2023,10 @@ void test_commit_durability(void)
RUN_TEST(test_legacy_sync_append_holds_committed_at_durable);
RUN_TEST(test_legacy_sync_append_lsn0_edge_valid);
RUN_TEST(test_commit_never_exceeds_durable_invariant);
/* task 5.1 — durable-majority commit wired end to end */
RUN_TEST(test_durable_majority_commit_end_to_end);
RUN_TEST(test_sole_durable_contributor_loss_survives);
RUN_TEST(test_durability_work_off_iomgr_preserves_liveness);
RUN_TEST(test_recover_verdict_clean_intact_log);
RUN_TEST(test_recover_verdict_empty_first_boot_clean);
RUN_TEST(test_recover_verdict_crc_break_first_entry_truncated);