wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha-commit-durability): demand-driven group-commit flusher + durable_lsn watermark (task 2.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 06:06:55 +00:00
parent 7e6f09f804
commit cc83caf5a0
4 changed files with 567 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

266
src/controller/testsuite/test_commit_durability.c
View file

@ -1,4 +1,6 @@
#define _GNU_SOURCE
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@ -172,6 +174,265 @@ static void test_durable_floor_survives_process_loss(void)
printf("test_durable_floor_survives_process_loss: PASS\n");
}
/* ================================================================== *
* task 2.1 — durable-LSN tracking via a group-commit flusher *
* *
* These tests pin the invariants of persist_t.durable_lsn, the *
* prefix watermark advanced by the demand-driven flusher (the *
* repurposed fsync_worker): *
* - durable_lsn never exceeds the synced tip (<= current_lsn - 1); *
* - durable_lsn is monotonic non-decreasing; *
* - one fdatasync advances durable_lsn past MULTIPLE appended LSNs *
* (batching / group-commit); *
* - durable_lsn advances ONLY after a successful fdatasync; *
* - under concurrent append load the invariants always hold. *
* ------------------------------------------------------------------ */
/* Spin until the flusher catches durable_lsn up to the appended tip,
* or a generous deadline elapses. Returns the observed durable_lsn. */
static uint64_t wait_durable_reaches(persist_t* p, uint64_t target_lsn,
int max_ms)
{
for (int i = 0; i < max_ms; i++) {
uint64_t d = persist_durable_lsn(p);
if (d >= target_lsn)
return d;
usleep(1000); /* 1 ms */
}
return persist_durable_lsn(p);
}
/* ------------------------------------------------------------------ *
* Test 4: a fresh store has durable_lsn == 0 (nothing synced yet). *
* ------------------------------------------------------------------ */
static void test_durable_lsn_initial_zero(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn0_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_UINT64(0, persist_durable_lsn(p));
teardown_durability_persist(p);
printf("test_durable_lsn_initial_zero: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 5 (core / batching, Req 1.1): several appends, then the *
* demand-driven flusher advances durable_lsn past ALL of them with a *
* single fdatasync — and never past the appended tip. Uses the fast *
* cadence so the append-signal (not the timer) drives the flush, but *
* the slow-cadence helper would converge too via the bounded fallback.*
* ------------------------------------------------------------------ */
static void test_durable_lsn_batches_multiple_appends(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn1_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* LSN 0 reserved → these take 1,2,3,4,5; current_lsn ends at 6. */
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
uint64_t tip = p->current_lsn - 1; /* 5 */
TEST_ASSERT_EQUAL_UINT64(5, tip);
uint64_t d = wait_durable_reaches(p, tip, 5000);
/* The flusher advanced durable_lsn past several appended LSNs with
* one fdatasync (batching), reaching the synced tip. */
TEST_ASSERT_EQUAL_UINT64(tip, d);
/* Invariant: never exceeds the synced tip. */
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
/* The on-disk seam agrees: those LSNs are genuinely fdatasync'd. */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_batches_multiple_appends: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 6 (advances-only-after-fdatasync + monotonic): the durable *
* floor reported by the on-disk seam must always be >= durable_lsn — *
* i.e. durable_lsn never claims an LSN whose bytes are not actually *
* on stable storage. Also asserts monotonic non-decreasing across *
* successive append+flush rounds. *
* ------------------------------------------------------------------ */
static void test_durable_lsn_never_exceeds_synced_tip(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn2_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
uint64_t prev = persist_durable_lsn(p);
for (int round = 0; round < 4; round++) {
for (int i = 0; i < 3; i++)
TEST_ASSERT_EQUAL_INT(0,
persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
uint64_t tip = p->current_lsn - 1;
uint64_t d = wait_durable_reaches(p, tip, 5000);
/* Monotonic non-decreasing. */
TEST_ASSERT_TRUE(d >= prev);
/* Never exceeds the synced tip. */
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
/* Advances only after a real fdatasync: the independent on-disk
* scan must confirm every claimed-durable LSN is on disk. */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
prev = d;
}
teardown_durability_persist(p);
printf("test_durable_lsn_never_exceeds_synced_tip: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 7 (UNDER LOAD, Req 1.6 / lost-wakeup): many appends from *
* multiple threads racing the flusher. Asserts the invariants hold *
* throughout (durable_lsn <= synced tip, never past disk) AND that *
* the flusher eventually catches durable_lsn up to the final tip — *
* i.e. no append is left permanently un-synced (no lost wakeup). *
* ------------------------------------------------------------------ */
#define LOAD_THREADS 4
#define LOAD_PER_THREAD 200
typedef struct {
persist_t* p;
int n;
} load_arg_t;
static void* load_appender(void* arg)
{
load_arg_t* a = arg;
for (int i = 0; i < a->n; i++) {
/* Ignore rc: a full-segment rotate or transient EIO is not the
* subject under test; the invariant checks below tolerate it. */
(void)persist_wal_async(a->p, WAL_JOB_UPDATE, "z", 1);
/* Sample the invariant mid-flight: durable_lsn must NEVER exceed
* the appended tip, even while a concurrent flush is in progress. */
uint64_t d = persist_durable_lsn(a->p);
uint64_t cur = atomic_load_explicit(
(const _Atomic uint64_t*)&a->p->current_lsn, memory_order_acquire);
/* current_lsn is read without the lock here purely as a coarse
* upper bound; a torn read can only make `cur` smaller, so the
* assertion stays conservative. durable_lsn is a prefix watermark
* that can never outrun the last assigned LSN. */
TEST_ASSERT_TRUE(d == 0 || d <= cur);
}
return NULL;
}
static void test_durable_lsn_invariants_under_load(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn3_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
pthread_t th[LOAD_THREADS];
load_arg_t args[LOAD_THREADS];
for (int i = 0; i < LOAD_THREADS; i++) {
args[i].p = p;
args[i].n = LOAD_PER_THREAD;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th[i], NULL,
load_appender, &args[i]));
}
for (int i = 0; i < LOAD_THREADS; i++)
pthread_join(th[i], NULL);
/* Final tip after all appends settled. */
uint64_t tip = p->current_lsn - 1;
/* No lost wakeup: the flusher must eventually advance durable_lsn up
* to the final appended tip (every entry gets synced). */
uint64_t d = wait_durable_reaches(p, tip, 10000);
TEST_ASSERT_EQUAL_UINT64(tip, d);
TEST_ASSERT_TRUE(d <= p->current_lsn - 1);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_invariants_under_load: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 8 (ERROR PATH, Req 1.1/1.2 — fail-loud): when fdatasync FAILS, *
* durable_lsn must NOT advance. Bytes that did not reach stable *
* storage must never be counted durable — otherwise the member would *
* contribute a non-durable LSN toward the commit majority. *
* *
* Mechanism: the XCE_TESTING fault-injection toggle forces every *
* fdatasync site to treat the sync as failed. We append entries while *
* the toggle is ON and assert durable_lsn stays put across the full *
* cadence window (the flusher runs but its advance is gated on rc==0). *
* Then we clear the toggle and confirm the flusher recovers — durable_ *
* lsn advances to the tip once syncs succeed again. *
* ------------------------------------------------------------------ */
static void test_durable_lsn_holds_on_fsync_failure(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_durlsn4_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* First, append one entry and let it become durable normally, so the
* watermark has a known non-zero baseline that must HOLD during the
* failure window. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
uint64_t baseline = wait_durable_reaches(p, 1, 5000);
TEST_ASSERT_EQUAL_UINT64(1, baseline);
/* Now inject fdatasync failure and append more entries. */
persist_test_set_force_fsync_fail(p, 1);
for (int i = 0; i < 5; i++)
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
uint64_t tip = p->current_lsn - 1; /* 6 */
TEST_ASSERT_EQUAL_UINT64(6, tip);
/* Give the flusher ample time to run several passes (cadence is 5s but
* the append signal fires it immediately) AND an explicit flush attempt.
* Under injected failure, durable_lsn MUST remain at the baseline — the
* new bytes are not on stable storage, so they are not durable. */
(void)persist_flush(p); /* also gated on rc==0 — must not advance either */
for (int i = 0; i < 200; i++) {
TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p));
usleep(1000); /* 1 ms; 200 ms total — many flusher signal cycles */
}
/* Still parked at the baseline after the window. */
TEST_ASSERT_EQUAL_UINT64(baseline, persist_durable_lsn(p));
TEST_ASSERT_TRUE(persist_durable_lsn(p) <= p->current_lsn - 1);
/* Recovery: clear the fault and confirm the flusher catches up — proving
* the failure path did not corrupt the watermark, only held it. */
persist_test_set_force_fsync_fail(p, 0);
/* Nudge the flusher with one more append so the level predicate signals. */
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
uint64_t new_tip = p->current_lsn - 1; /* 7 */
uint64_t d = wait_durable_reaches(p, new_tip, 5000);
TEST_ASSERT_EQUAL_UINT64(new_tip, d);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= d);
teardown_durability_persist(p);
printf("test_durable_lsn_holds_on_fsync_failure: PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
@ -183,6 +444,11 @@ void test_commit_durability(void)
RUN_TEST(test_durable_floor_empty);
RUN_TEST(test_durable_floor_after_flush);
RUN_TEST(test_durable_floor_survives_process_loss);
RUN_TEST(test_durable_lsn_initial_zero);
RUN_TEST(test_durable_lsn_batches_multiple_appends);
RUN_TEST(test_durable_lsn_never_exceeds_synced_tip);
RUN_TEST(test_durable_lsn_invariants_under_load);
RUN_TEST(test_durable_lsn_holds_on_fsync_failure);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif