lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): gate standby ACK on fdatasync success + persist-before-ACK regression test (task 2.3)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 06:45:21 +00:00
parent c41edc515e
commit e4c4968331
4 changed files with 155 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

31
src/controller/persist.c
View file

@ -730,8 +730,27 @@ int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
* persist_wal_async defers fsync to the background thread / persist_flush;
* here we sync inline because the entry feeds the primary's commit
* majority and must survive a standby restart. fdatasync is sufficient
* (entry bytes + segment metadata are pre-allocated by open_new_segment). */
fdatasync(p->wal_fd);
* (entry bytes + segment metadata are pre-allocated by open_new_segment).
*
* Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the
* append-path gate at task 2.1): the ACK is sent by the caller ONLY when
* this returns success, and the ACK claims the entry is durable. So if
* the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must
* NOT return success — we log loudly and return -EIO so the caller's
* existing error path (ha_apply_single_entry) does NOT apply or ACK. */
int fsrc = fdatasync(p->wal_fd);
#ifdef XCE_TESTING
atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1,
memory_order_relaxed);
#endif
if (fsrc != 0 || flusher_fsync_forced_fail(p)) {
int e = (fsrc != 0) ? errno : EIO;
ERROR("persist: persist_wal_write_replicated fdatasync FAILED for "
"lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning "
"error so the standby does NOT ACK", lsn, e, strerror(e));
pthread_mutex_unlock(&p->lock);
return -EIO;
}
/* Rotate segment when full (same threshold as the primary append). */
if (p->seg_offset >= WAL_SEGMENT_SIZE_BYTES) {
@ -799,6 +818,14 @@ void persist_test_set_force_fsync_fail(persist_t* p, int fail)
memory_order_release);
}
unsigned persist_test_replicated_fdatasync_count(persist_t* p)
{
if (!p)
return 0;
return atomic_load_explicit(&p->test_replicated_fdatasync_count,
memory_order_relaxed);
}
/* ------------------------------------------------------------------ *
* Durability-verification test seam (ha-commit-durability Req 3.2/3.3)*
* *

108
src/controller/testsuite/test_commit_durability.c
View file

@ -497,6 +497,112 @@ static void test_durable_advance_drives_commit_single_member(void)
printf("test_durable_advance_drives_commit_single_member: PASS\n");
}
/* ================================================================== *
* task 2.3 — standby persist-before-ACK regression (Req 1.3) *
* *
* Req 1.3: a standby durably persists a replicated entry to local *
* stable storage BEFORE sending its acknowledgment. The ACK is sent by *
* the CALLER (ha_apply_single_entry) only after *
* persist_wal_write_replicated returns success — so to pin "persist *
* before ACK" we prove a durable fdatasync ran INSIDE the persist call *
* before it returned. *
* *
* Mechanism: a disk re-read (persist_test_durable_lsn_on_disk) cannot *
* distinguish an fsync'd page from a page-cache-only page, so it CANNOT *
* prove the sync ran. Instead we read the XCE_TESTING fdatasync counter *
* incremented at the inline sync site: snapshot it before the call, *
* assert it incremented after — the sync provably ran within the call, *
* before the caller could ACK. The test FAILS if someone removed or *
* moved the fdatasync out of persist_wal_write_replicated. *
* ------------------------------------------------------------------ */
static void test_replicated_persist_syncs_before_return(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replsync_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Snapshot the sync counter, then write a replicated entry. The standby
* path must fdatasync inline BEFORE returning — so the counter must
* advance during the single call. */
unsigned before = persist_test_replicated_fdatasync_count(p);
int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_EQUAL_INT(0, rc);
unsigned after = persist_test_replicated_fdatasync_count(p);
TEST_ASSERT_TRUE_MESSAGE(
after > before,
"persist_wal_write_replicated must fdatasync inline (persist-before-"
"ACK): the sync counter did not advance during the call");
/* And the entry is genuinely on stable storage by the time the call
* returned (would survive an immediate process loss → recoverable ACK). */
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 1);
teardown_durability_persist(p);
printf("test_replicated_persist_syncs_before_return: PASS\n");
}
/* ------------------------------------------------------------------ *
* task 2.3 — forced standby-sync failure ⇒ error, NO success/ACK *
* *
* Closes the Req-1.3 hole: if the inline fdatasync FAILS, the standby *
* must NOT report success — otherwise the caller would ACK an entry *
* that is not durable (durable AFTER, not BEFORE, the ACK). With the *
* force-fail seam ON, persist_wal_write_replicated must return a *
* negative error; the caller (ha_apply_single_entry) already treats any *
* negative return as "do not apply, do not ACK". *
* *
* RED evidence: against the pre-2.3 code (which ignored the fdatasync *
* return at the standby path) this returned 0 (success) under forced *
* failure — a non-durable ACK. The rc-gate makes it return -EIO. *
* ------------------------------------------------------------------ */
static void test_replicated_sync_failure_no_ack(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_replsyncfail_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* Force the fdatasync to be treated as failed, then attempt a replicated
* write. The durable sync cannot succeed, so the call MUST fail (no ACK).
* A negative return is what ha_apply_single_entry routes to its
* "not applying/acking" path. */
persist_test_set_force_fsync_fail(p, 1);
int rc = persist_wal_write_replicated(p, /*lsn=*/1, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_TRUE_MESSAGE(
rc < 0,
"forced standby fdatasync failure must make persist_wal_write_"
"replicated FAIL (negative rc) so the standby does NOT ACK a "
"non-durable entry");
/* The sync was still attempted (counter advanced) — we fail loud on the
* RESULT, not by skipping the sync. */
TEST_ASSERT_TRUE(persist_test_replicated_fdatasync_count(p) >= 1);
/* Recovery: clear the fault; a fresh replicated write (new lsn, not the
* deduped lsn=1) now succeeds and is durable — proving the failure path
* did not corrupt persist state and the sync runs again on success.
* (lsn=1 already advanced current_lsn to 2, so re-using 1 would dedup;
* lsn=2 is the genuine next entry.) */
persist_test_set_force_fsync_fail(p, 0);
int rc2 = persist_wal_write_replicated(p, /*lsn=*/2, WAL_JOB_UPDATE,
"x", 1);
TEST_ASSERT_EQUAL_INT(0, rc2);
TEST_ASSERT_TRUE(persist_test_durable_lsn_on_disk(p) >= 2);
teardown_durability_persist(p);
printf("test_replicated_sync_failure_no_ack: PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
@ -514,6 +620,8 @@ void test_commit_durability(void)
RUN_TEST(test_durable_lsn_invariants_under_load);
RUN_TEST(test_durable_lsn_holds_on_fsync_failure);
RUN_TEST(test_durable_advance_drives_commit_single_member);
RUN_TEST(test_replicated_persist_syncs_before_return);
RUN_TEST(test_replicated_sync_failure_no_ack);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif