From f90395282c1e557716080e681c75898f11e381e4 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 18 Jun 2026 04:56:50 +0000 Subject: [PATCH] feat(ha-commit-durability): add XCE_TESTING durable-LSN-on-disk verification seam (task 1.2) --- .kiro/specs/ha-commit-durability/tasks.md | 3 +- src/controller/meson.build | 1 + src/controller/persist.c | 134 +++++++++++++ src/controller/testsuite/controller_test.c | 2 + .../testsuite/test_commit_durability.c | 189 ++++++++++++++++++ src/include/persist.h | 29 +++ 6 files changed, 357 insertions(+), 1 deletion(-) create mode 100644 src/controller/testsuite/test_commit_durability.c diff --git a/.kiro/specs/ha-commit-durability/tasks.md b/.kiro/specs/ha-commit-durability/tasks.md index 5a24a67e..2936a607 100644 --- a/.kiro/specs/ha-commit-durability/tasks.md +++ b/.kiro/specs/ha-commit-durability/tasks.md @@ -11,7 +11,7 @@ - _Requirements: 3.1_ - _Boundary: DurabilityConfig_ -- [ ] 1.2 Add the test-only durability-verification hook +- [x] 1.2 Add the test-only durability-verification hook - Add an `XCE_TESTING`-gated seam that lets a test assert an acknowledged operation is recoverable after an immediate ungraceful termination; compiled out of production builds. - Observable: in a test build the hook is callable and the production build links without it. - _Requirements: 3.2, 3.3_ @@ -121,3 +121,4 @@ ## Implementation Notes - 1.1: Config plumbing routes through `myconf_t.wal_sync_ms` + `persist_config_set_wal_sync_ms()` (0-sentinel = keep `DEFAULT_WAL_SYNC_MS`), NOT a new config.h constant (design File Structure Plan keeps `DEFAULT_WAL_SYNC_MS`). The override is applied at BOTH `persist_init` sites — `myapp_run` step-6 (production path) and `cluster_init_phase2` (guarded by `wal_path[0]=='\0'`, skipped in production). They are mutually exclusive; apply any new persist_config override at both sites or it is silently dropped. Pre-existing latent bug (do NOT fix here): `--sync-timeout-ms` is applied only in phase2, so it is ignored on the production startup path — relevant if a later task relies on `sync_timeout_ms`. +- 1.2: Test seam = `persist_test_durable_lsn_on_disk(persist_t*)` (decl `persist.h`, impl `persist.c`, all `#ifdef XCE_TESTING`) — re-opens WAL segment files from disk and CRC-validates the chain (read-only, state-free) to return the highest fully-persisted LSN; new test `testsuite/test_commit_durability.c` (registered in `meson.build` + `controller_test.c`). Production `xce_controller` links without it (proven via `nm`). DRY NOTE: ~115 lines of segment-scan/CRC-chain logic are duplicated from the static `replay_wal`; tasks 3.1 (recovery verdict) and 3.4 (durable floor from WAL tip) touch the SAME scan/CRC logic — consider extracting a shared read-only `wal_scan_durable_lsn`-style helper there and routing this seam through it (avoid a third copy). `XCE_TESTING` is defined only for `controller_unit_test` (`src/controller/meson.build:167`). diff --git a/src/controller/meson.build b/src/controller/meson.build index 7c4c04e2..d5f43058 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -131,6 +131,7 @@ controller_unit_test_src = [ 'testsuite/test_demotion_convergence.c', 'testsuite/test_convergence_verification.c', 'testsuite/test_coldstart_lsn0.c', + 'testsuite/test_commit_durability.c', 'commit.c', 'job.c', 'egress.c', diff --git a/src/controller/persist.c b/src/controller/persist.c index c63e125b..8e283b40 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -536,6 +536,140 @@ int persist_flush(persist_t* p) return rc; } +#ifdef XCE_TESTING +/* ------------------------------------------------------------------ * + * Durability-verification test seam (ha-commit-durability Req 3.2/3.3)* + * * + * Re-read + CRC-validate the WAL segments from stable storage and * + * return the highest fully-persisted LSN — independent of the * + * in-memory current_lsn / committed_lsn watermarks. Mirrors the * + * segment-scan + CRC-chain validation of replay_wal(), but is strictly * + * read-only: it applies no entries, fires no hooks, and mutates no * + * persist_t state, so it cannot change the externally observable * + * acknowledgment semantics (Req 3.3). * + * * + * Stops at the first corruption / short read / pre-allocated zero * + * region (exactly like recovery), so only entries genuinely on stable * + * storage are counted. Returns the highest such LSN, or 0 when nothing * + * is durable. * + * ------------------------------------------------------------------ */ +uint64_t persist_test_durable_lsn_on_disk(persist_t* p) +{ + if (!p) + return 0; + + DIR* d = opendir(p->wal_path); + if (!d) + return 0; /* no WAL dir → nothing durable */ + + /* Collect segment start LSNs (same bound as replay_wal). */ + uint64_t seg_lsns[4096]; + int nseg = 0; + + struct dirent* e; + while ((e = readdir(d)) != NULL && + nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { + if (strncmp(e->d_name, "seg-", 4) != 0) + continue; + uint64_t seg_start = 0; + if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) + continue; + seg_lsns[nseg++] = seg_start; + } + closedir(d); + + if (nseg == 0) + return 0; + + /* Insertion sort by start LSN (segment count is tiny). */ + for (int i = 1; i < nseg; i++) { + uint64_t key = seg_lsns[i]; + int j = i - 1; + while (j >= 0 && seg_lsns[j] > key) { + seg_lsns[j + 1] = seg_lsns[j]; + j--; + } + seg_lsns[j + 1] = key; + } + + uint64_t highest_durable = 0; + uint32_t last_crc = WAL_CRC_NONE; + + for (int si = 0; si < nseg; si++) { + char seg_path[4096]; + snprintf(seg_path, sizeof(seg_path), + WAL_SEG_FMT, p->wal_path, seg_lsns[si]); + + int fd = open(seg_path, O_RDONLY); + if (fd < 0) + continue; + + uint32_t seg_off = 0; + + for (;;) { + wal_entry_hdr_t hdr; + ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); + if (hn == 0) + break; /* clean end of segment */ + if (hn < (ssize_t)sizeof(hdr)) + break; /* truncated / pre-allocated zeros */ + + /* Pre-allocated zero region: no valid entry is all-zero. */ + if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) + break; + + /* CRC chain validation — identical rule to replay_wal. */ + if (seg_off == 0) { + if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && + hdr.prev_crc != last_crc) + break; + } else { + if (hdr.prev_crc != last_crc) + break; + } + + /* Verify entry CRC over header + payload. */ + uint32_t expected_crc = + crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); + + uint8_t* payload = NULL; + if (hdr.payload_len > 0) { + payload = malloc(hdr.payload_len); + if (!payload) { + close(fd); + return highest_durable; + } + ssize_t pn = pread(fd, payload, hdr.payload_len, + (off_t)(seg_off + sizeof(hdr))); + if (pn != (ssize_t)hdr.payload_len) { + free(payload); + break; + } + expected_crc = + crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); + } + + if (hdr.crc != expected_crc) { + free(payload); + break; + } + free(payload); + + /* Entry is intact and durable on disk. */ + last_crc = hdr.crc; + if (hdr.lsn > highest_durable) + highest_durable = hdr.lsn; + + seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); + } + + close(fd); + } + + return highest_durable; +} +#endif /* XCE_TESTING */ + /* ------------------------------------------------------------------ */ /* Checkpoint internals */ /* ------------------------------------------------------------------ */ diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index c0e0004c..4f6bceab 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -59,6 +59,7 @@ extern void test_egress_filter(void); extern void test_demotion_convergence(void); extern void test_convergence_verification(void); extern void test_coldstart_lsn0(void); +extern void test_commit_durability(void); void setUp(void) { return; } @@ -187,6 +188,7 @@ int main(int argc, char* argv[]) TEST(test_demotion_convergence); TEST(test_convergence_verification); TEST(test_coldstart_lsn0); + TEST(test_commit_durability); return UNITY_END(); } diff --git a/src/controller/testsuite/test_commit_durability.c b/src/controller/testsuite/test_commit_durability.c new file mode 100644 index 00000000..6c3b416a --- /dev/null +++ b/src/controller/testsuite/test_commit_durability.c @@ -0,0 +1,189 @@ +#define _GNU_SOURCE +#include +#include +#include +#include + +#include + +#include "config.h" +#include "persist.h" + +/* ------------------------------------------------------------------ * + * ha-commit-durability — task 1.2: durability-verification test seam * + * * + * These tests exercise persist_test_durable_lsn_on_disk(), the * + * XCE_TESTING-only seam that re-reads + CRC-validates the WAL from * + * stable storage and reports the highest fully-persisted LSN — * + * independent of the in-memory current_lsn watermark. This lets a * + * test assert an acknowledged operation is recoverable from disk after * + * an immediate ungraceful termination, without relying on the * + * background flush timer (Req 3.2). The seam is purely inspective: * + * it changes nothing observable about the acknowledgment contract * + * (Req 3.3). * + * * + * The seam is compiled ONLY into the controller_unit_test target * + * (-DXCE_TESTING). The production xce_controller target links without * + * it — see #ifdef XCE_TESTING in persist.c / declaration in persist.h. * + * ------------------------------------------------------------------ */ + +#ifdef XCE_TESTING + +/* Build a persist instance with a deliberately slow background flush + * cadence, so the only durability point is an explicit persist_flush() + * (mirrors make_fresh_persist in test_persist_recovery.c). */ +static persist_t* make_durability_persist(const char* tmpdir) +{ + persist_t* p = calloc(1, sizeof(persist_t)); + if (!p) + return NULL; + + persist_config_t cfg = persist_config_defaults(); + cfg.wal_sync_ms = 5000; /* slow background fsync — durability comes from + * the explicit flush, not the timer */ + cfg.snap_keep_n = 2; + + persist_set_workdir(p, tmpdir, "ctrl-test"); + + if (persist_init(p, &cfg, NULL) != 0) { + free(p); + return NULL; + } + if (persist_start(p) != 0) { + persist_fini(p); + free(p); + return NULL; + } + return p; +} + +static void teardown_durability_persist(persist_t* p) +{ + persist_stop(p); + persist_fini(p); + free(p); +} + +/* ------------------------------------------------------------------ * + * Test 1: an empty WAL has no durable entries (floor == 0). * + * An empty first-boot WAL is not corruption — it simply has nothing * + * persisted, so the seam reports 0. * + * ------------------------------------------------------------------ */ +static void test_durable_floor_empty(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_durfloor0_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + persist_t* p = make_durability_persist(tmp); + TEST_ASSERT_NOT_NULL(p); + + uint64_t floor = persist_test_durable_lsn_on_disk(p); + TEST_ASSERT_EQUAL_UINT64(0, floor); + + teardown_durability_persist(p); + printf("test_durable_floor_empty: PASS\n"); +} + +/* ------------------------------------------------------------------ * + * Test 2 (core Req 3.2): after append + explicit fdatasync, the seam * + * sees the entries on disk — the durable floor equals the highest * + * appended LSN, recoverable independent of in-memory state. * + * ------------------------------------------------------------------ */ +static void test_durable_floor_after_flush(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_durfloor1_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + persist_t* p = make_durability_persist(tmp); + TEST_ASSERT_NOT_NULL(p); + + /* Append three entries. LSN 0 is reserved, so these take LSN 1,2,3. + * current_lsn advances to 4 (one past the last appended). */ + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1)); + + /* Durability point: fdatasync the segment so the bytes are on stable + * storage (this is what an acknowledged-before-durable contract must + * have done before replying ok). */ + TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); + + /* The seam re-reads from disk and reports the highest intact LSN. */ + uint64_t floor = persist_test_durable_lsn_on_disk(p); + TEST_ASSERT_EQUAL_UINT64(p->current_lsn - 1, floor); + TEST_ASSERT_EQUAL_UINT64(3, floor); + + teardown_durability_persist(p); + printf("test_durable_floor_after_flush: PASS\n"); +} + +/* ------------------------------------------------------------------ * + * Test 3 (recoverable-after-ungraceful-kill, Req 3.2): a SECOND * + * persist instance opened over the same WAL dir — with NO in-memory * + * state from the writer — recovers exactly the durable floor the seam * + * reported. This is the "survives an immediate ungraceful * + * termination" assertion: durability lives on disk, not in the * + * volatile buffers of the process that wrote it. * + * ------------------------------------------------------------------ */ +static void test_durable_floor_survives_process_loss(void) +{ + char tmp[256]; + strcpy(tmp, "/tmp/xce_durfloor2_XXXXXX"); + TEST_ASSERT_NOT_NULL(mkdtemp(tmp)); + + uint64_t durable_floor = 0; + + /* Writer process: append, fdatasync, then "die" (we drop the instance + * without any further flush — the bytes are already durable). */ + { + persist_t* p = make_durability_persist(tmp); + TEST_ASSERT_NOT_NULL(p); + + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1)); + TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1)); + TEST_ASSERT_EQUAL_INT(0, persist_flush(p)); + + durable_floor = persist_test_durable_lsn_on_disk(p); + TEST_ASSERT_EQUAL_UINT64(2, durable_floor); + + teardown_durability_persist(p); + } + + /* "Restarted" process: a fresh persist_t with zero in-memory state, + * opened over the same on-disk WAL. Its durable floor — read straight + * from disk before any recover/replay — matches what the writer + * persisted. Acknowledged-before-durable ops therefore survive the + * loss of the acking process. */ + { + persist_t* p2 = make_durability_persist(tmp); + TEST_ASSERT_NOT_NULL(p2); + + /* current_lsn is still the fresh-boot value (1, LSN 0 reserved): + * the durable floor does NOT come from in-memory state. */ + uint64_t floor2 = persist_test_durable_lsn_on_disk(p2); + TEST_ASSERT_EQUAL_UINT64(durable_floor, floor2); + TEST_ASSERT_EQUAL_UINT64(2, floor2); + + teardown_durability_persist(p2); + } + + printf("test_durable_floor_survives_process_loss: PASS\n"); +} + +#endif /* XCE_TESTING */ + +/* ------------------------------------------------------------------ */ +/* Test suite entry point */ +/* ------------------------------------------------------------------ */ +void test_commit_durability(void) +{ +#ifdef XCE_TESTING + RUN_TEST(test_durable_floor_empty); + RUN_TEST(test_durable_floor_after_flush); + RUN_TEST(test_durable_floor_survives_process_loss); +#else + printf("test_commit_durability: skipped (no XCE_TESTING)\n"); +#endif +} diff --git a/src/include/persist.h b/src/include/persist.h index e67ce10a..f30b1cf2 100644 --- a/src/include/persist.h +++ b/src/include/persist.h @@ -184,6 +184,35 @@ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms); /* Force WAL fdatasync (for tests and clean shutdown) */ int persist_flush(persist_t* p); +#ifdef XCE_TESTING +/* ============================================================ + * Durability-verification test seam (ha-commit-durability Req 3.2, 3.3) + * + * persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL + * segments from STABLE STORAGE and return the highest fully-persisted + * LSN, independent of the in-memory current_lsn / committed_lsn + * watermarks. Lets a test assert that an acknowledged operation is + * recoverable from disk after an immediate, ungraceful termination — + * without relying on the background flush timer (Req 3.2): + * + * persist_wal_async(p, ...); // append to page cache + * persist_flush(p); // fdatasync → durable on disk + * lsn = persist_test_durable_lsn_on_disk(p); // highest durable LSN + * + * It scans the same on-disk format and CRC chain as recovery, stopping + * at the first corruption / short read / pre-allocated zero region, so + * it reports only entries that are genuinely on stable storage. It + * never mutates persist_t state and fires no hooks (Req 3.3 — the seam + * is purely inspective; the acknowledgment contract is unchanged). + * + * Returns the highest durable LSN, or 0 when nothing is durable (empty + * first-boot WAL, no segments). Compiled out of production builds: + * the xce_controller target is built without -DXCE_TESTING and links + * without this symbol. + * ============================================================ */ +uint64_t persist_test_durable_lsn_on_disk(persist_t* p); +#endif /* XCE_TESTING */ + /* Checkpoint */ int persist_checkpoint(persist_t* p);