feat(ha-commit-durability): add XCE_TESTING durable-LSN-on-disk verification seam (task 1.2)

.kiro/specs/ha-commit-durability/tasks.md

@@ -11,7 +11,7 @@
   - _Requirements: 3.1_
   - _Boundary: DurabilityConfig_
 
-- [ ] 1.2 Add the test-only durability-verification hook
+- [x] 1.2 Add the test-only durability-verification hook
   - Add an `XCE_TESTING`-gated seam that lets a test assert an acknowledged operation is recoverable after an immediate ungraceful termination; compiled out of production builds.
   - Observable: in a test build the hook is callable and the production build links without it.
   - _Requirements: 3.2, 3.3_
@@ -121,3 +121,4 @@
 
 ## Implementation Notes
 - 1.1: Config plumbing routes through `myconf_t.wal_sync_ms` + `persist_config_set_wal_sync_ms()` (0-sentinel = keep `DEFAULT_WAL_SYNC_MS`), NOT a new config.h constant (design File Structure Plan keeps `DEFAULT_WAL_SYNC_MS`). The override is applied at BOTH `persist_init` sites — `myapp_run` step-6 (production path) and `cluster_init_phase2` (guarded by `wal_path[0]=='\0'`, skipped in production). They are mutually exclusive; apply any new persist_config override at both sites or it is silently dropped. Pre-existing latent bug (do NOT fix here): `--sync-timeout-ms` is applied only in phase2, so it is ignored on the production startup path — relevant if a later task relies on `sync_timeout_ms`.
+- 1.2: Test seam = `persist_test_durable_lsn_on_disk(persist_t*)` (decl `persist.h`, impl `persist.c`, all `#ifdef XCE_TESTING`) — re-opens WAL segment files from disk and CRC-validates the chain (read-only, state-free) to return the highest fully-persisted LSN; new test `testsuite/test_commit_durability.c` (registered in `meson.build` + `controller_test.c`). Production `xce_controller` links without it (proven via `nm`). DRY NOTE: ~115 lines of segment-scan/CRC-chain logic are duplicated from the static `replay_wal`; tasks 3.1 (recovery verdict) and 3.4 (durable floor from WAL tip) touch the SAME scan/CRC logic — consider extracting a shared read-only `wal_scan_durable_lsn`-style helper there and routing this seam through it (avoid a third copy). `XCE_TESTING` is defined only for `controller_unit_test` (`src/controller/meson.build:167`).

src/controller/meson.build

@@ -131,6 +131,7 @@ controller_unit_test_src = [
   'testsuite/test_demotion_convergence.c',
   'testsuite/test_convergence_verification.c',
   'testsuite/test_coldstart_lsn0.c',
+  'testsuite/test_commit_durability.c',
   'commit.c',
   'job.c',
   'egress.c',

src/controller/persist.c

@@ -536,6 +536,140 @@ int persist_flush(persist_t* p)
     return rc;
 }
 
+#ifdef XCE_TESTING
+/* ------------------------------------------------------------------ *
+ * Durability-verification test seam (ha-commit-durability Req 3.2/3.3)*
+ *                                                                     *
+ * Re-read + CRC-validate the WAL segments from stable storage and      *
+ * return the highest fully-persisted LSN — independent of the          *
+ * in-memory current_lsn / committed_lsn watermarks.  Mirrors the       *
+ * segment-scan + CRC-chain validation of replay_wal(), but is strictly *
+ * read-only: it applies no entries, fires no hooks, and mutates no      *
+ * persist_t state, so it cannot change the externally observable        *
+ * acknowledgment semantics (Req 3.3).                                  *
+ *                                                                     *
+ * Stops at the first corruption / short read / pre-allocated zero       *
+ * region (exactly like recovery), so only entries genuinely on stable   *
+ * storage are counted.  Returns the highest such LSN, or 0 when nothing *
+ * is durable.                                                          *
+ * ------------------------------------------------------------------ */
+uint64_t persist_test_durable_lsn_on_disk(persist_t* p)
+{
+    if (!p)
+        return 0;
+
+    DIR* d = opendir(p->wal_path);
+    if (!d)
+        return 0; /* no WAL dir → nothing durable */
+
+    /* Collect segment start LSNs (same bound as replay_wal). */
+    uint64_t seg_lsns[4096];
+    int      nseg = 0;
+
+    struct dirent* e;
+    while ((e = readdir(d)) != NULL &&
+           nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
+        if (strncmp(e->d_name, "seg-", 4) != 0)
+            continue;
+        uint64_t seg_start = 0;
+        if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1)
+            continue;
+        seg_lsns[nseg++] = seg_start;
+    }
+    closedir(d);
+
+    if (nseg == 0)
+        return 0;
+
+    /* Insertion sort by start LSN (segment count is tiny). */
+    for (int i = 1; i < nseg; i++) {
+        uint64_t key = seg_lsns[i];
+        int      j   = i - 1;
+        while (j >= 0 && seg_lsns[j] > key) {
+            seg_lsns[j + 1] = seg_lsns[j];
+            j--;
+        }
+        seg_lsns[j + 1] = key;
+    }
+
+    uint64_t highest_durable = 0;
+    uint32_t last_crc        = WAL_CRC_NONE;
+
+    for (int si = 0; si < nseg; si++) {
+        char seg_path[4096];
+        snprintf(seg_path, sizeof(seg_path),
+                 WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
+
+        int fd = open(seg_path, O_RDONLY);
+        if (fd < 0)
+            continue;
+
+        uint32_t seg_off = 0;
+
+        for (;;) {
+            wal_entry_hdr_t hdr;
+            ssize_t         hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
+            if (hn == 0)
+                break; /* clean end of segment */
+            if (hn < (ssize_t)sizeof(hdr))
+                break; /* truncated / pre-allocated zeros */
+
+            /* Pre-allocated zero region: no valid entry is all-zero. */
+            if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
+                break;
+
+            /* CRC chain validation — identical rule to replay_wal. */
+            if (seg_off == 0) {
+                if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
+                    hdr.prev_crc != last_crc)
+                    break;
+            } else {
+                if (hdr.prev_crc != last_crc)
+                    break;
+            }
+
+            /* Verify entry CRC over header + payload. */
+            uint32_t expected_crc =
+                crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
+
+            uint8_t* payload = NULL;
+            if (hdr.payload_len > 0) {
+                payload = malloc(hdr.payload_len);
+                if (!payload) {
+                    close(fd);
+                    return highest_durable;
+                }
+                ssize_t pn = pread(fd, payload, hdr.payload_len,
+                                   (off_t)(seg_off + sizeof(hdr)));
+                if (pn != (ssize_t)hdr.payload_len) {
+                    free(payload);
+                    break;
+                }
+                expected_crc =
+                    crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
+            }
+
+            if (hdr.crc != expected_crc) {
+                free(payload);
+                break;
+            }
+            free(payload);
+
+            /* Entry is intact and durable on disk. */
+            last_crc = hdr.crc;
+            if (hdr.lsn > highest_durable)
+                highest_durable = hdr.lsn;
+
+            seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
+        }
+
+        close(fd);
+    }
+
+    return highest_durable;
+}
+#endif /* XCE_TESTING */
+
 /* ------------------------------------------------------------------ */
 /* Checkpoint internals                                                 */
 /* ------------------------------------------------------------------ */

src/controller/testsuite/controller_test.c

@@ -59,6 +59,7 @@ extern void test_egress_filter(void);
 extern void test_demotion_convergence(void);
 extern void test_convergence_verification(void);
 extern void test_coldstart_lsn0(void);
+extern void test_commit_durability(void);
 
 void setUp(void) { return; }
 
@@ -187,6 +188,7 @@ int main(int argc, char* argv[])
     TEST(test_demotion_convergence);
     TEST(test_convergence_verification);
     TEST(test_coldstart_lsn0);
+    TEST(test_commit_durability);
 
     return UNITY_END();
 }

src/controller/testsuite/test_commit_durability.c

@@ -0,0 +1,189 @@
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+
+#include "config.h"
+#include "persist.h"
+
+/* ------------------------------------------------------------------ *
+ * ha-commit-durability — task 1.2: durability-verification test seam  *
+ *                                                                     *
+ * These tests exercise persist_test_durable_lsn_on_disk(), the        *
+ * XCE_TESTING-only seam that re-reads + CRC-validates the WAL from     *
+ * stable storage and reports the highest fully-persisted LSN —         *
+ * independent of the in-memory current_lsn watermark.  This lets a     *
+ * test assert an acknowledged operation is recoverable from disk after *
+ * an immediate ungraceful termination, without relying on the          *
+ * background flush timer (Req 3.2).  The seam is purely inspective:    *
+ * it changes nothing observable about the acknowledgment contract      *
+ * (Req 3.3).                                                           *
+ *                                                                     *
+ * The seam is compiled ONLY into the controller_unit_test target       *
+ * (-DXCE_TESTING).  The production xce_controller target links without *
+ * it — see #ifdef XCE_TESTING in persist.c / declaration in persist.h. *
+ * ------------------------------------------------------------------ */
+
+#ifdef XCE_TESTING
+
+/* Build a persist instance with a deliberately slow background flush
+ * cadence, so the only durability point is an explicit persist_flush()
+ * (mirrors make_fresh_persist in test_persist_recovery.c). */
+static persist_t* make_durability_persist(const char* tmpdir)
+{
+    persist_t* p = calloc(1, sizeof(persist_t));
+    if (!p)
+        return NULL;
+
+    persist_config_t cfg = persist_config_defaults();
+    cfg.wal_sync_ms = 5000; /* slow background fsync — durability comes from
+                             * the explicit flush, not the timer */
+    cfg.snap_keep_n = 2;
+
+    persist_set_workdir(p, tmpdir, "ctrl-test");
+
+    if (persist_init(p, &cfg, NULL) != 0) {
+        free(p);
+        return NULL;
+    }
+    if (persist_start(p) != 0) {
+        persist_fini(p);
+        free(p);
+        return NULL;
+    }
+    return p;
+}
+
+static void teardown_durability_persist(persist_t* p)
+{
+    persist_stop(p);
+    persist_fini(p);
+    free(p);
+}
+
+/* ------------------------------------------------------------------ *
+ * Test 1: an empty WAL has no durable entries (floor == 0).           *
+ * An empty first-boot WAL is not corruption — it simply has nothing   *
+ * persisted, so the seam reports 0.                                   *
+ * ------------------------------------------------------------------ */
+static void test_durable_floor_empty(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_durfloor0_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    persist_t* p = make_durability_persist(tmp);
+    TEST_ASSERT_NOT_NULL(p);
+
+    uint64_t floor = persist_test_durable_lsn_on_disk(p);
+    TEST_ASSERT_EQUAL_UINT64(0, floor);
+
+    teardown_durability_persist(p);
+    printf("test_durable_floor_empty: PASS\n");
+}
+
+/* ------------------------------------------------------------------ *
+ * Test 2 (core Req 3.2): after append + explicit fdatasync, the seam  *
+ * sees the entries on disk — the durable floor equals the highest     *
+ * appended LSN, recoverable independent of in-memory state.           *
+ * ------------------------------------------------------------------ */
+static void test_durable_floor_after_flush(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_durfloor1_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    persist_t* p = make_durability_persist(tmp);
+    TEST_ASSERT_NOT_NULL(p);
+
+    /* Append three entries. LSN 0 is reserved, so these take LSN 1,2,3.
+     * current_lsn advances to 4 (one past the last appended). */
+    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
+    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
+    TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
+
+    /* Durability point: fdatasync the segment so the bytes are on stable
+     * storage (this is what an acknowledged-before-durable contract must
+     * have done before replying ok). */
+    TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
+
+    /* The seam re-reads from disk and reports the highest intact LSN. */
+    uint64_t floor = persist_test_durable_lsn_on_disk(p);
+    TEST_ASSERT_EQUAL_UINT64(p->current_lsn - 1, floor);
+    TEST_ASSERT_EQUAL_UINT64(3, floor);
+
+    teardown_durability_persist(p);
+    printf("test_durable_floor_after_flush: PASS\n");
+}
+
+/* ------------------------------------------------------------------ *
+ * Test 3 (recoverable-after-ungraceful-kill, Req 3.2): a SECOND        *
+ * persist instance opened over the same WAL dir — with NO in-memory    *
+ * state from the writer — recovers exactly the durable floor the seam  *
+ * reported.  This is the "survives an immediate ungraceful            *
+ * termination" assertion: durability lives on disk, not in the         *
+ * volatile buffers of the process that wrote it.                       *
+ * ------------------------------------------------------------------ */
+static void test_durable_floor_survives_process_loss(void)
+{
+    char tmp[256];
+    strcpy(tmp, "/tmp/xce_durfloor2_XXXXXX");
+    TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
+
+    uint64_t durable_floor = 0;
+
+    /* Writer process: append, fdatasync, then "die" (we drop the instance
+     * without any further flush — the bytes are already durable). */
+    {
+        persist_t* p = make_durability_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p);
+
+        TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "x", 1));
+        TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "y", 1));
+        TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
+
+        durable_floor = persist_test_durable_lsn_on_disk(p);
+        TEST_ASSERT_EQUAL_UINT64(2, durable_floor);
+
+        teardown_durability_persist(p);
+    }
+
+    /* "Restarted" process: a fresh persist_t with zero in-memory state,
+     * opened over the same on-disk WAL.  Its durable floor — read straight
+     * from disk before any recover/replay — matches what the writer
+     * persisted.  Acknowledged-before-durable ops therefore survive the
+     * loss of the acking process. */
+    {
+        persist_t* p2 = make_durability_persist(tmp);
+        TEST_ASSERT_NOT_NULL(p2);
+
+        /* current_lsn is still the fresh-boot value (1, LSN 0 reserved):
+         * the durable floor does NOT come from in-memory state. */
+        uint64_t floor2 = persist_test_durable_lsn_on_disk(p2);
+        TEST_ASSERT_EQUAL_UINT64(durable_floor, floor2);
+        TEST_ASSERT_EQUAL_UINT64(2, floor2);
+
+        teardown_durability_persist(p2);
+    }
+
+    printf("test_durable_floor_survives_process_loss: PASS\n");
+}
+
+#endif /* XCE_TESTING */
+
+/* ------------------------------------------------------------------ */
+/* Test suite entry point                                               */
+/* ------------------------------------------------------------------ */
+void test_commit_durability(void)
+{
+#ifdef XCE_TESTING
+    RUN_TEST(test_durable_floor_empty);
+    RUN_TEST(test_durable_floor_after_flush);
+    RUN_TEST(test_durable_floor_survives_process_loss);
+#else
+    printf("test_commit_durability: skipped (no XCE_TESTING)\n");
+#endif
+}

src/include/persist.h

@@ -184,6 +184,35 @@ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms);
 /* Force WAL fdatasync (for tests and clean shutdown) */
 int persist_flush(persist_t* p);
 
+#ifdef XCE_TESTING
+/* ============================================================
+ * Durability-verification test seam (ha-commit-durability Req 3.2, 3.3)
+ *
+ * persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL
+ * segments from STABLE STORAGE and return the highest fully-persisted
+ * LSN, independent of the in-memory current_lsn / committed_lsn
+ * watermarks.  Lets a test assert that an acknowledged operation is
+ * recoverable from disk after an immediate, ungraceful termination —
+ * without relying on the background flush timer (Req 3.2):
+ *
+ *     persist_wal_async(p, ...);     // append to page cache
+ *     persist_flush(p);              // fdatasync → durable on disk
+ *     lsn = persist_test_durable_lsn_on_disk(p);  // highest durable LSN
+ *
+ * It scans the same on-disk format and CRC chain as recovery, stopping
+ * at the first corruption / short read / pre-allocated zero region, so
+ * it reports only entries that are genuinely on stable storage.  It
+ * never mutates persist_t state and fires no hooks (Req 3.3 — the seam
+ * is purely inspective; the acknowledgment contract is unchanged).
+ *
+ * Returns the highest durable LSN, or 0 when nothing is durable (empty
+ * first-boot WAL, no segments).  Compiled out of production builds:
+ * the xce_controller target is built without -DXCE_TESTING and links
+ * without this symbol.
+ * ============================================================ */
+uint64_t persist_test_durable_lsn_on_disk(persist_t* p);
+#endif /* XCE_TESTING */
+
 /* Checkpoint */
 int persist_checkpoint(persist_t* p);