lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): WAL recovery integrity verdict (CLEAN/TRUNCATED) + recovery_truncated flag (task 3.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 07:43:01 +00:00
parent 526bca9679
commit dd4d70863b
4 changed files with 338 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

74
src/controller/persist.c
View file

@ -434,6 +434,11 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
* flusher thread after a successful fdatasync (ha-commit-durability 2.1). */
atomic_store(&p->durable_lsn, 0);
/* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1):
* a fresh store has no corruption. persist_recover overwrites this with the
* replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */
p->recovery_verdict = RECOVER_CLEAN;
/* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2):
* the commit engine registers it in commit_engine_init. Until then every
* flusher durable advance is a no-op wake. */
@ -1357,13 +1362,26 @@ int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payloa
* CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry
* in a segment). If prev_crc mismatches, stop at last good entry and log WARN.
*
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
* *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped
* on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a
* short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF,
* pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is
* about the STRUCTURE of the local log, not about being behind the cluster: a
* clean log that simply ends below the cluster high-water is still CLEAN.
*
* Returns number of entries replayed (>= 0) or negative errno on I/O error. */
static int replay_wal(persist_t* p, uint64_t from_lsn)
static int replay_wal(persist_t* p, uint64_t from_lsn,
recover_verdict_t* verdict_out)
{
/* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */
if (verdict_out)
*verdict_out = RECOVER_CLEAN;
/* --- Collect and sort segment filenames by start LSN --- */
DIR* d = opendir(p->wal_path);
if (!d)
return 0; /* no WAL dir → first boot, nothing to replay */
return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */
/* Collect up to 4096 segment names */
uint64_t seg_lsns[4096];
@ -1415,16 +1433,27 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
wal_entry_hdr_t hdr;
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
if (hn == 0)
break; /* clean end of segment */
break; /* clean end of segment (CLEAN) */
if (hn < (ssize_t)sizeof(hdr)) {
/* A partial header (or read error) is a SHORT READ — the log
* stops mid-entry, so it cannot be reconstructed contiguously
* (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */
if (hn < 0)
WARN("persist_recover: read error in %s at offset %u: %s",
seg_path, seg_off, strerror(errno));
break; /* truncated or pre-allocated zeros */
else
WARN("persist_recover: short header read in %s at offset %u "
"(got %zd of %zu bytes); stopping replay",
seg_path, seg_off, hn, sizeof(hdr));
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
/* Detect pre-allocated zero region (no valid entry starts with
* all-zero header: lsn == 0 && type == 0 && payload_len == 0) */
* all-zero header: lsn == 0 && type == 0 && payload_len == 0).
* This is the normal fallocate'd tail after the last valid entry —
* a clean stop, NOT corruption (stays RECOVER_CLEAN). */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break;
@ -1438,6 +1467,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
WARN("persist_recover: CRC chain break at start of %s "
"(prev_crc=%08x expected=%08x); stopping replay",
seg_path, hdr.prev_crc, last_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
} else {
@ -1445,6 +1476,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
WARN("persist_recover: CRC chain break in %s at offset %u "
"(prev_crc=%08x expected=%08x); stopping replay at last good entry",
seg_path, seg_off, hdr.prev_crc, last_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
}
@ -1464,6 +1497,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
if (pn != (ssize_t)hdr.payload_len) {
WARN("persist_recover: short payload read in %s at offset %u",
seg_path, seg_off);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
free(payload);
break;
}
@ -1474,6 +1509,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
WARN("persist_recover: CRC mismatch in %s at offset %u "
"(got=%08x expected=%08x); stopping replay",
seg_path, seg_off, hdr.crc, expected_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
free(payload);
break;
}
@ -1571,15 +1608,40 @@ int persist_recover(persist_t* p)
}
/* --- Step 3: replay WAL entries after checkpoint_lsn --- */
int replayed = replay_wal(p, p->checkpoint_lsn);
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
* replay distinguishes a clean log end from a corruption-truncated end.
* We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do
* NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean
* replay over a loaded snapshot-sentinel (no entries below the sentinel) is
* CLEAN — the sentinel established the committed position; replay only adds
* entries above it. */
recover_verdict_t verdict = RECOVER_CLEAN;
int replayed = replay_wal(p, p->checkpoint_lsn, &verdict);
if (replayed < 0)
return replayed;
p->recovery_verdict = verdict;
if (verdict == RECOVER_TRUNCATED)
WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally "
"corrupt (CRC break / mismatch / short read); member marked "
"recovery-truncated (non-promotable until caught up)");
/* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */
return 0;
}
/* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */
recover_verdict_t persist_recovery_verdict(persist_t* p)
{
return p ? p->recovery_verdict : RECOVER_CLEAN;
}
bool persist_recovery_truncated(persist_t* p)
{
return p && p->recovery_verdict == RECOVER_TRUNCATED;
}
/* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
* entry is already durable on local stable storage (ha-commit-durability task
* 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the

219
src/controller/testsuite/test_commit_durability.c
View file

@ -1,4 +1,6 @@
#define _GNU_SOURCE
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
@ -11,6 +13,7 @@
#include "commit.h"
#include "config.h"
#include "persist.h"
#include "wal_types.h"
/* ------------------------------------------------------------------ *
* ha-commit-durability — task 1.2: durability-verification test seam *
@ -775,6 +778,218 @@ static void test_commit_never_exceeds_durable_invariant(void)
printf("test_commit_never_exceeds_durable_invariant: PASS\n");
}
/* ================================================================== *
* task 3.1 — recovery integrity verdict (Req 2.1) *
* *
* persist_recover/replay_wal must distinguish a CLEAN log end (EOF, *
* pre-allocated-zero tail, snapshot-sentinel + clean replay, empty *
* first-boot WAL) from a corruption-TRUNCATED end (CRC chain break, *
* CRC mismatch, short/partial entry read). A corruption-truncated *
* recovery is NEVER reported as a clean recovery and sets a *
* member-level recovery_truncated flag the promotion gate (4.3) *
* consumes. 3.1 only SETS the verdict/flag — it does NOT FATAL and *
* does NOT fence (3.3 / 4.3 own those). *
* *
* NB the verdict is about STRUCTURAL integrity of the local log, not *
* about being behind the cluster: a clean log that ends below the *
* cluster high-water is still CLEAN (the freshness/promotion fence *
* catches "behind", not this verdict). *
* ------------------------------------------------------------------ */
/* The persist paths derive as <workdir>/<name>/data/wal; make_durability_persist
* uses name "ctrl-test" (see persist_set_workdir call above). */
#define DUR_WAL_SEG_FMT "%s/ctrl-test/data/wal/seg-%016" PRIx64 ".wal"
/* Open the first WAL segment (starts at LSN 1; LSN 0 reserved) for the
* given tmpdir and return its fd, or -1. */
static int open_first_seg_rw(const char* tmpdir)
{
char seg_path[512];
/* The first-ever entry is LSN 1, so the first segment is seg-...0001. */
snprintf(seg_path, sizeof(seg_path), DUR_WAL_SEG_FMT, tmpdir,
(uint64_t)1);
return open(seg_path, O_RDWR);
}
/* Open a SECOND persist instance over the same on-disk WAL with NO
* in-memory writer state, run recovery, and return the recovery verdict
* (via persist_recovery_truncated). The writer must already be torn down
* so the segments are closed and on disk. */
/* ------------------------------------------------------------------ *
* Test 3.1a: an intact log yields RECOVER_CLEAN (flag NOT set). *
* ------------------------------------------------------------------ */
static void test_recover_verdict_clean_intact_log(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recclean_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: append + flush three entries, then "die". */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Recover into a fresh instance: an intact CRC chain must be CLEAN. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"an intact CRC chain must recover CLEAN");
TEST_ASSERT_FALSE_MESSAGE(
persist_recovery_truncated(p2),
"a clean recovery must NOT set the recovery_truncated flag");
/* current_lsn advanced past the three replayed entries. */
TEST_ASSERT_EQUAL_UINT64(4, p2->current_lsn);
teardown_durability_persist(p2);
}
printf("test_recover_verdict_clean_intact_log: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1b: an empty first-boot WAL is CLEAN, not corruption. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_empty_first_boot_clean(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recempty_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
/* No appends: a first-boot WAL has nothing to replay — that is a clean
* (empty) recovery, never a corruption-truncated one. */
int rc = persist_recover(p);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p),
"an empty first-boot WAL must be CLEAN (no entries != corruption)");
TEST_ASSERT_FALSE(persist_recovery_truncated(p));
teardown_durability_persist(p);
printf("test_recover_verdict_empty_first_boot_clean: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1c (core RED): a CRC break injected at the FIRST entry yields *
* RECOVER_TRUNCATED and SETS the recovery_truncated flag. *
* *
* We append+flush two entries, then corrupt the on-disk `crc` field of *
* the FIRST entry (LSN 1). On recovery the CRC verification at LSN 1 *
* fails, replay stops before any clean boundary, and the verdict must *
* be TRUNCATED — never silently accepted as a complete clean recovery. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_crc_break_first_entry_truncated(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recbreak_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: two durable entries, then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Corrupt the FIRST entry's CRC field on disk: read its header at
* offset 0, flip the crc, write it back. This breaks the chain at
* LSN 1 — replay cannot reach any clean boundary. */
{
int fd = open_first_seg_rw(tmp);
TEST_ASSERT_TRUE_MESSAGE(fd >= 0, "first WAL segment must exist");
wal_entry_hdr_t hdr;
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pread(fd, &hdr, sizeof(hdr), 0));
hdr.crc ^= 0xDEADBEEFu; /* corrupt the entry CRC */
TEST_ASSERT_EQUAL_INT((int)sizeof(hdr),
(int)pwrite(fd, &hdr, sizeof(hdr), 0));
fsync(fd);
close(fd);
}
/* Recover: the corruption at LSN 1 must yield TRUNCATED + set the flag. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
/* persist_recover still returns 0 (no I/O error); the VERDICT is the
* signal — 3.1 classifies, it does not FATAL (that is task 3.3). */
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_TRUNCATED, persist_recovery_verdict(p2),
"a CRC break at the first entry must yield RECOVER_TRUNCATED");
TEST_ASSERT_TRUE_MESSAGE(
persist_recovery_truncated(p2),
"a corruption-truncated recovery must SET the recovery_truncated "
"flag (consumed by the promotion gate, task 4.3)");
teardown_durability_persist(p2);
}
printf("test_recover_verdict_crc_break_first_entry_truncated: PASS\n");
}
/* ------------------------------------------------------------------ *
* Test 3.1d (snapshot-sentinel edge): a member that recovered from a *
* 16-byte LSN-sentinel snapshot has a committed position at the *
* sentinel LSN but NO log entries below it. A clean replay over that *
* sentinel snapshot must classify CLEAN, never TRUNCATED — the absence *
* of sub-sentinel entries is normal, not corruption (design Open *
* Questions / snapshot-recovery edge). *
* *
* We checkpoint (writes the sentinel snapshot + trims the WAL), append *
* + flush one fresh entry above the checkpoint, then recover in a fresh *
* instance: snapshot-sentinel + clean replay ⇒ CLEAN. *
* ------------------------------------------------------------------ */
static void test_recover_verdict_snapshot_sentinel_clean(void)
{
char tmp[256];
strcpy(tmp, "/tmp/xce_recsnap_XXXXXX");
TEST_ASSERT_NOT_NULL(mkdtemp(tmp));
/* Writer: a couple of entries, a checkpoint (sentinel snapshot), then a
* further entry above the checkpoint, all flushed; then die. */
{
persist_t* p = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p);
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "a", 1));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "b", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
/* Sentinel snapshot at the current committed/checkpoint position. */
TEST_ASSERT_EQUAL_INT(0, persist_checkpoint(p));
TEST_ASSERT_EQUAL_INT(0, persist_wal_async(p, WAL_JOB_UPDATE, "c", 1));
TEST_ASSERT_EQUAL_INT(0, persist_flush(p));
teardown_durability_persist(p);
}
/* Recover: load_snapshot sets checkpoint_lsn from the sentinel, replay
* picks up entries above it cleanly — verdict CLEAN, flag NOT set. */
{
persist_t* p2 = make_durability_persist(tmp);
TEST_ASSERT_NOT_NULL(p2);
int rc = persist_recover(p2);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_INT_MESSAGE(
RECOVER_CLEAN, persist_recovery_verdict(p2),
"snapshot-sentinel + clean replay must be CLEAN, not TRUNCATED");
TEST_ASSERT_FALSE(persist_recovery_truncated(p2));
teardown_durability_persist(p2);
}
printf("test_recover_verdict_snapshot_sentinel_clean: PASS\n");
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
@ -797,6 +1012,10 @@ void test_commit_durability(void)
RUN_TEST(test_legacy_sync_append_holds_committed_at_durable);
RUN_TEST(test_legacy_sync_append_lsn0_edge_valid);
RUN_TEST(test_commit_never_exceeds_durable_invariant);
RUN_TEST(test_recover_verdict_clean_intact_log);
RUN_TEST(test_recover_verdict_empty_first_boot_clean);
RUN_TEST(test_recover_verdict_crc_break_first_entry_truncated);
RUN_TEST(test_recover_verdict_snapshot_sentinel_clean);
#else
printf("test_commit_durability: skipped (no XCE_TESTING)\n");
#endif

49
src/include/persist.h
View file

@ -2,6 +2,7 @@
#include <pthread.h>
#include <stdarg.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
@ -12,6 +13,28 @@
/* Forward declaration */
typedef struct cluster_s cluster_t;
/* ============================================================
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1)
*
* The outcome of WAL replay at startup, distinguishing a structurally
* intact log from one whose chain is broken. It is about the STRUCTURE
* of the local log, NOT about being behind the cluster: a clean log that
* simply ends below the cluster's committed high-water mark is still
* RECOVER_CLEAN (the freshness/promotion fence handles "behind"). A
* corruption-truncated recovery is NEVER reported as a clean recovery.
* ============================================================ */
typedef enum {
RECOVER_CLEAN = 0, /* WAL forms a valid CRC chain to a clean entry
* boundary: clean EOF, pre-allocated-zero tail,
* snapshot-sentinel + clean replay, or an empty
* first-boot WAL (no entries != corruption). */
RECOVER_TRUNCATED = 1, /* replay stopped on structural corruption: a CRC
* chain break, a CRC mismatch, or a short/partial
* entry read — the log cannot be reconstructed
* contiguously. Marks the member non-promotable
* (consumed by the promotion gate, task 4.3). */
} recover_verdict_t;
/* ============================================================
* Persistence configuration (Part 2.6)
* ============================================================ */
@ -97,6 +120,22 @@ struct persist_s {
uint64_t total_wal_bytes;
char seg_path[4096];
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1).
* Set by persist_recover/replay_wal at startup: RECOVER_CLEAN when the
* local WAL forms a valid CRC chain to a clean entry boundary (incl. an
* empty first-boot WAL, a snapshot-sentinel + clean replay, and a
* pre-allocated-zero tail), RECOVER_TRUNCATED when replay stopped on a
* structural corruption — a CRC chain break, a CRC mismatch, or a
* short/partial entry read — so the on-disk log cannot be reconstructed
* contiguously. A node that cleanly ends BELOW the cluster high-water is
* still CLEAN here (the freshness/promotion fence catches "behind", task
* 4.2/4.3 — not this verdict). 3.1 only CLASSIFIES + sets this; it does
* NOT FATAL (task 3.3) and does NOT fence promotion (task 4.3, which
* CONSUMES this verdict). Defaults to RECOVER_CLEAN before recovery runs
* (a fresh store is clean). Written single-threaded during recovery
* before the node participates; read by the promotion gate thereafter. */
recover_verdict_t recovery_verdict;
/* Paths (derived from workdir + name via persist_set_workdir) */
char wal_path[4096];
char snap_path[4096];
@ -227,6 +266,16 @@ int persist_flush(persist_t* p);
* any thread. Returns 0 when nothing is yet durable. */
uint64_t persist_durable_lsn(persist_t* p);
/* Recovery-integrity verdict + flag (ha-commit-durability task 3.1, Req 2.1).
* persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED
* verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh
* store before recovery runs). persist_recovery_truncated is the convenience
* flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate
* (task 4.3) consumes to mark the member non-promotable. Both are pure reads
* of recovery state established single-threaded at startup. */
recover_verdict_t persist_recovery_verdict(persist_t* p);
bool persist_recovery_truncated(persist_t* p);
/* Register a durable-advance wake callback (ha-commit-durability task 2.2).
* The group-commit flusher calls fn(ctx) after it advances durable_lsn and
* after releasing p->lock, so a consumer (the commit engine) can re-evaluate