lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-commit-durability): WAL recovery integrity verdict (CLEAN/TRUNCATED) + recovery_truncated flag (task 3.1)

Browse source
This commit is contained in:
Rongsong Shen 2026-06-18 07:43:01 +00:00
parent 526bca9679
commit dd4d70863b
4 changed files with 338 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

74
src/controller/persist.c
View file

@ -434,6 +434,11 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
* flusher thread after a successful fdatasync (ha-commit-durability 2.1). */
atomic_store(&p->durable_lsn, 0);
/* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1):
* a fresh store has no corruption. persist_recover overwrites this with the
* replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */
p->recovery_verdict = RECOVER_CLEAN;
/* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2):
* the commit engine registers it in commit_engine_init. Until then every
* flusher durable advance is a no-op wake. */
@ -1357,13 +1362,26 @@ int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr, const void* payloa
* CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry
* in a segment). If prev_crc mismatches, stop at last good entry and log WARN.
*
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
* *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped
* on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a
* short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF,
* pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is
* about the STRUCTURE of the local log, not about being behind the cluster: a
* clean log that simply ends below the cluster high-water is still CLEAN.
*
* Returns number of entries replayed (>= 0) or negative errno on I/O error. */
static int replay_wal(persist_t* p, uint64_t from_lsn)
static int replay_wal(persist_t* p, uint64_t from_lsn,
recover_verdict_t* verdict_out)
{
/* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */
if (verdict_out)
*verdict_out = RECOVER_CLEAN;
/* --- Collect and sort segment filenames by start LSN --- */
DIR* d = opendir(p->wal_path);
if (!d)
return 0; /* no WAL dir → first boot, nothing to replay */
return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */
/* Collect up to 4096 segment names */
uint64_t seg_lsns[4096];
@ -1415,16 +1433,27 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
wal_entry_hdr_t hdr;
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
if (hn == 0)
break; /* clean end of segment */
break; /* clean end of segment (CLEAN) */
if (hn < (ssize_t)sizeof(hdr)) {
/* A partial header (or read error) is a SHORT READ — the log
* stops mid-entry, so it cannot be reconstructed contiguously
* (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */
if (hn < 0)
WARN("persist_recover: read error in %s at offset %u: %s",
seg_path, seg_off, strerror(errno));
break; /* truncated or pre-allocated zeros */
else
WARN("persist_recover: short header read in %s at offset %u "
"(got %zd of %zu bytes); stopping replay",
seg_path, seg_off, hn, sizeof(hdr));
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
/* Detect pre-allocated zero region (no valid entry starts with
* all-zero header: lsn == 0 && type == 0 && payload_len == 0) */
* all-zero header: lsn == 0 && type == 0 && payload_len == 0).
* This is the normal fallocate'd tail after the last valid entry —
* a clean stop, NOT corruption (stays RECOVER_CLEAN). */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break;
@ -1438,6 +1467,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
WARN("persist_recover: CRC chain break at start of %s "
"(prev_crc=%08x expected=%08x); stopping replay",
seg_path, hdr.prev_crc, last_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
} else {
@ -1445,6 +1476,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
WARN("persist_recover: CRC chain break in %s at offset %u "
"(prev_crc=%08x expected=%08x); stopping replay at last good entry",
seg_path, seg_off, hdr.prev_crc, last_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
}
@ -1464,6 +1497,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
if (pn != (ssize_t)hdr.payload_len) {
WARN("persist_recover: short payload read in %s at offset %u",
seg_path, seg_off);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
free(payload);
break;
}
@ -1474,6 +1509,8 @@ static int replay_wal(persist_t* p, uint64_t from_lsn)
WARN("persist_recover: CRC mismatch in %s at offset %u "
"(got=%08x expected=%08x); stopping replay",
seg_path, seg_off, hdr.crc, expected_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
free(payload);
break;
}
@ -1571,15 +1608,40 @@ int persist_recover(persist_t* p)
}
/* --- Step 3: replay WAL entries after checkpoint_lsn --- */
int replayed = replay_wal(p, p->checkpoint_lsn);
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
* replay distinguishes a clean log end from a corruption-truncated end.
* We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do
* NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean
* replay over a loaded snapshot-sentinel (no entries below the sentinel) is
* CLEAN — the sentinel established the committed position; replay only adds
* entries above it. */
recover_verdict_t verdict = RECOVER_CLEAN;
int replayed = replay_wal(p, p->checkpoint_lsn, &verdict);
if (replayed < 0)
return replayed;
p->recovery_verdict = verdict;
if (verdict == RECOVER_TRUNCATED)
WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally "
"corrupt (CRC break / mismatch / short read); member marked "
"recovery-truncated (non-promotable until caught up)");
/* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */
return 0;
}
/* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */
recover_verdict_t persist_recovery_verdict(persist_t* p)
{
return p ? p->recovery_verdict : RECOVER_CLEAN;
}
bool persist_recovery_truncated(persist_t* p)
{
return p && p->recovery_verdict == RECOVER_TRUNCATED;
}
/* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
* entry is already durable on local stable storage (ha-commit-durability task
* 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the