feat: implement persist_recover with snapshot load and WAL replay (P0-4)
- Add load_snapshot(): reads P0 sentinel ("XCESNAP\0" + lsn), validates magic,
extracts checkpoint_lsn
- Add replay_wal(): scans seg-*.wal files in LSN order, skips entries <= from_lsn,
verifies CRC chain, stops on mismatch, applies entries via apply_wal_entry() stub
- Add persist_recover(): find latest .capnp snapshot, load it, replay post-checkpoint
WAL, retention pass (no-op in P0)
- Fix trim_wal_segments(): compare against active segment start LSN (not checkpoint_lsn)
to avoid deleting the currently-open segment and losing post-checkpoint WAL entries
- Add restore_job() stub in job.c (P0-12a placeholder)
- Add worker_from_capnp() stub in worker.c (P0-12 placeholder)
- Declare both new functions in cluster.h and worker.h
- Add 5 recovery tests: empty state, snapshot lsn loading, WAL replay, corrupt
sentinel rejection, latest-snapshot selection; all 23 controller tests pass
This commit is contained in:
parent
e76f76a88a
commit
f905f891a8
8 changed files with 584 additions and 7 deletions
|
|
@ -359,10 +359,26 @@ static int write_snapshot(persist_t* p)
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* Delete WAL segment files whose start LSN is strictly less than trim_lsn.
|
||||
* Conservative: only trims segments that started before the checkpoint. */
|
||||
/* Delete WAL segment files whose start LSN is strictly less than the
|
||||
* currently-open segment's start LSN. We parse the active segment
|
||||
* filename (seg_path) to get its start LSN and never delete that file,
|
||||
* even if its start_lsn < checkpoint_lsn. */
|
||||
static void trim_wal_segments(persist_t* p)
|
||||
{
|
||||
/* Extract the start LSN of the currently-open segment from seg_path.
|
||||
* seg_path has the form "<wal_path>/seg-<hex16>.wal".
|
||||
* If no segment is open yet (wal_fd < 0), skip trimming. */
|
||||
if (p->wal_fd < 0 || p->seg_path[0] == '\0')
|
||||
return;
|
||||
|
||||
const char* seg_basename = strrchr(p->seg_path, '/');
|
||||
if (!seg_basename) return;
|
||||
seg_basename++; /* skip the '/' */
|
||||
|
||||
uint64_t active_seg_lsn = 0;
|
||||
if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1)
|
||||
return;
|
||||
|
||||
DIR* d = opendir(p->wal_path);
|
||||
if (!d) return;
|
||||
|
||||
|
|
@ -371,9 +387,8 @@ static void trim_wal_segments(persist_t* p)
|
|||
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
|
||||
uint64_t seg_start_lsn = 0;
|
||||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue;
|
||||
/* Only trim segments that start strictly before the checkpoint LSN.
|
||||
* The segment containing checkpoint_lsn is still the active segment. */
|
||||
if (seg_start_lsn < p->checkpoint_lsn) {
|
||||
/* Never delete the currently-open segment; only trim older ones. */
|
||||
if (seg_start_lsn < active_seg_lsn) {
|
||||
char seg_path[4096];
|
||||
snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name);
|
||||
unlink(seg_path);
|
||||
|
|
@ -466,12 +481,287 @@ done:
|
|||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Stubs — wired in later tasks */
|
||||
/* Recovery internals */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
/* P0 sentinel format: "XCESNAP\0" (8 bytes) + checkpoint_lsn (8 bytes LE).
|
||||
* Returns 0 on success and sets p->checkpoint_lsn / p->current_lsn.
|
||||
* Returns -EINVAL on bad magic, -EIO on short read, -errno on open failure. */
|
||||
static int load_snapshot(persist_t* p, const char* path, uint64_t lsn)
|
||||
{
|
||||
(void)lsn; /* lsn is encoded in the file itself — we re-read it */
|
||||
|
||||
int fd = open(path, O_RDONLY);
|
||||
if (fd < 0)
|
||||
return -errno;
|
||||
|
||||
char header[16];
|
||||
ssize_t n = read(fd, header, sizeof(header));
|
||||
close(fd);
|
||||
|
||||
if (n < 16)
|
||||
return -EIO;
|
||||
|
||||
if (memcmp(header, "XCESNAP\0", 8) != 0)
|
||||
return -EINVAL;
|
||||
|
||||
/* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch) */
|
||||
uint64_t stored_lsn;
|
||||
memcpy(&stored_lsn, header + 8, 8);
|
||||
|
||||
p->checkpoint_lsn = stored_lsn;
|
||||
p->current_lsn = stored_lsn;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* P0 stub: no job/worker deserialization yet — wired in Task 12+/15. */
|
||||
static int apply_wal_entry(persist_t* p, const wal_entry_hdr_t* hdr,
|
||||
const void* payload)
|
||||
{
|
||||
(void)p;
|
||||
(void)hdr;
|
||||
(void)payload;
|
||||
/* Full implementation in Task 12+ (hook wiring) and Task 15 (cluster wiring) */
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Read WAL segment files in LSN order; replay entries with lsn > from_lsn.
|
||||
*
|
||||
* Segment filename format: seg-<start_lsn_hex16>.wal
|
||||
* Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes).
|
||||
*
|
||||
* CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry
|
||||
* in a segment). If prev_crc mismatches, stop at last good entry and log WARN.
|
||||
*
|
||||
* Returns number of entries replayed (>= 0) or negative errno on I/O error. */
|
||||
static int replay_wal(persist_t* p, uint64_t from_lsn)
|
||||
{
|
||||
/* --- Collect and sort segment filenames by start LSN --- */
|
||||
DIR* d = opendir(p->wal_path);
|
||||
if (!d)
|
||||
return 0; /* no WAL dir → first boot, nothing to replay */
|
||||
|
||||
/* Collect up to 4096 segment names */
|
||||
uint64_t seg_lsns[4096];
|
||||
int nseg = 0;
|
||||
|
||||
struct dirent* e;
|
||||
while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) {
|
||||
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
|
||||
uint64_t seg_start = 0;
|
||||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue;
|
||||
seg_lsns[nseg++] = seg_start;
|
||||
}
|
||||
closedir(d);
|
||||
|
||||
if (nseg == 0)
|
||||
return 0;
|
||||
|
||||
/* Simple insertion sort — segment count is tiny in practice */
|
||||
for (int i = 1; i < nseg; i++) {
|
||||
uint64_t key = seg_lsns[i];
|
||||
int j = i - 1;
|
||||
while (j >= 0 && seg_lsns[j] > key) {
|
||||
seg_lsns[j + 1] = seg_lsns[j];
|
||||
j--;
|
||||
}
|
||||
seg_lsns[j + 1] = key;
|
||||
}
|
||||
|
||||
/* --- Replay entries from each segment --- */
|
||||
int replayed = 0;
|
||||
uint32_t last_crc = WAL_CRC_NONE;
|
||||
|
||||
for (int si = 0; si < nseg; si++) {
|
||||
char seg_path[4096];
|
||||
snprintf(seg_path, sizeof(seg_path),
|
||||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||||
|
||||
int fd = open(seg_path, O_RDONLY);
|
||||
if (fd < 0) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: cannot open segment %s: %s\n",
|
||||
seg_path, strerror(errno));
|
||||
continue;
|
||||
}
|
||||
|
||||
uint32_t seg_off = 0;
|
||||
|
||||
for (;;) {
|
||||
/* Read header */
|
||||
wal_entry_hdr_t hdr;
|
||||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
|
||||
if (hn == 0)
|
||||
break; /* clean end of segment */
|
||||
if (hn < (ssize_t)sizeof(hdr)) {
|
||||
if (hn < 0) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: read error in %s at offset %u: %s\n",
|
||||
seg_path, seg_off, strerror(errno));
|
||||
}
|
||||
break; /* truncated or pre-allocated zeros */
|
||||
}
|
||||
|
||||
/* Detect pre-allocated zero region (no valid entry starts with
|
||||
* all-zero header: lsn == 0 && type == 0 && payload_len == 0) */
|
||||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||||
break;
|
||||
|
||||
/* CRC chain validation: prev_crc must match last_crc in segment */
|
||||
if (seg_off == 0) {
|
||||
/* First entry in this segment: prev_crc must be WAL_CRC_NONE
|
||||
* (or we accept any value if last_crc is also WAL_CRC_NONE,
|
||||
* e.g. across a segment boundary after checkpoint trim). */
|
||||
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
|
||||
hdr.prev_crc != last_crc) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: CRC chain break at start of %s "
|
||||
"(prev_crc=%08x expected=%08x); stopping replay\n",
|
||||
seg_path, hdr.prev_crc, last_crc);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
if (hdr.prev_crc != last_crc) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: CRC chain break in %s at offset %u "
|
||||
"(prev_crc=%08x expected=%08x); stopping replay at last good entry\n",
|
||||
seg_path, seg_off, hdr.prev_crc, last_crc);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/* Verify entry CRC */
|
||||
uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||||
|
||||
uint8_t* payload = NULL;
|
||||
if (hdr.payload_len > 0) {
|
||||
payload = malloc(hdr.payload_len);
|
||||
if (!payload) {
|
||||
close(fd);
|
||||
return -ENOMEM;
|
||||
}
|
||||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||||
(off_t)(seg_off + sizeof(hdr)));
|
||||
if (pn != (ssize_t)hdr.payload_len) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: short payload read in %s at offset %u\n",
|
||||
seg_path, seg_off);
|
||||
free(payload);
|
||||
break;
|
||||
}
|
||||
expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
|
||||
}
|
||||
|
||||
if (hdr.crc != expected_crc) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: CRC mismatch in %s at offset %u "
|
||||
"(got=%08x expected=%08x); stopping replay\n",
|
||||
seg_path, seg_off, hdr.crc, expected_crc);
|
||||
free(payload);
|
||||
break;
|
||||
}
|
||||
|
||||
/* Advance segment-local CRC */
|
||||
last_crc = hdr.crc;
|
||||
|
||||
/* Skip entries already covered by the snapshot */
|
||||
if (hdr.lsn <= from_lsn) {
|
||||
free(payload);
|
||||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Skip WAL_CHECKPOINT markers (local bookkeeping only) */
|
||||
if (hdr.type == WAL_CHECKPOINT) {
|
||||
free(payload);
|
||||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||||
/* Advance current_lsn past the marker */
|
||||
if (hdr.lsn + 1 > p->current_lsn)
|
||||
p->current_lsn = hdr.lsn + 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Unknown future entry types: log and skip */
|
||||
if (hdr.type > WAL_CHECKPOINT) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: WARNING: unknown WAL entry type %u at LSN %lu; skipping\n",
|
||||
hdr.type, (unsigned long)hdr.lsn);
|
||||
free(payload);
|
||||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Apply the entry (P0 stub: no-op) */
|
||||
apply_wal_entry(p, &hdr, payload);
|
||||
free(payload);
|
||||
|
||||
/* Advance current_lsn to one past this entry */
|
||||
if (hdr.lsn + 1 > p->current_lsn)
|
||||
p->current_lsn = hdr.lsn + 1;
|
||||
|
||||
replayed++;
|
||||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
|
||||
/* Update last_crc so the next WAL write chains correctly */
|
||||
pthread_mutex_lock(&p->lock);
|
||||
p->last_crc = last_crc;
|
||||
pthread_mutex_unlock(&p->lock);
|
||||
|
||||
return replayed;
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Public: persist_recover */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
int persist_recover(persist_t* p)
|
||||
{
|
||||
(void)p;
|
||||
/* --- Step 1: find latest snapshot (highest LSN filename) --- */
|
||||
char snap_file[4096] = "";
|
||||
uint64_t snap_lsn = 0;
|
||||
|
||||
DIR* d = opendir(p->snap_path);
|
||||
if (d) {
|
||||
struct dirent* e;
|
||||
while ((e = readdir(d)) != NULL) {
|
||||
size_t len = strlen(e->d_name);
|
||||
if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0)
|
||||
continue;
|
||||
uint64_t lsn = 0;
|
||||
if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1)
|
||||
continue;
|
||||
if (lsn > snap_lsn) {
|
||||
snap_lsn = lsn;
|
||||
snprintf(snap_file, sizeof(snap_file),
|
||||
"%s/%s", p->snap_path, e->d_name);
|
||||
}
|
||||
}
|
||||
closedir(d);
|
||||
}
|
||||
|
||||
/* --- Step 2: load snapshot (if any) --- */
|
||||
if (snap_lsn > 0 && snap_file[0] != '\0') {
|
||||
int rc = load_snapshot(p, snap_file, snap_lsn);
|
||||
if (rc != 0) {
|
||||
fprintf(stderr,
|
||||
"persist_recover: failed to load snapshot %s: %s\n",
|
||||
snap_file, strerror(-rc));
|
||||
return rc;
|
||||
}
|
||||
/* checkpoint_lsn and current_lsn set by load_snapshot */
|
||||
}
|
||||
|
||||
/* --- Step 3: replay WAL entries after checkpoint_lsn --- */
|
||||
int replayed = replay_wal(p, p->checkpoint_lsn);
|
||||
if (replayed < 0)
|
||||
return replayed;
|
||||
|
||||
/* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue