lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/persist.c
Rongsong Shen 50d016d905 feat(raft-apply-decoupling): point restart replay at apply_committed_entry seam (task 2.3) 
Rewire replay_wal()'s per-entry apply call from apply_wal_entry() to
apply_committed_entry() (task 2.2's shared dispatch seam), so restart
replay and the future live commit-advance pump (task 6.1) both
produce materialized state deterministically from the committed log.
apply_wal_entry() is not retired -- it remains ha.c's live standby
catch-up/replicated-apply dispatch, untouched.

Documents a real hazard: replay now fires AFTER_EVENT hooks for the
full post-checkpoint WAL range on every restart, including entries
already applied+notified in a prior process life. Traced and
confirmed inert today for both hook consumers (scheduler registers
post-boot, apiserver gates notify on a connected endpoint empty at
boot); flagged in tasks.md and in-line at the call site for
re-verification before tasks 4.1/6.1/6.2/6.3 make this path broadly
live.
2026-07-04 00:32:46 +00:00

4088 lines
169 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#define _GNU_SOURCE
#include "persist.h"
#include "config.h"
#include "ctrl_metrics.h"
#include "hook.h"
#include "xce/cluster_def.h"
#include "xce/errnox.h"
#include "xce/types.h"
#include "controller.capnp.h"
#include "xlist.h"
#include "cluster.h"
#include "commit.h" /* commit_submit RETAIN registration (barrier-rollout 1.2) */
#include <assert.h>
#include <stdbool.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/statfs.h>
#include <time.h>
#include <unistd.h>
#include <zlib.h>
/* restore_* functions defined in job.c / worker.c / queue.c */
extern int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload,
size_t len, uint64_t entry_lsn);
extern int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state);
extern int worker_from_capnp(cluster_t* c,
const void* capnp_payload, size_t len);
extern int restore_queue_state(cluster_t* c,
const char* name, uint32_t status);
/* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */
/* Serialize xce_job_t to a malloc'd Cap'n Proto buffer.
* Caller must free(*out) on success. */
static int capnp_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Job cj;
Job_ptr jp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Job(cs, &cj, (xce_job_t*)j);
jp = new_Job(cs);
write_Job(&cj, jp);
capn_setp(capn_root(&c), 0, jp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
/* Serialize xce_worker_t to a malloc'd Cap'n Proto buffer. */
static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out_len)
{
struct capn c;
struct capn_segment* cs;
struct Worker cw;
Worker_ptr wp;
capn_init_malloc(&c);
cs = capn_root(&c).seg;
encode_Worker(cs, &cw, (xce_worker_t*)w);
wp = new_Worker(cs);
write_Worker(&cw, wp);
capn_setp(capn_root(&c), 0, wp.p);
*out_len = capn_size(&c);
*out = malloc(*out_len);
if (!*out) { capn_free(&c); return -ENOMEM; }
capn_write_mem(&c, *out, *out_len, 0);
capn_free(&c);
return 0;
}
/* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */
#define WORKER_WAL_HDR_SIZE sizeof(uint32_t)
/* ------------------------------------------------------------------ */
/* Constants */
/* ------------------------------------------------------------------ */
#define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
/* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */
#define NFS_SUPER_MAGIC 0x6969
/* ------------------------------------------------------------------ */
/* Internal helpers */
/* ------------------------------------------------------------------ */
static uint64_t now_us(void)
{
struct timespec ts;
clock_gettime(CLOCK_REALTIME, &ts);
return (uint64_t)ts.tv_sec * 1000000ULL + (uint64_t)(ts.tv_nsec / 1000);
}
/* Recursive mkdir — equivalent to mkdir -p */
static int mkdir_p(const char* path, mode_t mode)
{
char tmp[4096];
char* p;
size_t len;
snprintf(tmp, sizeof(tmp), "%s", path);
len = strlen(tmp);
if (len > 0 && tmp[len - 1] == '/')
tmp[len - 1] = '\0';
for (p = tmp + 1; *p; p++) {
if (*p == '/') {
*p = '\0';
if (mkdir(tmp, mode) != 0 && errno != EEXIST)
return -1;
*p = '/';
}
}
if (mkdir(tmp, mode) != 0 && errno != EEXIST)
return -1;
return 0;
}
/* Open (and pre-allocate) a new WAL segment named after current_lsn */
static int open_new_segment(persist_t* p)
{
snprintf(p->seg_path, sizeof(p->seg_path),
WAL_SEG_FMT, p->wal_path, p->current_lsn);
int fd = open(p->seg_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
if (fd < 0)
return -errno;
/* Pre-allocate segment space to avoid fragmentation and metadata updates
* on every write. posix_fallocate is best-effort — ignore errors. */
posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes);
p->wal_fd = fd;
p->seg_offset = 0;
return 0;
}
/* Close current segment and open a fresh one */
static int rotate_segment(persist_t* p)
{
if (p->wal_fd >= 0) {
fdatasync(p->wal_fd);
close(p->wal_fd);
p->wal_fd = -1;
}
return open_new_segment(p);
}
/* ------------------------------------------------------------------ */
/* Demand-driven group-commit flusher (ha-commit-durability task 2.1) */
/* */
/* Repurposes the old fixed-interval fsync timer into a flusher driven */
/* by the append-vs-synced gap. On each pass it batches one fdatasync */
/* over all entries appended since the last sync, advances durable_lsn */
/* to the synced tip, and wakes the commit evaluation. --wal-sync-ms */
/* now bounds the MAX batching latency (a no-further-append entry is */
/* still synced within the cadence), not a durability gap. */
/* */
/* LOST-WAKEUP / CORRECTNESS REASONING (gating item): */
/* */
/* The "work pending" condition is LEVEL-triggered on two persistent */
/* variables, NOT an edge: pending ⇔ durable_lsn < current_lsn - 1 */
/* (flusher_work_pending below, evaluated under p->lock). Because it */
/* is a comparison of stored state — not a transient flag consumed on */
/* read — a signal that races the flusher's wait can never be lost: */
/* */
/* - persist_wal_async advances current_lsn under p->lock, THEN */
/* signals fsync_cond. If the flusher is mid-flush when the append */
/* lands, current_lsn has already moved past the tip the flusher */
/* captured, so after that flush durable_lsn < current_lsn - 1 */
/* STILL holds; the flusher re-checks the level under p->lock on */
/* its next loop iteration and flushes again (it does NOT wait */
/* first). The missed signal is therefore harmless — the level */
/* itself carries the pending work forward. */
/* - If the append's signal arrives while the flusher sits in */
/* pthread_cond_wait, it wakes it. If it arrives in the tiny window */
/* after the flusher decided "no work" but before it re-enters the */
/* wait, the flusher re-evaluates the level under p->lock before */
/* waiting (we re-check inside the fsync_mutex-held region by */
/* reading current_lsn/durable_lsn under p->lock first), so it */
/* will not block while work is pending. */
/* */
/* "NEVER durable before its fdatasync": the flusher snapshots */
/* tip = current_lsn - 1 AND calls fdatasync WHILE HOLDING p->lock. */
/* persist_wal_async also pwrites under p->lock, so no append can */
/* extend the segment between the tip snapshot and the fdatasync — by */
/* the time fdatasync returns, every byte for LSNs <= tip is on stable */
/* storage. durable_lsn is only then advanced to tip (monotonic max). */
/* Releasing p->lock and broadcasting commit-eval happens after the */
/* advance, so a WAIT caller that observes durable_lsn >= its LSN is */
/* guaranteed the bytes are synced. */
/* */
/* Lock discipline: p->lock and fsync_mutex are NEVER held together */
/* (fsync_mutex guards only the cond wait; the flush section takes */
/* p->lock alone). This keeps fdatasync off any lock the iomgr / */
/* heartbeat path needs and preserves liveness (Req 1.6). */
/* ------------------------------------------------------------------ */
/* Test-only fault-injection probe (ha-commit-durability task 2.1 error path).
* Returns true when the test has forced fdatasync to be treated as failed.
* Compiled to a constant `false` in production (no -DXCE_TESTING), so the
* fail-loud gate below collapses to the pure rc != 0 check with zero overhead
* and no dependency on the test-only struct field. */
static inline bool flusher_fsync_forced_fail(persist_t* p)
{
#ifdef XCE_TESTING
return atomic_load_explicit(&p->test_force_fsync_fail,
memory_order_acquire) != 0;
#else
(void)p;
return false;
#endif
}
/* Returns true iff there is un-synced appended data (level predicate).
* MUST be called with p->lock held. current_lsn - 1 is the appended tip
* (current_lsn is the NEXT lsn to assign); LSN 0 reserved, so a fresh
* store has current_lsn == 1 → tip 0 → no pending work. */
static bool flusher_work_pending_locked(persist_t* p)
{
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
return dur < tip;
}
/* Perform one group-commit flush: snapshot the appended tip, fdatasync the
* live segment, and advance durable_lsn to the synced tip. Takes p->lock
* for the whole snapshot+sync+advance so no concurrent append can extend
* the segment between the tip read and the fdatasync. Returns the new
* durable_lsn (unchanged if there was nothing to sync). */
static uint64_t flusher_flush_once(persist_t* p)
{
pthread_mutex_lock(&p->lock);
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
if (dur >= tip || p->wal_fd < 0) {
/* Nothing new to sync (or no segment yet). Still flush the fd for a
* clean-shutdown / fallback pass, but do not advance the watermark.
* The watermark is already <= tip here, so a failed fallback sync
* cannot make it claim more than is durable — but we still surface
* the error loudly so a degrading disk is never silent (Req 1.1). */
if (p->wal_fd >= 0) {
int rc = fdatasync(p->wal_fd);
if (rc != 0 || flusher_fsync_forced_fail(p)) {
int e = (rc != 0) ? errno : EIO;
ERROR("persist: flusher fallback fdatasync FAILED (errno=%d %s)"
" — durable_lsn held at %" PRIu64 "; WAL not advanced",
e, strerror(e), dur);
}
}
pthread_mutex_unlock(&p->lock);
return dur;
}
/* fdatasync under p->lock: appends also hold p->lock to pwrite, so the
* segment cannot grow between this tip snapshot and the sync. Every byte
* for LSNs <= tip is therefore on stable storage when fdatasync returns
* SUCCESSFULLY. */
int rc = fdatasync(p->wal_fd);
/* FAIL-LOUD GATE (Req 1.1/1.2): advance durable_lsn ONLY on a successful
* sync. On EIO/ENOSPC the bytes are NOT on stable storage, so leaving the
* watermark unchanged keeps the non-durable LSNs from being counted toward
* the commit majority. We do NOT retry into an advanced watermark — the
* level predicate stays pending and the next pass retries the sync. */
if (rc != 0 || flusher_fsync_forced_fail(p)) {
int e = (rc != 0) ? errno : EIO;
ERROR("persist: flusher fdatasync FAILED (errno=%d %s) — durable_lsn "
"held at %" PRIu64 " (tip=%" PRIu64 " NOT made durable); commit "
"majority will not count these LSNs",
e, strerror(e), dur, tip);
pthread_mutex_unlock(&p->lock);
return dur;
}
/* Monotonic advance to the synced tip — never past it (the invariant
* durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1
* was read under this same lock). One fdatasync advances durable_lsn past
* however many LSNs were appended since the last sync (batching). */
if (tip > dur)
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
#ifdef XCE_TESTING
/* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6):
* a liveness test asserts this is the dedicated flusher thread, never the
* caller/iomgr/heartbeat thread. Written under p->lock; read after the
* flusher has caught up, so no extra synchronization is needed. */
p->test_last_fdatasync_thread = pthread_self();
atomic_store_explicit(&p->test_last_fdatasync_thread_set, 1,
memory_order_release);
#endif
pthread_mutex_unlock(&p->lock);
return tip;
}
static void* fsync_worker(void* arg)
{
persist_t* p = arg;
pthread_mutex_lock(&p->fsync_mutex);
while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) {
/* Re-evaluate the level predicate BEFORE waiting so a signal that
* raced in just before we entered the wait is not needed for
* correctness — if work is already pending we skip the wait entirely
* and flush. (Reads current_lsn/durable_lsn under p->lock.) */
pthread_mutex_lock(&p->lock);
bool pending = flusher_work_pending_locked(p);
pthread_mutex_unlock(&p->lock);
if (!pending) {
/* No pending work: wait for an append-signal OR the cadence
* deadline (bounded fallback flush so a lone no-further-append
* entry is still synced within wal_sync_ms — Req 3.1). */
long ms = (long)p->config.wal_sync_ms;
struct timespec dl;
clock_gettime(CLOCK_REALTIME, &dl);
dl.tv_sec += ms / 1000;
dl.tv_nsec += (ms % 1000) * 1000000L;
if (dl.tv_nsec >= 1000000000L) {
dl.tv_sec += 1;
dl.tv_nsec -= 1000000000L;
}
pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl);
if (atomic_load_explicit(&p->fsync_stop, memory_order_acquire))
break;
}
/* Flush section: never hold fsync_mutex across the fdatasync (the
* flush takes p->lock alone). Release fsync_mutex, flush, re-acquire.
* After the flush we loop and re-check the level — if a new append
* landed during the flush, the level is still pending and we flush
* again without needing its signal (lost-wakeup-safe). */
pthread_mutex_unlock(&p->fsync_mutex);
uint64_t before = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
uint64_t after = flusher_flush_once(p);
/* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced
* INSIDE flusher_flush_once (atomic store with release) BEFORE we
* return here, and flusher_flush_once has already RELEASED p->lock.
* Now — holding NO persist lock — invoke the registered durable-advance
* callback so the commit engine re-evaluates its majority and wakes any
* COMMIT_MODE_WAIT caller whose entry just became durable locally.
*
* LOST-WAKEUP SAFETY: the wake fires strictly AFTER the durable_lsn
* release-store, and the commit engine's predicate re-reads durable_lsn
* via an acquire load under ce->lock — so a WAIT caller either (a) sees
* the advanced watermark on its own re-check, or (b) is blocked in
* pthread_cond_wait and is woken by this broadcast; it can never both
* miss the watermark AND miss the wake. We only fire the callback when
* durable_lsn actually advanced (after > before): a no-op flush (or a
* failed fdatasync that held the watermark) needs no commit re-eval,
* avoiding spurious wakes.
*
* LOCK ORDERING (cycle-free): the flusher releases p->lock inside
* flusher_flush_once, THEN the callback takes ce->lock (commit side).
* The commit side only ever takes ce->lock and reads persist atomics —
* it never takes p->lock while holding ce->lock. So the only ordering
* is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no
* deadlock cycle. The callback explicitly MUST NOT take p->lock. */
if (after > before && p->on_durable_advance != NULL) {
p->on_durable_advance(p->durable_cb_ctx);
}
pthread_mutex_lock(&p->fsync_mutex);
}
pthread_mutex_unlock(&p->fsync_mutex);
return NULL;
}
/* ------------------------------------------------------------------ *
* WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) *
* ------------------------------------------------------------------ */
/* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN
* term index (R1.5). Grows the index on demand; LSNs replay/append contiguously
* so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous
* LSN (should not happen on the contiguous WAL) is zero-filled by realloc-then-
* memset, which read back as "absent" until/unless written. The raw term
* (including a legacy 0) is stored verbatim; the legacy-0 -> WAL_LEGACY_TERM
* mapping is applied only at READ-BACK (persist_entry_term), keeping "present
* legacy-0" distinguishable from "absent".
*
* Callers: the live append path (holds p->lock) and the single-threaded
* recovery path (replay_wal — no other thread touches the index before the node
* participates, so no lock is required there). Best-effort: an allocation
* failure is logged loudly and leaves the index unextended (read-back then
* reports the LSN absent) — it never aborts and never corrupts existing
* entries. */
static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term)
{
if (!p || lsn == 0)
return; /* LSN 0 is reserved — never an entry */
if (lsn > p->term_index_cap) {
/* Grow geometrically to amortize; never below the requested LSN. */
uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64;
if (new_cap < lsn)
new_cap = lsn;
uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t));
if (!grown) {
ERROR("persist: term-index realloc failed (lsn=%" PRIu64
", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read "
"back as absent", lsn, p->term_index_cap, new_cap);
return;
}
/* Zero-fill the newly grown region so any unwritten slot reads absent. */
memset(grown + p->term_index_cap, 0,
(new_cap - p->term_index_cap) * sizeof(uint64_t));
p->term_index = grown;
p->term_index_cap = new_cap;
}
p->term_index[lsn - 1] = term;
if (lsn > p->term_index_count)
p->term_index_count = lsn;
}
/* entry_term_locked: the lock-free core of persist_entry_term — read the
* recorded Raft term for `lsn` from the in-memory term index, applying the
* legacy-0 rule (R1.8: a present term-0 entry maps to WAL_LEGACY_TERM; an absent
* LSN returns 0). Caller MUST hold p->lock. Used by the append path
* (persist_wal_async) to source the AppendEntries prev_log_term while already
* under the lock — calling persist_entry_term there would re-acquire p->lock
* and deadlock. */
static uint64_t entry_term_locked(const persist_t* p, uint64_t lsn)
{
if (!p || lsn == 0)
return 0;
if (lsn <= p->term_index_count && p->term_index != NULL) {
uint64_t raw = p->term_index[lsn - 1];
return (raw == 0) ? WAL_LEGACY_TERM : raw;
}
return 0;
}
/* Shrink the in-memory per-LSN term index so that every LSN > `keep_lsn` reads
* back as absent (ha-term-safety task 2.2). Called from the truncate path after
* the on-disk entries above keep_lsn are removed, keeping the term index
* consistent with the WAL: a later persist_entry_term on a truncated LSN must
* return 0. The backing allocation is left as-is (only the live count moves);
* the zeroed-on-grow contract means a later re-append at those LSNs overwrites
* cleanly. Caller must hold p->lock. */
static void truncate_term_index_locked(persist_t* p, uint64_t keep_lsn)
{
if (!p)
return;
if (keep_lsn < p->term_index_count)
p->term_index_count = keep_lsn;
}
void persist_set_term(persist_t* p, uint64_t term)
{
if (!p)
return;
/* p->term is read on the append path under p->lock; take the lock so the
* stamped value is consistent with the append that reads it. */
pthread_mutex_lock(&p->lock);
p->term = term;
pthread_mutex_unlock(&p->lock);
}
uint64_t persist_entry_term(const persist_t* p, uint64_t lsn)
{
if (!p || lsn == 0)
return 0;
/* Take p->lock to read consistently against concurrent record_entry_term
* on the append path. Cast away const — the lock is logically a mutable
* sub-object of an otherwise-const read. */
persist_t* mp = (persist_t*)p;
pthread_mutex_lock(&mp->lock);
/* Legacy-zero rule (R1.8) lives in entry_term_locked: a present entry
* carrying term 0 maps to the single lowest-known real Raft term, applied
* uniformly so two legacy-0 entries at the same LSN match; absent LSNs
* return 0. */
uint64_t result = entry_term_locked(mp, lsn);
pthread_mutex_unlock(&mp->lock);
return result;
}
/* persist_reconcile_checkpoint_term — raft-ha-core-safety task 3.2 (Req
* 3.1/5.1), Snapshot Boundary Reconcile. See persist.h for the full contract.
* Reuses record_entry_term (same translation unit) to authoritatively
* overwrite whatever term (if any) the local term index held at
* checkpoint_lsn — this is a genuine reconcile-overwrite, not an idempotent
* skip, because a pre-snapshot divergent entry may have left a stale local
* term at this exact LSN. Only the checkpoint LSN's own term is reconciled:
* post-install Raft semantics mean no AppendEntries prevLogIndex/prevLogTerm
* check against this member ever references an LSN below the checkpoint
* again (the checkpoint is the new log-start boundary), and the WAL bytes
* below it were compacted away, so there is no data to reconstruct a full
* per-LSN history from even if it were needed. */
void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn,
uint64_t checkpoint_term)
{
if (!p || checkpoint_lsn == 0)
return;
pthread_mutex_lock(&p->lock);
record_entry_term(p, checkpoint_lsn, checkpoint_term);
pthread_mutex_unlock(&p->lock);
}
/* ------------------------------------------------------------------ */
/* Public API */
/* ------------------------------------------------------------------ */
void persist_set_workdir(persist_t* p, const char* workdir, const char* name)
{
char effective[4096];
snprintf(effective, sizeof(effective), "%s/%s", workdir, name);
snprintf(p->wal_path, sizeof(p->wal_path), "%s/data/wal", effective);
snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective);
}
int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
{
p->cluster = c;
p->config = cfg ? *cfg : persist_config_defaults();
p->wal_fd = -1;
p->last_replicated_crc = WAL_CRC_NONE;
p->last_crc = WAL_CRC_NONE;
/* LSN 0 is RESERVED as the "none / empty" sentinel — it is NEVER assigned
* to a real WAL entry. A fresh store therefore starts current_lsn at 1, so
* the first-ever appended entry (e.g. the promotion WAL_NOOP) is lsn 1 and
* lsn 0 unambiguously means "nothing here":
* - committed_lsn == 0 → nothing committed (the lsn-1 NOOP is committable
* 0 → 1, vs. the old lsn-0 entry which could never advance committed_lsn);
* - writable_lsn == 0 → own-term gate off (the gate armed at lsn 1 is
* effective, vs. lsn 0 which read as "gate off" on a cold-start leader);
* - standby_lsn == 0 → empty standby (the streamer's `hdr.lsn > 0`
* filter streams the lsn-1 entry, closing the cold-start WAL hole that
* dropped lsn 0 on `0 > 0`).
* persist_recover() overwrites current_lsn from the durable snapshot/WAL tip
* (load_snapshot / replay_wal), so this fresh default only applies before any
* recovery — i.e. a brand-new store with no snapshot and no WAL. */
p->current_lsn = 1;
/* Enforce minimum value for max_wal_records to prevent runaway checkpointing */
if (p->config.max_wal_records < 100) {
ERROR("persist: max_wal_records=%u too low (minimum 100) — setting to 100",
p->config.max_wal_records);
p->config.max_wal_records = 100;
}
/* Enforce minimum value for wal_segment_bytes to prevent a rotation storm
* (must comfortably fit at least one max-payload entry: header + up to
* MAX_WAL_PAYLOAD_BYTES). */
if (p->config.wal_segment_bytes < 4096) {
ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096",
p->config.wal_segment_bytes);
p->config.wal_segment_bytes = 4096;
}
/* Initialise atomic checkpoint-trigger counters */
atomic_store(&p->wal_record_count, 0);
atomic_store(&p->checkpoint_in_progress, 0);
/* Durable watermark starts at 0 ("nothing fdatasync'd"): mirrors the
* LSN-0-reserved committed_lsn convention. It advances only on the
* flusher thread after a successful fdatasync (ha-commit-durability 2.1). */
atomic_store(&p->durable_lsn, 0);
/* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1):
* a fresh store has no corruption. persist_recover overwrites this with the
* replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */
p->recovery_verdict = RECOVER_CLEAN;
/* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh
* store is promotable. Set by persist_recover_loud_fence_if_unrecoverable
* on an unrecoverable boot; cleared on catch-up (never self-exits). */
atomic_store(&p->recovery_unrecoverable, false);
/* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2):
* the commit engine registers it in commit_engine_init. Until then every
* flusher durable advance is a no-op wake. */
p->on_durable_advance = NULL;
p->durable_cb_ctx = NULL;
/* Truncation wake hook starts unregistered (raft-apply-decoupling task
* 1.4): the commit engine registers it in commit_engine_init. Until
* then every truncation is a no-op wake. */
p->on_truncate = NULL;
p->truncate_cb_ctx = NULL;
/* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries
* recorded yet. Grown on demand by record_entry_term during replay/append;
* freed in persist_fini. */
p->term_index = NULL;
p->term_index_count = 0;
p->term_index_cap = 0;
/* Create WAL and snapshot directories */
if (mkdir_p(p->wal_path, 0700) != 0) return -errno;
if (mkdir_p(p->snap_path, 0700) != 0) return -errno;
/* Reject NFS mounts — fsync durability is not guaranteed over NFS */
struct statfs sfs;
if (statfs(p->wal_path, &sfs) == 0 &&
(unsigned long)sfs.f_type == (unsigned long)NFS_SUPER_MAGIC) {
ERROR("persist: WAL path %s is on NFS — unsafe for fsync durability",
p->wal_path);
return -EREMOTE;
}
pthread_mutex_init(&p->lock, NULL);
pthread_mutex_init(&p->checkpoint_mutex, NULL);
pthread_cond_init(&p->checkpoint_done, NULL);
pthread_mutex_init(&p->fsync_mutex, NULL);
pthread_cond_init(&p->fsync_cond, NULL);
repl_init(&p->repl); /* P2-4: fan-out replicator lifecycle */
WARN("WAL/snapshot stored unencrypted in %s — "
"ensure filesystem is encrypted if job payloads contain sensitive data",
p->wal_path);
return 0;
}
int persist_start(persist_t* p)
{
ctrl_metrics_init();
/* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */
if (p->cluster != NULL) {
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun, p);
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun, p);
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun, p);
}
atomic_store_explicit(&p->fsync_stop, 0, memory_order_release);
int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p);
if (rc == 0)
atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release);
return rc;
}
void persist_stop(persist_t* p)
{
if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) {
/* Set the stop flag and wake the timer wait under fsync_mutex so the
* worker cannot miss the signal between its flag check and the wait. */
pthread_mutex_lock(&p->fsync_mutex);
atomic_store_explicit(&p->fsync_stop, 1, memory_order_release);
pthread_cond_signal(&p->fsync_cond);
pthread_mutex_unlock(&p->fsync_mutex);
pthread_join(p->fsync_thread, NULL);
atomic_store_explicit(&p->fsync_thread_running, 0, memory_order_release);
}
pthread_mutex_lock(&p->lock);
if (p->wal_fd >= 0) {
/* Final synchronous flush-or-exit on stop: fdatasync everything
* appended so far and advance durable_lsn to the synced tip so a
* post-stop read reflects the durable state (the flusher thread is
* already joined and will not advance it again).
*
* FAIL-LOUD GATE (Req 1.1/1.2): advance the watermark ONLY when the
* final sync SUCCEEDS — on EIO/ENOSPC the bytes are not on stable
* storage, so durable_lsn must stay where it is and the failure is
* surfaced loudly. */
int rc = fdatasync(p->wal_fd);
if (rc == 0 && !flusher_fsync_forced_fail(p)) {
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
if (tip > dur)
atomic_store_explicit(&p->durable_lsn, tip,
memory_order_release);
} else {
int e = (rc != 0) ? errno : EIO;
ERROR("persist: final fdatasync on stop FAILED (errno=%d %s) — "
"durable_lsn left unchanged; appended-but-unsynced entries "
"are NOT durable",
e, strerror(e));
}
close(p->wal_fd);
p->wal_fd = -1;
}
pthread_mutex_unlock(&p->lock);
}
void persist_fini(persist_t* p)
{
repl_fini(&p->repl); /* P2-4: fan-out replicator lifecycle */
pthread_mutex_destroy(&p->lock);
pthread_mutex_destroy(&p->checkpoint_mutex);
pthread_cond_destroy(&p->checkpoint_done);
pthread_mutex_destroy(&p->fsync_mutex);
pthread_cond_destroy(&p->fsync_cond);
/* Free the in-memory per-LSN term index (ha-term-safety 1.3). */
free(p->term_index);
p->term_index = NULL;
p->term_index_count = 0;
p->term_index_cap = 0;
}
/* Forward declaration — defined after checkpoint internals */
static void schedule_checkpoint(persist_t* persist);
int persist_wal_async(persist_t* p, uint32_t type,
const void* payload, uint32_t len)
{
/* WAL_CHECKPOINT is a local segment marker, never written via this path */
if (type == WAL_CHECKPOINT)
return -EINVAL;
pthread_mutex_lock(&p->lock);
/* Open first segment lazily */
if (p->wal_fd < 0) {
if (open_new_segment(p) != 0) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
/* Build entry header */
wal_entry_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.term = p->term;
hdr.lsn = p->current_lsn++;
hdr.timestamp = now_us();
hdr.type = type;
hdr.payload_len = len;
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
/* CRC32 over WAL_HDR_CRC_SIZE bytes of header (excludes crc + prev_crc
* fields themselves) then over the payload */
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
if (payload && len > 0)
c = crc32(c, (const Bytef*)payload, len);
hdr.crc = c;
/* Write header */
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
if (hw != (ssize_t)sizeof(hdr)) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
/* Write payload (if any) */
if (payload && len > 0) {
ssize_t pw = pwrite(p->wal_fd, payload, len,
(off_t)(p->seg_offset + sizeof(hdr)));
if (pw != (ssize_t)len) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
p->seg_offset += entry_size;
p->total_wal_bytes += entry_size;
p->last_crc = hdr.crc;
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
* 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */
record_entry_term(p, hdr.lsn, hdr.term);
/* Replicate to standbys — never replicate WAL_CHECKPOINT entries.
* last_replicated_crc is distinct from last_crc:
* last_crc includes checkpoint markers; last_replicated_crc skips them.
* This maintains a continuous CRC chain for standbys (design doc Part 2.3).
* Note: WAL_CHECKPOINT is blocked by the assert at function entry, so
* every entry that reaches here is a replicable entry. */
p->last_replicated_crc = hdr.crc;
/* ha-term-safety task 3.4: source the AppendEntries prev_log_term (term of
* the entry at hdr.lsn-1) under p->lock via the lock-free accessor, so the
* live broadcast carries the leader's term + prev tuple and the standby
* routes it through the library (raft_on_append_entries_request). 0 when
* hdr.lsn == 1 (no predecessor). */
uint64_t prev_log_term = entry_term_locked(p, hdr.lsn - 1);
repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term);
/* Rotate segment when full */
if (p->seg_offset >= p->config.wal_segment_bytes) {
if (rotate_segment(p) != 0) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
pthread_mutex_unlock(&p->lock);
/* Signal the group-commit flusher (ha-commit-durability 2.1): current_lsn
* advanced and the entry's bytes are pwritten, so durable_lsn < tip now
* holds (the level predicate). The signal is sent AFTER releasing p->lock
* and is level-triggered — if it races the flusher and is "lost", the
* flusher re-checks durable_lsn < current_lsn-1 under p->lock and flushes
* anyway (see fsync_worker). We do NOT fdatasync inline on this path; the
* flusher batches the sync across all pending appends. */
pthread_mutex_lock(&p->fsync_mutex);
pthread_cond_signal(&p->fsync_cond);
pthread_mutex_unlock(&p->fsync_mutex);
/* WAL record-count checkpoint trigger: schedule an async checkpoint once
* the record count reaches the configured threshold. The fetch_add is
* outside the lock — atomic_fetch_add is sufficient for the counter; the
* checkpoint itself is guarded by its own CAS inside schedule_checkpoint. */
if (atomic_fetch_add(&p->wal_record_count, 1) + 1 >=
p->config.max_wal_records) {
schedule_checkpoint(p);
}
return 0;
}
/* Lock-held core of persist_truncate_wal_above (defined below): assumes the
* caller already holds p->lock and never touches the lock itself. The
* replicated-write conflict guard reuses it to repair a divergent entry while
* still under p->lock (the follower-AE-conflict truncate contract). */
static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn);
/* read_entry_meta_locked: scan the on-disk WAL for the entry at `lsn` and
* return its (payload_len, payload-only CRC, raw stored term). The
* replicated-write conflict guard uses this to compare a re-streamed entry
* against the copy already on disk WITHOUT trusting the LSN alone. pcrc is
* computed as crc32(0, payload, len) — the SAME payload-only fingerprint
* walcompare/waldump compare across members (NOT the chained header CRC); an
* empty (PLEN=0) entry yields len=0, pcrc=0. Reads through the page cache so
* a written-but-not-yet-fdatasync'd entry is still visible. Caller MUST hold
* p->lock. Returns 0 if the entry is found, -1 if absent or unreadable. */
static int read_entry_meta_locked(persist_t* p, uint64_t lsn,
uint32_t* out_len, uint32_t* out_pcrc,
uint64_t* out_term)
{
if (!p || lsn == 0)
return -1;
DIR* d = opendir(p->wal_path);
if (!d)
return -1;
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
if (strncmp(e->d_name, "seg-", 4) != 0)
continue;
uint64_t sl = 0;
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
continue;
seg_lsns[nseg++] = sl;
}
closedir(d);
if (nseg == 0)
return -1;
/* Ascending sort by start LSN (segment count is tiny). */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) {
seg_lsns[j + 1] = seg_lsns[j];
j--;
}
seg_lsns[j + 1] = key;
}
int rc = -1;
for (int si = 0; si < nseg && rc != 0; si++) {
/* Segments are LSN-ordered; one whose start LSN already exceeds the
* target cannot contain it (nor can any later one). */
if (seg_lsns[si] > lsn)
break;
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0)
continue;
uint32_t off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
if (hn < (ssize_t)sizeof(hdr))
break; /* short read / EOF / pre-allocated zeros */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break;
if (hdr.lsn == lsn) {
uint32_t pc = 0;
if (hdr.payload_len > 0) {
uint8_t* pl = malloc(hdr.payload_len);
if (!pl)
break; /* leave rc == -1: unreadable → conservative */
ssize_t pn = pread(fd, pl, hdr.payload_len,
(off_t)(off + sizeof(hdr)));
if (pn == (ssize_t)hdr.payload_len)
pc = crc32(0L, (const Bytef*)pl, hdr.payload_len);
free(pl);
if (pn != (ssize_t)hdr.payload_len)
break; /* unreadable body → conservative */
}
if (out_len)
*out_len = hdr.payload_len;
if (out_pcrc)
*out_pcrc = pc;
if (out_term)
*out_term = hdr.term;
rc = 0;
break;
}
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
return rc;
}
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
const void* payload, uint32_t len)
{
if (!p)
return -EINVAL;
/* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */
if (type == WAL_CHECKPOINT)
return -EINVAL;
/* Reject oversized payloads BEFORE touching the segment. */
if (len > MAX_WAL_PAYLOAD_BYTES)
return -EMSGSIZE;
pthread_mutex_lock(&p->lock);
/* Tip-repair bookkeeping (set only on the TIP-conflict repair branch below):
* the truncate-and-reappend repair transiently lowers committed_lsn to
* lsn-1, so we snapshot the true committed boundary here and restore it
* EXACTLY after the re-append re-establishes the entry at `lsn` (never let a
* same-(term,lsn) payload repair regress the committed boundary — commit
* monotonicity / State Machine Safety). */
int repaired_tip = 0;
uint64_t repaired_committed_save = 0;
/* Payload-aware conflict guard (raft-ha-core-safety task 2.2): this lsn is
* already persisted (current_lsn advanced past it). The legacy guard was
* (lsn)-only — it no-op'd ANY re-stream, including a corrective full-payload
* write over a previously-stored empty (PLEN=0) body, which is exactly the
* cross-term catch-up payload-loss signature (a committed LSN that carries
* the right (term,lsn) but an empty payload). Compare (payload-crc, len)
* against the copy already on disk and act on the difference:
* (c) identical payload → idempotent no-op (keep).
* (b) incoming EMPTY over a stored entry → never degrade (keep stored);
* an empty body is never the durable form of a known entry.
* (a) incoming NON-EMPTY whose payload differs from the stored copy at an
* existing (term,lsn) → a genuine payload conflict whose handling
* depends on WHERE the conflict sits relative to the stored tip:
* · AT THE TIP (lsn == current_lsn - 1, nothing stored above) →
* REPAIR: truncate just this one entry and re-append the
* authoritative payload at the same (term,lsn). Because nothing
* is stored above it, no distinct committed entry is destroyed,
* and the committed boundary is restored verbatim afterwards.
* · BELOW THE TIP (lsn < current_lsn - 1, committed entries stored
* ABOVE the conflict) → MUST NOT truncate: dropping [lsn..tip]
* would delete committed entries above the conflict AND roll
* committed_lsn/durable_lsn backward — a State Machine Safety /
* commit-monotonicity violation. Instead keep the stored entry
* untouched, raise a loud non-promotable fence, and let catch-up
* reconverge. The real source fix (the streamer must stop
* shipping an empty body) is task 2.4; the guard's only job here
* is to NOT make things worse (never drop a committed entry,
* never regress a boundary) and to surface the corruption loudly.
* An unreadable stored entry stays a conservative no-op — never truncate a
* committed LSN we cannot read back. */
if (lsn + 1 <= p->current_lsn) {
uint32_t inc_pcrc =
(payload && len > 0) ? crc32(0L, (const Bytef*)payload, len) : 0;
uint32_t st_len = 0;
uint32_t st_pcrc = 0;
uint64_t st_term = 0;
int have = read_entry_meta_locked(p, lsn, &st_len, &st_pcrc,
&st_term);
int payload_matches =
(have == 0 && st_len == len && st_pcrc == inc_pcrc);
/* (c) identical payload, (b) empty incoming, or an unreadable stored
* entry → leave the stored entry untouched. */
if (payload_matches || len == 0 || have != 0) {
pthread_mutex_unlock(&p->lock);
return 0;
}
/* (a) non-empty incoming whose payload differs at an existing (term,lsn)
* — a genuine payload conflict. Branch on tip vs below-tip. */
uint64_t tip_lsn = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
if (lsn != tip_lsn) {
/* BELOW-TIP conflict: committed entries are stored ABOVE `lsn`.
* Truncating here would delete them and roll committed_lsn/
* durable_lsn backward (State Machine Safety / commit-monotonicity
* violation — bug-class the round-1 review rejected). Do NOT
* truncate, do NOT lower any boundary, do NOT degrade the stored
* entry. Surface loudly and raise the sticky non-promotable fence
* (reuse the recovery_unrecoverable mechanism: loud, stays ALIVE,
* non-promotable, self-heals via persist_clear_recovery_unrecoverable
* once catch-up reaches a committed-consistent state — never
* self-exits, per the controller invariant). The source fix lives
* in task 2.4 (the streamer must stop shipping an empty payload). */
ERROR("persist: persist_wal_write_replicated payload CONFLICT BELOW "
"TIP at lsn=%" PRIu64 " (stored_term=%" PRIu64
" stored_len=%u stored_pcrc=%08x vs incoming_len=%u "
"incoming_pcrc=%08x); current_lsn=%" PRIu64 " committed_lsn=%"
PRIu64 " — refusing to truncate committed entries above the "
"conflict (would regress the committed boundary). Keeping the "
"stored entry, raising the non-promotable fence and staying "
"ALIVE; awaiting a correct re-stream (source fix: task 2.4)",
lsn, st_term, st_len, st_pcrc, len, inc_pcrc, p->current_lsn,
atomic_load_explicit(&p->committed_lsn, memory_order_acquire));
atomic_store_explicit(&p->recovery_unrecoverable, true,
memory_order_release);
pthread_mutex_unlock(&p->lock);
return 0; /* no-op the write: keep stored, never degrade/regress */
}
/* TIP conflict: `lsn` is the highest stored entry, so nothing is stored
* above it — truncate-and-reappend replaces EXACTLY this one divergent
* entry. truncate_wal_above_locked(lsn-1) drops it and sets
* current_lsn = lsn, so control falls through to the append below and
* re-persists the authoritative entry at this exact LSN. Snapshot the
* committed boundary first: the truncate transiently rewrites
* committed_lsn to lsn-1, and we restore it verbatim after the
* re-append so a same-(term,lsn) payload repair never regresses (nor
* spuriously advances) the committed boundary. */
repaired_committed_save =
atomic_load_explicit(&p->committed_lsn, memory_order_acquire);
int trc = truncate_wal_above_locked(p, lsn - 1);
if (trc != 0) {
ERROR("persist: persist_wal_write_replicated repair-truncate FAILED "
"for lsn=%" PRIu64 " (rc=%d) — divergent entry NOT repaired",
lsn, trc);
pthread_mutex_unlock(&p->lock);
return trc;
}
repaired_tip = 1;
/* fall through to the contiguity guard + append (lsn == current_lsn). */
}
/* Contiguity guard (bug-850): REFUSE a forward GAP. current_lsn is the
* next-expected LSN, so a replicated entry with lsn > current_lsn would
* leave [current_lsn .. lsn-1] missing AND advance current_lsn past that
* hole — after which the idempotency guard above silently DROPS the later
* in-order catch-up of the prefix (the failover prefix-loss mechanism).
* Append only the contiguous next entry (lsn == current_lsn); reject a gap
* without writing or acking so the ACK-driven catch-up continuation
* re-streams from current_lsn and the log fills in order. This is Raft
* log-matching: append entry N only when the log already holds N-1. A
* behind standby that keeps receiving live broadcasts simply drops the
* out-of-order ones until catch-up backfills the gap (no loss). -EAGAIN is
* the "retry via catch-up" signal the apply path treats as a quiet skip. */
if (lsn > p->current_lsn) {
pthread_mutex_unlock(&p->lock);
return -EAGAIN;
}
/* Open first segment lazily. open_new_segment names the file from
* current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is
* 1 (reserve-lsn-0), so the segment is seg-...001.wal. */
if (p->wal_fd < 0) {
if (open_new_segment(p) != 0) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
/* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not
* current_lsn++). term isn't streamed, so use our best-effort p->term.
* The CRC chain is SELF-computed (the streamed entry carries no crc):
* recovery validates an intra-segment chain (prev_crc == last_crc), so the
* standby's own writes must chain to each other — cross-node CRC equality
* is neither required nor possible. */
wal_entry_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.term = p->term;
hdr.lsn = lsn;
hdr.timestamp = now_us();
hdr.type = type;
hdr.payload_len = len;
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
if (payload && len > 0)
c = crc32(c, (const Bytef*)payload, len);
hdr.crc = c;
/* Write header */
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
if (hw != (ssize_t)sizeof(hdr)) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
/* Write payload (if any) */
if (payload && len > 0) {
ssize_t pw = pwrite(p->wal_fd, payload, len,
(off_t)(p->seg_offset + sizeof(hdr)));
if (pw != (ssize_t)len) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
p->seg_offset += entry_size;
p->total_wal_bytes += entry_size;
p->last_crc = hdr.crc;
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
* 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed
* envelope does not yet carry the leader's term, so this records the local
* best-effort p->term (the design's known hazard closed by task 2.x, which
* carries the leader's term on the envelope; this task only provides the
* read-back seam over whatever term was persisted). */
record_entry_term(p, hdr.lsn, hdr.term);
/* Advance current_lsn to one past the given lsn (max-idempotent — mirrors
* apply_wal_entry's monotonic advance). */
if (lsn + 1 > p->current_lsn)
p->current_lsn = lsn + 1;
/* Durability: fsync the segment BEFORE returning so the caller's ACK
* implies the entry is on stable storage (persist-before-ACK).
* persist_wal_async defers fsync to the background thread / persist_flush;
* here we sync inline because the entry feeds the primary's commit
* majority and must survive a standby restart. fdatasync is sufficient
* (entry bytes + segment metadata are pre-allocated by open_new_segment).
*
* Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the
* append-path gate at task 2.1): the ACK is sent by the caller ONLY when
* this returns success, and the ACK claims the entry is durable. So if
* the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must
* NOT return success — we log loudly and return -EIO so the caller's
* existing error path (ha_apply_single_entry) does NOT apply or ACK. */
int fsrc = fdatasync(p->wal_fd);
#ifdef XCE_TESTING
atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1,
memory_order_relaxed);
#endif
if (fsrc != 0 || flusher_fsync_forced_fail(p)) {
int e = (fsrc != 0) ? errno : EIO;
ERROR("persist: persist_wal_write_replicated fdatasync FAILED for "
"lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning "
"error so the standby does NOT ACK", lsn, e, strerror(e));
pthread_mutex_unlock(&p->lock);
return -EIO;
}
/* bug-828 secondary: the entry is now fdatasync'd durable, so advance
* durable_lsn to the synced tip immediately (mirror persist_flush). The
* 10ms cadence flusher would otherwise advance it lazily, leaving this
* standby's ADVERTISED durable position (persist_durable_floor →
* RequestVoteReply.peerLastLogLsn) lagging its own synced tip during the
* vote window. A standby that has durably persisted the committed prefix
* must advertise it at once so a clean failover's safe-commit-from-votes
* computation sees the true majority-held LSN. Monotonic max; preserves
* durable_lsn <= current_lsn - 1 (tip read under this same p->lock, after
* the current_lsn advance above). Reached ONLY past the fail-loud rc-gate
* — never advances on an fdatasync error. */
{
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
if (tip > dur)
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
}
/* Tip-repair committed-boundary restore (raft-ha-core-safety task 2.2): a
* TIP-conflict repair above truncated [lsn..tip] (rewriting committed_lsn to
* lsn-1) and has now re-appended + fdatasync'd the authoritative entry at the
* SAME (term,lsn). Restore the committed boundary VERBATIM so the repair is
* commit-boundary-neutral: a same-(term,lsn) payload correction must never
* regress committed_lsn (State Machine Safety / commit monotonicity) nor
* spuriously advance it (truncate would otherwise have raised it to lsn-1).
* Reached only past the fail-loud fdatasync gate, with current_lsn re-advanced
* to lsn+1 and durable_lsn at the synced tip, so the invariant
* committed_lsn <= min(current_lsn - 1, durable_lsn) holds (saved value was
* itself <= the pre-repair tip == lsn). The whole repair is serialized under
* p->lock, so no external observer ever sees the transient lowered value. */
if (repaired_tip) {
atomic_store_explicit(&p->committed_lsn, repaired_committed_save,
memory_order_release);
}
/* Rotate segment when full (same threshold as the primary append). */
if (p->seg_offset >= p->config.wal_segment_bytes) {
if (rotate_segment(p) != 0) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
/* NOTE: no repl_broadcast — standbys never re-replicate. */
pthread_mutex_unlock(&p->lock);
return 0;
}
int persist_flush(persist_t* p)
{
pthread_mutex_lock(&p->lock);
int rc = (p->wal_fd >= 0) ? fdatasync(p->wal_fd) : 0;
/* persist_flush is a synchronous durability point (tests, clean shutdown):
* it fdatasync'd everything appended so far, so advance durable_lsn to the
* synced tip too. Without this, an explicit-flush-only path (slow flusher
* cadence) would leave durable_lsn lagging even though the bytes are on
* disk. Monotonic max; preserves durable_lsn <= current_lsn - 1 (tip read
* under the same p->lock). Gated on rc == 0 (fail-loud, Req 1.1/1.2): a
* failed flush must not advance the watermark. */
if (rc == 0 && !flusher_fsync_forced_fail(p)) {
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
if (tip > dur)
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
} else if (rc != 0 || flusher_fsync_forced_fail(p)) {
int e = (rc != 0) ? errno : EIO;
ERROR("persist: persist_flush fdatasync FAILED (errno=%d %s) — "
"durable_lsn not advanced; flushed bytes are NOT durable",
e, strerror(e));
}
pthread_mutex_unlock(&p->lock);
return rc;
}
uint64_t persist_durable_lsn(persist_t* p)
{
return atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
}
uint64_t persist_durable_floor(persist_t* p)
{
/* The durable freshness floor (durable last-log index) IS the durable
* watermark: persist_recover re-derives it from the recovered WAL tip
* (ha-commit-durability task 3.4), and the flusher advances it on every
* successful fdatasync — so a thin wrapper avoids a redundant field. NULL
* guard keeps the accessor safe for the election consumers (task 4.2). */
return p ? atomic_load_explicit(&p->durable_lsn, memory_order_acquire) : 0;
}
void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx)
{
if (!p)
return;
/* Set ctx before fn so the flusher (which checks fn != NULL) never sees a
* live callback with a stale/uninitialised ctx. Registration happens in
* commit_engine_init, before the engine is reachable by any ACK path. */
p->durable_cb_ctx = ctx;
p->on_durable_advance = fn;
}
/* raft-apply-decoupling task 1.4: registration counterpart of
* persist_set_durable_cb -- see persist.h for the full contract. */
void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn),
void* ctx)
{
if (!p)
return;
p->truncate_cb_ctx = ctx;
p->on_truncate = fn;
}
#ifdef XCE_TESTING
void persist_test_set_force_fsync_fail(persist_t* p, int fail)
{
if (p)
atomic_store_explicit(&p->test_force_fsync_fail, fail ? 1 : 0,
memory_order_release);
}
void persist_test_set_force_job_encode_fail_at(persist_t* p, int index)
{
if (p)
atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1,
memory_order_relaxed);
}
unsigned persist_test_replicated_fdatasync_count(persist_t* p)
{
if (!p)
return 0;
return atomic_load_explicit(&p->test_replicated_fdatasync_count,
memory_order_relaxed);
}
pthread_t persist_test_last_fdatasync_thread(persist_t* p)
{
/* If no flusher fdatasync has advanced durable_lsn yet, return the
* caller's own id so an unprimed read never spuriously looks off-thread.
* After a flush, return the recorded flusher thread id (Req 1.6). */
if (!p ||
atomic_load_explicit(&p->test_last_fdatasync_thread_set,
memory_order_acquire) == 0)
return pthread_self();
return p->test_last_fdatasync_thread;
}
/* Force a fresh WAL segment at the current LSN (test seam — see persist.h).
* Takes p->lock for the same single-writer serialization the append path uses,
* so the segment swap cannot race a concurrent write. */
int persist_test_force_new_segment(persist_t* p)
{
if (!p)
return -EINVAL;
pthread_mutex_lock(&p->lock);
int rc = rotate_segment(p);
pthread_mutex_unlock(&p->lock);
return rc;
}
/* ------------------------------------------------------------------ *
* Durability-verification test seam (ha-commit-durability Req 3.2/3.3)*
* *
* Re-read + CRC-validate the WAL segments from stable storage and *
* return the highest fully-persisted LSN — independent of the *
* in-memory current_lsn / committed_lsn watermarks. Mirrors the *
* segment-scan + CRC-chain validation of replay_wal(), but is strictly *
* read-only: it applies no entries, fires no hooks, and mutates no *
* persist_t state, so it cannot change the externally observable *
* acknowledgment semantics (Req 3.3). *
* *
* Stops at the first corruption / short read / pre-allocated zero *
* region (exactly like recovery), so only entries genuinely on stable *
* storage are counted. Returns the highest such LSN, or 0 when nothing *
* is durable. *
* ------------------------------------------------------------------ */
uint64_t persist_test_durable_lsn_on_disk(persist_t* p)
{
if (!p)
return 0;
DIR* d = opendir(p->wal_path);
if (!d)
return 0; /* no WAL dir → nothing durable */
/* Collect segment start LSNs (same bound as replay_wal). */
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
if (strncmp(e->d_name, "seg-", 4) != 0)
continue;
uint64_t seg_start = 0;
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1)
continue;
seg_lsns[nseg++] = seg_start;
}
closedir(d);
if (nseg == 0)
return 0;
/* Insertion sort by start LSN (segment count is tiny). */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) {
seg_lsns[j + 1] = seg_lsns[j];
j--;
}
seg_lsns[j + 1] = key;
}
uint64_t highest_durable = 0;
uint32_t last_crc = WAL_CRC_NONE;
for (int si = 0; si < nseg; si++) {
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0)
continue;
uint32_t seg_off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
if (hn == 0)
break; /* clean end of segment */
if (hn < (ssize_t)sizeof(hdr))
break; /* truncated / pre-allocated zeros */
/* Pre-allocated zero region: no valid entry is all-zero. */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break;
/* CRC chain validation — identical rule to replay_wal. */
if (seg_off == 0) {
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
hdr.prev_crc != last_crc)
break;
} else {
if (hdr.prev_crc != last_crc)
break;
}
/* Verify entry CRC over header + payload. */
uint32_t expected_crc =
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
uint8_t* payload = NULL;
if (hdr.payload_len > 0) {
payload = malloc(hdr.payload_len);
if (!payload) {
close(fd);
return highest_durable;
}
ssize_t pn = pread(fd, payload, hdr.payload_len,
(off_t)(seg_off + sizeof(hdr)));
if (pn != (ssize_t)hdr.payload_len) {
free(payload);
break;
}
expected_crc =
crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
}
if (hdr.crc != expected_crc) {
free(payload);
break;
}
free(payload);
/* Entry is intact and durable on disk. */
last_crc = hdr.crc;
if (hdr.lsn > highest_durable)
highest_durable = hdr.lsn;
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
return highest_durable;
}
#endif /* XCE_TESTING */
/* ------------------------------------------------------------------ */
/* Checkpoint internals */
/* ------------------------------------------------------------------ */
/* Write a WAL_CHECKPOINT marker to the current segment.
* Never calls repl_broadcast() — checkpoint markers are local only. */
static int persist_write_checkpoint_marker(persist_t* p)
{
pthread_mutex_lock(&p->lock);
if (p->wal_fd < 0) {
if (open_new_segment(p) != 0) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
}
wal_entry_hdr_t hdr;
memset(&hdr, 0, sizeof(hdr));
hdr.term = p->term;
hdr.lsn = p->current_lsn++;
hdr.timestamp = now_us();
hdr.type = WAL_CHECKPOINT;
hdr.payload_len = 0;
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
hdr.crc = c;
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
if (hw != (ssize_t)sizeof(hdr)) {
pthread_mutex_unlock(&p->lock);
return -EIO;
}
fdatasync(p->wal_fd);
p->seg_offset += (uint32_t)sizeof(hdr);
p->total_wal_bytes += sizeof(hdr);
p->last_crc = hdr.crc;
/* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */
/* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS
* checkpoint markers (local bookkeeping, not a logical Raft entry), so the
* recovered index never holds a checkpoint LSN; the live index must match
* (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which
* never compares against a checkpoint marker. */
pthread_mutex_unlock(&p->lock);
return 0;
}
/* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2/3.2, Req
* 1.1-1.4). Physical layout appended after the existing legacy header
* (design.md "Checkpoint File Layout"):
* [uint32_t format_version]
* [uint32_t job_count]
* [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }]
* [uint32_t queue_count]
* [queue_count * { uint32_t name_len; char name[name_len]; uint32_t status }] */
#define SNAPSHOT_FORMAT_VERSION 1u
typedef struct {
uint8_t* buf;
uint32_t len;
} snap_job_frame_t;
static void snap_free_job_frames(xlist_t* frames)
{
for (int i = 0; i < frames->count; i++) {
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i);
if (fr) {
free(fr->buf);
free(fr);
}
}
xlist_fini(frames);
}
/* Collect + encode every retained job into `frames`, one malloc'd
* snap_job_frame_t per job, under a point-in-time-consistent walk
* (design.md "Existing Architecture Analysis" / Requirement 1.2): the
* jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes
* concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is
* ADDITIONALLY taken nested around just that job's encode call (excludes a
* concurrent per-entity writer for exactly the duration of its encode --
* the table lock alone does not do this). No lock is held during the
* disk write itself: this function only builds in-memory frames; the
* caller writes them to disk after the table walk has fully released both
* locks.
*
* Filters via job_should_retain (Requirement 1.1) so the checkpoint's
* notion of "live" matches the existing GC retention policy.
*
* Returns 0 on success (frames populated; `frames` is always left
* xlist_init'd -- even zero results -- so the caller can always
* xlist_fini it) or a negative errno on the first encode failure, in which
* case every frame collected so far has already been freed and `frames`
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
* entity content behind). */
static int collect_job_frames(persist_t* p, xlist_t* frames)
{
cluster_t* c = p->cluster;
int rc = 0;
int retained_index = 0;
xlist_init(frames, 64);
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
* zero jobs is not a failure. */
if (c == NULL || c->jobmgr == NULL)
return 0;
lock(c->jobmgr, RTTI_LOCK_READ);
iterator_t it;
int irc = iter_make(&it, c->jobmgr);
if (irc < 0) {
unlock(c->jobmgr);
return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */
}
for (;;) {
job_t* j = NULL;
int nrc = iter_next(&it, (void**)&j);
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
break;
if (j == NULL)
continue;
if (!job_should_retain(j, &p->config))
continue;
uint8_t* buf = NULL;
size_t sz = 0;
int enc_rc;
lock(j, RTTI_LOCK_READ);
#ifdef XCE_TESTING
int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at,
memory_order_relaxed);
if (forced > 0 && (forced - 1) == retained_index)
enc_rc = -EIO; /* simulated mid-loop encode failure */
else
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
#else
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
#endif
unlock(j);
retained_index++;
if (enc_rc != 0) {
rc = enc_rc;
break;
}
snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr));
if (fr == NULL) {
free(buf);
rc = -ENOMEM;
break;
}
fr->buf = buf;
fr->len = (uint32_t)sz;
if (xlist_store(frames, fr) < 0) {
free(buf);
free(fr);
rc = -ENOMEM;
break;
}
}
iter_destroy(&it);
unlock(c->jobmgr);
if (rc != 0)
snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */
return rc;
}
/* Queue admin-status frame (ha-entity-snapshots task 3.2, Req 1.1). Unlike
* jobs, a queue's on-disk frame carries no capnp payload at all -- it is a
* plain (name, status) pair, the entirety of restore_queue_state's contract
* (queue.c:208). `name` is a malloc'd, NUL-terminated copy; the on-disk
* frame itself carries no NUL (design.md "Checkpoint File Layout":
* `{ uint32_t name_len; char name[name_len]; uint32_t status }`). */
typedef struct {
char* name;
uint32_t status;
} snap_queue_frame_t;
static void snap_free_queue_frames(xlist_t* frames)
{
for (int i = 0; i < frames->count; i++) {
snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(frames, i);
if (fr) {
free(fr->name);
free(fr);
}
}
xlist_fini(frames);
}
/* Collect every queue in `frames`, one malloc'd snap_queue_frame_t per
* queue, under the same point-in-time-consistent walk discipline as
* collect_job_frames above (design.md "Existing Architecture Analysis" /
* Requirement 1.2): the qmgr table's own RTTI_LOCK_READ is held for the
* ENTIRE walk, and each queue's own RTTI_LOCK_READ is ADDITIONALLY taken
* nested around just that queue's (name, status) read -- mirroring
* restore_queue_state's own per-entity lock pattern on the load side. No
* lock is held during the disk write itself.
*
* Unlike collect_job_frames, no retention predicate is applied: a queue's
* mere presence in the live qmgr table already means config provisioned
* it (design.md "write_snapshot (extended)" Responsibilities), so every
* queue found is included.
*
* Returns 0 on success (frames populated; `frames` is always left
* xlist_init'd, even for zero results, so the caller can always xlist_fini
* it) or a negative errno on the first failure (name allocation), in which
* case every frame collected so far has already been freed and `frames`
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
* entity content behind). */
static int collect_queue_frames(persist_t* p, xlist_t* frames)
{
cluster_t* c = p->cluster;
int rc = 0;
xlist_init(frames, 64);
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
* zero queues is not a failure. */
if (c == NULL || c->qmgr == NULL)
return 0;
lock(c->qmgr, RTTI_LOCK_READ);
iterator_t it;
int irc = iter_make(&it, c->qmgr);
if (irc < 0) {
unlock(c->qmgr);
return 0; /* nothing to walk -- mirrors collect_queues' empty-table handling */
}
for (;;) {
queue_t* q = NULL;
int nrc = iter_next(&it, (void**)&q);
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
break;
if (q == NULL)
continue;
char* name;
uint32_t status;
lock(q, RTTI_LOCK_READ);
name = q->queue.queue ? strdup(q->queue.queue) : NULL;
status = q->queue.status;
unlock(q);
if (name == NULL) {
rc = -ENOMEM;
break;
}
snap_queue_frame_t* fr = (snap_queue_frame_t*)malloc(sizeof(*fr));
if (fr == NULL) {
free(name);
rc = -ENOMEM;
break;
}
fr->name = name;
fr->status = status;
if (xlist_store(frames, fr) < 0) {
free(name);
free(fr);
rc = -ENOMEM;
break;
}
}
iter_destroy(&it);
unlock(c->qmgr);
if (rc != 0)
snap_free_queue_frames(frames); /* Requirement 1.3: discard, not partial */
return rc;
}
/* Write an entity-bearing checkpoint: legacy header (unchanged) + entity
* payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any
* failure at any step -- collect, encode, write, sync -- leaves NO file at
* the final path; the tmp file is always unlinked on error. This is a
* retriable-next-checkpoint-cycle failure, never a crash/abort (controller
* never self-terminates on an operational condition). */
static int write_snapshot(persist_t* p)
{
char tmp_path[4096];
char final_path[4096];
snprintf(tmp_path, sizeof(tmp_path),
"%s/snap-%016" PRIx64 ".tmp", p->snap_path, p->checkpoint_lsn);
snprintf(final_path, sizeof(final_path),
"%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn);
/* Collect + encode entities BEFORE opening the tmp file: a collection
* failure (Requirement 1.3) then never creates a tmp file at all. */
xlist_t job_frames;
int rc = collect_job_frames(p, &job_frames);
if (rc != 0)
return rc;
xlist_t queue_frames;
rc = collect_queue_frames(p, &queue_frames);
if (rc != 0) {
snap_free_job_frames(&job_frames);
return rc;
}
int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600);
if (fd < 0) {
snap_free_job_frames(&job_frames);
snap_free_queue_frames(&queue_frames);
return -errno;
}
/* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference
* (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1).
* The config-ref tail is OPTIONAL on read-back, so an older 16-byte
* sentinel still loads (load_snapshot reads the tail best-effort). The
* tag is little-endian on this arch, matching the checkpoint_lsn field. */
char header[16 + 1 + 8 + 32];
memset(header, 0, sizeof(header));
memcpy(header, "XCESNAP\0", 8);
memcpy(header + 8, &p->checkpoint_lsn, 8);
header[16] = (char)(p->cfgref_present ? 1 : 0);
memcpy(header + 17, &p->cfgref_tag, 8);
memcpy(header + 25, p->cfgref_sha, 32);
if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header))
rc = -EIO;
/* Entity payload: format_version, then job frames (design.md "Checkpoint
* File Layout"). No table/entity lock is held here -- collect_job_frames
* already released both before returning. */
if (rc == 0) {
uint32_t fmt = SNAPSHOT_FORMAT_VERSION;
if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt))
rc = -EIO;
}
if (rc == 0) {
uint32_t job_count = (uint32_t)job_frames.count;
if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
rc = -EIO;
}
for (int i = 0; rc == 0 && i < job_frames.count; i++) {
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i);
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
rc = -EIO;
break;
}
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
rc = -EIO;
break;
}
}
/* Queue trailer (ha-entity-snapshots task 3.2, Req 1.1, 1.2): plain
* (name, status) frames, no capnp -- matches restore_queue_state's
* exact contract (design.md "Checkpoint File Layout"). */
if (rc == 0) {
uint32_t queue_count = (uint32_t)queue_frames.count;
if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count))
rc = -EIO;
}
for (int i = 0; rc == 0 && i < queue_frames.count; i++) {
snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(&queue_frames, i);
uint32_t name_len = (uint32_t)strlen(fr->name);
if (write(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len)) {
rc = -EIO;
break;
}
if (name_len > 0 &&
write(fd, fr->name, name_len) != (ssize_t)name_len) {
rc = -EIO;
break;
}
if (write(fd, &fr->status, sizeof(fr->status)) != (ssize_t)sizeof(fr->status)) {
rc = -EIO;
break;
}
}
if (rc == 0 && fdatasync(fd) != 0)
rc = -errno;
close(fd);
snap_free_job_frames(&job_frames);
snap_free_queue_frames(&queue_frames);
if (rc != 0) {
unlink(tmp_path);
return rc;
}
if (rename(tmp_path, final_path) != 0) {
int saved = errno;
unlink(tmp_path);
return -saved;
}
return 0;
}
/* scan_latest_committed_config_ref (ha-term-safety task 4.2 remediation r1):
* walk the local WAL segments and capture the HIGHEST-LSN WAL_CONFIG_MARKER
* whose LSN is <= committed_lsn into p->cfgref_{present,tag,sha}. This is the
* SAME committed-prefix marker config_committed_marker derives, cached durably
* so a later checkpoint can trim the marker's segment without losing the
* derived committed-config reference (compaction safety). The marker payload is
* a fixed 40-byte reference (8-byte big-endian version_tag + 32-byte sha256 —
* mirrors config_marker_encode; the config BYTES are never in the WAL), decoded
* inline here so persist stays free of a config_repl layering dependency. */
static void scan_latest_committed_config_ref(persist_t* p)
{
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
memory_order_acquire);
if (committed == 0)
return; /* nothing committed → keep any prior cached ref unchanged */
DIR* d = opendir(p->wal_path);
if (!d)
return;
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
uint64_t sl = 0;
if (sscanf(e->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
seg_lsns[nseg++] = sl;
}
closedir(d);
if (nseg == 0)
return;
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
seg_lsns[j + 1] = key;
}
for (int si = 0; si < nseg; si++) {
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path,
seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0)
continue;
uint32_t off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
if (n < (ssize_t)sizeof(hdr)) break;
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
if (hdr.lsn > committed) break; /* past committed prefix */
if (hdr.type == WAL_CONFIG_MARKER &&
hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) {
uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN];
ssize_t pr = pread(fd, buf, sizeof(buf),
(off_t)(off + sizeof(hdr)));
if (pr == (ssize_t)sizeof(buf)) {
p->cfgref_tag =
((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) |
((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) |
((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) |
((uint64_t)buf[6] << 8) | ((uint64_t)buf[7]);
memcpy(p->cfgref_sha, buf + 8, 32);
p->cfgref_present = 1; /* later markers overwrite — latest wins */
}
}
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
}
/* Delete WAL segment files whose start LSN is strictly less than the
* active segment's start LSN AND strictly less than the trim floor.
*
* Bug-1147 (stopgap, raft-drive-async task 3.2): the trim floor is the
* minimum of the active-segment LSN, every active standby's last acked LSN,
* and every recently-disconnected slot's last_disconnected_lsn (strategy-A
* retention discipline). This ensures we NEVER delete a WAL segment that a
* live standby still needs for catch-up (the bug-795/828 committed-job-loss
* family). It does NOT fix the empty/far-behind join case (cursor=0) —
* that requires entity-bearing snapshots (separate spec).
*
* Lock ordering (verified 2026-06-26): persist_checkpoint takes p->lock only
* to snapshot checkpoint_lsn, then RELEASES it before calling
* trim_wal_segments — so this function runs with NO p->lock held.
* repl_trim_lsn and repl_expire_disconnect_slots take only r->lock
* (p->repl.lock); neither ever acquires p->lock or ce->lock, so there is no
* p->lock↔repl.lock (or ce->lock↔repl.lock) inversion. */
static void trim_wal_segments(persist_t* p)
{
/* Extract the start LSN of the currently-open segment from seg_path.
* seg_path has the form "<wal_path>/seg-<hex16>.wal".
* If no segment is open yet (wal_fd < 0), skip trimming. */
if (p->wal_fd < 0 || p->seg_path[0] == '\0')
return;
const char* seg_basename = strrchr(p->seg_path, '/');
if (!seg_basename) return;
seg_basename++; /* skip the '/' */
uint64_t active_seg_lsn = 0;
if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1)
return;
/* Expire stale disconnected slots so their last_disconnected_lsn guards
* don't hold WAL segments indefinitely (slot guard expires at
* last_disconnect_time + snap_interval_s). */
repl_expire_disconnect_slots(&p->repl, time(NULL),
p->config.snap_interval_s);
/* Compute the trim floor: min(active_seg_lsn, all active standby
* acked_lsn, all valid disconnected-slot last_disconnected_lsn).
* Any segment with seg_start_lsn BELOW this floor is disposable. */
uint64_t floor = repl_trim_lsn(&p->repl, active_seg_lsn,
p->config.snap_interval_s);
DIR* d = opendir(p->wal_path);
if (!d) return;
struct dirent* e;
while ((e = readdir(d)) != NULL) {
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
uint64_t seg_start_lsn = 0;
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue;
/* Only delete segments below BOTH the active segment AND the
* standby-cursor floor (bug-1147 guard). floor <= active_seg_lsn
* by construction (repl_trim_lsn caps at checkpoint_lsn=active_seg_lsn),
* so this check subsumes the old < active_seg_lsn guard. */
if (seg_start_lsn < floor) {
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name);
unlink(seg_path);
}
}
closedir(d);
}
/* Snapshot filename comparison helper for qsort — sorts ascending by name
* (which sorts ascending by LSN since filenames embed the LSN as hex). */
static int snap_name_cmp(const void* a, const void* b)
{
return strcmp(*(const char* const*)a, *(const char* const*)b);
}
/* Keep only the snap_keep_n most recent snapshot files; delete the rest. */
/* True when `lsn` is the checkpoint position an in-flight peer snapshot
* transfer currently references (ha-entity-snapshots task 4.1/4.3, Req
* 3.1-3.3) — checked directly against snapshot_inflight_lsn[], not gated on
* snapshot_inflight[] first: the LSN clears before the flag in every
* ha_snapshot_send_dispatch/snapshot_lane_fn path, so a stale flag=1 can
* momentarily coexist with an already-cleared LSN=0 (see task 4.1's
* Implementation Note). A checkpoint's LSN is never 0, so "nonzero and
* matches" is the correct, race-free protection signal on its own. No
* cluster (p->cluster == NULL, e.g. standalone lib-level tests) means no HA
* layer exists to be mid-transfer — nothing is protected. */
static bool snapshot_lsn_in_flight(persist_t* p, uint64_t lsn)
{
if (p->cluster == NULL || lsn == 0)
return false;
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
if (atomic_load_explicit(&p->cluster->ha.snapshot_inflight_lsn[i],
memory_order_acquire) == lsn)
return true;
}
return false;
}
static void rotate_snapshots(persist_t* p)
{
DIR* d = opendir(p->snap_path);
if (!d) return;
/* Collect all .capnp snapshot filenames */
char* names[1024];
int n = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL && n < (int)(sizeof(names) / sizeof(names[0]))) {
size_t len = strlen(e->d_name);
if (len <= 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue;
names[n] = strdup(e->d_name);
if (names[n]) n++;
}
closedir(d);
if (n == 0) return;
/* Sort ascending — oldest first */
qsort(names, (size_t)n, sizeof(names[0]), snap_name_cmp);
/* Delete all but the snap_keep_n newest, skipping any file an in-flight
* peer transfer is still reading from (task 4.3) — it is retried on the
* next rotation pass once that transfer completes. */
int keep = (int)p->config.snap_keep_n;
int delete_up_to = n - keep; /* first index to keep */
for (int i = 0; i < delete_up_to && i < n; i++) {
uint64_t lsn = 0;
if (sscanf(names[i], "snap-%" SCNx64 ".capnp", &lsn) == 1
&& snapshot_lsn_in_flight(p, lsn))
continue;
char path[4096];
snprintf(path, sizeof(path), "%s/%s", p->snap_path, names[i]);
unlink(path);
}
for (int i = 0; i < n; i++)
free(names[i]);
}
/* ------------------------------------------------------------------ */
/* Async checkpoint scheduling (schedule_checkpoint + task fn) */
/* ------------------------------------------------------------------ */
/* checkpoint_task_fn: executed by delaymgr worker thread.
*
* The CAS in schedule_checkpoint already holds the "in_progress" slot (=1).
* We must clear it before calling persist_checkpoint(), which has its own
* atomic_exchange guard. After persist_checkpoint() returns (it resets the
* flag to 0), we also reset wal_record_count so the counter starts fresh. */
static int checkpoint_task_fn(void* arg)
{
persist_t* persist = (persist_t*)arg;
/* Release the "scheduled" slot so persist_checkpoint's guard can acquire */
atomic_store(&persist->checkpoint_in_progress, 0);
persist_checkpoint(persist);
/* Reset the record counter so the next checkpoint window starts clean.
* Done after persist_checkpoint returns — the flag is already 0 by now. */
atomic_store(&persist->wal_record_count, 0);
return 0;
}
/* schedule_checkpoint: idempotent async checkpoint request via delaymgr.
* Safe to call from any thread. Uses atomic CAS to ensure at most one
* checkpoint is scheduled or running at a time.
*
* If cluster is NULL (unit tests without a live delaymgr) the function
* falls back to a synchronous checkpoint. */
static void schedule_checkpoint(persist_t* persist)
{
int expected = 0;
if (!atomic_compare_exchange_strong(&persist->checkpoint_in_progress,
&expected, 1))
return; /* already scheduled or running — idempotency guard */
if (persist->cluster == NULL) {
/* Unit-test / no-cluster path: run synchronously and reset. */
atomic_store(&persist->checkpoint_in_progress, 0);
persist_checkpoint(persist);
atomic_store(&persist->wal_record_count, 0);
return;
}
/* Schedule via delaymgr with 1 s coalescing delay, single execution */
delay_task(&persist->cluster->delaymgr, 1, 1,
checkpoint_task_fn, persist);
}
/* ------------------------------------------------------------------ */
/* Public: persist_checkpoint */
/* ------------------------------------------------------------------ */
int persist_checkpoint(persist_t* p)
{
/* Guard against concurrent checkpoints */
if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0)
return -EBUSY;
int rc = 0;
/* 1. Write WAL_CHECKPOINT marker (local only — never replicated) */
rc = persist_write_checkpoint_marker(p);
if (rc != 0)
goto done;
/* 2. Record checkpoint_lsn = current position after the marker */
pthread_mutex_lock(&p->lock);
p->checkpoint_lsn = p->current_lsn;
pthread_mutex_unlock(&p->lock);
/* 3a. Capture the latest committed config-marker reference BEFORE trimming
* (ha-term-safety task 4.2 r1): the trim below can unlink the segment
* holding a committed WAL_CONFIG_MARKER, so the derived committed config
* must be cached durably (into the sentinel) before that happens. */
scan_latest_committed_config_ref(p);
/* 3b. Serialize in-memory state to a snapshot file
* (P0: no live kvmgrs — writes sentinel header + cached config ref) */
rc = write_snapshot(p);
if (rc != 0)
goto done;
/* 4. Trim old WAL segments before the checkpoint */
trim_wal_segments(p);
/* 5. Rotate old snapshots — keep only snap_keep_n most recent */
rotate_snapshots(p);
done:
atomic_store(&p->checkpoint_in_progress, 0);
return rc;
}
/* ------------------------------------------------------------------ */
/* Recovery internals */
/* ------------------------------------------------------------------ */
/* Parse and restore every job frame then every queue frame from the entity
* payload beginning at the current `fd` offset (design.md "Checkpoint File
* Layout"): [job_count][job_count * {payload_len; capnp Job bytes}]
* [queue_count][queue_count * {name_len; name; status}].
*
* `file_size` is the checkpoint file's total size (from fstat) and
* `consumed` is the number of bytes already read from it (legacy header +
* format_version) — every length-prefixed frame's length is bounds-checked
* against the ACTUAL remaining bytes in the file before being trusted for a
* read or allocation, never blindly followed (a corrupt/attacker-controlled
* length must fail fast, not drive an oversized read).
*
* Restores in file order: every job via restore_job, then every queue via
* restore_queue_state (design.md sequence diagram). Returns 0 only if every
* frame restored; on the FIRST failure of any kind (short read, oversized
* length, decode error, or restore error) returns immediately with a
* negative errno — the caller MUST NOT commit any position change on a
* non-zero return (Requirement 2.3).
*
* restore_job's -EEXIST is treated as success, not a failure: it is job.c's
* own documented idempotent-replay guard (job already present), needed so a
* retried load — after an earlier partial attempt already created some jobs
* before failing on a later one — can continue past already-restored entries
* rather than aborting on them again. restore_queue_state has no such
* idempotent-duplicate case (re-applying the same status is already a
* harmless overwrite), so any non-zero return from it is a genuine failure. */
static int load_snapshot_entities(int fd, off_t file_size, off_t consumed,
persist_t* p, uint64_t stored_lsn)
{
uint32_t job_count = 0;
if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
return -EIO;
consumed += (off_t)sizeof(job_count);
if (job_count > 0 && (p->cluster == NULL || p->cluster->jobmgr == NULL))
return -EINVAL; /* entities present but nowhere to restore them into */
for (uint32_t i = 0; i < job_count; i++) {
uint32_t payload_len = 0;
if (read(fd, &payload_len, sizeof(payload_len)) !=
(ssize_t)sizeof(payload_len))
return -EIO;
consumed += (off_t)sizeof(payload_len);
if (payload_len == 0 || (off_t)payload_len > file_size - consumed)
return -EIO;
uint8_t* buf = (uint8_t*)malloc(payload_len);
if (buf == NULL)
return -ENOMEM;
if (read(fd, buf, payload_len) != (ssize_t)payload_len) {
free(buf);
return -EIO;
}
consumed += (off_t)payload_len;
int rrc = restore_job(p->cluster, 0, buf, payload_len, stored_lsn);
free(buf);
if (rrc != 0 && rrc != -EEXIST)
return rrc;
}
uint32_t queue_count = 0;
if (read(fd, &queue_count, sizeof(queue_count)) !=
(ssize_t)sizeof(queue_count))
return -EIO;
consumed += (off_t)sizeof(queue_count);
if (queue_count > 0 && (p->cluster == NULL || p->cluster->qmgr == NULL))
return -EINVAL; /* entities present but nowhere to restore them into */
for (uint32_t i = 0; i < queue_count; i++) {
uint32_t name_len = 0;
if (read(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len))
return -EIO;
consumed += (off_t)sizeof(name_len);
if (name_len == 0 || (off_t)name_len > file_size - consumed)
return -EIO;
char* name = (char*)malloc((size_t)name_len + 1);
if (name == NULL)
return -ENOMEM;
if (read(fd, name, name_len) != (ssize_t)name_len) {
free(name);
return -EIO;
}
name[name_len] = '\0';
consumed += (off_t)name_len;
uint32_t status = 0;
if (read(fd, &status, sizeof(status)) != (ssize_t)sizeof(status)) {
free(name);
return -EIO;
}
consumed += (off_t)sizeof(status);
int qrc = restore_queue_state(p->cluster, name, status);
free(name);
if (qrc != 0)
return qrc;
}
return 0;
}
/* Load a snapshot: legacy header (magic + checkpoint_lsn + optional
* config-ref tail), then — for an entity-bearing checkpoint — every job and
* queue frame that follows it (ha-entity-snapshots task 5.2, Requirements
* 2.1-2.4, 5.1, 5.2). See design.md "load_snapshot (extended)" and its
* sequence diagram for the exact contract this implements.
*
* Ordering (Requirement 2.1, 2.2, 2.3 — restore FIRST, validate FULLY, THEN
* advance position, never the reverse, never partial): the header and any
* config-ref tail are parsed into LOCAL variables only. Format is then
* detected by whether any bytes remain past the header (design.md's stated
* discriminator — not a flag bit). A short file (nothing remains) is a
* legacy, pre-feature checkpoint: not an error, so its recorded position is
* committed immediately and *out_entities_restored is false. Otherwise every
* job then every queue frame is restored via load_snapshot_entities() before
* ANYTHING is written to p->checkpoint_lsn/p->current_lsn/p->cfgref_*; on
* any restore failure this function returns non-zero with p's fields
* PROVABLY untouched (the single commit point below is only ever reached
* after load_snapshot_entities() returns 0). *out_entities_restored is only
* set on a rc==0 return, matching persist.h's documented contract.
*
* Returns 0 on success, -EINVAL on bad magic / unrecognized entity format,
* -EIO on a short/corrupt read, -ENOMEM on allocation failure, -errno on
* open/fstat failure, or a restore_job/restore_queue_state error code. */
static int load_snapshot(persist_t* p, const char* path, uint64_t lsn,
bool* out_entities_restored)
{
(void)lsn; /* lsn is encoded in the file itself — we re-read it */
int fd = open(path, O_RDONLY);
if (fd < 0)
return -errno;
struct stat st;
if (fstat(fd, &st) != 0) {
int saved = errno;
close(fd);
return -saved;
}
/* magic(8) + checkpoint_lsn(8) + optional config-ref tail(1+8+32). */
char header[16 + 1 + 8 + 32];
ssize_t n = read(fd, header, sizeof(header));
if (n < 16) {
close(fd);
return -EIO;
}
if (memcmp(header, "XCESNAP\0", 8) != 0) {
close(fd);
return -EINVAL;
}
/* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch).
* Held LOCALLY -- not yet written to p-> -- until the single commit
* point below (legacy short-circuit, or after every entity restores). */
uint64_t stored_lsn;
memcpy(&stored_lsn, header + 8, 8);
/* Optional committed-config reference (ha-term-safety task 4.2 r1):
* present only when the sentinel carries the 41-byte tail. Also held
* locally until the same commit point. */
int cfgref_present = 0;
uint64_t cfgref_tag = 0;
uint8_t cfgref_sha[32];
memset(cfgref_sha, 0, sizeof(cfgref_sha));
if (n >= 16 + 1 + 8 + 32 && header[16] != 0) {
cfgref_present = 1;
memcpy(&cfgref_tag, header + 17, 8);
memcpy(cfgref_sha, header + 25, 32);
}
off_t consumed = (off_t)n;
/* Legacy-format detection (Requirement 2.4, 5.1, 5.2): "does more data
* exist" is the discriminator (design.md), not a flag bit -- a file
* ending exactly at the legacy header is unambiguously a pre-feature
* checkpoint. This is an expected rollout shape, not corruption: commit
* the position it records (existing, preserved legacy behavior) and
* report entities_restored=false so the caller falls back to normal
* catch-up rather than treating this checkpoint as fully complete. */
if (st.st_size <= consumed) {
close(fd);
p->checkpoint_lsn = stored_lsn;
p->current_lsn = stored_lsn;
if (cfgref_present) {
p->cfgref_present = 1;
p->cfgref_tag = cfgref_tag;
memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha));
}
if (out_entities_restored)
*out_entities_restored = false;
return 0;
}
/* Entity-bearing checkpoint: format_version, then job frames, then queue
* frames (design.md "Checkpoint File Layout"). */
uint32_t fmt = 0;
if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) {
close(fd);
return -EIO;
}
consumed += (off_t)sizeof(fmt);
/* An unrecognized format_version is corruption or a future layout this
* binary cannot read -- fail loud rather than silently treating it as
* legacy (task 5.1 review note: a version mismatch must never be masked
* as an expected legacy shape, or real corruption goes undetected). */
if (fmt != SNAPSHOT_FORMAT_VERSION) {
close(fd);
return -EINVAL;
}
int rc = load_snapshot_entities(fd, st.st_size, consumed, p, stored_lsn);
close(fd);
if (rc != 0)
return rc; /* Requirement 2.3: p->*_lsn / p->cfgref_* untouched */
/* Single commit point (Requirement 2.1, 2.2): reached ONLY after every
* job and queue frame restored successfully above. */
p->checkpoint_lsn = stored_lsn;
p->current_lsn = stored_lsn;
if (cfgref_present) {
p->cfgref_present = 1;
p->cfgref_tag = cfgref_tag;
memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha));
}
if (out_entities_restored)
*out_entities_restored = true;
return 0;
}
/* Public wrapper for load_snapshot — exposed via persist.h for ha.c (Task
* 3.5c). load_snapshot() above now owns the full restore-before-advance
* implementation (ha-entity-snapshots task 5.2): this wrapper is a thin
* pass-through, since load_snapshot() already gates *out_entities_restored
* and the p->checkpoint_lsn/current_lsn commit on entity-restore success
* (Requirements 2.1-2.4, 5.1, 5.2). */
int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn,
bool *out_entities_restored)
{
return load_snapshot(p, path, lsn, out_entities_restored);
}
/* P2-2: Standby WAL replay — apply a single replicated entry to in-memory state.
*
* No hooks fire on standbys; the hook system is empty until promotion.
*
* Returns 0 on success (including skip for checkpoint / unknown types),
* negative errno on error (NULL args, oversized payload). */
int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
const void* payload, uint32_t payload_len)
{
if (!p)
return -EINVAL;
/* Reject oversized payloads BEFORE any allocation */
if (payload_len > MAX_WAL_PAYLOAD_BYTES) {
ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u",
payload_len, MAX_WAL_PAYLOAD_BYTES);
return -EMSGSIZE;
}
/* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain
* (p->last_crc), and current_lsn are owned exclusively by the physical
* layer — persist_wal_write_replicated on the live replicated path, and
* replay_wal/persist_recover on recovery. This function MUST NOT touch
* them. (bug-778: it previously set p->last_crc from a synthetic crc=0
* header, breaking the on-disk CRC chain and cold-restart recovery.) */
/* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it
* via replication. Skip silently. */
if (type == WAL_CHECKPOINT)
return 0;
/* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward
* compatibility — an older controller can boot on WAL written by a newer
* one without hard-failing on unrecognised entry types. */
if (type > WAL_ENTRY_TYPE_MAX) {
WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u"
" (lsn=%" PRIu64 ") — skipping for forward compatibility",
type, WAL_ENTRY_TYPE_MAX, lsn);
return 0;
}
/* Advance the current_lsn high-water (monotonic, idempotent). This is a
* write-cursor high-water that is safe to bump from multiple paths. The
* CRC chain link (p->last_crc) is deliberately NOT touched here — it has a
* single owner: the physical writer (persist_wal_write_replicated) and
* replay (persist_recover). (bug-778) */
pthread_mutex_lock(&p->lock);
if (lsn + 1 > p->current_lsn)
p->current_lsn = lsn + 1;
pthread_mutex_unlock(&p->lock);
if (!p->cluster)
return 0; /* unit tests without a full cluster — skip entity restore */
switch (type) {
case WAL_JOB_SUBMIT:
/* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN
* through so the restored job is tagged for the egress watermark
* (ha-convergence-recovery 3.3). */
restore_job(p->cluster, 0, payload, payload_len, lsn);
break;
case WAL_JOB_START:
case WAL_JOB_FINISH:
case WAL_JOB_FAIL:
case WAL_JOB_STOP:
case WAL_JOB_RESUME:
case WAL_JOB_UPDATE:
case WAL_JOB_CKPT:
case WAL_JOB_PREASON:
case WAL_JOB_RUSAGE:
/* Payload: wal_jobstate_t {jobid, state} */
if (payload_len >= sizeof(wal_jobstate_t)) {
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
restore_job_state(p->cluster, js->jobid, js->state);
}
break;
case WAL_WORKER_CTRL:
/* Payload: [uint32_t detail][capnp worker] */
if (payload_len > WORKER_WAL_HDR_SIZE)
worker_from_capnp(p->cluster, payload, payload_len);
break;
case WAL_QUEUE_CTRL:
/* Payload: [wal_queuehdr_t][name NUL-terminated] */
if (payload_len > sizeof(wal_queuehdr_t)) {
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
const char* name = (const char*)(payload + sizeof(wal_queuehdr_t));
restore_queue_state(p->cluster, name, qh->status);
}
break;
case WAL_NOOP:
/* Internal promotion no-op: empty payload, no entity restore, no
* hooks. current_lsn is advanced by the physical layer (writer /
* replay), not here — recognised explicitly rather than via default. */
break;
case WAL_CONFIG_MARKER:
/* Config-change marker (ha-term-safety task 4.2): a {version_tag,
* sha256} reference, NEVER the config bytes (R7.4). It carries NO
* entity state, so replay performs no restore here — the committed
* configuration is DERIVED from the latest committed marker on demand
* (config_committed_marker), and the bytes are obtained + hash-verified
* out-of-band before activation (task 4.3). Recognised explicitly so
* the per-LSN term index (built above this switch's caller) records its
* term and on_get_entry_term / (term,lsn) reconcile treat it as an
* ordinary log entry. */
break;
default:
break;
}
return 0;
}
/* raft-apply-decoupling task 2.2 — see persist.h for the full contract doc
* (routing table, the job-state-family hook-mapping deviation, and what this
* function deliberately does NOT own). Not yet wired into any live call
* site — replay (2.3) and the live apply pump (6.1) will call this later. */
int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn,
const void* payload, uint32_t len)
{
if (!p)
return -EINVAL;
if (!p->cluster)
return 0; /* unit tests without a full cluster — nothing to dispatch into */
cluster_t* c = p->cluster;
switch (wal_type) {
case WAL_JOB_SUBMIT: {
int rc = restore_job(c, 0, payload, len, lsn);
if (rc == -EEXIST)
return 0; /* idempotent re-apply (replay / resync) — no hook */
if (rc < 0)
return rc;
/* Recover the jobid via a cheap raw capnp field read (read_Job only
* reads the fixed uint64 field — no allocation, unlike decode_Job)
* so the just-constructed job can be looked up for the notification
* hook without changing restore_job's own contract. */
struct capn jc_ctx;
capn_init_mem(&jc_ctx, (const uint8_t*)payload, len, 0);
Job_ptr jp;
jp.p = capn_getp(capn_root(&jc_ctx), 0, 1);
struct Job cj;
read_Job(&cj, jp);
uint64_t jobid = cj.jobid;
capn_free(&jc_ctx);
job_t* j = NULL;
lock(c->jobmgr, RTTI_LOCK_READ);
key_find(c->jobmgr, (void*)jobid, (void**)&j);
unlock(c->jobmgr);
if (!j)
return 0; /* defensive: should not happen after a successful restore */
void* jptr = j;
xlist_t jlist = { .count = 1, .caps = 1, .delta = 1, .data = &jptr };
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_NEW, c, &jlist);
return 0;
}
case WAL_JOB_START:
case WAL_JOB_FINISH:
case WAL_JOB_FAIL:
case WAL_JOB_STOP:
case WAL_JOB_RESUME:
case WAL_JOB_UPDATE:
case WAL_JOB_RUSAGE:
case WAL_JOB_CKPT:
case WAL_JOB_PREASON: {
/* Every job-state fact type shares one wire shape — wal_jobstate_t
* {jobid, state} — and therefore one guarded core. See persist.h's
* doc comment on apply_committed_entry for why this does not
* dispatch to the type-specific apply_job_{start,finish,fail}_
* transition() cores (their richer parameter structs are not
* present on this wire format). */
if (len < sizeof(wal_jobstate_t))
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
job_t* j = NULL;
lock(c->jobmgr, RTTI_LOCK_READ);
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
unlock(c->jobmgr);
if (!j)
return -ENOENT;
lock(j, RTTI_LOCK_WRITE);
job_transition_result_t result =
apply_job_update_transition(j, (int)js->state);
unlock(j);
if (result != JOB_TRANSITION_APPLIED)
return 0; /* rejected-backward / no-op-duplicate — no hook */
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
HOOK_DETAIL_JOB_UPDATE, c, j);
return 0;
}
case WAL_QUEUE_CTRL: {
if (len <= sizeof(wal_queuehdr_t))
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
const char* name =
(const char*)((const uint8_t*)payload + sizeof(wal_queuehdr_t));
/* restore_queue_state's -ENOENT ("queue not yet built from config")
* and -EINVAL (bad name) are both benign skips here, matching
* apply_wal_entry's existing behavior of never fencing on this
* entry type's restore outcome. */
if (restore_queue_state(c, name, qh->status) < 0)
return 0;
queue_t* q = NULL;
lock(c->qmgr, RTTI_LOCK_READ);
key_find(c->qmgr, (void*)name, (void**)&q);
unlock(c->qmgr);
if (!q)
return 0;
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, HOOK_AFTER_EVENT,
HOOK_DETAIL_QUEUE_UPDATE, c, q);
return 0;
}
case WAL_WORKER_CTRL: {
if (len <= WORKER_WAL_HDR_SIZE)
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
uint32_t hook_detail;
memcpy(&hook_detail, payload, sizeof(hook_detail));
int rc = worker_from_capnp(c, payload, len);
if (rc == -EEXIST)
return 0; /* idempotent re-apply (replay / resync) — no hook */
if (rc < 0)
return rc;
/* Recover the worker's name via a cheap raw capnp field read
* (read_Worker's capn_get_text is a borrowed, NUL-terminated pointer
* into the decode buffer — no allocation, unlike decode_Worker) so
* the just-restored worker can be looked up for the notification
* hook without changing worker_from_capnp's own contract. */
const uint8_t* wpayload = (const uint8_t*)payload + WORKER_WAL_HDR_SIZE;
size_t wlen = len - WORKER_WAL_HDR_SIZE;
struct capn wc_ctx;
capn_init_mem(&wc_ctx, wpayload, wlen, 0);
Worker_ptr wp;
wp.p = capn_getp(capn_root(&wc_ctx), 0, 1);
struct Worker cw;
read_Worker(&cw, wp);
worker_t* w = NULL;
lock(c->workermgr, RTTI_LOCK_READ);
key_find(c->workermgr, (void*)cw.name.str, (void**)&w);
unlock(c->workermgr);
capn_free(&wc_ctx);
if (!w)
return 0; /* defensive: should not happen after a successful restore */
switch (hook_detail) {
case HOOK_DETAIL_WORKER_JOIN:
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c,
w->host.name, w);
break;
case HOOK_DETAIL_WORKER_UPDATE:
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_UPDATE, c, w);
break;
case HOOK_DETAIL_WORKER_DEAD:
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w);
break;
default:
break; /* unrecognized discriminator — mutation already applied,
* notification skipped rather than guessed at */
}
return 0;
}
default:
/* WAL_NOOP / WAL_CONFIG_MARKER / WAL_CHECKPOINT / unknown — no
* entity effect at this seam, no hook (unchanged from
* apply_wal_entry's existing treatment of these types). */
return 0;
}
}
/* Read WAL segment files in LSN order; replay entries with lsn > from_lsn.
*
* Segment filename format: seg-<start_lsn_hex16>.wal
* Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes).
*
* CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry
* in a segment). If prev_crc mismatches, stop at last good entry and log WARN.
*
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
* *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped
* on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a
* short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF,
* pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is
* about the STRUCTURE of the local log, not about being behind the cluster: a
* clean log that simply ends below the cluster high-water is still CLEAN.
*
* Returns number of entries replayed (>= 0) or negative errno on I/O error. */
static int replay_wal(persist_t* p, uint64_t from_lsn,
recover_verdict_t* verdict_out)
{
/* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */
if (verdict_out)
*verdict_out = RECOVER_CLEAN;
/* --- Collect and sort segment filenames by start LSN --- */
DIR* d = opendir(p->wal_path);
if (!d)
return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */
/* Collect up to 4096 segment names */
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent* e;
while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) {
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
uint64_t seg_start = 0;
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue;
seg_lsns[nseg++] = seg_start;
}
closedir(d);
if (nseg == 0)
return 0;
/* Simple insertion sort — segment count is tiny in practice */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) {
seg_lsns[j + 1] = seg_lsns[j];
j--;
}
seg_lsns[j + 1] = key;
}
/* --- Replay entries from each segment --- */
int replayed = 0;
uint32_t last_crc = WAL_CRC_NONE;
for (int si = 0; si < nseg; si++) {
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0) {
WARN("persist_recover: cannot open segment %s: %s",
seg_path, strerror(errno));
continue;
}
uint32_t seg_off = 0;
for (;;) {
/* Read header */
wal_entry_hdr_t hdr;
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
if (hn == 0)
break; /* clean end of segment (CLEAN) */
if (hn < (ssize_t)sizeof(hdr)) {
/* A partial header (or read error) is a SHORT READ — the log
* stops mid-entry, so it cannot be reconstructed contiguously
* (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */
if (hn < 0)
WARN("persist_recover: read error in %s at offset %u: %s",
seg_path, seg_off, strerror(errno));
else
WARN("persist_recover: short header read in %s at offset %u "
"(got %zd of %zu bytes); stopping replay",
seg_path, seg_off, hn, sizeof(hdr));
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
/* Detect pre-allocated zero region (no valid entry starts with
* all-zero header: lsn == 0 && type == 0 && payload_len == 0).
* This is the normal fallocate'd tail after the last valid entry —
* a clean stop, NOT corruption (stays RECOVER_CLEAN). */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break;
/* CRC chain validation: prev_crc must match last_crc in segment */
if (seg_off == 0) {
/* First entry in this segment: prev_crc must be WAL_CRC_NONE
* (or we accept any value if last_crc is also WAL_CRC_NONE,
* e.g. across a segment boundary after checkpoint trim). */
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
hdr.prev_crc != last_crc) {
WARN("persist_recover: CRC chain break at start of %s "
"(prev_crc=%08x expected=%08x); stopping replay",
seg_path, hdr.prev_crc, last_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
} else {
if (hdr.prev_crc != last_crc) {
WARN("persist_recover: CRC chain break in %s at offset %u "
"(prev_crc=%08x expected=%08x); stopping replay at last good entry",
seg_path, seg_off, hdr.prev_crc, last_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
break;
}
}
/* Verify entry CRC */
uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
uint8_t* payload = NULL;
if (hdr.payload_len > 0) {
payload = malloc(hdr.payload_len);
if (!payload) {
close(fd);
return -ENOMEM;
}
ssize_t pn = pread(fd, payload, hdr.payload_len,
(off_t)(seg_off + sizeof(hdr)));
if (pn != (ssize_t)hdr.payload_len) {
WARN("persist_recover: short payload read in %s at offset %u",
seg_path, seg_off);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
free(payload);
break;
}
expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
}
if (hdr.crc != expected_crc) {
WARN("persist_recover: CRC mismatch in %s at offset %u "
"(got=%08x expected=%08x); stopping replay",
seg_path, seg_off, hdr.crc, expected_crc);
if (verdict_out)
*verdict_out = RECOVER_TRUNCATED;
free(payload);
break;
}
/* Advance segment-local CRC */
last_crc = hdr.crc;
/* Record this entry's term at its LSN for recovery read-back
* (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical
* entry — including snapshot-covered ones below from_lsn — so the
* index spans the full LSN range the reconcile/promotion logic may
* query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and
* unknown future types (type > WAL_CHECKPOINT): they are skipped
* below and are never logical Raft entries, so the live append path
* does not record them either (kept symmetric). The raw hdr.term is
* stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM
* mapping is applied only at read-back. */
if (hdr.type < WAL_CHECKPOINT)
record_entry_term(p, hdr.lsn, hdr.term);
/* Skip entries already covered by the snapshot */
if (hdr.lsn <= from_lsn) {
free(payload);
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
continue;
}
/* Skip WAL_CHECKPOINT markers (local bookkeeping only) */
if (hdr.type == WAL_CHECKPOINT) {
free(payload);
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
/* Advance current_lsn past the marker */
if (hdr.lsn + 1 > p->current_lsn)
p->current_lsn = hdr.lsn + 1;
continue;
}
/* Unknown future entry types: log and skip */
if (hdr.type > WAL_CHECKPOINT) {
WARN("persist_recover: unknown WAL entry type %u at LSN %lu; skipping",
hdr.type, (unsigned long)hdr.lsn);
free(payload);
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
continue;
}
/* Logical entity restore via the shared apply seam
* (raft-apply-decoupling task 2.3, design.md "Apply Seam"): the
* SAME apply_committed_entry() dispatch the live commit-advance
* apply pump will use (task 6.1), so a node's materialized state
* is the deterministic result of its committed log regardless of
* whether it got there via restart replay or live apply
* (Req 3.1-3.4). apply_wal_entry() (this replay loop's previous
* per-entry dispatch) is NOT retired by this change — it remains
* the live standby catch-up/replicated-apply dispatch used
* elsewhere (ha.c); it is simply no longer this loop's call. The
* CRC chain + current_lsn remain owned by this replay loop
* (current_lsn advanced below; last_crc at line ~1601) — neither
* dispatch function ever touched them (bug-778).
*
* KNOWN HAZARD (tracked in tasks.md Implementation Notes, task
* 2.3): this replays the FULL post-checkpoint WAL range on every
* restart (last_applied_lsn is not persisted, deliberately
* recomputed here), so apply_committed_entry()'s AFTER_EVENT
* hooks fire again for entries already applied+notified in a
* prior process life. Confirmed inert today: scheduler hooks
* aren't registered until a post-boot network JOIN
* (on_scheduler_join, sched.c), and apiserver hooks — though
* registered before this runs — gate their notify on a
* connected endpoint that is empty at first-boot replay time.
* MUST be re-verified before tasks 4.1/6.1/6.2/6.3 make this
* path broadly live. */
apply_committed_entry(p, hdr.type, hdr.lsn, payload,
hdr.payload_len);
free(payload);
/* Advance current_lsn to one past this entry */
if (hdr.lsn + 1 > p->current_lsn)
p->current_lsn = hdr.lsn + 1;
replayed++;
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
/* Update last_crc so the next WAL write chains correctly */
pthread_mutex_lock(&p->lock);
p->last_crc = last_crc;
pthread_mutex_unlock(&p->lock);
return replayed;
}
/* ------------------------------------------------------------------ */
/* Public: persist_recover */
/* ------------------------------------------------------------------ */
int persist_recover(persist_t* p)
{
/* --- Step 1: find latest snapshot (highest LSN filename) --- */
char snap_file[4096] = "";
uint64_t snap_lsn = 0;
DIR* d = opendir(p->snap_path);
if (d) {
struct dirent* e;
while ((e = readdir(d)) != NULL) {
size_t len = strlen(e->d_name);
if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0)
continue;
uint64_t lsn = 0;
if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1)
continue;
if (lsn > snap_lsn) {
snap_lsn = lsn;
snprintf(snap_file, sizeof(snap_file),
"%s/%s", p->snap_path, e->d_name);
}
}
closedir(d);
}
/* --- Step 2: load snapshot (if any) --- */
if (snap_lsn > 0 && snap_file[0] != '\0') {
/* Local recovery doesn't need the entities-restored indicator to
* decide behavior: the WAL replay in Step 3 applies everything
* above checkpoint_lsn regardless, unchanged by ha-entity-snapshots.
* It IS consumed for observability parity with the wire receive
* path (ha_handle_snapshot_chunk, task 6.1) — a rolling-upgrade
* restart against a pre-feature (legacy, position-only) local
* checkpoint is an expected rollout state, not an error. */
bool entities_restored = false;
int rc = load_snapshot(p, snap_file, snap_lsn, &entities_restored);
if (rc != 0) {
ERROR("persist_recover: failed to load snapshot %s: %s",
snap_file, strerror(-rc));
return rc;
}
/* checkpoint_lsn and current_lsn set by load_snapshot */
if (entities_restored) {
INFO("persist_recover: snapshot %s loaded lsn=%" PRIu64
" with full job/queue entity restoration", snap_file,
p->checkpoint_lsn);
} else {
INFO("persist_recover: snapshot %s lsn=%" PRIu64 " is a legacy "
"position-only checkpoint (no entity content) — continuing "
"via normal WAL replay", snap_file, p->checkpoint_lsn);
}
}
/* --- Step 3: replay WAL entries after checkpoint_lsn --- */
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
* replay distinguishes a clean log end from a corruption-truncated end.
* We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do
* NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean
* replay over a loaded snapshot-sentinel (no entries below the sentinel) is
* CLEAN — the sentinel established the committed position; replay only adds
* entries above it. */
recover_verdict_t verdict = RECOVER_CLEAN;
int replayed = replay_wal(p, p->checkpoint_lsn, &verdict);
if (replayed < 0)
return replayed;
p->recovery_verdict = verdict;
if (verdict == RECOVER_TRUNCATED)
WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally "
"corrupt (CRC break / mismatch / short read); member marked "
"recovery-truncated (non-promotable until caught up)");
/* --- Step 3b: re-derive the durable floor from the recovered WAL tip
* (ha-commit-durability task 3.4, Req 2.3) ---
*
* Publish the member's durable floor (the durable last-log index consumed
* by the Raft freshness gate, task 4.2) as the highest INTACT replayed LSN.
* replay_wal advanced current_lsn to one past the last intact entry — incl.
* stopping at a corruption boundary on TRUNCATED — so current_lsn - 1 is
* exactly that intact WAL tip, and a snapshot-sentinel load leaves it at the
* sentinel/replayed tip. We store it into durable_lsn because the recovered
* on-disk entries are GENUINELY durable (each was CRC-validated and read
* from stable storage by replay_wal), so the freshness advertisement is now
* backed by durable state rather than a volatile in-memory position.
*
* Crux (Req 2.3 — a truncated member cannot over-advertise): this is an
* ASSIGN to the recovered tip, NOT a max() with whatever durable_lsn held.
* A fresh recover starts from durable_lsn=0 (persist_init), so on a
* truncating restart the floor lands at the LOWER intact tip — never a
* stale higher in-memory value and never the commit position
* (committed_lsn). The recovered tip is, by construction, <= current_lsn-1,
* preserving the durable_lsn <= current_lsn-1 invariant. The post-election
* safe-commit point continues to come from voter replies, not this floor
* (CLAUDE.md). */
uint64_t recovered_tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
atomic_store_explicit(&p->durable_lsn, recovered_tip, memory_order_release);
/* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */
return 0;
}
/* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */
recover_verdict_t persist_recovery_verdict(persist_t* p)
{
return p ? p->recovery_verdict : RECOVER_CLEAN;
}
bool persist_recovery_truncated(persist_t* p)
{
return p && p->recovery_verdict == RECOVER_TRUNCATED;
}
/* ------------------------------------------------------------------ *
* Boot-policy loud-fence decision (ha-commit-durability task 3.3, Req 2.4;
* never-self-exit rework 2026-06-18).
*
* Implements the FENCE_UNRECOVERABLE terminal of the Boot Policy Decision Table
* (design.md). The decision is a PURE function of its inputs; the action (loud
* operator-visible log + set the sticky non-promotable flag) is separated into
* persist_recover_loud_fence_if_unrecoverable so the decision can be unit-tested
* in isolation. A reachable healthy peer NEVER yields FENCE_UNRECOVERABLE
* (rows 3, 5); it is reserved for a member that cannot recover locally AND
* cannot catch up (no healthy peer, rows 4, 4b).
*
* CRITICAL: this terminal is a LOUD, PERMANENT, NON-promotable fence — the
* process STAYS ALIVE and keeps retrying catch-up. There is NO abort()/exit():
* a controller MUST NEVER self-terminate as a designed response to an
* operational condition (user invariant, 2026-06-18). A self-exited controller
* needs manual admin recovery and degrades quorum meanwhile; one that stays
* alive and keeps retrying self-heals when a peer returns (Req 2.6).
* ------------------------------------------------------------------ */
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in)
{
if (in == NULL)
return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */
bool truncated = (in->verdict == RECOVER_TRUNCATED);
/* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster
* high-water mark is a normal promotable member — no fence. */
if (!truncated && !in->durable_below_hwm)
return RECOVER_DECISION_ELIGIBLE;
/* From here the member is truncated OR behind the HWM — it must NOT lead
* as-is. A truncated log is by definition not provably at/above the HWM
* (design.md note), so it is treated as behind for fence purposes. */
/* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain
* eligibility (task 4.3). A reachable peer NEVER yields the unrecoverable
* fence — even for a truncated log or a legitimate snapshot-sentinel. If
* catch-up has already succeeded the member is eligible; otherwise it stays
* fenced (4.3 drives the catch-up + release). */
if (in->healthy_peer_reachable) {
if (in->catch_up_succeeded)
return RECOVER_DECISION_ELIGIBLE;
return RECOVER_DECISION_FENCE;
}
/* No healthy peer. If catch-up did not reach a committed-consistent state,
* the member can neither recover locally nor catch up right now — rows 4 &
* 4b: a LOUD, PERMANENT, NON-promotable fence (NOT process exit). It stays
* alive and keeps retrying; the fence releases when a peer returns
* (Req 2.4/2.6). */
if (!in->catch_up_succeeded)
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
/* No peer but catch-up is reported succeeded (e.g. already at/above after a
* local catch-up): eligible. */
return RECOVER_DECISION_ELIGIBLE;
}
bool persist_recovery_unrecoverable(persist_t* p)
{
return p && atomic_load_explicit(&p->recovery_unrecoverable,
memory_order_acquire);
}
void persist_clear_recovery_unrecoverable(persist_t* p)
{
if (!p)
return;
/* Clear only on a genuine transition so the loud "fence released" log fires
* once, not on every catch-up tick. */
if (atomic_exchange_explicit(&p->recovery_unrecoverable, false,
memory_order_acq_rel)) {
INFO("persist_recover: unrecoverable fence RELEASED — member reached a "
"committed-consistent state via catch-up; promotable again "
"(ha-commit-durability Req 2.6)");
}
}
recover_decision_t
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
const recover_policy_inputs_t* in)
{
recover_decision_t decision = persist_recover_decision(in);
if (decision != RECOVER_DECISION_FENCE_UNRECOVERABLE)
return decision;
/* Loud, operator-visible alert: log the truncation boundary
* (durable_floor_lsn) and the cluster high-water mark so an operator sees
* exactly where the local log stops short of the cluster's committed state.
* The member STAYS ALIVE, stays non-promotable, and keeps retrying catch-up
* (Req 2.4) — it NEVER self-exits. */
const char* why = (in->verdict == RECOVER_TRUNCATED)
? "recovery-truncated log (structural corruption)"
: "log behind the cluster committed high-water mark";
ERROR("persist_recover: UNRECOVERABLE — %s and no healthy peer reachable to "
"catch up; truncation boundary durable_floor_lsn=%" PRIu64
" < cluster committed high-water hwm=%" PRIu64 ". Staying ALIVE and "
"PERMANENTLY non-promotable (refusing to serve divergent state) and "
"RETRYING catch-up indefinitely — will self-heal when a healthy peer "
"returns; operator attention may be needed "
"(ha-commit-durability Req 2.4)",
why, in->durable_floor_lsn, in->cluster_hwm_lsn);
/* Set the sticky non-promotable flag so the promotion gate refuses
* leadership as belt-and-suspenders until catch-up clears it (Req 2.4/2.6).
* p may be NULL for pure-classification callers that only want the loud log. */
if (p != NULL)
atomic_store_explicit(&p->recovery_unrecoverable, true,
memory_order_release);
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
}
/* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
* entry is already durable on local stable storage (ha-commit-durability task
* 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the
* commit engine's durable self-cap (commit.c majority_candidate_locked): a
* majority ACK no longer implies the local copy is fsync'd, because group-
* commit (task 2.1) deferred the append fsync to the demand-driven flusher. So
* advancing committed_lsn here on the ACK alone would reintroduce the bug-661
* ACK-without-durability hazard — committed could be durable on a single
* member and lost across failover. We gate the store on written_lsn <=
* durable_lsn so committed_lsn ⇒ durable-on-this-member on every path.
* committed_lsn is monotonic, so a max() store; the durable cap can only delay
* the advance until the flusher syncs the entry (the durable-advance commit
* wake re-drives the engine path; the legacy path is re-evaluated on the next
* sync_append). */
static void sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn)
{
uint64_t durable = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
/* DURABILITY PRECONDITION: never advance committed past what is durable
* locally. The LSN-0 first-entry edge stays valid (0 <= 0). */
if (written_lsn > durable)
return;
uint64_t cur = atomic_load_explicit(&p->committed_lsn,
memory_order_acquire);
if (written_lsn <= cur)
return; /* monotonic: committed_lsn never regresses */
atomic_store_explicit(&p->committed_lsn, written_lsn, memory_order_release);
#ifdef XCE_TESTING
/* INVARIANT (debug guard): committed_lsn must never exceed durable_lsn on
* any commit advance. A future caller reintroducing ACK-without-durability
* trips this. */
assert(atomic_load_explicit(&p->committed_lsn, memory_order_acquire)
<= atomic_load_explicit(&p->durable_lsn, memory_order_acquire));
#endif
}
int persist_wal_sync_append(persist_t* p, uint32_t type,
const void* payload, uint32_t len,
int* sync_rc_out)
{
/* ha-term-safety task 3.1: the primary's WAL_JOB_SUBMIT append is routed
* through the raft library (ha_primary_log_append → raft_log_append) so the
* library stamps the term + assigns the LSN (firing on_persist_entry → the
* WAL write), keeping p->current_lsn co-tracked with s->log_len; the existing
* majority-commit barrier below (repl_wait_majority_ack) is UNCHANGED and runs
* after the append. Non-raft road / engine-less tests (library not LEADER)
* fall back inside ha_primary_log_append to persist_wal_async. Standalone
* persist (p->cluster == NULL, lib-level tests) keeps the direct append. */
int rc;
if (p->cluster != NULL)
rc = ha_primary_log_append(p->cluster, type, payload, len, NULL);
else
rc = persist_wal_async(p, type, payload, len);
if (rc != 0) {
if (sync_rc_out)
*sync_rc_out = rc;
return rc;
}
/* Majority-commit replication (Req 5.1–5.3; ha-commit-engine Req 1.1–1.3).
* persist_wal_async appends to the page cache and signals the demand-driven
* flusher; it no longer fsyncs inline (group-commit, task 2.1). Durability
* is therefore established by the flusher advancing durable_lsn, and a
* commit position advances only once the entry is durable locally — the
* engine path via the durable self-cap (commit.c), the legacy path below
* via sync_append_legacy_commit_store's written_lsn <= durable_lsn gate.
* We use current_lsn - 1 because persist_wal_async already incremented it. */
uint64_t written_lsn;
pthread_mutex_lock(&p->lock);
written_lsn = p->current_lsn - 1;
pthread_mutex_unlock(&p->lock);
int sync_err;
/* Engine-wired path (ha-commit-engine task 5.3): delegate the commit
* decision to the commit engine through the walAck adapter. The engine
* is the single committed_lsn writer (no store here), and its
* n_peers<=1 self-quorum replaces the old has_standbys → -ENODEV
* degraded special case (Req 6.3).
*
* LSN-0 edge (ACCEPTED interim behavior — spec Implementation Notes):
* written_lsn is passed unconditionally, but the FIRST-EVER WAL entry
* of a fresh cluster is LSN 0 and commit_submit rejects lsn==0, so the
* adapter's entry_lsn != 0 guard routes that one entry to the legacy
* branch below. This happens once per cluster lifetime;
* ha-single-leader-role's promotion no-op entry eliminates it in
* practice. */
if (p->repl.ce != NULL && written_lsn != 0) {
sync_err = repl_wait_majority_ack(&p->repl, written_lsn,
p->config.sync_timeout_ms,
(uint8_t)OPCode_walAck);
if (sync_err != 0)
CTRL_METRIC_INC(ctrl_repl_degraded_count);
} else {
/* Residual non-engine branch: engine-less (ce == NULL) unit-test
* contexts plus the LSN-0 first entry. The legacy connectivity-
* based counting was retired by ha-commit-barrier-rollout task 2.2:
* repl_wait_majority_ack now confirms LSN 0 immediately (documented
* once-per-lifetime edge) and surfaces engine-less walAck waits as
* -EINVAL via commit_submit, so the committed_lsn store below only
* fires for the LSN-0 case (a no-op store of 0). */
mylock_lock(&p->repl.lock);
int has_standbys = (p->repl.nstandbys > 0);
mylock_unlock(&p->repl.lock);
if (!has_standbys) {
/* No standbys registered → degraded mode (no quorum to wait for) */
sync_err = -ENODEV;
} else {
sync_err = repl_wait_majority_ack(&p->repl, written_lsn,
p->config.sync_timeout_ms,
(uint8_t)OPCode_walAck);
if (sync_err == 0)
/* Durable-gated commit advance (task 2.4): advance committed_lsn
* only if written_lsn is durable locally — never on the ACK
* alone (group-commit removed the inline append fsync). */
sync_append_legacy_commit_store(p, written_lsn);
else
CTRL_METRIC_INC(ctrl_repl_degraded_count);
}
}
if (sync_rc_out)
*sync_rc_out = sync_err;
/* sync_err < 0 — replication failed.
* -ENODEV: legacy degraded mode (no standbys, engine-less) — acceptable
* for forward progress.
* -ETIMEDOUT: no per-entry majority within the deadline — the write
* must be rejected to preserve the majority-consistency
* invariant.
* -XCE_ERR_NOT_READY: engine admission cap exhausted — retriable
* rejection.
* Timeout and admission rejection propagate to the job-submit error
* path (ha-commit-engine Req 1.1, 4.4). */
if (sync_err == -ETIMEDOUT || sync_err == -XCE_ERR_NOT_READY)
return sync_err;
return 0;
}
#ifdef XCE_TESTING
/* Test seam (task 2.4): invoke the EXACT durable-gated commit store that the
* legacy sync_append branch uses on a majority ACK, so a test can prove the
* ACK-without-durability hazard is closed without standing up the replication
* wire. Pre-fix this branch stored committed_lsn = written_lsn unconditionally
* (over-advancing past durable_lsn). */
void persist_test_sync_append_legacy_commit_store(persist_t* p,
uint64_t written_lsn)
{
sync_append_legacy_commit_store(p, written_lsn);
}
#endif /* XCE_TESTING */
void persist_log_entry_metadata(const wal_entry_hdr_t* hdr)
{
TRACE("WAL entry: type=%u lsn=%lu term=%lu payload_len=%u crc=%08x",
hdr->type,
(unsigned long)hdr->lsn,
(unsigned long)hdr->term,
hdr->payload_len,
hdr->crc);
}
/* ------------------------------------------------------------------ */
/* Hook functions — AFTER_EVENT handlers that write WAL entries */
/* ------------------------------------------------------------------ */
/* Map HOOK_DETAIL_* values to WAL entry types.
* Returns 0 for details that have no WAL representation (skip). */
static uint32_t detail_to_wal_type(int detail)
{
switch (detail) {
case HOOK_DETAIL_JOB_NEW: return WAL_JOB_SUBMIT;
case HOOK_DETAIL_JOB_START: return WAL_JOB_START;
case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH;
case HOOK_DETAIL_JOB_FAIL: return WAL_JOB_FAIL;
case HOOK_DETAIL_JOB_STOP: return WAL_JOB_STOP;
case HOOK_DETAIL_JOB_RESUME: return WAL_JOB_RESUME;
case HOOK_DETAIL_JOB_UPDATE: return WAL_JOB_UPDATE;
case HOOK_DETAIL_JOB_CKPT: return WAL_JOB_CKPT;
case HOOK_DETAIL_JOB_RUSAGE_UPDATE: return WAL_JOB_RUSAGE;
case HOOK_DETAIL_SETPREASON: return WAL_JOB_PREASON;
case HOOK_DETAIL_CLRPREASON: return WAL_JOB_PREASON;
case HOOK_DETAIL_QUEUE_UPDATE: return WAL_QUEUE_CTRL;
case HOOK_DETAIL_WORKER_JOIN: return WAL_WORKER_CTRL;
case HOOK_DETAIL_WORKER_UPDATE: return WAL_WORKER_CTRL;
case HOOK_DETAIL_WORKER_DEAD: return WAL_WORKER_CTRL;
default: return 0; /* unknown — skip */
}
}
/* Write-path commit classification (ha-commit-barrier-rollout task 1.2).
* Pure lookup over the VERIFIED table (design.md §Write-Path
* Classification); `detail` disambiguates WAL_WORKER_CTRL only.
* Unclassified types fail fast under XCE_TESTING (abort) and fall back to
* RETAIN in production — never dropped, never falsely confirmed (Req 2.2). */
wal_commit_mode_t wal_classify(uint32_t wal_type, int detail)
{
switch (wal_type) {
case WAL_JOB_SUBMIT:
/* client job submit — request-shaped; keeps the in-hook sync path
* (reference implementation, handle_jobsubmit) */
return WAL_MODE_WAIT;
case WAL_QUEUE_CTRL:
/* client queue control — request-shaped; the handler barrier owes
* commit completion before the requester reply (tasks 2-3) */
return WAL_MODE_WAIT;
case WAL_WORKER_CTRL:
/* the only detail-disambiguated type: JOIN/UPDATE have a requester
* awaiting confirmation (WAIT); DEAD is a controller-detected fact.
* Unexpected details take the fact-safe RETAIN side. */
if (detail == HOOK_DETAIL_WORKER_JOIN ||
detail == HOOK_DETAIL_WORKER_UPDATE)
return WAL_MODE_WAIT;
return WAL_MODE_RETAIN;
case WAL_JOB_START:
case WAL_JOB_FINISH:
case WAL_JOB_FAIL:
case WAL_JOB_STOP:
case WAL_JOB_RESUME:
case WAL_JOB_UPDATE:
case WAL_JOB_RUSAGE:
case WAL_JOB_CKPT:
case WAL_JOB_PREASON:
/* worker/scheduler-reported facts — retained until committed.
* FAIL/UPDATE are dual-origin; their request-shaped side is
* realized by the handler barrier in do_jobcontrol_op
* (research.md §2 call-site verification). */
return WAL_MODE_RETAIN;
case WAL_NOOP:
/* internal promotion no-op — replicated like any committed entry,
* retained until committed (ha-commit-barrier-rollout pre-declared
* row: promotion | internal | RETAIN; guard forces classification) */
return WAL_MODE_RETAIN;
case WAL_CONFIG_MARKER:
/* config-change marker (ha-term-safety task 4.2): a committable
* (term,lsn) log entry like a job-state fact — replicated and retained
* until committed; it commits through the SAME barrier as job state
* (R7.1/R7.11), so it is RETAIN, NOT a no-op or a LOCAL marker. The
* requester barrier on a config upgrade is owned by the upgrade path
* (do_upgrade_config_with_bytes), not the per-entry WAIT hook. */
return WAL_MODE_RETAIN;
case WAL_CHECKPOINT:
/* local segment marker — never replicated, by design */
return WAL_MODE_LOCAL;
default:
ERROR("wal_classify: unclassified WAL type %u (detail %d) — "
"RETAIN fallback; add it to the classification table",
wal_type, detail);
#ifdef XCE_TESTING
assert(0 && "unclassified WAL type — add it to wal_classify");
#endif
return WAL_MODE_RETAIN;
}
}
/* Commit barrier helper (ha-commit-barrier-rollout task 2.2, design.md
* §Commit Barrier Helper; Req 1.1, 1.2, 1.3).
*
* Handler-callable WAIT completion on the appended watermark: snapshot the
* last-appended LSN, return 0 when the commit index already covers it,
* else block in the commit engine until coverage or deadline. The engine
* result is returned UNMODIFIED (0 / -ETIMEDOUT / -XCE_ERR_NOT_READY /
* -EINVAL) so callers map it to their own error replies.
*
* Lock discipline: p->lock is held only for the current_lsn snapshot and
* is released BEFORE commit_submit(WAIT) — the engine blocks on ce->lock
* + condvar, and p->lock must never be held across it (engine
* lock-ordering rule). */
int persist_commit_barrier(persist_t* p, uint32_t timeout_ms)
{
uint64_t watermark;
if (p == NULL)
return -EINVAL;
/* Snapshot the last-appended watermark under p->lock. current_lsn is
* the NEXT LSN to assign (persist.h invariant: committed_lsn <=
* current_lsn - 1), so the highest appended entry is current_lsn - 1
* (resolved contract, spec Implementation Notes 2.1). current_lsn ==
* 0 means nothing was ever appended — vacuously satisfied. */
pthread_mutex_lock(&p->lock);
if (p->current_lsn == 0) {
pthread_mutex_unlock(&p->lock);
return 0;
}
watermark = p->current_lsn - 1;
pthread_mutex_unlock(&p->lock);
/* Already covered — committed_lsn is monotone, so coverage observed
* here can never regress. This check also absorbs the LSN-0 first
* entry (watermark 0 vs committed_lsn 0): that once-per-lifetime entry
* has no quorum contract (ha-commit-engine Implementation Notes) and
* commit_submit would reject lsn == 0 anyway. */
if (atomic_load_explicit(&p->committed_lsn, memory_order_acquire) >=
watermark)
return 0;
/* Engine not wired: legacy/bootstrap contexts have no engine commit
* contract — there is nothing to wait on, so the barrier is a no-op.
* Production always wires the engine (cluster_init_phase2). */
if (p->repl.ce == NULL)
return 0;
/* timeout_ms == 0 borrows the configured sync deadline (Req 1.2). */
return commit_submit(p->repl.ce, watermark, COMMIT_MODE_WAIT,
timeout_ms != 0 ? timeout_ms
: p->config.sync_timeout_ms);
}
/* Classified append at the hook seam (barrier-rollout task 1.2).
*
* WAIT (queue/worker control): the append stays async here — the handler
* barrier (tasks 2-3) owes commit completion before the requester reply.
* WAL_JOB_SUBMIT never reaches this helper (it keeps the in-hook sync
* path in persist_job_hook_fun).
*
* RETAIN (facts): append + broadcast, then register the appended LSN with
* the commit engine. commit_submit(COMMIT_MODE_RETAIN) is registration-
* free bookkeeping (commit.c — returns 0 immediately; commit state is
* derived from registry watermarks at ACK time), so the call is contract
* documentation: the entry stays pending purely by lsn > committed_lsn
* and commits later via covering ACKs. Commit-progress results are NEVER
* propagated to hook callers — only the local append rc is (Req 3.1).
*
* Lock context: persist_wal_async takes/releases p->lock internally; the
* appended-LSN read below re-takes p->lock briefly (same discipline as
* persist_wal_sync_append) and releases it BEFORE commit_submit, so no
* p->lock is ever held across ce->lock (engine lock-ordering rule). */
static int wal_hook_append_classified(persist_t* p, uint32_t wal_type,
wal_commit_mode_t mode,
const void* payload, uint32_t len)
{
uint64_t appended_lsn;
int rc;
/* ha-term-safety task 3.1: route the primary's local append through the raft
* library (ha_primary_log_append → raft_log_append) so the LIBRARY stamps the
* live election term and assigns the LSN (firing on_persist_entry → the WAL
* write), keeping p->current_lsn co-tracked with s->log_len. The library is
* the SINGLE primary append entry — no append bypasses it on the raft road.
* On the non-raft road (library not LEADER) ha_primary_log_append falls back
* to persist_wal_async, so engine-less / init-time contexts are unchanged.
* When p->cluster is NULL (persist used standalone in lib tests) keep the
* direct append (no cluster → no raft state to drive). */
if (p->cluster != NULL)
rc = ha_primary_log_append(p->cluster, wal_type, payload, len, NULL);
else
rc = persist_wal_async(p, wal_type, payload, len);
if (rc != 0 || mode != WAL_MODE_RETAIN)
return rc;
if (p->repl.ce == NULL)
return rc; /* engine not wired (bootstrap / engine-less contexts) */
/* current_lsn - 1 is the LSN persist_wal_async just assigned (it
* increments under p->lock). A concurrent append may bump it further;
* that is benign — contiguous advancement means the higher watermark
* still covers this entry. LSN-0 edge: commit_submit rejects lsn == 0
* (first-ever entry of a fresh cluster — accepted interim behavior,
* spec Implementation Notes). */
pthread_mutex_lock(&p->lock);
appended_lsn = p->current_lsn - 1;
pthread_mutex_unlock(&p->lock);
if (appended_lsn != 0)
(void)commit_submit(p->repl.ce, appended_lsn, COMMIT_MODE_RETAIN, 0);
return rc; /* local append rc only — facts have no commit timeout */
}
/* Explicit controller-side raft step for a job submission (raft-apply-decoupling
* item #1 / bug-1306): WAL-append each job in `jlist` and block on the
* majority-commit barrier, returning the commit verdict DIRECTLY to the caller
* (handle_jobsubmit) instead of through the xce_run_hooks fan-out (which returns
* only the last hook's rc and so clobbered this -ETIMEDOUT with a later notify
* hook's 0 — the committed-job-loss root cause).
*
* Returns 0 when every job's entry committed on a majority; the FIRST negative
* rc otherwise (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY:
* admission cap; -ENOMEM: encode failure; or a local-write error). The caller
* MUST reject the submission on a negative rc (never reply success) to preserve
* the Commit Durability invariant. On -ETIMEDOUT / -XCE_ERR_NOT_READY the entry
* WAS appended locally and the LSN advanced, so j->submit_lsn is still tagged —
* the entry remains a pending, uncommitted tail a new-term leader may discard. */
int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist)
{
int rc = 0;
if (p == NULL || jlist == NULL)
return -EINVAL;
for (int i = 0; i < jlist->count && rc == 0; i++) {
job_t* j = (job_t*)xlist_get(jlist, i);
uint8_t* buf = NULL;
size_t sz = 0;
if (capnp_encode_job(&j->job, &buf, &sz) != 0) {
rc = -ENOMEM;
break;
}
/* ha-convergence-recovery (3.3): snapshot the LSN this submit entry
* will receive BEFORE the append. persist_wal_async assigns
* hdr.lsn = current_lsn then increments, so the pre-append current_lsn
* is exactly the written entry's LSN. */
pthread_mutex_lock(&p->lock);
uint64_t submit_lsn = p->current_lsn;
pthread_mutex_unlock(&p->lock);
int sync_rc = 0;
rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz,
&sync_rc);
free(buf);
/* Tag the job with its submit entry's LSN whenever the LOCAL WAL write
* succeeded. persist_wal_sync_append returns the persist_wal_async
* error only on local-write failure; on a replication-degraded outcome
* (0 / -ETIMEDOUT / -ENODEV / -XCE_ERR_NOT_READY) the entry was appended
* locally and the LSN advanced, so the egress tag is set. */
if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY)
j->submit_lsn = submit_lsn;
}
return rc;
}
/* P0 payload: just the detail value (4 bytes).
* Full Cap'n Proto serialization is wired in Task 15. */
int persist_job_hook_fun(int type, int situation, int detail,
void* cls, va_list ap)
{
(void)type;
/* Write to WAL only after the state change is complete.
* BEFORE_EVENT has a different va_list layout and must be ignored. */
if (situation != HOOK_AFTER_EVENT)
return 0;
persist_t* p = cls;
uint32_t wal_type = detail_to_wal_type(detail);
wal_commit_mode_t mode;
if (wal_type == 0)
return 0;
mode = wal_classify(wal_type, detail);
if (mode == WAL_MODE_WAIT) {
/* WAL_JOB_SUBMIT durability (WAL append + majority-commit barrier) was
* MOVED OUT of this hook into the explicit raft step
* persist_submit_wal_and_commit(), called directly by handle_jobsubmit
* (raft-apply-decoupling item #1 / bug-1306). Rationale: xce_run_hooks
* returns only the LAST-registered hook's rc, so a later JOB AFTER_EVENT
* subscriber (sched/apiserver, returning 0) CLOBBERED this branch's
* -ETIMEDOUT — handle_jobsubmit then replied `jobsubmit ok` for an
* entry that never reached majority, and a new-term leader legally
* discarded it (committed-job-loss). The commit verdict must be read
* from the protocol step directly, never as a hook return. This branch
* is now a no-op; the AFTER_EVENT hook chain is notification-only. */
return 0;
}
/* All other job events: payload = wal_jobstate_t {jobid, state}.
* State is read from the job_t after the AFTER_EVENT hook updates it. */
wal_jobstate_t js = {0};
switch (detail) {
case HOOK_DETAIL_JOB_START: {
/* args: cluster_t*, job_pidinfo_t*, job_t* */
(void)va_arg(ap, cluster_t*);
(void)va_arg(ap, void*);
job_t* j = va_arg(ap, job_t*);
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
break;
}
case HOOK_DETAIL_JOB_EXIT: {
/* args: cluster_t*, xce_jobfinish_t*, job_t* */
(void)va_arg(ap, cluster_t*);
(void)va_arg(ap, void*);
job_t* j = va_arg(ap, job_t*);
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
break;
}
case HOOK_DETAIL_JOB_FAIL: {
/* args: cluster_t*, job_failinfo_t*, job_t* */
(void)va_arg(ap, cluster_t*);
(void)va_arg(ap, void*);
job_t* j = va_arg(ap, job_t*);
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
break;
}
default: {
/* STOP, RESUME, UPDATE, CKPT, PREASON, RUSAGE — args: cluster_t*, job_t* */
(void)va_arg(ap, cluster_t*);
job_t* j = va_arg(ap, job_t*);
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
break;
}
}
/* mode == WAL_MODE_RETAIN — fact-shaped job state transition */
return wal_hook_append_classified(p, wal_type, mode, &js, sizeof(js));
}
int persist_queue_hook_fun(int type, int situation, int detail,
void* cls, va_list ap)
{
(void)type;
if (situation != HOOK_AFTER_EVENT)
return 0;
persist_t* p = cls;
uint32_t wal_type = detail_to_wal_type(detail);
wal_commit_mode_t mode;
if (wal_type == 0)
return 0;
mode = wal_classify(wal_type, detail);
/* args: cluster_t*, queue_t* */
(void)va_arg(ap, cluster_t*);
queue_t* q = va_arg(ap, queue_t*);
/* Payload: [wal_queuehdr_t][queue name NUL-terminated] */
const char* name = q->queue.queue ? q->queue.queue : "";
size_t name_len = strlen(name) + 1;
size_t total = sizeof(wal_queuehdr_t) + name_len;
uint8_t* pkt = malloc(total);
if (!pkt) return -ENOMEM;
wal_queuehdr_t hdr = { .status = q->queue.status };
memcpy(pkt, &hdr, sizeof(hdr));
memcpy(pkt + sizeof(hdr), name, name_len);
/* mode == WAL_MODE_WAIT — the append stays async; the handler barrier
* owes commit completion before the requester reply (task 3) */
int rc = wal_hook_append_classified(p, wal_type, mode, pkt,
(uint32_t)total);
free(pkt);
return rc;
}
int persist_worker_hook_fun(int type, int situation, int detail,
void* cls, va_list ap)
{
(void)type;
if (situation != HOOK_AFTER_EVENT)
return 0;
persist_t* p = cls;
uint32_t wal_type = detail_to_wal_type(detail);
wal_commit_mode_t mode;
if (wal_type == 0)
return 0;
mode = wal_classify(wal_type, detail);
/* WAL_WORKER_CTRL payload layout: [uint32_t detail][capnp worker OR name] */
uint32_t disc = (uint32_t)detail;
uint8_t* buf = NULL;
size_t capnp_sz = 0;
int rc = 0;
/* All worker detail types: args are cluster_t* then worker_t*.
* JOIN uses a unified signature with an extra char* name between them;
* DEAD and UPDATE use (cluster_t*, worker_t*) directly. */
(void)va_arg(ap, cluster_t*);
if (detail == HOOK_DETAIL_WORKER_JOIN)
(void)va_arg(ap, char*); /* skip name — present only for JOIN */
worker_t* w = va_arg(ap, worker_t*);
if (!w) return 0; /* guard: failed join produced no worker slot */
rc = capnp_encode_worker(&w->host, &buf, &capnp_sz);
if (rc != 0) return rc;
/* Build [disc | capnp bytes] in one allocation */
size_t total = WORKER_WAL_HDR_SIZE + capnp_sz;
uint8_t* pkt = malloc(total);
if (!pkt) { free(buf); return -ENOMEM; }
memcpy(pkt, &disc, WORKER_WAL_HDR_SIZE);
memcpy(pkt + WORKER_WAL_HDR_SIZE, buf, capnp_sz);
free(buf);
/* JOIN/UPDATE → WAIT (handler barrier owes completion, task 3);
* DEAD → RETAIN (controller-detected fact, registered with the engine) */
rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total);
free(pkt);
return rc;
}
/* ============================================================
* GC retention predicates (P0-5a / P0-8 / P0-13)
*
* Placed in persist.c so the unit test binary (which links only
* persist.c from the controller sources) can access them without
* pulling in RTTI-dependent job.c / worker.c.
* ============================================================ */
bool job_is_terminal(const job_t* j)
{
return (j->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0;
}
bool job_should_retain(const job_t* j, const persist_config_t* cfg)
{
time_t now = time(NULL);
/* TRANSIENT jobs: auto-terminate when TTL elapses after failover.
* transient_since == 0 means the timestamp was never set — do not GC. */
if (j->job.state & JOB_STAT_TRANSIENT) {
if (j->job.transient_since > 0 &&
now > (time_t)j->job.transient_since + (time_t)cfg->transient_job_timeout_s) {
return false;
}
return true;
}
/* Non-terminal jobs are always retained */
if (!job_is_terminal(j)) return true;
/* Terminal jobs: retain until finished_retain_s expires */
return (j->job.finish_time + (time_t)cfg->finished_retain_s >= now);
}
bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg)
{
time_t now = time(NULL);
/* HIDDEN workers: admin explicitly removed — clean up at next checkpoint */
if (w->status & XCE_WORKER_HIDDEN) return false;
/* UNAVAILABLE workers: retain until worker_retain_s expires.
* dead_since == 0 means not yet set — do not GC. */
if (w->status & XCE_WORKER_UNAVAIL) {
if (w->dead_since > 0 &&
now > w->dead_since + (time_t)cfg->worker_retain_s) {
return false;
}
}
return true;
}
/* ============================================================
* WAL truncation (task 7.5)
*
* Remove all WAL entries with LSN > cutoff_lsn.
* Idempotent: calling twice with same cutoff_lsn returns 0 both times.
* ============================================================ */
/* Clamp durable_lsn down to `tip` when the log tip recedes below it (truncate
* / divergent-suffix discard). Without this, durable_lsn could exceed
* current_lsn - 1 after a truncate, violating its invariant and letting the
* commit majority count self at an LSN whose bytes were just removed. MUST be
* called with p->lock held (mirrors the committed_lsn clamp in the truncate
* paths). durable_lsn only ever decreases here — the sole exception to its
* monotonic-non-decreasing rule (ha-commit-durability 2.1 State Management). */
static void clamp_durable_lsn_locked(persist_t* p, uint64_t tip)
{
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
if (dur > tip)
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
}
int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn)
{
pthread_mutex_lock(&p->lock);
uint64_t before = p->current_lsn;
int rc = truncate_wal_above_locked(p, cutoff_lsn);
uint64_t after = p->current_lsn;
pthread_mutex_unlock(&p->lock);
/* raft-apply-decoupling task 1.4: fire the truncation wake only when a
* truncation actually removed entries above cutoff_lsn (current_lsn
* strictly decreased -- both truncate_wal_above_locked branches set
* current_lsn = cutoff_lsn + 1 on a real truncation) -- mirrors the
* durable-advance wake's own before/after guard (see fsync_worker
* above) so a no-op truncate (nothing above cutoff_lsn, or an I/O
* error before current_lsn is touched) never spuriously fails a
* continuation whose entry is still safely in the log. Called AFTER
* releasing p->lock: the callback (the commit engine) takes only
* ce->lock, preserving the cycle-free p->lock -> ce->lock edge already
* established by the durable-advance wake. */
if (rc == 0 && after < before && p->on_truncate != NULL) {
p->on_truncate(p->truncate_cb_ctx, cutoff_lsn);
}
return rc;
}
/* Lock-held core of persist_truncate_wal_above (see the forward declaration and
* the conflict-guard reuse near persist_wal_write_replicated). Caller MUST hold
* p->lock; this function never touches the lock so it is reusable from another
* already-locked path. */
static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn)
{
/* No-op: nothing written yet, or cutoff covers all written entries */
if (p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn) {
return 0;
}
/* --- Collect segment filenames and sort by start LSN --- */
uint64_t seg_lsns[4096];
int nseg = 0;
DIR* d = opendir(p->wal_path);
if (d) {
struct dirent* e;
while ((e = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
if (strncmp(e->d_name, "seg-", 4) != 0)
continue;
uint64_t sl = 0;
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
continue;
seg_lsns[nseg++] = sl;
}
closedir(d);
}
if (nseg == 0) {
return 0;
}
/* Insertion sort ascending by start LSN */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) {
seg_lsns[j + 1] = seg_lsns[j];
j--;
}
seg_lsns[j + 1] = key;
}
/* --- Find segment containing cutoff_lsn --- */
int cutoff_seg_idx = -1;
for (int i = 0; i < nseg; i++) {
if (seg_lsns[i] <= cutoff_lsn)
cutoff_seg_idx = i;
else
break;
}
if (cutoff_seg_idx < 0) {
/* cutoff_lsn is below the oldest retained segment's start LSN, so EVERY
* on-disk entry has LSN > cutoff_lsn and must be removed (the function's
* contract: "remove all entries with LSN > cutoff_lsn"). This is the
* demotion-convergence case (ha-convergence-recovery Req 1.2): a demoted
* ex-primary whose committed boundary sits below its entire retained log
* must discard all of it and collapse the tip to cutoff_lsn + 1. Delete
* every segment, close the active fd, and reset the in-memory tip. */
for (int i = 0; i < nseg; i++) {
char del_path[4096];
snprintf(del_path, sizeof(del_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[i]);
unlink(del_path);
}
if (p->wal_fd >= 0) {
fdatasync(p->wal_fd);
close(p->wal_fd);
p->wal_fd = -1;
}
p->seg_offset = 0;
p->seg_path[0] = '\0';
p->current_lsn = cutoff_lsn + 1;
p->last_crc = WAL_CRC_NONE;
atomic_store_explicit(&p->committed_lsn, cutoff_lsn,
memory_order_release);
clamp_durable_lsn_locked(p, cutoff_lsn);
/* Keep the per-LSN term index consistent: drop terms for removed LSNs. */
truncate_term_index_locked(p, cutoff_lsn);
return 0;
}
/* --- Scan segment to find byte offset of first entry with lsn > cutoff --- */
char cutoff_seg_path[4096];
snprintf(cutoff_seg_path, sizeof(cutoff_seg_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]);
int scan_fd = open(cutoff_seg_path, O_RDONLY);
if (scan_fd < 0) {
return -errno;
}
uint32_t scan_off = 0;
uint32_t trunc_off = 0;
uint32_t new_last_crc = WAL_CRC_NONE;
bool found_cut = false;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t n = pread(scan_fd, &hdr, sizeof(hdr), (off_t)scan_off);
if (n < (ssize_t)sizeof(hdr))
break; /* short read or EOF */
/* Pre-allocated zero region: type==0 is not a valid WAL type (min=1) */
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
break;
if (hdr.lsn > cutoff_lsn) {
trunc_off = scan_off;
found_cut = true;
break;
}
new_last_crc = hdr.crc;
scan_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(scan_fd);
if (!found_cut)
trunc_off = scan_off; /* end of valid data in segment */
/* --- Close the current wal_fd --- */
if (p->wal_fd >= 0) {
fdatasync(p->wal_fd);
close(p->wal_fd);
p->wal_fd = -1;
}
/* --- Truncate the cutoff segment at the computed offset --- */
int new_fd = open(cutoff_seg_path, O_RDWR);
if (new_fd < 0) {
return -errno;
}
if (ftruncate(new_fd, (off_t)trunc_off) != 0) {
int saved = errno;
close(new_fd);
return -saved;
}
fdatasync(new_fd);
/* --- Delete all segments after the cutoff segment --- */
for (int i = cutoff_seg_idx + 1; i < nseg; i++) {
char del_path[4096];
snprintf(del_path, sizeof(del_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[i]);
unlink(del_path);
}
/* --- Update persist_t state under lock --- */
p->wal_fd = new_fd;
p->seg_offset = trunc_off;
snprintf(p->seg_path, sizeof(p->seg_path),
WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]);
p->current_lsn = cutoff_lsn + 1;
p->last_crc = new_last_crc;
atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release);
clamp_durable_lsn_locked(p, cutoff_lsn);
/* Keep the per-LSN term index consistent: drop terms for removed LSNs. */
truncate_term_index_locked(p, cutoff_lsn);
return 0;
}
Reference in a new issue View git blame Copy permalink