Rongsong Shen
16747aa96d
replay_wal() (persist.c ~3902) tallied *new_job_submits_out only for hdr.type == WAL_JOB_SUBMIT. WAL_JOB_ARRAY_SUBMIT entries were dispatched and correctly materialized N real jobmgr entries via apply_committed_entry()'s array case, but never counted -- persist_verify_apply_correctness() then saw real jobmgr growth exceed the undercounted expectation and declared APPLY_DIVERGENT, engaging a PERMANENT, non-self-healing fence (distinct from task 4.2's self-healing Live Apply Resource Fence). Live-reproduced during task 6.1's soak attempt on 2 of 3 nodes in a single small-array (N=2) sub-round via two different trigger paths. Deterministic, not size-dependent -- the first array submission to a live cluster would fence any node that restarts or hits an election-path recovery fallback before its next checkpoint, shrinking the promotable pool. A pre-existing comment at persist.c ~3863-3874 (predating this spec) had already flagged this exact code region as needing re-verification before tasks 4.1/6.1+ went live; this is that re-verification, and it failed. Fix adds a WAL_JOB_ARRAY_SUBMIT branch tallying apply_rc (the genuine per-record element count apply_committed_entry()'s array case already returns) when apply_rc > 0, mirroring the existing dedup-safe pattern for WAL_JOB_SUBMIT. Traced the full return-value space of restore_job_array()/apply_committed_entry()'s array case to confirm every tallied value is provably equal to real jobmgr growth -- the fix cannot overcount and so cannot mask a genuine divergence. Verified via a from-scratch live 3-node HA soak re-run (--array-soak), independently by both implementer and reviewer: zero APPLY_DIVERGENT across all three original trigger paths, one-WAL-record/one-commit- barrier confirmed for N=2 and N=10000, per-element post-failover status/cancel/completion confirmed. Task 6.0a of job-array-single-wal-record (Requirements 3.1, 3.2, 4.1, 9.4). Inserted mid-run after live soak testing surfaced the bug; user confirmed sequencing (fix now, blocking task 6.1+).
5953 lines
257 KiB
C
5953 lines
257 KiB
C
#define _GNU_SOURCE
|
||
#include "persist.h"
|
||
#include "config.h"
|
||
#include "ctrl_metrics.h"
|
||
#include "hook.h"
|
||
#include "xce/cluster_def.h"
|
||
#include "xce/errnox.h"
|
||
#include "xce/types.h"
|
||
#include "controller.capnp.h"
|
||
#include "xlist.h"
|
||
#include "cluster.h"
|
||
#include "commit.h" /* commit_submit RETAIN registration (barrier-rollout 1.2) */
|
||
|
||
#include <assert.h>
|
||
#include <stdbool.h>
|
||
#include <dirent.h>
|
||
#include <errno.h>
|
||
#include <fcntl.h>
|
||
#include <inttypes.h>
|
||
#include <pthread.h>
|
||
#include <stdatomic.h>
|
||
#include <stdint.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <sys/stat.h>
|
||
#include <sys/statfs.h>
|
||
#include <time.h>
|
||
#include <unistd.h>
|
||
#include <zlib.h>
|
||
|
||
/* restore_* functions defined in job.c / worker.c / queue.c */
|
||
extern int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload,
|
||
size_t len, uint64_t entry_lsn);
|
||
extern int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state);
|
||
extern int worker_from_capnp(cluster_t* c,
|
||
const void* capnp_payload, size_t len);
|
||
extern int restore_queue_state(cluster_t* c,
|
||
const char* name, uint32_t status);
|
||
|
||
/* raft-apply-decoupling task 3.2: pending command-apply hand-off list node
|
||
* (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is
|
||
* the consumer). Declared here (rather than persist.h) so the list stays
|
||
* an implementation detail — persist.h exposes only the opaque `void*`
|
||
* head field plus persist_register_pending_apply/persist_has_pending_apply.
|
||
* See persist.h's doc comment on persist_register_pending_apply for the
|
||
* full contract. */
|
||
typedef struct pending_command_apply {
|
||
uint64_t lsn;
|
||
pending_apply_notify_fn on_applied;
|
||
void* ctx;
|
||
struct pending_command_apply* next;
|
||
} pending_command_apply_t;
|
||
|
||
/* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */
|
||
|
||
/* Serialize xce_job_t to a malloc'd Cap'n Proto buffer.
|
||
* Caller must free(*out) on success. */
|
||
static int capnp_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct Job cj;
|
||
Job_ptr jp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_Job(cs, &cj, (xce_job_t*)j);
|
||
jp = new_Job(cs);
|
||
write_Job(&cj, jp);
|
||
capn_setp(capn_root(&c), 0, jp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* Serialize xce_worker_t to a malloc'd Cap'n Proto buffer. */
|
||
static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct Worker cw;
|
||
Worker_ptr wp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_Worker(cs, &cw, (xce_worker_t*)w);
|
||
wp = new_Worker(cs);
|
||
write_Worker(&cw, wp);
|
||
capn_setp(capn_root(&c), 0, wp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */
|
||
#define WORKER_WAL_HDR_SIZE sizeof(uint32_t)
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Constants */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
#define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal"
|
||
|
||
/* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */
|
||
#define NFS_SUPER_MAGIC 0x6969
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Internal helpers */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
static uint64_t now_us(void)
|
||
{
|
||
struct timespec ts;
|
||
clock_gettime(CLOCK_REALTIME, &ts);
|
||
return (uint64_t)ts.tv_sec * 1000000ULL + (uint64_t)(ts.tv_nsec / 1000);
|
||
}
|
||
|
||
/* Recursive mkdir — equivalent to mkdir -p */
|
||
static int mkdir_p(const char* path, mode_t mode)
|
||
{
|
||
char tmp[4096];
|
||
char* p;
|
||
size_t len;
|
||
|
||
snprintf(tmp, sizeof(tmp), "%s", path);
|
||
len = strlen(tmp);
|
||
if (len > 0 && tmp[len - 1] == '/')
|
||
tmp[len - 1] = '\0';
|
||
|
||
for (p = tmp + 1; *p; p++) {
|
||
if (*p == '/') {
|
||
*p = '\0';
|
||
if (mkdir(tmp, mode) != 0 && errno != EEXIST)
|
||
return -1;
|
||
*p = '/';
|
||
}
|
||
}
|
||
if (mkdir(tmp, mode) != 0 && errno != EEXIST)
|
||
return -1;
|
||
return 0;
|
||
}
|
||
|
||
/* Open (and pre-allocate) a new WAL segment named after current_lsn */
|
||
static int open_new_segment(persist_t* p)
|
||
{
|
||
snprintf(p->seg_path, sizeof(p->seg_path),
|
||
WAL_SEG_FMT, p->wal_path, p->current_lsn);
|
||
|
||
int fd = open(p->seg_path, O_RDWR | O_CREAT | O_TRUNC, 0600);
|
||
if (fd < 0)
|
||
return -errno;
|
||
|
||
/* Pre-allocate segment space to avoid fragmentation and metadata updates
|
||
* on every write. posix_fallocate is best-effort — ignore errors. */
|
||
posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes);
|
||
|
||
p->wal_fd = fd;
|
||
p->seg_offset = 0;
|
||
return 0;
|
||
}
|
||
|
||
/* Close current segment and open a fresh one */
|
||
static int rotate_segment(persist_t* p)
|
||
{
|
||
if (p->wal_fd >= 0) {
|
||
fdatasync(p->wal_fd);
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
return open_new_segment(p);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Demand-driven group-commit flusher (ha-commit-durability task 2.1) */
|
||
/* */
|
||
/* Repurposes the old fixed-interval fsync timer into a flusher driven */
|
||
/* by the append-vs-synced gap. On each pass it batches one fdatasync */
|
||
/* over all entries appended since the last sync, advances durable_lsn */
|
||
/* to the synced tip, and wakes the commit evaluation. --wal-sync-ms */
|
||
/* now bounds the MAX batching latency (a no-further-append entry is */
|
||
/* still synced within the cadence), not a durability gap. */
|
||
/* */
|
||
/* LOST-WAKEUP / CORRECTNESS REASONING (gating item): */
|
||
/* */
|
||
/* The "work pending" condition is LEVEL-triggered on two persistent */
|
||
/* variables, NOT an edge: pending ⇔ durable_lsn < current_lsn - 1 */
|
||
/* (flusher_work_pending below, evaluated under p->lock). Because it */
|
||
/* is a comparison of stored state — not a transient flag consumed on */
|
||
/* read — a signal that races the flusher's wait can never be lost: */
|
||
/* */
|
||
/* - persist_wal_async advances current_lsn under p->lock, THEN */
|
||
/* signals fsync_cond. If the flusher is mid-flush when the append */
|
||
/* lands, current_lsn has already moved past the tip the flusher */
|
||
/* captured, so after that flush durable_lsn < current_lsn - 1 */
|
||
/* STILL holds; the flusher re-checks the level under p->lock on */
|
||
/* its next loop iteration and flushes again (it does NOT wait */
|
||
/* first). The missed signal is therefore harmless — the level */
|
||
/* itself carries the pending work forward. */
|
||
/* - If the append's signal arrives while the flusher sits in */
|
||
/* pthread_cond_wait, it wakes it. If it arrives in the tiny window */
|
||
/* after the flusher decided "no work" but before it re-enters the */
|
||
/* wait, the flusher re-evaluates the level under p->lock before */
|
||
/* waiting (we re-check inside the fsync_mutex-held region by */
|
||
/* reading current_lsn/durable_lsn under p->lock first), so it */
|
||
/* will not block while work is pending. */
|
||
/* */
|
||
/* "NEVER durable before its fdatasync": the flusher snapshots */
|
||
/* tip = current_lsn - 1 AND calls fdatasync WHILE HOLDING p->lock. */
|
||
/* persist_wal_async also pwrites under p->lock, so no append can */
|
||
/* extend the segment between the tip snapshot and the fdatasync — by */
|
||
/* the time fdatasync returns, every byte for LSNs <= tip is on stable */
|
||
/* storage. durable_lsn is only then advanced to tip (monotonic max). */
|
||
/* Releasing p->lock and broadcasting commit-eval happens after the */
|
||
/* advance, so a WAIT caller that observes durable_lsn >= its LSN is */
|
||
/* guaranteed the bytes are synced. */
|
||
/* */
|
||
/* Lock discipline: p->lock and fsync_mutex are NEVER held together */
|
||
/* (fsync_mutex guards only the cond wait; the flush section takes */
|
||
/* p->lock alone). This keeps fdatasync off any lock the iomgr / */
|
||
/* heartbeat path needs and preserves liveness (Req 1.6). */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Test-only fault-injection probe (ha-commit-durability task 2.1 error path).
|
||
* Returns true when the test has forced fdatasync to be treated as failed.
|
||
* Compiled to a constant `false` in production (no -DXCE_TESTING), so the
|
||
* fail-loud gate below collapses to the pure rc != 0 check with zero overhead
|
||
* and no dependency on the test-only struct field. */
|
||
static inline bool flusher_fsync_forced_fail(persist_t* p)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
return atomic_load_explicit(&p->test_force_fsync_fail,
|
||
memory_order_acquire) != 0;
|
||
#else
|
||
(void)p;
|
||
return false;
|
||
#endif
|
||
}
|
||
|
||
/* Returns true iff there is un-synced appended data (level predicate).
|
||
* MUST be called with p->lock held. current_lsn - 1 is the appended tip
|
||
* (current_lsn is the NEXT lsn to assign); LSN 0 reserved, so a fresh
|
||
* store has current_lsn == 1 → tip 0 → no pending work. */
|
||
static bool flusher_work_pending_locked(persist_t* p)
|
||
{
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
return dur < tip;
|
||
}
|
||
|
||
/* Perform one group-commit flush: snapshot the appended tip, fdatasync the
|
||
* live segment, and advance durable_lsn to the synced tip. Takes p->lock
|
||
* for the whole snapshot+sync+advance so no concurrent append can extend
|
||
* the segment between the tip read and the fdatasync. Returns the new
|
||
* durable_lsn (unchanged if there was nothing to sync). */
|
||
static uint64_t flusher_flush_once(persist_t* p)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
|
||
if (dur >= tip || p->wal_fd < 0) {
|
||
/* Nothing new to sync (or no segment yet). Still flush the fd for a
|
||
* clean-shutdown / fallback pass, but do not advance the watermark.
|
||
* The watermark is already <= tip here, so a failed fallback sync
|
||
* cannot make it claim more than is durable — but we still surface
|
||
* the error loudly so a degrading disk is never silent (Req 1.1). */
|
||
if (p->wal_fd >= 0) {
|
||
int rc = fdatasync(p->wal_fd);
|
||
if (rc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: flusher fallback fdatasync FAILED (errno=%d %s)"
|
||
" — durable_lsn held at %" PRIu64 "; WAL not advanced",
|
||
e, strerror(e), dur);
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&p->lock);
|
||
return dur;
|
||
}
|
||
|
||
/* fdatasync under p->lock: appends also hold p->lock to pwrite, so the
|
||
* segment cannot grow between this tip snapshot and the sync. Every byte
|
||
* for LSNs <= tip is therefore on stable storage when fdatasync returns
|
||
* SUCCESSFULLY. */
|
||
int rc = fdatasync(p->wal_fd);
|
||
|
||
/* FAIL-LOUD GATE (Req 1.1/1.2): advance durable_lsn ONLY on a successful
|
||
* sync. On EIO/ENOSPC the bytes are NOT on stable storage, so leaving the
|
||
* watermark unchanged keeps the non-durable LSNs from being counted toward
|
||
* the commit majority. We do NOT retry into an advanced watermark — the
|
||
* level predicate stays pending and the next pass retries the sync. */
|
||
if (rc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: flusher fdatasync FAILED (errno=%d %s) — durable_lsn "
|
||
"held at %" PRIu64 " (tip=%" PRIu64 " NOT made durable); commit "
|
||
"majority will not count these LSNs",
|
||
e, strerror(e), dur, tip);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return dur;
|
||
}
|
||
|
||
/* Monotonic advance to the synced tip — never past it (the invariant
|
||
* durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1
|
||
* was read under this same lock). One fdatasync advances durable_lsn past
|
||
* however many LSNs were appended since the last sync (batching). */
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6):
|
||
* a liveness test asserts this is the dedicated flusher thread, never the
|
||
* caller/iomgr/heartbeat thread. Written under p->lock; read after the
|
||
* flusher has caught up, so no extra synchronization is needed. */
|
||
p->test_last_fdatasync_thread = pthread_self();
|
||
atomic_store_explicit(&p->test_last_fdatasync_thread_set, 1,
|
||
memory_order_release);
|
||
#endif
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
return tip;
|
||
}
|
||
|
||
static void* fsync_worker(void* arg)
|
||
{
|
||
persist_t* p = arg;
|
||
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) {
|
||
/* Re-evaluate the level predicate BEFORE waiting so a signal that
|
||
* raced in just before we entered the wait is not needed for
|
||
* correctness — if work is already pending we skip the wait entirely
|
||
* and flush. (Reads current_lsn/durable_lsn under p->lock.) */
|
||
pthread_mutex_lock(&p->lock);
|
||
bool pending = flusher_work_pending_locked(p);
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (!pending) {
|
||
/* No pending work: wait for an append-signal OR the cadence
|
||
* deadline (bounded fallback flush so a lone no-further-append
|
||
* entry is still synced within wal_sync_ms — Req 3.1). */
|
||
long ms = (long)p->config.wal_sync_ms;
|
||
struct timespec dl;
|
||
clock_gettime(CLOCK_REALTIME, &dl);
|
||
dl.tv_sec += ms / 1000;
|
||
dl.tv_nsec += (ms % 1000) * 1000000L;
|
||
if (dl.tv_nsec >= 1000000000L) {
|
||
dl.tv_sec += 1;
|
||
dl.tv_nsec -= 1000000000L;
|
||
}
|
||
pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl);
|
||
|
||
if (atomic_load_explicit(&p->fsync_stop, memory_order_acquire))
|
||
break;
|
||
}
|
||
|
||
/* Flush section: never hold fsync_mutex across the fdatasync (the
|
||
* flush takes p->lock alone). Release fsync_mutex, flush, re-acquire.
|
||
* After the flush we loop and re-check the level — if a new append
|
||
* landed during the flush, the level is still pending and we flush
|
||
* again without needing its signal (lost-wakeup-safe). */
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
uint64_t before = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
uint64_t after = flusher_flush_once(p);
|
||
|
||
/* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced
|
||
* INSIDE flusher_flush_once (atomic store with release) BEFORE we
|
||
* return here, and flusher_flush_once has already RELEASED p->lock.
|
||
* Now — holding NO persist lock — invoke the registered durable-advance
|
||
* callback so the commit engine re-evaluates its majority and wakes any
|
||
* COMMIT_MODE_WAIT caller whose entry just became durable locally.
|
||
*
|
||
* LOST-WAKEUP SAFETY: the wake fires strictly AFTER the durable_lsn
|
||
* release-store, and the commit engine's predicate re-reads durable_lsn
|
||
* via an acquire load under ce->lock — so a WAIT caller either (a) sees
|
||
* the advanced watermark on its own re-check, or (b) is blocked in
|
||
* pthread_cond_wait and is woken by this broadcast; it can never both
|
||
* miss the watermark AND miss the wake. We only fire the callback when
|
||
* durable_lsn actually advanced (after > before): a no-op flush (or a
|
||
* failed fdatasync that held the watermark) needs no commit re-eval,
|
||
* avoiding spurious wakes.
|
||
*
|
||
* LOCK ORDERING (cycle-free): the flusher releases p->lock inside
|
||
* flusher_flush_once, THEN the callback takes ce->lock (commit side).
|
||
* The commit side only ever takes ce->lock and reads persist atomics —
|
||
* it never takes p->lock while holding ce->lock. So the only ordering
|
||
* is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no
|
||
* deadlock cycle. The callback explicitly MUST NOT take p->lock. */
|
||
if (after > before && p->on_durable_advance != NULL) {
|
||
p->on_durable_advance(p->durable_cb_ctx);
|
||
}
|
||
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
}
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
return NULL;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ *
|
||
* WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) *
|
||
* ------------------------------------------------------------------ */
|
||
|
||
/* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN
|
||
* term index (R1.5). Grows the index on demand; LSNs replay/append contiguously
|
||
* so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous
|
||
* LSN (should not happen on the contiguous WAL) is zero-filled by realloc-then-
|
||
* memset, which read back as "absent" until/unless written. The raw term
|
||
* (including a legacy 0) is stored verbatim; the legacy-0 -> WAL_LEGACY_TERM
|
||
* mapping is applied only at READ-BACK (persist_entry_term), keeping "present
|
||
* legacy-0" distinguishable from "absent".
|
||
*
|
||
* Callers: the live append path (holds p->lock) and the single-threaded
|
||
* recovery path (replay_wal — no other thread touches the index before the node
|
||
* participates, so no lock is required there). Best-effort: an allocation
|
||
* failure is logged loudly and leaves the index unextended (read-back then
|
||
* reports the LSN absent) — it never aborts and never corrupts existing
|
||
* entries. */
|
||
static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return; /* LSN 0 is reserved — never an entry */
|
||
|
||
if (lsn > p->term_index_cap) {
|
||
/* Grow geometrically to amortize; never below the requested LSN. */
|
||
uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64;
|
||
if (new_cap < lsn)
|
||
new_cap = lsn;
|
||
uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t));
|
||
if (!grown) {
|
||
ERROR("persist: term-index realloc failed (lsn=%" PRIu64
|
||
", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read "
|
||
"back as absent", lsn, p->term_index_cap, new_cap);
|
||
return;
|
||
}
|
||
/* Zero-fill the newly grown region so any unwritten slot reads absent. */
|
||
memset(grown + p->term_index_cap, 0,
|
||
(new_cap - p->term_index_cap) * sizeof(uint64_t));
|
||
p->term_index = grown;
|
||
p->term_index_cap = new_cap;
|
||
}
|
||
|
||
p->term_index[lsn - 1] = term;
|
||
if (lsn > p->term_index_count)
|
||
p->term_index_count = lsn;
|
||
}
|
||
|
||
/* entry_term_locked: the lock-free core of persist_entry_term — read the
|
||
* recorded Raft term for `lsn` from the in-memory term index, applying the
|
||
* legacy-0 rule (R1.8: a present term-0 entry maps to WAL_LEGACY_TERM; an absent
|
||
* LSN returns 0). Caller MUST hold p->lock. Used by the append path
|
||
* (persist_wal_async) to source the AppendEntries prev_log_term while already
|
||
* under the lock — calling persist_entry_term there would re-acquire p->lock
|
||
* and deadlock. */
|
||
static uint64_t entry_term_locked(const persist_t* p, uint64_t lsn)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return 0;
|
||
if (lsn <= p->term_index_count && p->term_index != NULL) {
|
||
uint64_t raw = p->term_index[lsn - 1];
|
||
return (raw == 0) ? WAL_LEGACY_TERM : raw;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/* Shrink the in-memory per-LSN term index so that every LSN > `keep_lsn` reads
|
||
* back as absent (ha-term-safety task 2.2). Called from the truncate path after
|
||
* the on-disk entries above keep_lsn are removed, keeping the term index
|
||
* consistent with the WAL: a later persist_entry_term on a truncated LSN must
|
||
* return 0. The backing allocation is left as-is (only the live count moves);
|
||
* the zeroed-on-grow contract means a later re-append at those LSNs overwrites
|
||
* cleanly. Caller must hold p->lock. */
|
||
static void truncate_term_index_locked(persist_t* p, uint64_t keep_lsn)
|
||
{
|
||
if (!p)
|
||
return;
|
||
if (keep_lsn < p->term_index_count)
|
||
p->term_index_count = keep_lsn;
|
||
}
|
||
|
||
void persist_set_term(persist_t* p, uint64_t term)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* p->term is read on the append path under p->lock; take the lock so the
|
||
* stamped value is consistent with the append that reads it. */
|
||
pthread_mutex_lock(&p->lock);
|
||
p->term = term;
|
||
pthread_mutex_unlock(&p->lock);
|
||
}
|
||
|
||
uint64_t persist_entry_term(const persist_t* p, uint64_t lsn)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return 0;
|
||
|
||
/* Take p->lock to read consistently against concurrent record_entry_term
|
||
* on the append path. Cast away const — the lock is logically a mutable
|
||
* sub-object of an otherwise-const read. */
|
||
persist_t* mp = (persist_t*)p;
|
||
pthread_mutex_lock(&mp->lock);
|
||
|
||
/* Legacy-zero rule (R1.8) lives in entry_term_locked: a present entry
|
||
* carrying term 0 maps to the single lowest-known real Raft term, applied
|
||
* uniformly so two legacy-0 entries at the same LSN match; absent LSNs
|
||
* return 0. */
|
||
uint64_t result = entry_term_locked(mp, lsn);
|
||
|
||
pthread_mutex_unlock(&mp->lock);
|
||
return result;
|
||
}
|
||
|
||
uint64_t persist_entry_term_locked(const persist_t* p, uint64_t lsn)
|
||
{
|
||
return entry_term_locked(p, lsn);
|
||
}
|
||
|
||
/* persist_reconcile_checkpoint_term — raft-ha-core-safety task 3.2 (Req
|
||
* 3.1/5.1), Snapshot Boundary Reconcile. See persist.h for the full contract.
|
||
* Reuses record_entry_term (same translation unit) to authoritatively
|
||
* overwrite whatever term (if any) the local term index held at
|
||
* checkpoint_lsn — this is a genuine reconcile-overwrite, not an idempotent
|
||
* skip, because a pre-snapshot divergent entry may have left a stale local
|
||
* term at this exact LSN. Only the checkpoint LSN's own term is reconciled:
|
||
* post-install Raft semantics mean no AppendEntries prevLogIndex/prevLogTerm
|
||
* check against this member ever references an LSN below the checkpoint
|
||
* again (the checkpoint is the new log-start boundary), and the WAL bytes
|
||
* below it were compacted away, so there is no data to reconstruct a full
|
||
* per-LSN history from even if it were needed. */
|
||
void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn,
|
||
uint64_t checkpoint_term)
|
||
{
|
||
if (!p || checkpoint_lsn == 0)
|
||
return;
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
record_entry_term(p, checkpoint_lsn, checkpoint_term);
|
||
pthread_mutex_unlock(&p->lock);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Public API */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
void persist_set_workdir(persist_t* p, const char* workdir, const char* name)
|
||
{
|
||
char effective[4096];
|
||
snprintf(effective, sizeof(effective), "%s/%s", workdir, name);
|
||
snprintf(p->wal_path, sizeof(p->wal_path), "%s/data/wal", effective);
|
||
snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective);
|
||
}
|
||
|
||
int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
|
||
{
|
||
p->cluster = c;
|
||
p->config = cfg ? *cfg : persist_config_defaults();
|
||
p->wal_fd = -1;
|
||
p->last_replicated_crc = WAL_CRC_NONE;
|
||
p->last_crc = WAL_CRC_NONE;
|
||
|
||
/* LSN 0 is RESERVED as the "none / empty" sentinel — it is NEVER assigned
|
||
* to a real WAL entry. A fresh store therefore starts current_lsn at 1, so
|
||
* the first-ever appended entry (e.g. the promotion WAL_NOOP) is lsn 1 and
|
||
* lsn 0 unambiguously means "nothing here":
|
||
* - committed_lsn == 0 → nothing committed (the lsn-1 NOOP is committable
|
||
* 0 → 1, vs. the old lsn-0 entry which could never advance committed_lsn);
|
||
* - writable_lsn == 0 → own-term gate off (the gate armed at lsn 1 is
|
||
* effective, vs. lsn 0 which read as "gate off" on a cold-start leader);
|
||
* - standby_lsn == 0 → empty standby (the streamer's `hdr.lsn > 0`
|
||
* filter streams the lsn-1 entry, closing the cold-start WAL hole that
|
||
* dropped lsn 0 on `0 > 0`).
|
||
* persist_recover() overwrites current_lsn from the durable snapshot/WAL tip
|
||
* (load_snapshot / replay_wal), so this fresh default only applies before any
|
||
* recovery — i.e. a brand-new store with no snapshot and no WAL. */
|
||
p->current_lsn = 1;
|
||
|
||
/* Enforce minimum value for max_wal_records to prevent runaway checkpointing */
|
||
if (p->config.max_wal_records < 100) {
|
||
ERROR("persist: max_wal_records=%u too low (minimum 100) — setting to 100",
|
||
p->config.max_wal_records);
|
||
p->config.max_wal_records = 100;
|
||
}
|
||
|
||
/* Enforce minimum value for wal_segment_bytes to prevent a rotation storm
|
||
* (must comfortably fit at least one max-payload entry: header + up to
|
||
* MAX_WAL_PAYLOAD_BYTES). */
|
||
if (p->config.wal_segment_bytes < 4096) {
|
||
ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096",
|
||
p->config.wal_segment_bytes);
|
||
p->config.wal_segment_bytes = 4096;
|
||
}
|
||
|
||
/* Initialise atomic checkpoint-trigger counters */
|
||
atomic_store(&p->wal_record_count, 0);
|
||
atomic_store(&p->checkpoint_in_progress, 0);
|
||
|
||
/* Durable watermark starts at 0 ("nothing fdatasync'd"): mirrors the
|
||
* LSN-0-reserved committed_lsn convention. It advances only on the
|
||
* flusher thread after a successful fdatasync (ha-commit-durability 2.1). */
|
||
atomic_store(&p->durable_lsn, 0);
|
||
|
||
/* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1):
|
||
* a fresh store has no corruption. persist_recover overwrites this with the
|
||
* replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */
|
||
p->recovery_verdict = RECOVER_CLEAN;
|
||
|
||
/* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh
|
||
* store is promotable. Set by persist_recover_loud_fence_if_unrecoverable
|
||
* on an unrecoverable boot; cleared on catch-up (never self-exits). */
|
||
atomic_store(&p->recovery_unrecoverable, false);
|
||
|
||
/* Post-replay apply-correctness verdict defaults to CLEAN (raft-apply-
|
||
* decoupling task 5.1, Req 8.2): a fresh store has nothing to diverge
|
||
* from. persist_recover overwrites both after every replay. */
|
||
p->apply_verdict = APPLY_CLEAN;
|
||
atomic_store(&p->last_applied_lsn, 0);
|
||
|
||
/* Live Apply Resource Fence starts clear (job-array-single-wal-record
|
||
* task 4.2, Req 9.4): a fresh store has nothing stuck failing to apply.
|
||
* Set by persist_mark_apply_resource_fenced on a resource-exhaustion
|
||
* apply failure; cleared by persist_clear_apply_resource_fence the
|
||
* instant a retry of the same LSN succeeds -- see persist_t's field doc
|
||
* comment for why this is deliberately separate from
|
||
* recovery_unrecoverable just above. */
|
||
atomic_store(&p->apply_resource_fenced_lsn, 0);
|
||
|
||
/* bug-1416: defaults to true (nothing to gate) so every fixture that
|
||
* calls persist_init() but never persist_recover() -- most of this
|
||
* spec's own test suite -- sees identical, ungated apply-pump/live-
|
||
* replication behavior to before this flag existed. Only persist_
|
||
* recover() ever sets this false, for its own single-threaded
|
||
* replay+verify duration (see the field's doc comment in persist.h). */
|
||
atomic_store(&p->recovery_complete, true);
|
||
|
||
/* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2):
|
||
* the commit engine registers it in commit_engine_init. Until then every
|
||
* flusher durable advance is a no-op wake. */
|
||
p->on_durable_advance = NULL;
|
||
p->durable_cb_ctx = NULL;
|
||
|
||
/* Truncation wake hook starts unregistered (raft-apply-decoupling task
|
||
* 1.4): the commit engine registers it in commit_engine_init. Until
|
||
* then every truncation is a no-op wake. */
|
||
p->on_truncate = NULL;
|
||
p->truncate_cb_ctx = NULL;
|
||
|
||
/* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries
|
||
* recorded yet. Grown on demand by record_entry_term during replay/append;
|
||
* freed in persist_fini. */
|
||
p->term_index = NULL;
|
||
p->term_index_count = 0;
|
||
p->term_index_cap = 0;
|
||
|
||
/* Pending command-apply hand-off list starts empty (raft-apply-
|
||
* decoupling task 3.2): no committed-outcome continuations have
|
||
* registered yet. Freed in persist_fini. */
|
||
p->pending_applies = NULL;
|
||
pthread_mutex_init(&p->pending_applies_lock, NULL);
|
||
|
||
/* Ordered apply pump starts idle (raft-apply-decoupling task 6.1): no
|
||
* walk scheduled or running yet. Freed/destroyed in persist_fini. */
|
||
pthread_mutex_init(&p->apply_pump_lock, NULL);
|
||
atomic_store(&p->apply_pump_scheduled, 0);
|
||
|
||
/* Apply-pump segment-lookup cache starts invalid (raft-apply-decoupling
|
||
* task 7.4 perf-fix): no segment/fd cached yet. -1 is always a safe
|
||
* fallback trigger for apply_pump_read_entry's fast path (see persist.h
|
||
* doc comment). */
|
||
p->apply_pump_cache_fd = -1;
|
||
p->apply_pump_cache_seg_lsn = 0;
|
||
p->apply_pump_cache_off = 0;
|
||
p->apply_pump_cache_next_lsn = 0;
|
||
|
||
/* Create WAL and snapshot directories */
|
||
if (mkdir_p(p->wal_path, 0700) != 0) return -errno;
|
||
if (mkdir_p(p->snap_path, 0700) != 0) return -errno;
|
||
|
||
/* Reject NFS mounts — fsync durability is not guaranteed over NFS */
|
||
struct statfs sfs;
|
||
if (statfs(p->wal_path, &sfs) == 0 &&
|
||
(unsigned long)sfs.f_type == (unsigned long)NFS_SUPER_MAGIC) {
|
||
ERROR("persist: WAL path %s is on NFS — unsafe for fsync durability",
|
||
p->wal_path);
|
||
return -EREMOTE;
|
||
}
|
||
|
||
pthread_mutex_init(&p->lock, NULL);
|
||
pthread_mutex_init(&p->checkpoint_mutex, NULL);
|
||
pthread_cond_init(&p->checkpoint_done, NULL);
|
||
pthread_mutex_init(&p->fsync_mutex, NULL);
|
||
pthread_cond_init(&p->fsync_cond, NULL);
|
||
repl_init(&p->repl); /* P2-4: fan-out replicator lifecycle */
|
||
|
||
WARN("WAL/snapshot stored unencrypted in %s — "
|
||
"ensure filesystem is encrypted if job payloads contain sensitive data",
|
||
p->wal_path);
|
||
return 0;
|
||
}
|
||
|
||
int persist_start(persist_t* p)
|
||
{
|
||
ctrl_metrics_init();
|
||
|
||
/* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */
|
||
if (p->cluster != NULL) {
|
||
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||
(hook_fn_t)persist_job_hook_fun, p);
|
||
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
|
||
(hook_fn_t)persist_queue_hook_fun, p);
|
||
xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
(hook_fn_t)persist_worker_hook_fun, p);
|
||
}
|
||
|
||
atomic_store_explicit(&p->fsync_stop, 0, memory_order_release);
|
||
int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p);
|
||
if (rc == 0)
|
||
atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release);
|
||
return rc;
|
||
}
|
||
|
||
void persist_stop(persist_t* p)
|
||
{
|
||
if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) {
|
||
/* Set the stop flag and wake the timer wait under fsync_mutex so the
|
||
* worker cannot miss the signal between its flag check and the wait. */
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
atomic_store_explicit(&p->fsync_stop, 1, memory_order_release);
|
||
pthread_cond_signal(&p->fsync_cond);
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
pthread_join(p->fsync_thread, NULL);
|
||
atomic_store_explicit(&p->fsync_thread_running, 0, memory_order_release);
|
||
}
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
if (p->wal_fd >= 0) {
|
||
/* Final synchronous flush-or-exit on stop: fdatasync everything
|
||
* appended so far and advance durable_lsn to the synced tip so a
|
||
* post-stop read reflects the durable state (the flusher thread is
|
||
* already joined and will not advance it again).
|
||
*
|
||
* FAIL-LOUD GATE (Req 1.1/1.2): advance the watermark ONLY when the
|
||
* final sync SUCCEEDS — on EIO/ENOSPC the bytes are not on stable
|
||
* storage, so durable_lsn must stay where it is and the failure is
|
||
* surfaced loudly. */
|
||
int rc = fdatasync(p->wal_fd);
|
||
if (rc == 0 && !flusher_fsync_forced_fail(p)) {
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip,
|
||
memory_order_release);
|
||
} else {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: final fdatasync on stop FAILED (errno=%d %s) — "
|
||
"durable_lsn left unchanged; appended-but-unsynced entries "
|
||
"are NOT durable",
|
||
e, strerror(e));
|
||
}
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
pthread_mutex_unlock(&p->lock);
|
||
}
|
||
|
||
void persist_fini(persist_t* p)
|
||
{
|
||
repl_fini(&p->repl); /* P2-4: fan-out replicator lifecycle */
|
||
pthread_mutex_destroy(&p->lock);
|
||
pthread_mutex_destroy(&p->checkpoint_mutex);
|
||
pthread_cond_destroy(&p->checkpoint_done);
|
||
pthread_mutex_destroy(&p->fsync_mutex);
|
||
pthread_cond_destroy(&p->fsync_cond);
|
||
|
||
/* Free the in-memory per-LSN term index (ha-term-safety 1.3). */
|
||
free(p->term_index);
|
||
p->term_index = NULL;
|
||
p->term_index_count = 0;
|
||
p->term_index_cap = 0;
|
||
|
||
/* Free any pending command-apply hand-off entries never drained by a
|
||
* (not-yet-built) task 6 apply pump (raft-apply-decoupling task 3.2).
|
||
* Each node's `ctx` is opaque to persist.c (owned by the registering
|
||
* caller, e.g. job.c's jobsubmit_async_ctx_t) — freeing the node here
|
||
* does NOT free ctx; a caller shutting down with entries still pending
|
||
* is responsible for its own ctx lifetime (test harnesses that tear
|
||
* down mid-flight must account for this). */
|
||
pending_command_apply_t* pn =
|
||
(pending_command_apply_t*)p->pending_applies;
|
||
while (pn != NULL) {
|
||
pending_command_apply_t* next = pn->next;
|
||
free(pn);
|
||
pn = next;
|
||
}
|
||
p->pending_applies = NULL;
|
||
pthread_mutex_destroy(&p->pending_applies_lock);
|
||
|
||
/* raft-apply-decoupling task 6.1: no walk can still be in flight by the
|
||
* time persist_fini runs (the caller owns quiescing any vtask pool
|
||
* first, same precondition every other persist_t resource here already
|
||
* assumes). */
|
||
pthread_mutex_destroy(&p->apply_pump_lock);
|
||
|
||
/* raft-apply-decoupling task 7.4 perf-fix: close any cached segment fd
|
||
* left open by apply_pump_read_entry. No walk can be in flight here
|
||
* (same precondition as apply_pump_lock's destroy just above), so no
|
||
* lock is needed. */
|
||
if (p->apply_pump_cache_fd >= 0) {
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
}
|
||
|
||
/* Forward declaration — defined after checkpoint internals */
|
||
static void schedule_checkpoint(persist_t* persist);
|
||
|
||
int persist_wal_async(persist_t* p, uint32_t type,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
/* WAL_CHECKPOINT is a local segment marker, never written via this path */
|
||
if (type == WAL_CHECKPOINT)
|
||
return -EINVAL;
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
/* Open first segment lazily */
|
||
if (p->wal_fd < 0) {
|
||
if (open_new_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
/* Build entry header */
|
||
wal_entry_hdr_t hdr;
|
||
memset(&hdr, 0, sizeof(hdr));
|
||
hdr.term = p->term;
|
||
hdr.lsn = p->current_lsn++;
|
||
hdr.timestamp = now_us();
|
||
hdr.type = type;
|
||
hdr.payload_len = len;
|
||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||
|
||
/* CRC32 over WAL_HDR_CRC_SIZE bytes of header (excludes crc + prev_crc
|
||
* fields themselves) then over the payload */
|
||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
if (payload && len > 0)
|
||
c = crc32(c, (const Bytef*)payload, len);
|
||
hdr.crc = c;
|
||
|
||
/* Write header */
|
||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||
if (hw != (ssize_t)sizeof(hdr)) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
|
||
/* Write payload (if any) */
|
||
if (payload && len > 0) {
|
||
ssize_t pw = pwrite(p->wal_fd, payload, len,
|
||
(off_t)(p->seg_offset + sizeof(hdr)));
|
||
if (pw != (ssize_t)len) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
|
||
p->seg_offset += entry_size;
|
||
p->total_wal_bytes += entry_size;
|
||
p->last_crc = hdr.crc;
|
||
|
||
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
|
||
* 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */
|
||
record_entry_term(p, hdr.lsn, hdr.term);
|
||
|
||
/* Replicate to standbys — never replicate WAL_CHECKPOINT entries.
|
||
* last_replicated_crc is distinct from last_crc:
|
||
* last_crc includes checkpoint markers; last_replicated_crc skips them.
|
||
* This maintains a continuous CRC chain for standbys (design doc Part 2.3).
|
||
* Note: WAL_CHECKPOINT is blocked by the assert at function entry, so
|
||
* every entry that reaches here is a replicable entry. */
|
||
p->last_replicated_crc = hdr.crc;
|
||
/* ha-term-safety task 3.4: source the AppendEntries prev_log_term (term of
|
||
* the entry at hdr.lsn-1) under p->lock via the lock-free accessor, so the
|
||
* live broadcast carries the leader's term + prev tuple and the standby
|
||
* routes it through the library (raft_on_append_entries_request). 0 when
|
||
* hdr.lsn == 1 (no predecessor). */
|
||
uint64_t prev_log_term = entry_term_locked(p, hdr.lsn - 1);
|
||
repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term);
|
||
|
||
/* Rotate segment when full */
|
||
if (p->seg_offset >= p->config.wal_segment_bytes) {
|
||
if (rotate_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* Signal the group-commit flusher (ha-commit-durability 2.1): current_lsn
|
||
* advanced and the entry's bytes are pwritten, so durable_lsn < tip now
|
||
* holds (the level predicate). The signal is sent AFTER releasing p->lock
|
||
* and is level-triggered — if it races the flusher and is "lost", the
|
||
* flusher re-checks durable_lsn < current_lsn-1 under p->lock and flushes
|
||
* anyway (see fsync_worker). We do NOT fdatasync inline on this path; the
|
||
* flusher batches the sync across all pending appends. */
|
||
pthread_mutex_lock(&p->fsync_mutex);
|
||
pthread_cond_signal(&p->fsync_cond);
|
||
pthread_mutex_unlock(&p->fsync_mutex);
|
||
|
||
/* WAL record-count checkpoint trigger: schedule an async checkpoint once
|
||
* the record count reaches the configured threshold. The fetch_add is
|
||
* outside the lock — atomic_fetch_add is sufficient for the counter; the
|
||
* checkpoint itself is guarded by its own CAS inside schedule_checkpoint. */
|
||
if (atomic_fetch_add(&p->wal_record_count, 1) + 1 >=
|
||
p->config.max_wal_records) {
|
||
schedule_checkpoint(p);
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* Lock-held core of persist_truncate_wal_above (defined below): assumes the
|
||
* caller already holds p->lock and never touches the lock itself. The
|
||
* replicated-write conflict guard reuses it to repair a divergent entry while
|
||
* still under p->lock (the follower-AE-conflict truncate contract). */
|
||
static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn);
|
||
|
||
/* read_entry_meta_locked: scan the on-disk WAL for the entry at `lsn` and
|
||
* return its (payload_len, payload-only CRC, raw stored term). The
|
||
* replicated-write conflict guard uses this to compare a re-streamed entry
|
||
* against the copy already on disk WITHOUT trusting the LSN alone. pcrc is
|
||
* computed as crc32(0, payload, len) — the SAME payload-only fingerprint
|
||
* walcompare/waldump compare across members (NOT the chained header CRC); an
|
||
* empty (PLEN=0) entry yields len=0, pcrc=0. Reads through the page cache so
|
||
* a written-but-not-yet-fdatasync'd entry is still visible. Caller MUST hold
|
||
* p->lock. Returns 0 if the entry is found, -1 if absent or unreadable. */
|
||
static int read_entry_meta_locked(persist_t* p, uint64_t lsn,
|
||
uint32_t* out_len, uint32_t* out_pcrc,
|
||
uint64_t* out_term)
|
||
{
|
||
if (!p || lsn == 0)
|
||
return -1;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return -1;
|
||
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return -1;
|
||
|
||
/* Ascending sort by start LSN (segment count is tiny). */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
int rc = -1;
|
||
for (int si = 0; si < nseg && rc != 0; si++) {
|
||
/* Segments are LSN-ordered; one whose start LSN already exceeds the
|
||
* target cannot contain it (nor can any later one). */
|
||
if (seg_lsns[si] > lsn)
|
||
break;
|
||
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (hn < (ssize_t)sizeof(hdr))
|
||
break; /* short read / EOF / pre-allocated zeros */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
if (hdr.lsn == lsn) {
|
||
uint32_t pc = 0;
|
||
if (hdr.payload_len > 0) {
|
||
uint8_t* pl = malloc(hdr.payload_len);
|
||
if (!pl)
|
||
break; /* leave rc == -1: unreadable → conservative */
|
||
ssize_t pn = pread(fd, pl, hdr.payload_len,
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pn == (ssize_t)hdr.payload_len)
|
||
pc = crc32(0L, (const Bytef*)pl, hdr.payload_len);
|
||
free(pl);
|
||
if (pn != (ssize_t)hdr.payload_len)
|
||
break; /* unreadable body → conservative */
|
||
}
|
||
if (out_len)
|
||
*out_len = hdr.payload_len;
|
||
if (out_pcrc)
|
||
*out_pcrc = pc;
|
||
if (out_term)
|
||
*out_term = hdr.term;
|
||
rc = 0;
|
||
break;
|
||
}
|
||
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(fd);
|
||
}
|
||
return rc;
|
||
}
|
||
|
||
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
/* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */
|
||
if (type == WAL_CHECKPOINT)
|
||
return -EINVAL;
|
||
|
||
/* Reject oversized payloads BEFORE touching the segment. */
|
||
if (len > MAX_WAL_PAYLOAD_BYTES)
|
||
return -EMSGSIZE;
|
||
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
/* Tip-repair bookkeeping (set only on the TIP-conflict repair branch below):
|
||
* the truncate-and-reappend repair transiently lowers committed_lsn to
|
||
* lsn-1, so we snapshot the true committed boundary here and restore it
|
||
* EXACTLY after the re-append re-establishes the entry at `lsn` (never let a
|
||
* same-(term,lsn) payload repair regress the committed boundary — commit
|
||
* monotonicity / State Machine Safety). */
|
||
int repaired_tip = 0;
|
||
uint64_t repaired_committed_save = 0;
|
||
|
||
/* Payload-aware conflict guard (raft-ha-core-safety task 2.2): this lsn is
|
||
* already persisted (current_lsn advanced past it). The legacy guard was
|
||
* (lsn)-only — it no-op'd ANY re-stream, including a corrective full-payload
|
||
* write over a previously-stored empty (PLEN=0) body, which is exactly the
|
||
* cross-term catch-up payload-loss signature (a committed LSN that carries
|
||
* the right (term,lsn) but an empty payload). Compare (payload-crc, len)
|
||
* against the copy already on disk and act on the difference:
|
||
* (c) identical payload → idempotent no-op (keep).
|
||
* (b) incoming EMPTY over a stored entry → never degrade (keep stored);
|
||
* an empty body is never the durable form of a known entry.
|
||
* (a) incoming NON-EMPTY whose payload differs from the stored copy at an
|
||
* existing (term,lsn) → a genuine payload conflict whose handling
|
||
* depends on WHERE the conflict sits relative to the stored tip:
|
||
* · AT THE TIP (lsn == current_lsn - 1, nothing stored above) →
|
||
* REPAIR: truncate just this one entry and re-append the
|
||
* authoritative payload at the same (term,lsn). Because nothing
|
||
* is stored above it, no distinct committed entry is destroyed,
|
||
* and the committed boundary is restored verbatim afterwards.
|
||
* · BELOW THE TIP (lsn < current_lsn - 1, committed entries stored
|
||
* ABOVE the conflict) → MUST NOT truncate: dropping [lsn..tip]
|
||
* would delete committed entries above the conflict AND roll
|
||
* committed_lsn/durable_lsn backward — a State Machine Safety /
|
||
* commit-monotonicity violation. Instead keep the stored entry
|
||
* untouched, raise a loud non-promotable fence, and let catch-up
|
||
* reconverge. The real source fix (the streamer must stop
|
||
* shipping an empty body) is task 2.4; the guard's only job here
|
||
* is to NOT make things worse (never drop a committed entry,
|
||
* never regress a boundary) and to surface the corruption loudly.
|
||
* An unreadable stored entry stays a conservative no-op — never truncate a
|
||
* committed LSN we cannot read back. */
|
||
if (lsn + 1 <= p->current_lsn) {
|
||
uint32_t inc_pcrc =
|
||
(payload && len > 0) ? crc32(0L, (const Bytef*)payload, len) : 0;
|
||
uint32_t st_len = 0;
|
||
uint32_t st_pcrc = 0;
|
||
uint64_t st_term = 0;
|
||
int have = read_entry_meta_locked(p, lsn, &st_len, &st_pcrc,
|
||
&st_term);
|
||
|
||
int payload_matches =
|
||
(have == 0 && st_len == len && st_pcrc == inc_pcrc);
|
||
|
||
/* (c) identical payload, (b) empty incoming, or an unreadable stored
|
||
* entry → leave the stored entry untouched. */
|
||
if (payload_matches || len == 0 || have != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* (a) non-empty incoming whose payload differs at an existing (term,lsn)
|
||
* — a genuine payload conflict. Branch on tip vs below-tip. */
|
||
uint64_t tip_lsn = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
|
||
if (lsn != tip_lsn) {
|
||
/* BELOW-TIP conflict: committed entries are stored ABOVE `lsn`.
|
||
* Truncating here would delete them and roll committed_lsn/
|
||
* durable_lsn backward (State Machine Safety / commit-monotonicity
|
||
* violation — bug-class the round-1 review rejected). Do NOT
|
||
* truncate, do NOT lower any boundary, do NOT degrade the stored
|
||
* entry. Surface loudly and raise the sticky non-promotable fence
|
||
* (reuse the recovery_unrecoverable mechanism: loud, stays ALIVE,
|
||
* non-promotable, self-heals via persist_clear_recovery_unrecoverable
|
||
* once catch-up reaches a committed-consistent state — never
|
||
* self-exits, per the controller invariant). The source fix lives
|
||
* in task 2.4 (the streamer must stop shipping an empty payload). */
|
||
ERROR("persist: persist_wal_write_replicated payload CONFLICT BELOW "
|
||
"TIP at lsn=%" PRIu64 " (stored_term=%" PRIu64
|
||
" stored_len=%u stored_pcrc=%08x vs incoming_len=%u "
|
||
"incoming_pcrc=%08x); current_lsn=%" PRIu64 " committed_lsn=%"
|
||
PRIu64 " — refusing to truncate committed entries above the "
|
||
"conflict (would regress the committed boundary). Keeping the "
|
||
"stored entry, raising the non-promotable fence and staying "
|
||
"ALIVE; awaiting a correct re-stream (source fix: task 2.4)",
|
||
lsn, st_term, st_len, st_pcrc, len, inc_pcrc, p->current_lsn,
|
||
atomic_load_explicit(&p->committed_lsn, memory_order_acquire));
|
||
atomic_store_explicit(&p->recovery_unrecoverable, true,
|
||
memory_order_release);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0; /* no-op the write: keep stored, never degrade/regress */
|
||
}
|
||
|
||
/* TIP conflict: `lsn` is the highest stored entry, so nothing is stored
|
||
* above it — truncate-and-reappend replaces EXACTLY this one divergent
|
||
* entry. truncate_wal_above_locked(lsn-1) drops it and sets
|
||
* current_lsn = lsn, so control falls through to the append below and
|
||
* re-persists the authoritative entry at this exact LSN. Snapshot the
|
||
* committed boundary first: the truncate transiently rewrites
|
||
* committed_lsn to lsn-1, and we restore it verbatim after the
|
||
* re-append so a same-(term,lsn) payload repair never regresses (nor
|
||
* spuriously advances) the committed boundary. */
|
||
repaired_committed_save =
|
||
atomic_load_explicit(&p->committed_lsn, memory_order_acquire);
|
||
int trc = truncate_wal_above_locked(p, lsn - 1);
|
||
if (trc != 0) {
|
||
ERROR("persist: persist_wal_write_replicated repair-truncate FAILED "
|
||
"for lsn=%" PRIu64 " (rc=%d) — divergent entry NOT repaired",
|
||
lsn, trc);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return trc;
|
||
}
|
||
repaired_tip = 1;
|
||
/* fall through to the contiguity guard + append (lsn == current_lsn). */
|
||
}
|
||
|
||
/* Contiguity guard (bug-850): REFUSE a forward GAP. current_lsn is the
|
||
* next-expected LSN, so a replicated entry with lsn > current_lsn would
|
||
* leave [current_lsn .. lsn-1] missing AND advance current_lsn past that
|
||
* hole — after which the idempotency guard above silently DROPS the later
|
||
* in-order catch-up of the prefix (the failover prefix-loss mechanism).
|
||
* Append only the contiguous next entry (lsn == current_lsn); reject a gap
|
||
* without writing or acking so the ACK-driven catch-up continuation
|
||
* re-streams from current_lsn and the log fills in order. This is Raft
|
||
* log-matching: append entry N only when the log already holds N-1. A
|
||
* behind standby that keeps receiving live broadcasts simply drops the
|
||
* out-of-order ones until catch-up backfills the gap (no loss). -EAGAIN is
|
||
* the "retry via catch-up" signal the apply path treats as a quiet skip. */
|
||
if (lsn > p->current_lsn) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EAGAIN;
|
||
}
|
||
|
||
/* Open first segment lazily. open_new_segment names the file from
|
||
* current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is
|
||
* 1 (reserve-lsn-0), so the segment is seg-...001.wal. */
|
||
if (p->wal_fd < 0) {
|
||
if (open_new_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
/* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not
|
||
* current_lsn++). term isn't streamed, so use our best-effort p->term.
|
||
* The CRC chain is SELF-computed (the streamed entry carries no crc):
|
||
* recovery validates an intra-segment chain (prev_crc == last_crc), so the
|
||
* standby's own writes must chain to each other — cross-node CRC equality
|
||
* is neither required nor possible. */
|
||
wal_entry_hdr_t hdr;
|
||
memset(&hdr, 0, sizeof(hdr));
|
||
hdr.term = p->term;
|
||
hdr.lsn = lsn;
|
||
hdr.timestamp = now_us();
|
||
hdr.type = type;
|
||
hdr.payload_len = len;
|
||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||
|
||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
if (payload && len > 0)
|
||
c = crc32(c, (const Bytef*)payload, len);
|
||
hdr.crc = c;
|
||
|
||
/* Write header */
|
||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||
if (hw != (ssize_t)sizeof(hdr)) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
|
||
/* Write payload (if any) */
|
||
if (payload && len > 0) {
|
||
ssize_t pw = pwrite(p->wal_fd, payload, len,
|
||
(off_t)(p->seg_offset + sizeof(hdr)));
|
||
if (pw != (ssize_t)len) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
uint32_t entry_size = (uint32_t)(sizeof(hdr) + len);
|
||
p->seg_offset += entry_size;
|
||
p->total_wal_bytes += entry_size;
|
||
p->last_crc = hdr.crc;
|
||
|
||
/* Record this entry's term at its LSN for recovery read-back (ha-term-safety
|
||
* 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed
|
||
* envelope does not yet carry the leader's term, so this records the local
|
||
* best-effort p->term (the design's known hazard closed by task 2.x, which
|
||
* carries the leader's term on the envelope; this task only provides the
|
||
* read-back seam over whatever term was persisted). */
|
||
record_entry_term(p, hdr.lsn, hdr.term);
|
||
|
||
/* Advance current_lsn to one past the given lsn (max-idempotent — mirrors
|
||
* apply_wal_entry's monotonic advance). */
|
||
if (lsn + 1 > p->current_lsn)
|
||
p->current_lsn = lsn + 1;
|
||
|
||
/* Durability: fsync the segment BEFORE returning so the caller's ACK
|
||
* implies the entry is on stable storage (persist-before-ACK).
|
||
* persist_wal_async defers fsync to the background thread / persist_flush;
|
||
* here we sync inline because the entry feeds the primary's commit
|
||
* majority and must survive a standby restart. fdatasync is sufficient
|
||
* (entry bytes + segment metadata are pre-allocated by open_new_segment).
|
||
*
|
||
* Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the
|
||
* append-path gate at task 2.1): the ACK is sent by the caller ONLY when
|
||
* this returns success, and the ACK claims the entry is durable. So if
|
||
* the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must
|
||
* NOT return success — we log loudly and return -EIO so the caller's
|
||
* existing error path (ha_apply_single_entry) does NOT apply or ACK. */
|
||
int fsrc = fdatasync(p->wal_fd);
|
||
#ifdef XCE_TESTING
|
||
atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1,
|
||
memory_order_relaxed);
|
||
#endif
|
||
if (fsrc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (fsrc != 0) ? errno : EIO;
|
||
ERROR("persist: persist_wal_write_replicated fdatasync FAILED for "
|
||
"lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning "
|
||
"error so the standby does NOT ACK", lsn, e, strerror(e));
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
|
||
/* bug-828 secondary: the entry is now fdatasync'd durable, so advance
|
||
* durable_lsn to the synced tip immediately (mirror persist_flush). The
|
||
* 10ms cadence flusher would otherwise advance it lazily, leaving this
|
||
* standby's ADVERTISED durable position (persist_durable_floor →
|
||
* RequestVoteReply.peerLastLogLsn) lagging its own synced tip during the
|
||
* vote window. A standby that has durably persisted the committed prefix
|
||
* must advertise it at once so a clean failover's safe-commit-from-votes
|
||
* computation sees the true majority-held LSN. Monotonic max; preserves
|
||
* durable_lsn <= current_lsn - 1 (tip read under this same p->lock, after
|
||
* the current_lsn advance above). Reached ONLY past the fail-loud rc-gate
|
||
* — never advances on an fdatasync error. */
|
||
{
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
}
|
||
|
||
/* Tip-repair committed-boundary restore (raft-ha-core-safety task 2.2): a
|
||
* TIP-conflict repair above truncated [lsn..tip] (rewriting committed_lsn to
|
||
* lsn-1) and has now re-appended + fdatasync'd the authoritative entry at the
|
||
* SAME (term,lsn). Restore the committed boundary VERBATIM so the repair is
|
||
* commit-boundary-neutral: a same-(term,lsn) payload correction must never
|
||
* regress committed_lsn (State Machine Safety / commit monotonicity) nor
|
||
* spuriously advance it (truncate would otherwise have raised it to lsn-1).
|
||
* Reached only past the fail-loud fdatasync gate, with current_lsn re-advanced
|
||
* to lsn+1 and durable_lsn at the synced tip, so the invariant
|
||
* committed_lsn <= min(current_lsn - 1, durable_lsn) holds (saved value was
|
||
* itself <= the pre-repair tip == lsn). The whole repair is serialized under
|
||
* p->lock, so no external observer ever sees the transient lowered value. */
|
||
if (repaired_tip) {
|
||
atomic_store_explicit(&p->committed_lsn, repaired_committed_save,
|
||
memory_order_release);
|
||
}
|
||
|
||
/* Rotate segment when full (same threshold as the primary append). */
|
||
if (p->seg_offset >= p->config.wal_segment_bytes) {
|
||
if (rotate_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
/* NOTE: no repl_broadcast — standbys never re-replicate. */
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
|
||
int persist_flush(persist_t* p)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
int rc = (p->wal_fd >= 0) ? fdatasync(p->wal_fd) : 0;
|
||
|
||
/* persist_flush is a synchronous durability point (tests, clean shutdown):
|
||
* it fdatasync'd everything appended so far, so advance durable_lsn to the
|
||
* synced tip too. Without this, an explicit-flush-only path (slow flusher
|
||
* cadence) would leave durable_lsn lagging even though the bytes are on
|
||
* disk. Monotonic max; preserves durable_lsn <= current_lsn - 1 (tip read
|
||
* under the same p->lock). Gated on rc == 0 (fail-loud, Req 1.1/1.2): a
|
||
* failed flush must not advance the watermark. */
|
||
if (rc == 0 && !flusher_fsync_forced_fail(p)) {
|
||
uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
if (tip > dur)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
} else if (rc != 0 || flusher_fsync_forced_fail(p)) {
|
||
int e = (rc != 0) ? errno : EIO;
|
||
ERROR("persist: persist_flush fdatasync FAILED (errno=%d %s) — "
|
||
"durable_lsn not advanced; flushed bytes are NOT durable",
|
||
e, strerror(e));
|
||
}
|
||
pthread_mutex_unlock(&p->lock);
|
||
return rc;
|
||
}
|
||
|
||
uint64_t persist_durable_lsn(persist_t* p)
|
||
{
|
||
return atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
}
|
||
|
||
uint64_t persist_durable_floor(persist_t* p)
|
||
{
|
||
/* The durable freshness floor (durable last-log index) IS the durable
|
||
* watermark: persist_recover re-derives it from the recovered WAL tip
|
||
* (ha-commit-durability task 3.4), and the flusher advances it on every
|
||
* successful fdatasync — so a thin wrapper avoids a redundant field. NULL
|
||
* guard keeps the accessor safe for the election consumers (task 4.2). */
|
||
return p ? atomic_load_explicit(&p->durable_lsn, memory_order_acquire) : 0;
|
||
}
|
||
|
||
void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* Set ctx before fn so the flusher (which checks fn != NULL) never sees a
|
||
* live callback with a stale/uninitialised ctx. Registration happens in
|
||
* commit_engine_init, before the engine is reachable by any ACK path. */
|
||
p->durable_cb_ctx = ctx;
|
||
p->on_durable_advance = fn;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 1.4: registration counterpart of
|
||
* persist_set_durable_cb -- see persist.h for the full contract. */
|
||
void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn),
|
||
void* ctx)
|
||
{
|
||
if (!p)
|
||
return;
|
||
p->truncate_cb_ctx = ctx;
|
||
p->on_truncate = fn;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
void persist_test_set_force_fsync_fail(persist_t* p, int fail)
|
||
{
|
||
if (p)
|
||
atomic_store_explicit(&p->test_force_fsync_fail, fail ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
void persist_test_set_force_job_encode_fail_at(persist_t* p, int index)
|
||
{
|
||
if (p)
|
||
atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1,
|
||
memory_order_relaxed);
|
||
}
|
||
|
||
unsigned persist_test_replicated_fdatasync_count(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return 0;
|
||
return atomic_load_explicit(&p->test_replicated_fdatasync_count,
|
||
memory_order_relaxed);
|
||
}
|
||
|
||
pthread_t persist_test_last_fdatasync_thread(persist_t* p)
|
||
{
|
||
/* If no flusher fdatasync has advanced durable_lsn yet, return the
|
||
* caller's own id so an unprimed read never spuriously looks off-thread.
|
||
* After a flush, return the recorded flusher thread id (Req 1.6). */
|
||
if (!p ||
|
||
atomic_load_explicit(&p->test_last_fdatasync_thread_set,
|
||
memory_order_acquire) == 0)
|
||
return pthread_self();
|
||
return p->test_last_fdatasync_thread;
|
||
}
|
||
|
||
/* Force a fresh WAL segment at the current LSN (test seam — see persist.h).
|
||
* Takes p->lock for the same single-writer serialization the append path uses,
|
||
* so the segment swap cannot race a concurrent write. */
|
||
int persist_test_force_new_segment(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
pthread_mutex_lock(&p->lock);
|
||
int rc = rotate_segment(p);
|
||
pthread_mutex_unlock(&p->lock);
|
||
return rc;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ *
|
||
* Durability-verification test seam (ha-commit-durability Req 3.2/3.3)*
|
||
* *
|
||
* Re-read + CRC-validate the WAL segments from stable storage and *
|
||
* return the highest fully-persisted LSN — independent of the *
|
||
* in-memory current_lsn / committed_lsn watermarks. Mirrors the *
|
||
* segment-scan + CRC-chain validation of replay_wal(), but is strictly *
|
||
* read-only: it applies no entries, fires no hooks, and mutates no *
|
||
* persist_t state, so it cannot change the externally observable *
|
||
* acknowledgment semantics (Req 3.3). *
|
||
* *
|
||
* Stops at the first corruption / short read / pre-allocated zero *
|
||
* region (exactly like recovery), so only entries genuinely on stable *
|
||
* storage are counted. Returns the highest such LSN, or 0 when nothing *
|
||
* is durable. *
|
||
* ------------------------------------------------------------------ */
|
||
uint64_t persist_test_durable_lsn_on_disk(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return 0;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return 0; /* no WAL dir → nothing durable */
|
||
|
||
/* Collect segment start LSNs (same bound as replay_wal). */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t seg_start = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = seg_start;
|
||
}
|
||
closedir(d);
|
||
|
||
if (nseg == 0)
|
||
return 0;
|
||
|
||
/* Insertion sort by start LSN (segment count is tiny). */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
uint64_t highest_durable = 0;
|
||
uint32_t last_crc = WAL_CRC_NONE;
|
||
|
||
for (int si = 0; si < nseg; si++) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t seg_off = 0;
|
||
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
|
||
if (hn == 0)
|
||
break; /* clean end of segment */
|
||
if (hn < (ssize_t)sizeof(hdr))
|
||
break; /* truncated / pre-allocated zeros */
|
||
|
||
/* Pre-allocated zero region: no valid entry is all-zero. */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
/* CRC chain validation — identical rule to replay_wal. */
|
||
if (seg_off == 0) {
|
||
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
|
||
hdr.prev_crc != last_crc)
|
||
break;
|
||
} else {
|
||
if (hdr.prev_crc != last_crc)
|
||
break;
|
||
}
|
||
|
||
/* Verify entry CRC over header + payload. */
|
||
uint32_t expected_crc =
|
||
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
|
||
uint8_t* payload = NULL;
|
||
if (hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
close(fd);
|
||
return highest_durable;
|
||
}
|
||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||
(off_t)(seg_off + sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
free(payload);
|
||
break;
|
||
}
|
||
expected_crc =
|
||
crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
|
||
}
|
||
|
||
if (hdr.crc != expected_crc) {
|
||
free(payload);
|
||
break;
|
||
}
|
||
free(payload);
|
||
|
||
/* Entry is intact and durable on disk. */
|
||
last_crc = hdr.crc;
|
||
if (hdr.lsn > highest_durable)
|
||
highest_durable = hdr.lsn;
|
||
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
|
||
close(fd);
|
||
}
|
||
|
||
return highest_durable;
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Checkpoint internals */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Write a WAL_CHECKPOINT marker to the current segment.
|
||
* Never calls repl_broadcast() — checkpoint markers are local only. */
|
||
static int persist_write_checkpoint_marker(persist_t* p)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
|
||
if (p->wal_fd < 0) {
|
||
if (open_new_segment(p) != 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
wal_entry_hdr_t hdr;
|
||
memset(&hdr, 0, sizeof(hdr));
|
||
hdr.term = p->term;
|
||
hdr.lsn = p->current_lsn++;
|
||
hdr.timestamp = now_us();
|
||
hdr.type = WAL_CHECKPOINT;
|
||
hdr.payload_len = 0;
|
||
hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc;
|
||
|
||
uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
hdr.crc = c;
|
||
|
||
ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset);
|
||
if (hw != (ssize_t)sizeof(hdr)) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return -EIO;
|
||
}
|
||
fdatasync(p->wal_fd);
|
||
|
||
p->seg_offset += (uint32_t)sizeof(hdr);
|
||
p->total_wal_bytes += sizeof(hdr);
|
||
p->last_crc = hdr.crc;
|
||
/* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */
|
||
/* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS
|
||
* checkpoint markers (local bookkeeping, not a logical Raft entry), so the
|
||
* recovered index never holds a checkpoint LSN; the live index must match
|
||
* (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which
|
||
* never compares against a checkpoint marker. */
|
||
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2/3.2, Req
|
||
* 1.1-1.4). Physical layout appended after the existing legacy header
|
||
* (design.md "Checkpoint File Layout"):
|
||
* [uint32_t format_version]
|
||
* [uint32_t job_count]
|
||
* [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }]
|
||
* [uint32_t queue_count]
|
||
* [queue_count * { uint32_t name_len; char name[name_len]; uint32_t status }] */
|
||
#define SNAPSHOT_FORMAT_VERSION 1u
|
||
|
||
typedef struct {
|
||
uint8_t* buf;
|
||
uint32_t len;
|
||
} snap_job_frame_t;
|
||
|
||
static void snap_free_job_frames(xlist_t* frames)
|
||
{
|
||
for (int i = 0; i < frames->count; i++) {
|
||
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i);
|
||
if (fr) {
|
||
free(fr->buf);
|
||
free(fr);
|
||
}
|
||
}
|
||
xlist_fini(frames);
|
||
}
|
||
|
||
/* Collect + encode every retained job into `frames`, one malloc'd
|
||
* snap_job_frame_t per job, under a point-in-time-consistent walk
|
||
* (design.md "Existing Architecture Analysis" / Requirement 1.2): the
|
||
* jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes
|
||
* concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is
|
||
* ADDITIONALLY taken nested around just that job's encode call (excludes a
|
||
* concurrent per-entity writer for exactly the duration of its encode --
|
||
* the table lock alone does not do this). No lock is held during the
|
||
* disk write itself: this function only builds in-memory frames; the
|
||
* caller writes them to disk after the table walk has fully released both
|
||
* locks.
|
||
*
|
||
* Filters via job_should_retain (Requirement 1.1) so the checkpoint's
|
||
* notion of "live" matches the existing GC retention policy.
|
||
*
|
||
* Returns 0 on success (frames populated; `frames` is always left
|
||
* xlist_init'd -- even zero results -- so the caller can always
|
||
* xlist_fini it) or a negative errno on the first encode failure, in which
|
||
* case every frame collected so far has already been freed and `frames`
|
||
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
|
||
* entity content behind). */
|
||
static int collect_job_frames(persist_t* p, xlist_t* frames)
|
||
{
|
||
cluster_t* c = p->cluster;
|
||
int rc = 0;
|
||
int retained_index = 0;
|
||
|
||
xlist_init(frames, 64);
|
||
|
||
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
|
||
* zero jobs is not a failure. */
|
||
if (c == NULL || c->jobmgr == NULL)
|
||
return 0;
|
||
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, c->jobmgr);
|
||
if (irc < 0) {
|
||
unlock(c->jobmgr);
|
||
return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */
|
||
}
|
||
|
||
for (;;) {
|
||
job_t* j = NULL;
|
||
int nrc = iter_next(&it, (void**)&j);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (j == NULL)
|
||
continue;
|
||
if (!job_should_retain(j, &p->config))
|
||
continue;
|
||
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int enc_rc;
|
||
|
||
lock(j, RTTI_LOCK_READ);
|
||
#ifdef XCE_TESTING
|
||
int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at,
|
||
memory_order_relaxed);
|
||
if (forced > 0 && (forced - 1) == retained_index)
|
||
enc_rc = -EIO; /* simulated mid-loop encode failure */
|
||
else
|
||
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
|
||
#else
|
||
enc_rc = capnp_encode_job(&j->job, &buf, &sz);
|
||
#endif
|
||
unlock(j);
|
||
|
||
retained_index++;
|
||
|
||
if (enc_rc != 0) {
|
||
rc = enc_rc;
|
||
break;
|
||
}
|
||
|
||
snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr));
|
||
if (fr == NULL) {
|
||
free(buf);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
fr->buf = buf;
|
||
fr->len = (uint32_t)sz;
|
||
if (xlist_store(frames, fr) < 0) {
|
||
free(buf);
|
||
free(fr);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(c->jobmgr);
|
||
|
||
if (rc != 0)
|
||
snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Queue admin-status frame (ha-entity-snapshots task 3.2, Req 1.1). Unlike
|
||
* jobs, a queue's on-disk frame carries no capnp payload at all -- it is a
|
||
* plain (name, status) pair, the entirety of restore_queue_state's contract
|
||
* (queue.c:208). `name` is a malloc'd, NUL-terminated copy; the on-disk
|
||
* frame itself carries no NUL (design.md "Checkpoint File Layout":
|
||
* `{ uint32_t name_len; char name[name_len]; uint32_t status }`). */
|
||
typedef struct {
|
||
char* name;
|
||
uint32_t status;
|
||
} snap_queue_frame_t;
|
||
|
||
static void snap_free_queue_frames(xlist_t* frames)
|
||
{
|
||
for (int i = 0; i < frames->count; i++) {
|
||
snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(frames, i);
|
||
if (fr) {
|
||
free(fr->name);
|
||
free(fr);
|
||
}
|
||
}
|
||
xlist_fini(frames);
|
||
}
|
||
|
||
/* Collect every queue in `frames`, one malloc'd snap_queue_frame_t per
|
||
* queue, under the same point-in-time-consistent walk discipline as
|
||
* collect_job_frames above (design.md "Existing Architecture Analysis" /
|
||
* Requirement 1.2): the qmgr table's own RTTI_LOCK_READ is held for the
|
||
* ENTIRE walk, and each queue's own RTTI_LOCK_READ is ADDITIONALLY taken
|
||
* nested around just that queue's (name, status) read -- mirroring
|
||
* restore_queue_state's own per-entity lock pattern on the load side. No
|
||
* lock is held during the disk write itself.
|
||
*
|
||
* Unlike collect_job_frames, no retention predicate is applied: a queue's
|
||
* mere presence in the live qmgr table already means config provisioned
|
||
* it (design.md "write_snapshot (extended)" Responsibilities), so every
|
||
* queue found is included.
|
||
*
|
||
* Returns 0 on success (frames populated; `frames` is always left
|
||
* xlist_init'd, even for zero results, so the caller can always xlist_fini
|
||
* it) or a negative errno on the first failure (name allocation), in which
|
||
* case every frame collected so far has already been freed and `frames`
|
||
* is empty (Requirement 1.3: a mid-walk failure must not leave partial
|
||
* entity content behind). */
|
||
static int collect_queue_frames(persist_t* p, xlist_t* frames)
|
||
{
|
||
cluster_t* c = p->cluster;
|
||
int rc = 0;
|
||
|
||
xlist_init(frames, 64);
|
||
|
||
/* No live cluster back-pointer (e.g. a bare unit-test persist_t) --
|
||
* zero queues is not a failure. */
|
||
if (c == NULL || c->qmgr == NULL)
|
||
return 0;
|
||
|
||
lock(c->qmgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, c->qmgr);
|
||
if (irc < 0) {
|
||
unlock(c->qmgr);
|
||
return 0; /* nothing to walk -- mirrors collect_queues' empty-table handling */
|
||
}
|
||
|
||
for (;;) {
|
||
queue_t* q = NULL;
|
||
int nrc = iter_next(&it, (void**)&q);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (q == NULL)
|
||
continue;
|
||
|
||
char* name;
|
||
uint32_t status;
|
||
|
||
lock(q, RTTI_LOCK_READ);
|
||
name = q->queue.queue ? strdup(q->queue.queue) : NULL;
|
||
status = q->queue.status;
|
||
unlock(q);
|
||
|
||
if (name == NULL) {
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
|
||
snap_queue_frame_t* fr = (snap_queue_frame_t*)malloc(sizeof(*fr));
|
||
if (fr == NULL) {
|
||
free(name);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
fr->name = name;
|
||
fr->status = status;
|
||
if (xlist_store(frames, fr) < 0) {
|
||
free(name);
|
||
free(fr);
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(c->qmgr);
|
||
|
||
if (rc != 0)
|
||
snap_free_queue_frames(frames); /* Requirement 1.3: discard, not partial */
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Write an entity-bearing checkpoint: legacy header (unchanged) + entity
|
||
* payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any
|
||
* failure at any step -- collect, encode, write, sync -- leaves NO file at
|
||
* the final path; the tmp file is always unlinked on error. This is a
|
||
* retriable-next-checkpoint-cycle failure, never a crash/abort (controller
|
||
* never self-terminates on an operational condition). */
|
||
static int write_snapshot(persist_t* p)
|
||
{
|
||
char tmp_path[4096];
|
||
char final_path[4096];
|
||
|
||
snprintf(tmp_path, sizeof(tmp_path),
|
||
"%s/snap-%016" PRIx64 ".tmp", p->snap_path, p->checkpoint_lsn);
|
||
snprintf(final_path, sizeof(final_path),
|
||
"%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn);
|
||
|
||
/* Collect + encode entities BEFORE opening the tmp file: a collection
|
||
* failure (Requirement 1.3) then never creates a tmp file at all. */
|
||
xlist_t job_frames;
|
||
int rc = collect_job_frames(p, &job_frames);
|
||
if (rc != 0)
|
||
return rc;
|
||
|
||
xlist_t queue_frames;
|
||
rc = collect_queue_frames(p, &queue_frames);
|
||
if (rc != 0) {
|
||
snap_free_job_frames(&job_frames);
|
||
return rc;
|
||
}
|
||
|
||
int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600);
|
||
if (fd < 0) {
|
||
snap_free_job_frames(&job_frames);
|
||
snap_free_queue_frames(&queue_frames);
|
||
return -errno;
|
||
}
|
||
|
||
/* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference
|
||
* (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1).
|
||
* The config-ref tail is OPTIONAL on read-back, so an older 16-byte
|
||
* sentinel still loads (load_snapshot reads the tail best-effort). The
|
||
* tag is little-endian on this arch, matching the checkpoint_lsn field. */
|
||
char header[16 + 1 + 8 + 32];
|
||
memset(header, 0, sizeof(header));
|
||
memcpy(header, "XCESNAP\0", 8);
|
||
memcpy(header + 8, &p->checkpoint_lsn, 8);
|
||
header[16] = (char)(p->cfgref_present ? 1 : 0);
|
||
memcpy(header + 17, &p->cfgref_tag, 8);
|
||
memcpy(header + 25, p->cfgref_sha, 32);
|
||
|
||
if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header))
|
||
rc = -EIO;
|
||
|
||
/* Entity payload: format_version, then job frames (design.md "Checkpoint
|
||
* File Layout"). No table/entity lock is held here -- collect_job_frames
|
||
* already released both before returning. */
|
||
if (rc == 0) {
|
||
uint32_t fmt = SNAPSHOT_FORMAT_VERSION;
|
||
if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt))
|
||
rc = -EIO;
|
||
}
|
||
|
||
if (rc == 0) {
|
||
uint32_t job_count = (uint32_t)job_frames.count;
|
||
if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
|
||
rc = -EIO;
|
||
}
|
||
|
||
for (int i = 0; rc == 0 && i < job_frames.count; i++) {
|
||
snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i);
|
||
if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* Queue trailer (ha-entity-snapshots task 3.2, Req 1.1, 1.2): plain
|
||
* (name, status) frames, no capnp -- matches restore_queue_state's
|
||
* exact contract (design.md "Checkpoint File Layout"). */
|
||
if (rc == 0) {
|
||
uint32_t queue_count = (uint32_t)queue_frames.count;
|
||
if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count))
|
||
rc = -EIO;
|
||
}
|
||
|
||
for (int i = 0; rc == 0 && i < queue_frames.count; i++) {
|
||
snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(&queue_frames, i);
|
||
uint32_t name_len = (uint32_t)strlen(fr->name);
|
||
if (write(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len)) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
if (name_len > 0 &&
|
||
write(fd, fr->name, name_len) != (ssize_t)name_len) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
if (write(fd, &fr->status, sizeof(fr->status)) != (ssize_t)sizeof(fr->status)) {
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (rc == 0 && fdatasync(fd) != 0)
|
||
rc = -errno;
|
||
|
||
close(fd);
|
||
snap_free_job_frames(&job_frames);
|
||
snap_free_queue_frames(&queue_frames);
|
||
|
||
if (rc != 0) {
|
||
unlink(tmp_path);
|
||
return rc;
|
||
}
|
||
|
||
if (rename(tmp_path, final_path) != 0) {
|
||
int saved = errno;
|
||
unlink(tmp_path);
|
||
return -saved;
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/* scan_latest_committed_config_ref (ha-term-safety task 4.2 remediation r1):
|
||
* walk the local WAL segments and capture the HIGHEST-LSN WAL_CONFIG_MARKER
|
||
* whose LSN is <= committed_lsn into p->cfgref_{present,tag,sha}. This is the
|
||
* SAME committed-prefix marker config_committed_marker derives, cached durably
|
||
* so a later checkpoint can trim the marker's segment without losing the
|
||
* derived committed-config reference (compaction safety). The marker payload is
|
||
* a fixed 40-byte reference (8-byte big-endian version_tag + 32-byte sha256 —
|
||
* mirrors config_marker_encode; the config BYTES are never in the WAL), decoded
|
||
* inline here so persist stays free of a config_repl layering dependency. */
|
||
static void scan_latest_committed_config_ref(persist_t* p)
|
||
{
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
if (committed == 0)
|
||
return; /* nothing committed → keep any prior cached ref unchanged */
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return;
|
||
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return;
|
||
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
for (int si = 0; si < nseg; si++) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path,
|
||
seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (n < (ssize_t)sizeof(hdr)) break;
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
|
||
if (hdr.lsn > committed) break; /* past committed prefix */
|
||
|
||
if (hdr.type == WAL_CONFIG_MARKER &&
|
||
hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) {
|
||
uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN];
|
||
ssize_t pr = pread(fd, buf, sizeof(buf),
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pr == (ssize_t)sizeof(buf)) {
|
||
p->cfgref_tag =
|
||
((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) |
|
||
((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) |
|
||
((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) |
|
||
((uint64_t)buf[6] << 8) | ((uint64_t)buf[7]);
|
||
memcpy(p->cfgref_sha, buf + 8, 32);
|
||
p->cfgref_present = 1; /* later markers overwrite — latest wins */
|
||
}
|
||
}
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(fd);
|
||
}
|
||
}
|
||
|
||
/* Delete WAL segment files whose start LSN is strictly less than the
|
||
* active segment's start LSN AND strictly less than the trim floor.
|
||
*
|
||
* Bug-1147 (stopgap, raft-drive-async task 3.2): the trim floor is the
|
||
* minimum of the active-segment LSN, every active standby's last acked LSN,
|
||
* and every recently-disconnected slot's last_disconnected_lsn (strategy-A
|
||
* retention discipline). This ensures we NEVER delete a WAL segment that a
|
||
* live standby still needs for catch-up (the bug-795/828 committed-job-loss
|
||
* family). It does NOT fix the empty/far-behind join case (cursor=0) —
|
||
* that requires entity-bearing snapshots (separate spec).
|
||
*
|
||
* Lock ordering (verified 2026-06-26): persist_checkpoint takes p->lock only
|
||
* to snapshot checkpoint_lsn, then RELEASES it before calling
|
||
* trim_wal_segments — so this function runs with NO p->lock held.
|
||
* repl_trim_lsn and repl_expire_disconnect_slots take only r->lock
|
||
* (p->repl.lock); neither ever acquires p->lock or ce->lock, so there is no
|
||
* p->lock↔repl.lock (or ce->lock↔repl.lock) inversion. */
|
||
static void trim_wal_segments(persist_t* p)
|
||
{
|
||
/* Extract the start LSN of the currently-open segment from seg_path.
|
||
* seg_path has the form "<wal_path>/seg-<hex16>.wal".
|
||
* If no segment is open yet (wal_fd < 0), skip trimming. */
|
||
if (p->wal_fd < 0 || p->seg_path[0] == '\0')
|
||
return;
|
||
|
||
const char* seg_basename = strrchr(p->seg_path, '/');
|
||
if (!seg_basename) return;
|
||
seg_basename++; /* skip the '/' */
|
||
|
||
uint64_t active_seg_lsn = 0;
|
||
if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1)
|
||
return;
|
||
|
||
/* Expire stale disconnected slots so their last_disconnected_lsn guards
|
||
* don't hold WAL segments indefinitely (slot guard expires at
|
||
* last_disconnect_time + snap_interval_s). */
|
||
repl_expire_disconnect_slots(&p->repl, time(NULL),
|
||
p->config.snap_interval_s);
|
||
|
||
/* Compute the trim floor: min(active_seg_lsn, all active standby
|
||
* acked_lsn, all valid disconnected-slot last_disconnected_lsn).
|
||
* Any segment with seg_start_lsn BELOW this floor is disposable. */
|
||
uint64_t floor = repl_trim_lsn(&p->repl, active_seg_lsn,
|
||
p->config.snap_interval_s);
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d) return;
|
||
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
|
||
uint64_t seg_start_lsn = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue;
|
||
/* Only delete segments below BOTH the active segment AND the
|
||
* standby-cursor floor (bug-1147 guard). floor <= active_seg_lsn
|
||
* by construction (repl_trim_lsn caps at checkpoint_lsn=active_seg_lsn),
|
||
* so this check subsumes the old < active_seg_lsn guard. */
|
||
if (seg_start_lsn < floor) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name);
|
||
unlink(seg_path);
|
||
}
|
||
}
|
||
closedir(d);
|
||
}
|
||
|
||
/* Snapshot filename comparison helper for qsort — sorts ascending by name
|
||
* (which sorts ascending by LSN since filenames embed the LSN as hex). */
|
||
static int snap_name_cmp(const void* a, const void* b)
|
||
{
|
||
return strcmp(*(const char* const*)a, *(const char* const*)b);
|
||
}
|
||
|
||
/* Keep only the snap_keep_n most recent snapshot files; delete the rest. */
|
||
/* True when `lsn` is the checkpoint position an in-flight peer snapshot
|
||
* transfer currently references (ha-entity-snapshots task 4.1/4.3, Req
|
||
* 3.1-3.3) — checked directly against snapshot_inflight_lsn[], not gated on
|
||
* snapshot_inflight[] first: the LSN clears before the flag in every
|
||
* ha_snapshot_send_dispatch/snapshot_lane_fn path, so a stale flag=1 can
|
||
* momentarily coexist with an already-cleared LSN=0 (see task 4.1's
|
||
* Implementation Note). A checkpoint's LSN is never 0, so "nonzero and
|
||
* matches" is the correct, race-free protection signal on its own. No
|
||
* cluster (p->cluster == NULL, e.g. standalone lib-level tests) means no HA
|
||
* layer exists to be mid-transfer — nothing is protected. */
|
||
static bool snapshot_lsn_in_flight(persist_t* p, uint64_t lsn)
|
||
{
|
||
if (p->cluster == NULL || lsn == 0)
|
||
return false;
|
||
|
||
for (int i = 0; i < MAX_HA_CANDIDATES; i++) {
|
||
if (atomic_load_explicit(&p->cluster->ha.snapshot_inflight_lsn[i],
|
||
memory_order_acquire) == lsn)
|
||
return true;
|
||
}
|
||
return false;
|
||
}
|
||
|
||
static void rotate_snapshots(persist_t* p)
|
||
{
|
||
DIR* d = opendir(p->snap_path);
|
||
if (!d) return;
|
||
|
||
/* Collect all .capnp snapshot filenames */
|
||
char* names[1024];
|
||
int n = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL && n < (int)(sizeof(names) / sizeof(names[0]))) {
|
||
size_t len = strlen(e->d_name);
|
||
if (len <= 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue;
|
||
names[n] = strdup(e->d_name);
|
||
if (names[n]) n++;
|
||
}
|
||
closedir(d);
|
||
|
||
if (n == 0) return;
|
||
|
||
/* Sort ascending — oldest first */
|
||
qsort(names, (size_t)n, sizeof(names[0]), snap_name_cmp);
|
||
|
||
/* Delete all but the snap_keep_n newest, skipping any file an in-flight
|
||
* peer transfer is still reading from (task 4.3) — it is retried on the
|
||
* next rotation pass once that transfer completes. */
|
||
int keep = (int)p->config.snap_keep_n;
|
||
int delete_up_to = n - keep; /* first index to keep */
|
||
|
||
for (int i = 0; i < delete_up_to && i < n; i++) {
|
||
uint64_t lsn = 0;
|
||
if (sscanf(names[i], "snap-%" SCNx64 ".capnp", &lsn) == 1
|
||
&& snapshot_lsn_in_flight(p, lsn))
|
||
continue;
|
||
|
||
char path[4096];
|
||
snprintf(path, sizeof(path), "%s/%s", p->snap_path, names[i]);
|
||
unlink(path);
|
||
}
|
||
|
||
for (int i = 0; i < n; i++)
|
||
free(names[i]);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Async checkpoint scheduling (schedule_checkpoint + task fn) */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* checkpoint_task_fn: executed by delaymgr worker thread.
|
||
*
|
||
* The CAS in schedule_checkpoint already holds the "in_progress" slot (=1).
|
||
* We must clear it before calling persist_checkpoint(), which has its own
|
||
* atomic_exchange guard. After persist_checkpoint() returns (it resets the
|
||
* flag to 0), we also reset wal_record_count so the counter starts fresh. */
|
||
static int checkpoint_task_fn(void* arg)
|
||
{
|
||
persist_t* persist = (persist_t*)arg;
|
||
|
||
/* Release the "scheduled" slot so persist_checkpoint's guard can acquire */
|
||
atomic_store(&persist->checkpoint_in_progress, 0);
|
||
|
||
persist_checkpoint(persist);
|
||
|
||
/* Reset the record counter so the next checkpoint window starts clean.
|
||
* Done after persist_checkpoint returns — the flag is already 0 by now. */
|
||
atomic_store(&persist->wal_record_count, 0);
|
||
return 0;
|
||
}
|
||
|
||
/* schedule_checkpoint: idempotent async checkpoint request via delaymgr.
|
||
* Safe to call from any thread. Uses atomic CAS to ensure at most one
|
||
* checkpoint is scheduled or running at a time.
|
||
*
|
||
* If cluster is NULL (unit tests without a live delaymgr) the function
|
||
* falls back to a synchronous checkpoint. */
|
||
static void schedule_checkpoint(persist_t* persist)
|
||
{
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong(&persist->checkpoint_in_progress,
|
||
&expected, 1))
|
||
return; /* already scheduled or running — idempotency guard */
|
||
|
||
if (persist->cluster == NULL) {
|
||
/* Unit-test / no-cluster path: run synchronously and reset. */
|
||
atomic_store(&persist->checkpoint_in_progress, 0);
|
||
persist_checkpoint(persist);
|
||
atomic_store(&persist->wal_record_count, 0);
|
||
return;
|
||
}
|
||
|
||
/* Schedule via delaymgr with 1 s coalescing delay, single execution */
|
||
delay_task(&persist->cluster->delaymgr, 1, 1,
|
||
checkpoint_task_fn, persist);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Public: persist_checkpoint */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
int persist_checkpoint(persist_t* p)
|
||
{
|
||
/* Guard against concurrent checkpoints */
|
||
if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0)
|
||
return -EBUSY;
|
||
|
||
int rc = 0;
|
||
|
||
/* 1. Write WAL_CHECKPOINT marker (local only — never replicated) */
|
||
rc = persist_write_checkpoint_marker(p);
|
||
if (rc != 0)
|
||
goto done;
|
||
|
||
/* 2. Record checkpoint_lsn = the APPLIED cursor (last_applied_lsn), not
|
||
* the physical WAL append tip (current_lsn). write_snapshot() below
|
||
* serializes live jobmgr state, which only reflects entries the
|
||
* apply pump has actually dispatched -- NOT every entry durably
|
||
* appended/committed, since apply is decoupled from commit
|
||
* (raft-apply-decoupling) and can lag behind current_lsn under load.
|
||
* Stamping current_lsn here mislabels the snapshot: on restart,
|
||
* last_applied_lsn is re-seeded FROM checkpoint_lsn and replay
|
||
* resumes strictly above it, so any committed-but-not-yet-applied
|
||
* entries between the true apply cursor and current_lsn at
|
||
* checkpoint time are silently skipped forever, even though jobmgr
|
||
* never actually contained them (persist.h last_applied_lsn doc).
|
||
* Held under apply_pump_lock together with 3a/3b so the label and
|
||
* the jobmgr walk it describes are captured atomically w.r.t. the
|
||
* live apply pump -- the pump can only ever move last_applied_lsn
|
||
* and jobmgr forward, never backward, so freezing it here rules out
|
||
* the label ever claiming more than jobmgr actually holds. */
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
|
||
p->checkpoint_lsn = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_acquire);
|
||
|
||
/* 3a. Capture the latest committed config-marker reference BEFORE trimming
|
||
* (ha-term-safety task 4.2 r1): the trim below can unlink the segment
|
||
* holding a committed WAL_CONFIG_MARKER, so the derived committed config
|
||
* must be cached durably (into the sentinel) before that happens. */
|
||
scan_latest_committed_config_ref(p);
|
||
|
||
/* 3b. Serialize in-memory state to a snapshot file
|
||
* (P0: no live kvmgrs — writes sentinel header + cached config ref) */
|
||
rc = write_snapshot(p);
|
||
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
|
||
if (rc != 0)
|
||
goto done;
|
||
|
||
/* 4. Trim old WAL segments before the checkpoint */
|
||
trim_wal_segments(p);
|
||
|
||
/* 5. Rotate old snapshots — keep only snap_keep_n most recent */
|
||
rotate_snapshots(p);
|
||
|
||
done:
|
||
atomic_store(&p->checkpoint_in_progress, 0);
|
||
return rc;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Recovery internals */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Parse and restore every job frame then every queue frame from the entity
|
||
* payload beginning at the current `fd` offset (design.md "Checkpoint File
|
||
* Layout"): [job_count][job_count * {payload_len; capnp Job bytes}]
|
||
* [queue_count][queue_count * {name_len; name; status}].
|
||
*
|
||
* `file_size` is the checkpoint file's total size (from fstat) and
|
||
* `consumed` is the number of bytes already read from it (legacy header +
|
||
* format_version) — every length-prefixed frame's length is bounds-checked
|
||
* against the ACTUAL remaining bytes in the file before being trusted for a
|
||
* read or allocation, never blindly followed (a corrupt/attacker-controlled
|
||
* length must fail fast, not drive an oversized read).
|
||
*
|
||
* Restores in file order: every job via restore_job, then every queue via
|
||
* restore_queue_state (design.md sequence diagram). Returns 0 only if every
|
||
* frame restored; on the FIRST failure of any kind (short read, oversized
|
||
* length, decode error, or restore error) returns immediately with a
|
||
* negative errno — the caller MUST NOT commit any position change on a
|
||
* non-zero return (Requirement 2.3).
|
||
*
|
||
* restore_job's -EEXIST is treated as success, not a failure: it is job.c's
|
||
* own documented idempotent-replay guard (job already present), needed so a
|
||
* retried load — after an earlier partial attempt already created some jobs
|
||
* before failing on a later one — can continue past already-restored entries
|
||
* rather than aborting on them again. restore_queue_state has no such
|
||
* idempotent-duplicate case (re-applying the same status is already a
|
||
* harmless overwrite), so any non-zero return from it is a genuine failure. */
|
||
static int load_snapshot_entities(int fd, off_t file_size, off_t consumed,
|
||
persist_t* p, uint64_t stored_lsn)
|
||
{
|
||
uint32_t job_count = 0;
|
||
if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(job_count);
|
||
|
||
if (job_count > 0 && (p->cluster == NULL || p->cluster->jobmgr == NULL))
|
||
return -EINVAL; /* entities present but nowhere to restore them into */
|
||
|
||
for (uint32_t i = 0; i < job_count; i++) {
|
||
uint32_t payload_len = 0;
|
||
if (read(fd, &payload_len, sizeof(payload_len)) !=
|
||
(ssize_t)sizeof(payload_len))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(payload_len);
|
||
|
||
if (payload_len == 0 || (off_t)payload_len > file_size - consumed)
|
||
return -EIO;
|
||
|
||
uint8_t* buf = (uint8_t*)malloc(payload_len);
|
||
if (buf == NULL)
|
||
return -ENOMEM;
|
||
if (read(fd, buf, payload_len) != (ssize_t)payload_len) {
|
||
free(buf);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)payload_len;
|
||
|
||
int rrc = restore_job(p->cluster, 0, buf, payload_len, stored_lsn);
|
||
free(buf);
|
||
if (rrc != 0 && rrc != -EEXIST)
|
||
return rrc;
|
||
}
|
||
|
||
uint32_t queue_count = 0;
|
||
if (read(fd, &queue_count, sizeof(queue_count)) !=
|
||
(ssize_t)sizeof(queue_count))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(queue_count);
|
||
|
||
if (queue_count > 0 && (p->cluster == NULL || p->cluster->qmgr == NULL))
|
||
return -EINVAL; /* entities present but nowhere to restore them into */
|
||
|
||
for (uint32_t i = 0; i < queue_count; i++) {
|
||
uint32_t name_len = 0;
|
||
if (read(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len))
|
||
return -EIO;
|
||
consumed += (off_t)sizeof(name_len);
|
||
|
||
if (name_len == 0 || (off_t)name_len > file_size - consumed)
|
||
return -EIO;
|
||
|
||
char* name = (char*)malloc((size_t)name_len + 1);
|
||
if (name == NULL)
|
||
return -ENOMEM;
|
||
if (read(fd, name, name_len) != (ssize_t)name_len) {
|
||
free(name);
|
||
return -EIO;
|
||
}
|
||
name[name_len] = '\0';
|
||
consumed += (off_t)name_len;
|
||
|
||
uint32_t status = 0;
|
||
if (read(fd, &status, sizeof(status)) != (ssize_t)sizeof(status)) {
|
||
free(name);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)sizeof(status);
|
||
|
||
int qrc = restore_queue_state(p->cluster, name, status);
|
||
free(name);
|
||
if (qrc != 0)
|
||
return qrc;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* Load a snapshot: legacy header (magic + checkpoint_lsn + optional
|
||
* config-ref tail), then — for an entity-bearing checkpoint — every job and
|
||
* queue frame that follows it (ha-entity-snapshots task 5.2, Requirements
|
||
* 2.1-2.4, 5.1, 5.2). See design.md "load_snapshot (extended)" and its
|
||
* sequence diagram for the exact contract this implements.
|
||
*
|
||
* Ordering (Requirement 2.1, 2.2, 2.3 — restore FIRST, validate FULLY, THEN
|
||
* advance position, never the reverse, never partial): the header and any
|
||
* config-ref tail are parsed into LOCAL variables only. Format is then
|
||
* detected by whether any bytes remain past the header (design.md's stated
|
||
* discriminator — not a flag bit). A short file (nothing remains) is a
|
||
* legacy, pre-feature checkpoint: not an error, so its recorded position is
|
||
* committed immediately and *out_entities_restored is false. Otherwise every
|
||
* job then every queue frame is restored via load_snapshot_entities() before
|
||
* ANYTHING is written to p->checkpoint_lsn/p->current_lsn/p->cfgref_*; on
|
||
* any restore failure this function returns non-zero with p's fields
|
||
* PROVABLY untouched (the single commit point below is only ever reached
|
||
* after load_snapshot_entities() returns 0). *out_entities_restored is only
|
||
* set on a rc==0 return, matching persist.h's documented contract.
|
||
*
|
||
* Returns 0 on success, -EINVAL on bad magic / unrecognized entity format,
|
||
* -EIO on a short/corrupt read, -ENOMEM on allocation failure, -errno on
|
||
* open/fstat failure, or a restore_job/restore_queue_state error code. */
|
||
static int load_snapshot(persist_t* p, const char* path, uint64_t lsn,
|
||
bool* out_entities_restored)
|
||
{
|
||
(void)lsn; /* lsn is encoded in the file itself — we re-read it */
|
||
|
||
int fd = open(path, O_RDONLY);
|
||
if (fd < 0)
|
||
return -errno;
|
||
|
||
struct stat st;
|
||
if (fstat(fd, &st) != 0) {
|
||
int saved = errno;
|
||
close(fd);
|
||
return -saved;
|
||
}
|
||
|
||
/* magic(8) + checkpoint_lsn(8) + optional config-ref tail(1+8+32). */
|
||
char header[16 + 1 + 8 + 32];
|
||
ssize_t n = read(fd, header, sizeof(header));
|
||
|
||
if (n < 16) {
|
||
close(fd);
|
||
return -EIO;
|
||
}
|
||
|
||
if (memcmp(header, "XCESNAP\0", 8) != 0) {
|
||
close(fd);
|
||
return -EINVAL;
|
||
}
|
||
|
||
/* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch).
|
||
* Held LOCALLY -- not yet written to p-> -- until the single commit
|
||
* point below (legacy short-circuit, or after every entity restores). */
|
||
uint64_t stored_lsn;
|
||
memcpy(&stored_lsn, header + 8, 8);
|
||
|
||
/* Optional committed-config reference (ha-term-safety task 4.2 r1):
|
||
* present only when the sentinel carries the 41-byte tail. Also held
|
||
* locally until the same commit point. */
|
||
int cfgref_present = 0;
|
||
uint64_t cfgref_tag = 0;
|
||
uint8_t cfgref_sha[32];
|
||
memset(cfgref_sha, 0, sizeof(cfgref_sha));
|
||
if (n >= 16 + 1 + 8 + 32 && header[16] != 0) {
|
||
cfgref_present = 1;
|
||
memcpy(&cfgref_tag, header + 17, 8);
|
||
memcpy(cfgref_sha, header + 25, 32);
|
||
}
|
||
|
||
off_t consumed = (off_t)n;
|
||
|
||
/* Legacy-format detection (Requirement 2.4, 5.1, 5.2): "does more data
|
||
* exist" is the discriminator (design.md), not a flag bit -- a file
|
||
* ending exactly at the legacy header is unambiguously a pre-feature
|
||
* checkpoint. This is an expected rollout shape, not corruption: commit
|
||
* the position it records (existing, preserved legacy behavior) and
|
||
* report entities_restored=false so the caller falls back to normal
|
||
* catch-up rather than treating this checkpoint as fully complete. */
|
||
if (st.st_size <= consumed) {
|
||
close(fd);
|
||
p->checkpoint_lsn = stored_lsn;
|
||
p->current_lsn = stored_lsn;
|
||
if (cfgref_present) {
|
||
p->cfgref_present = 1;
|
||
p->cfgref_tag = cfgref_tag;
|
||
memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha));
|
||
}
|
||
if (out_entities_restored)
|
||
*out_entities_restored = false;
|
||
return 0;
|
||
}
|
||
|
||
/* Entity-bearing checkpoint: format_version, then job frames, then queue
|
||
* frames (design.md "Checkpoint File Layout"). */
|
||
uint32_t fmt = 0;
|
||
if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) {
|
||
close(fd);
|
||
return -EIO;
|
||
}
|
||
consumed += (off_t)sizeof(fmt);
|
||
|
||
/* An unrecognized format_version is corruption or a future layout this
|
||
* binary cannot read -- fail loud rather than silently treating it as
|
||
* legacy (task 5.1 review note: a version mismatch must never be masked
|
||
* as an expected legacy shape, or real corruption goes undetected). */
|
||
if (fmt != SNAPSHOT_FORMAT_VERSION) {
|
||
close(fd);
|
||
return -EINVAL;
|
||
}
|
||
|
||
int rc = load_snapshot_entities(fd, st.st_size, consumed, p, stored_lsn);
|
||
close(fd);
|
||
if (rc != 0)
|
||
return rc; /* Requirement 2.3: p->*_lsn / p->cfgref_* untouched */
|
||
|
||
/* Single commit point (Requirement 2.1, 2.2): reached ONLY after every
|
||
* job and queue frame restored successfully above. */
|
||
p->checkpoint_lsn = stored_lsn;
|
||
p->current_lsn = stored_lsn;
|
||
if (cfgref_present) {
|
||
p->cfgref_present = 1;
|
||
p->cfgref_tag = cfgref_tag;
|
||
memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha));
|
||
}
|
||
if (out_entities_restored)
|
||
*out_entities_restored = true;
|
||
return 0;
|
||
}
|
||
|
||
/* Public wrapper for load_snapshot — exposed via persist.h for ha.c (Task
|
||
* 3.5c). load_snapshot() above now owns the full restore-before-advance
|
||
* implementation (ha-entity-snapshots task 5.2): this wrapper is a thin
|
||
* pass-through, since load_snapshot() already gates *out_entities_restored
|
||
* and the p->checkpoint_lsn/current_lsn commit on entity-restore success
|
||
* (Requirements 2.1-2.4, 5.1, 5.2). */
|
||
int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn,
|
||
bool *out_entities_restored)
|
||
{
|
||
return load_snapshot(p, path, lsn, out_entities_restored);
|
||
}
|
||
|
||
/* P2-2: Standby WAL replay — apply a single replicated entry to in-memory state.
|
||
*
|
||
* No hooks fire on standbys; the hook system is empty until promotion.
|
||
*
|
||
* Returns 0 on success (including skip for checkpoint / unknown types),
|
||
* negative errno on error (NULL args, oversized payload). */
|
||
int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
|
||
const void* payload, uint32_t payload_len)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
/* Reject oversized payloads BEFORE any allocation */
|
||
if (payload_len > MAX_WAL_PAYLOAD_BYTES) {
|
||
ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u",
|
||
payload_len, MAX_WAL_PAYLOAD_BYTES);
|
||
return -EMSGSIZE;
|
||
}
|
||
|
||
/* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain
|
||
* (p->last_crc), and current_lsn are owned exclusively by the physical
|
||
* layer — persist_wal_write_replicated on the live replicated path, and
|
||
* replay_wal/persist_recover on recovery. This function MUST NOT touch
|
||
* them. (bug-778: it previously set p->last_crc from a synthetic crc=0
|
||
* header, breaking the on-disk CRC chain and cold-restart recovery.) */
|
||
|
||
/* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it
|
||
* via replication. Skip silently. */
|
||
if (type == WAL_CHECKPOINT)
|
||
return 0;
|
||
|
||
/* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward
|
||
* compatibility — an older controller can boot on WAL written by a newer
|
||
* one without hard-failing on unrecognised entry types. */
|
||
if (type > WAL_ENTRY_TYPE_MAX) {
|
||
WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u"
|
||
" (lsn=%" PRIu64 ") — skipping for forward compatibility",
|
||
type, WAL_ENTRY_TYPE_MAX, lsn);
|
||
return 0;
|
||
}
|
||
|
||
/* Advance the current_lsn high-water (monotonic, idempotent). This is a
|
||
* write-cursor high-water that is safe to bump from multiple paths. The
|
||
* CRC chain link (p->last_crc) is deliberately NOT touched here — it has a
|
||
* single owner: the physical writer (persist_wal_write_replicated) and
|
||
* replay (persist_recover). (bug-778) */
|
||
pthread_mutex_lock(&p->lock);
|
||
if (lsn + 1 > p->current_lsn)
|
||
p->current_lsn = lsn + 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (!p->cluster)
|
||
return 0; /* unit tests without a full cluster — skip entity restore */
|
||
|
||
switch (type) {
|
||
case WAL_JOB_SUBMIT:
|
||
/* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN
|
||
* through so the restored job is tagged for the egress watermark
|
||
* (ha-convergence-recovery 3.3). */
|
||
restore_job(p->cluster, 0, payload, payload_len, lsn);
|
||
break;
|
||
|
||
case WAL_JOB_ARRAY_SUBMIT:
|
||
/* job-array-single-wal-record task 3.3 (design.md "Array Apply
|
||
* Expansion"): the array-record analog of WAL_JOB_SUBMIT's
|
||
* restore_job() call directly above, dispatched from this SEPARATE,
|
||
* still-live standby catch-up / replicated-apply seam so it can
|
||
* never diverge from apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT
|
||
* case (Req 3.1, 3.2, 4.1). Mirrors the single-job case's exact
|
||
* minimalism: no xlist_t, no hook fire — this dispatcher is
|
||
* deliberately hook-free (standby catch-up does not notify the
|
||
* local scheduler/API server; that is the primary's job, done via
|
||
* apply_committed_entry()). out_new_jobs is not needed here, so
|
||
* NULL is passed (restore_job_array()'s out_new_jobs parameter is
|
||
* optional — see test_rja_out_new_jobs_is_optional,
|
||
* test_job_array_apply.c). The return value is intentionally
|
||
* discarded, matching every other case in this switch today (task
|
||
* 4.1 — Live Apply Resource Fence — is what later makes this
|
||
* function's callers respect a negative rc; not this task's
|
||
* boundary). */
|
||
restore_job_array(p->cluster, payload, payload_len, lsn, NULL);
|
||
break;
|
||
|
||
case WAL_JOB_START:
|
||
case WAL_JOB_FINISH:
|
||
case WAL_JOB_FAIL:
|
||
case WAL_JOB_STOP:
|
||
case WAL_JOB_RESUME:
|
||
case WAL_JOB_UPDATE:
|
||
case WAL_JOB_CKPT:
|
||
case WAL_JOB_PREASON:
|
||
case WAL_JOB_RUSAGE:
|
||
/* Payload: wal_jobstate_t {jobid, state} */
|
||
if (payload_len >= sizeof(wal_jobstate_t)) {
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
restore_job_state(p->cluster, js->jobid, js->state);
|
||
}
|
||
break;
|
||
|
||
case WAL_WORKER_CTRL:
|
||
/* Payload: [uint32_t detail][capnp worker] */
|
||
if (payload_len > WORKER_WAL_HDR_SIZE)
|
||
worker_from_capnp(p->cluster, payload, payload_len);
|
||
break;
|
||
|
||
case WAL_QUEUE_CTRL:
|
||
/* Payload: [wal_queuehdr_t][name NUL-terminated] */
|
||
if (payload_len > sizeof(wal_queuehdr_t)) {
|
||
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
|
||
const char* name = (const char*)(payload + sizeof(wal_queuehdr_t));
|
||
restore_queue_state(p->cluster, name, qh->status);
|
||
}
|
||
break;
|
||
|
||
case WAL_NOOP:
|
||
/* Internal promotion no-op: empty payload, no entity restore, no
|
||
* hooks. current_lsn is advanced by the physical layer (writer /
|
||
* replay), not here — recognised explicitly rather than via default. */
|
||
break;
|
||
|
||
case WAL_CONFIG_MARKER:
|
||
/* Config-change marker (ha-term-safety task 4.2): a {version_tag,
|
||
* sha256} reference, NEVER the config bytes (R7.4). It carries NO
|
||
* entity state, so replay performs no restore here — the committed
|
||
* configuration is DERIVED from the latest committed marker on demand
|
||
* (config_committed_marker), and the bytes are obtained + hash-verified
|
||
* out-of-band before activation (task 4.3). Recognised explicitly so
|
||
* the per-LSN term index (built above this switch's caller) records its
|
||
* term and on_get_entry_term / (term,lsn) reconcile treat it as an
|
||
* ordinary log entry. */
|
||
break;
|
||
|
||
default:
|
||
break;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 2.2 — see persist.h for the full contract doc
|
||
* (routing table, the job-state-family hook-mapping deviation, and what this
|
||
* function deliberately does NOT own). Not yet wired into any live call
|
||
* site — replay (2.3) and the live apply pump (6.1) will call this later. */
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1 fix): thread-local "this call
|
||
* stack is inside apply_committed_entry's own post-mutation notification"
|
||
* guard. persist_job_hook_fun / persist_queue_hook_fun / persist_worker_hook_fun
|
||
* are AFTER_EVENT subscribers that (for RETAIN/fact-shaped entry types)
|
||
* durably append a NEW WAL entry every time they fire — the correct behavior
|
||
* for their ORIGINAL, still-live purpose (job_ctrl's mark_job_update/
|
||
* mark_job_fail synchronous command path; the legacy queue/worker control
|
||
* command handlers) which mutate live state and rely on this exact hook
|
||
* firing to durably record that mutation.
|
||
*
|
||
* apply_committed_entry() now ALSO fires these same AFTER_EVENT hooks (job
|
||
* facts always as the generic HOOK_DETAIL_JOB_UPDATE, per task 2.2's
|
||
* documented wire-format deviation; queue/worker control at their native
|
||
* detail) to deliver its own post-mutation NOTIFICATION — but that entry is
|
||
* by definition already durably committed (it just got applied FROM the
|
||
* WAL), so re-appending it would silently duplicate the entry on every live
|
||
* apply. This is fixable, correctly, ONLY by provenance (which caller fired
|
||
* the hook), NOT by hook `detail` value: HOOK_DETAIL_JOB_UPDATE in
|
||
* particular is genuinely dual-origin (job_ctrl's SIGCONT/SIGSTOP path uses
|
||
* the identical detail for its own, still-legitimate live append) — an
|
||
* unconditional detail-based filter would silently break job_ctrl's suspend/
|
||
* resume durability instead of fixing anything. A shared persist_t-wide flag
|
||
* would ALSO be wrong: it would race across threads (e.g. job_ctrl mutating
|
||
* a DIFFERENT job on the iomgr thread while this thread's apply walk is
|
||
* mid-hook for an unrelated fact) and could wrongly suppress or wrongly
|
||
* permit an append depending on scheduling. A thread-local flag, scoped
|
||
* narrowly around each xce_run_hooks() call this function makes, is race-free
|
||
* by construction: it is only ever observed by hook subscribers invoked
|
||
* SYNCHRONOUSLY on this exact call stack, from this exact function. */
|
||
static _Thread_local bool t_apply_seam_notifying = false;
|
||
|
||
int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
if (!p->cluster)
|
||
return 0; /* unit tests without a full cluster — nothing to dispatch into */
|
||
|
||
cluster_t* c = p->cluster;
|
||
|
||
switch (wal_type) {
|
||
case WAL_JOB_SUBMIT: {
|
||
/* Return value distinguishes a genuine creation from an idempotent
|
||
* no-op (bug-1419): 1 = restore_job() actually constructed a new
|
||
* job, 0 = the jobid already existed (duplicate WAL entry — see
|
||
* below), negative = a real error. replay_wal()'s new-job tally
|
||
* relies on this distinction; the live apply pump discards it.
|
||
*
|
||
* A duplicate WAL_JOB_SUBMIT for the same jobid is an EXPECTED,
|
||
* harmless occurrence, not a bug: a client's local commit-timeout
|
||
* retry can append a second entry for the same jobid while the
|
||
* first is still in flight during a quorum-loss window; the WAL
|
||
* entry that timed out is never retracted, so if it later achieves
|
||
* majority commit anyway, both entries end up committed. restore_job
|
||
* ()'s own -EEXIST here is what makes that safe to replay twice. */
|
||
int rc = restore_job(c, 0, payload, len, lsn);
|
||
if (rc == -EEXIST)
|
||
return 0; /* idempotent re-apply (replay / resync) — no hook */
|
||
if (rc < 0)
|
||
return rc;
|
||
|
||
struct capn jc_ctx;
|
||
capn_init_mem(&jc_ctx, (const uint8_t*)payload, len, 0);
|
||
Job_ptr jp;
|
||
jp.p = capn_getp(capn_root(&jc_ctx), 0, 1);
|
||
struct Job cj;
|
||
read_Job(&cj, jp);
|
||
uint64_t jobid = cj.jobid;
|
||
capn_free(&jc_ctx);
|
||
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return 1; /* defensive: should not happen after a successful restore */
|
||
|
||
void* jptr = j;
|
||
xlist_t jlist = { .count = 1, .caps = 1, .delta = 1, .data = &jptr };
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_NEW, c, &jlist);
|
||
t_apply_seam_notifying = false;
|
||
|
||
return 1;
|
||
}
|
||
|
||
case WAL_JOB_ARRAY_SUBMIT: {
|
||
/* job-array-single-wal-record task 3.2 (design.md "Array Apply
|
||
* Expansion"): the array-record analog of WAL_JOB_SUBMIT's
|
||
* restore_job() above, dispatched from this SAME shared seam so live
|
||
* apply, boot replay (both callers of this function), and (task
|
||
* 3.3) standby catch-up can never diverge in how they expand a
|
||
* committed array record (Req 3.1, 3.2).
|
||
*
|
||
* restore_job_array()'s return generalizes restore_job()'s
|
||
* three-way contract to N elements: a positive count = the number
|
||
* of genuinely NEW elements this call constructed (this is exactly,
|
||
* and only, what the hook fire below carries — Req 3.4); -EEXIST =
|
||
* every index in the record already existed (idempotent
|
||
* whole-record no-op, Req 3.3) — no hook; a negative errno = a
|
||
* genuine construction/resource-exhaustion failure (Req 9.4) — no
|
||
* hook, propagated so persist_apply_pump_run() (task 4.1) can halt
|
||
* and fence on it rather than silently skipping it. */
|
||
xlist_t out_new_jobs;
|
||
memset(&out_new_jobs, 0, sizeof(out_new_jobs));
|
||
int rc = restore_job_array(c, payload, len, lsn, &out_new_jobs);
|
||
if (rc == -EEXIST)
|
||
return 0; /* idempotent re-apply (replay / resync) — no hook */
|
||
if (rc < 0)
|
||
return rc;
|
||
|
||
/* Exactly ONE fire carrying every newly created element in one
|
||
* list — never once per element (Req 3.4). out_new_jobs is already
|
||
* the precise newly-created set restore_job_array() built; no
|
||
* re-derivation via key_find needed. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_NEW, c, &out_new_jobs);
|
||
t_apply_seam_notifying = false;
|
||
|
||
/* Only the list's own backing array is released here — ownership of
|
||
* every job_t* it holds already transferred to c->jobmgr inside
|
||
* restore_job_array()'s bulk-insert pass, mirroring
|
||
* test_job_array_apply.c's own xlist_fini(&out) usage after a
|
||
* successful call. */
|
||
xlist_fini(&out_new_jobs);
|
||
|
||
return rc;
|
||
}
|
||
|
||
case WAL_JOB_START:
|
||
case WAL_JOB_FINISH:
|
||
case WAL_JOB_STOP:
|
||
case WAL_JOB_RESUME:
|
||
case WAL_JOB_RUSAGE:
|
||
case WAL_JOB_CKPT:
|
||
case WAL_JOB_PREASON: {
|
||
/* Every job-state fact type in this group shares one wire shape —
|
||
* wal_jobstate_t {jobid, state} — and therefore one guarded core.
|
||
* Their payloads are deterministic/state-independent (a START is
|
||
* always JOB_STAT_RUN, a FINISH's outcome depends only on
|
||
* exit_code), so an absolute `state` baked at receipt time never
|
||
* goes stale — unlike WAL_JOB_FAIL/WAL_JOB_UPDATE below, which have
|
||
* their own dedicated cases. See persist.h's doc comment on
|
||
* apply_committed_entry for the full routing table and why this
|
||
* does not dispatch to the richer type-specific apply_job_{start,
|
||
* finish}_transition() cores (their parameter structs are not
|
||
* present on this wire format). */
|
||
if (len < sizeof(wal_jobstate_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return -ENOENT;
|
||
|
||
lock(j, RTTI_LOCK_WRITE);
|
||
job_transition_result_t result =
|
||
apply_job_update_transition(j, (int)js->state);
|
||
unlock(j);
|
||
|
||
if (result != JOB_TRANSITION_APPLIED)
|
||
return 0; /* rejected-backward / no-op-duplicate — no hook */
|
||
|
||
/* Hazard 1 guard (see t_apply_seam_notifying's doc comment above):
|
||
* HOOK_DETAIL_JOB_UPDATE is dual-origin — job_ctrl's mark_job_update
|
||
* (SIGCONT/SIGSTOP) fires the identical detail for its own live,
|
||
* still-legitimate append via persist_job_hook_fun. Without this
|
||
* guard, EVERY fact this apply pump applies would re-append itself
|
||
* as a duplicate, mis-typed WAL_JOB_UPDATE entry the moment the live
|
||
* pump (task 6.1) starts calling this function. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_JOB_FAIL: {
|
||
/* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3) fix for a job-state
|
||
* corruption bug: unlike the deterministic group above, a FAIL's
|
||
* resulting state depends on the job's CURRENT state (PERR merges
|
||
* onto whatever terminal state already holds). Dispatch on _pad
|
||
* (see wal_types.h's doc comment on wal_jobstate_t._pad):
|
||
* - WAL_JOB_FAIL_STAGE_FLAG set: a deferred fail-fact
|
||
* (record_job_fail_fact, job.c) — reconstruct a minimal
|
||
* job_failinfo_t from {jobid, stage} and call the REAL
|
||
* apply_job_fail_transition() against the job's ACTUAL CURRENT
|
||
* state at apply time, never a receipt-time snapshot.
|
||
* - _pad == 0: the legacy live path (job_ctrl's mark_job_fail) —
|
||
* `state` is already the correct, current, absolute value,
|
||
* computed synchronously under the same lock as the mutation
|
||
* (no staleness window) — an absolute overwrite is safe and
|
||
* unchanged. */
|
||
if (len < sizeof(wal_jobstate_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return -ENOENT;
|
||
|
||
lock(j, RTTI_LOCK_WRITE);
|
||
job_transition_result_t result;
|
||
if (js->_pad & WAL_JOB_FAIL_STAGE_FLAG) {
|
||
job_failinfo_t failinfo = {
|
||
.jobid = js->jobid,
|
||
.stage = js->_pad & ~WAL_JOB_FAIL_STAGE_FLAG,
|
||
.code = 0,
|
||
.detail = NULL,
|
||
};
|
||
result = apply_job_fail_transition(j, &failinfo);
|
||
} else {
|
||
result = apply_job_update_transition(j, (int)js->state);
|
||
}
|
||
unlock(j);
|
||
|
||
if (result != JOB_TRANSITION_APPLIED)
|
||
return 0; /* rejected-backward / no-op-duplicate — no hook */
|
||
|
||
/* Hazard 1 guard — see the identical guard above. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_JOB_UPDATE: {
|
||
/* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3): same class of fix as
|
||
* WAL_JOB_FAIL above. Dispatch on _pad:
|
||
* - WAL_JOB_UPDATE_POSTDONE_MARK: a deferred postdone fact
|
||
* (record_job_postdone_fact, job.c) — call
|
||
* apply_job_postdone_transition(), which re-derives the
|
||
* postdone merge (conditional EXIT backstop + OR JOB_STAT_PDONE)
|
||
* fresh against the job's ACTUAL CURRENT state at apply time.
|
||
* - _pad == 0: the legacy live path (job_ctrl's mark_job_update,
|
||
* SIGCONT/SIGSTOP) — `state` is already correct/absolute,
|
||
* computed synchronously under the same lock as the mutation —
|
||
* an absolute overwrite is safe and unchanged. */
|
||
if (len < sizeof(wal_jobstate_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_jobstate_t* js = (const wal_jobstate_t*)payload;
|
||
job_t* j = NULL;
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
key_find(c->jobmgr, (void*)js->jobid, (void**)&j);
|
||
unlock(c->jobmgr);
|
||
if (!j)
|
||
return -ENOENT;
|
||
|
||
lock(j, RTTI_LOCK_WRITE);
|
||
job_transition_result_t result =
|
||
(js->_pad == WAL_JOB_UPDATE_POSTDONE_MARK)
|
||
? apply_job_postdone_transition(j)
|
||
: apply_job_update_transition(j, (int)js->state);
|
||
unlock(j);
|
||
|
||
if (result != JOB_TRANSITION_APPLIED)
|
||
return 0; /* rejected-backward / no-op-duplicate — no hook */
|
||
|
||
/* Hazard 1 guard — see the identical guard above. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_JOB_UPDATE, c, j);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_QUEUE_CTRL: {
|
||
if (len <= sizeof(wal_queuehdr_t))
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload;
|
||
const char* name =
|
||
(const char*)((const uint8_t*)payload + sizeof(wal_queuehdr_t));
|
||
|
||
/* restore_queue_state's -ENOENT ("queue not yet built from config")
|
||
* and -EINVAL (bad name) are both benign skips here, matching
|
||
* apply_wal_entry's existing behavior of never fencing on this
|
||
* entry type's restore outcome. */
|
||
if (restore_queue_state(c, name, qh->status) < 0)
|
||
return 0;
|
||
|
||
queue_t* q = NULL;
|
||
lock(c->qmgr, RTTI_LOCK_READ);
|
||
key_find(c->qmgr, (void*)name, (void**)&q);
|
||
unlock(c->qmgr);
|
||
if (!q)
|
||
return 0;
|
||
|
||
/* Same double-append class as the job-fact guard above: the legacy
|
||
* queue-control command handler still mutates then relies on this
|
||
* exact AFTER_EVENT firing to durably append via
|
||
* persist_queue_hook_fun (WAL_QUEUE_CTRL is unconditionally
|
||
* appended on AFTER_EVENT there, not gated on WAL_MODE_WAIT like the
|
||
* job-submit case is) — guard so this live-pump re-notify does not
|
||
* re-append an already-committed entry. */
|
||
t_apply_seam_notifying = true;
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, HOOK_AFTER_EVENT,
|
||
HOOK_DETAIL_QUEUE_UPDATE, c, q);
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
case WAL_WORKER_CTRL: {
|
||
if (len <= WORKER_WAL_HDR_SIZE)
|
||
return 0; /* undersized payload — skip, mirrors apply_wal_entry */
|
||
|
||
uint32_t hook_detail;
|
||
memcpy(&hook_detail, payload, sizeof(hook_detail));
|
||
|
||
int rc = worker_from_capnp(c, payload, len);
|
||
if (rc == -EEXIST)
|
||
return 0; /* idempotent re-apply (replay / resync) — no hook */
|
||
if (rc < 0)
|
||
return rc;
|
||
|
||
/* Recover the worker's name via a cheap raw capnp field read
|
||
* (read_Worker's capn_get_text is a borrowed, NUL-terminated pointer
|
||
* into the decode buffer — no allocation, unlike decode_Worker) so
|
||
* the just-restored worker can be looked up for the notification
|
||
* hook without changing worker_from_capnp's own contract. */
|
||
const uint8_t* wpayload = (const uint8_t*)payload + WORKER_WAL_HDR_SIZE;
|
||
size_t wlen = len - WORKER_WAL_HDR_SIZE;
|
||
struct capn wc_ctx;
|
||
capn_init_mem(&wc_ctx, wpayload, wlen, 0);
|
||
Worker_ptr wp;
|
||
wp.p = capn_getp(capn_root(&wc_ctx), 0, 1);
|
||
struct Worker cw;
|
||
read_Worker(&cw, wp);
|
||
|
||
worker_t* w = NULL;
|
||
lock(c->workermgr, RTTI_LOCK_READ);
|
||
key_find(c->workermgr, (void*)cw.name.str, (void**)&w);
|
||
unlock(c->workermgr);
|
||
capn_free(&wc_ctx);
|
||
if (!w)
|
||
return 0; /* defensive: should not happen after a successful restore */
|
||
|
||
/* Same double-append class as the job-fact/queue guards above:
|
||
* persist_worker_hook_fun unconditionally appends on AFTER_EVENT for
|
||
* every worker detail (no WAL_MODE_WAIT gate), and the legacy worker
|
||
* join/update/dead command paths still rely on that firing for their
|
||
* own live durability — guard the whole re-notify switch so this
|
||
* live-pump call never re-appends an already-committed entry. */
|
||
t_apply_seam_notifying = true;
|
||
switch (hook_detail) {
|
||
case HOOK_DETAIL_WORKER_JOIN:
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c,
|
||
w->host.name, w);
|
||
break;
|
||
case HOOK_DETAIL_WORKER_UPDATE:
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_UPDATE, c, w);
|
||
break;
|
||
case HOOK_DETAIL_WORKER_DEAD:
|
||
xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w);
|
||
break;
|
||
default:
|
||
break; /* unrecognized discriminator — mutation already applied,
|
||
* notification skipped rather than guessed at */
|
||
}
|
||
t_apply_seam_notifying = false;
|
||
return 0;
|
||
}
|
||
|
||
default:
|
||
/* WAL_NOOP / WAL_CONFIG_MARKER / WAL_CHECKPOINT / unknown — no
|
||
* entity effect at this seam, no hook (unchanged from
|
||
* apply_wal_entry's existing treatment of these types). */
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/* ============================================================
|
||
* Ordered apply pump (raft-apply-decoupling task 6.1)
|
||
* ============================================================ */
|
||
|
||
/* Locate and read back the WAL entry at exactly `lsn` from the on-disk
|
||
* segment files, for the live apply pump to hand to apply_committed_entry()
|
||
* (mirrors read_entry_meta_locked's segment-scan idiom above, but returns
|
||
* the entry's type and a malloc'd copy of its payload instead of just
|
||
* metadata, and re-verifies the entry CRC — this is live-path plumbing, not
|
||
* a metadata probe).
|
||
*
|
||
* Does not take p->lock: every LSN this is ever called with is already
|
||
* majority-committed (durable and replicated), strictly below the physical
|
||
* append tip a concurrent persist_wal_async could be extending — so this
|
||
* never races the single-writer append path for the bytes it actually
|
||
* reads.
|
||
*
|
||
* Returns 0 and populates *out_type/*out_payload (malloc'd if payload_len >
|
||
* 0, caller frees; NULL when payload_len is 0)/*out_len on success.
|
||
* Returns -ENOENT if no segment contains this LSN (should not happen for a
|
||
* genuinely committed entry) or a negative errno on a read/CRC failure. */
|
||
static int apply_pump_read_entry(persist_t* p, uint64_t lsn,
|
||
uint32_t* out_type, uint8_t** out_payload,
|
||
uint32_t* out_len)
|
||
{
|
||
if (!p || lsn == 0 || !out_type || !out_payload || !out_len)
|
||
return -EINVAL;
|
||
|
||
/* Fast path (raft-apply-decoupling task 7.4 perf-fix): the sole caller
|
||
* (persist_apply_pump_run) calls this once per LSN in strict ascending
|
||
* order with no gaps in the common case, and holds apply_pump_lock
|
||
* across the entire walk -- so the cache fields below need no lock of
|
||
* their own here (see persist.h doc comment). A cache hit reads the next
|
||
* header directly at the remembered offset on the already-open fd,
|
||
* giving amortized O(1) per entry instead of a fresh
|
||
* opendir/readdir/segment-sort plus a scan-from-zero on every single
|
||
* call (the O(n^2) regression this fixes). ANY miss below just drops the
|
||
* cache and falls through to the unchanged slow path -- correctness
|
||
* never depends on the cache being right, only performance does. */
|
||
if (p->apply_pump_cache_fd >= 0 && p->apply_pump_cache_next_lsn == lsn) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(p->apply_pump_cache_fd, &hdr, sizeof(hdr),
|
||
(off_t)p->apply_pump_cache_off);
|
||
bool hit = (hn == (ssize_t)sizeof(hdr)) && (hdr.lsn == lsn);
|
||
uint8_t* payload = NULL;
|
||
|
||
if (hit && hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
hit = false;
|
||
} else {
|
||
ssize_t pn = pread(p->apply_pump_cache_fd, payload,
|
||
hdr.payload_len,
|
||
(off_t)(p->apply_pump_cache_off +
|
||
sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
free(payload);
|
||
payload = NULL;
|
||
hit = false;
|
||
}
|
||
}
|
||
}
|
||
if (hit) {
|
||
uint32_t expected_crc =
|
||
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
if (hdr.payload_len > 0)
|
||
expected_crc = crc32(expected_crc, (const Bytef*)payload,
|
||
hdr.payload_len);
|
||
if (hdr.crc != expected_crc) {
|
||
free(payload);
|
||
payload = NULL;
|
||
hit = false;
|
||
}
|
||
}
|
||
|
||
if (hit) {
|
||
*out_type = hdr.type;
|
||
*out_payload = payload;
|
||
*out_len = hdr.payload_len;
|
||
p->apply_pump_cache_off +=
|
||
(uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
p->apply_pump_cache_next_lsn = lsn + 1;
|
||
return 0;
|
||
}
|
||
|
||
/* Miss (short read/EOF meaning the entry rolled into the next
|
||
* segment, header mismatch, CRC failure, whatever): don't try to be
|
||
* clever about advancing to the next segment inline. Close and fall
|
||
* through to the EXISTING slow path unchanged -- it already
|
||
* correctly finds the right segment from scratch. */
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return -ENOENT;
|
||
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return -ENOENT;
|
||
|
||
/* Ascending sort by start LSN (segment count is tiny in practice). */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
int rc = -ENOENT;
|
||
for (int si = 0; si < nseg; si++) {
|
||
/* Segments are LSN-ordered; one whose start LSN already exceeds the
|
||
* target cannot contain it, nor can any later one. */
|
||
if (seg_lsns[si] > lsn)
|
||
break;
|
||
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path,
|
||
seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (hn < (ssize_t)sizeof(hdr))
|
||
break; /* short read / EOF / pre-allocated zero tail */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
if (hdr.lsn == lsn) {
|
||
uint8_t* payload = NULL;
|
||
uint32_t expected_crc =
|
||
crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
|
||
if (hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
rc = -ENOMEM;
|
||
break;
|
||
}
|
||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
free(payload);
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
expected_crc = crc32(expected_crc, (const Bytef*)payload,
|
||
hdr.payload_len);
|
||
}
|
||
|
||
if (hdr.crc != expected_crc) {
|
||
free(payload);
|
||
rc = -EIO;
|
||
break;
|
||
}
|
||
|
||
*out_type = hdr.type;
|
||
*out_payload = payload;
|
||
*out_len = hdr.payload_len;
|
||
rc = 0;
|
||
|
||
/* raft-apply-decoupling task 7.4 perf-fix: populate the
|
||
* segment-lookup cache with this fd positioned just past the
|
||
* entry we just returned, BEFORE closing it below, so the
|
||
* next expected LSN (lsn+1) hits the fast path above instead
|
||
* of re-scanning from segment start next time. Every OTHER
|
||
* branch (error paths, not-found-in-this-segment
|
||
* continuation) closes fd exactly as before -- caching only
|
||
* ever happens on this success branch. */
|
||
p->apply_pump_cache_fd = fd;
|
||
p->apply_pump_cache_seg_lsn = seg_lsns[si];
|
||
p->apply_pump_cache_off =
|
||
off + (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
p->apply_pump_cache_next_lsn = lsn + 1;
|
||
fd = -1; /* ownership transferred to the cache -- the
|
||
* close(fd) below becomes a guarded no-op. */
|
||
break;
|
||
}
|
||
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
if (fd >= 0)
|
||
close(fd);
|
||
|
||
if (rc == 0 || rc == -ENOMEM || rc == -EIO)
|
||
break; /* found it, or hit a genuine error inside the segment
|
||
* that should contain it */
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
int persist_apply_pump_run(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return -EINVAL;
|
||
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
|
||
int applied = 0;
|
||
for (;;) {
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t next = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_relaxed) + 1;
|
||
if (next > committed)
|
||
break; /* caught up as of this instant */
|
||
|
||
uint32_t wal_type = 0;
|
||
uint8_t* payload = NULL;
|
||
uint32_t len = 0;
|
||
int rc = apply_pump_read_entry(p, next, &wal_type, &payload, &len);
|
||
if (rc < 0) {
|
||
ERROR("persist_apply_pump_run: failed to read committed entry "
|
||
"lsn=%" PRIu64 " (rc=%d) -- stopping this pass; the entry "
|
||
"stays committed and will be retried, from the same "
|
||
"position, on the next kick",
|
||
next, rc);
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
return rc;
|
||
}
|
||
|
||
/* Strict order, no skipping or reordering: this is the single
|
||
* dispatch call every newly committed entry passes through here,
|
||
* regardless of its commit mode (WAIT command or RETAIN fact) — the
|
||
* loop only ever advances last_applied_lsn one entry at a time, in
|
||
* ascending order, so a burst of commands and facts that committed
|
||
* out of their original submission order still applies in strict
|
||
* commit (LSN) order (Req 3.3, 7.3). */
|
||
int apply_rc = apply_committed_entry(p, wal_type, next, payload, len);
|
||
if (apply_rc < 0) {
|
||
/* job-array-single-wal-record task 4.1 (design.md "Live Apply
|
||
* Resource Fence"): this return value was previously discarded
|
||
* -- (void)apply_committed_entry(...) -- with last_applied_lsn
|
||
* advancing unconditionally, silently and permanently losing
|
||
* any apply failure. A negative rc here is never a content-
|
||
* derived failure (Req 9.1-9.3's pre-commit validation already
|
||
* made those impossible post-commit); it is a node-local
|
||
* resource-exhaustion failure (e.g. OOM mid array expansion,
|
||
* Req 9.4) that must not be skipped. Mirror the
|
||
* apply_pump_read_entry failure branch immediately above: stop
|
||
* the walk exactly at this LSN, do not advance
|
||
* last_applied_lsn, and do not take this LSN's pending-apply
|
||
* hand-off (persist_take_pending_apply/on_applied) -- the entry
|
||
* never actually applied, so that hand-off must not fire for
|
||
* it. The entry stays committed and will be retried, from the
|
||
* same position, on the next kick. This is a GENERAL fix: every
|
||
* WAL type's apply outcome is now respected, not just
|
||
* WAL_JOB_ARRAY_SUBMIT's. Under NORMAL operation every other
|
||
* existing type's apply case returns 0 or positive, so this
|
||
* branch is a behavior no-op for them in practice; the
|
||
* fact-type cases' pre-existing "-ENOENT: referenced job not
|
||
* found" branch (persist.h's dispatch table) is one other real,
|
||
* pre-existing source of a negative return here, and now also
|
||
* correctly halts the walk here instead of being silently
|
||
* discarded, exactly as this fix intends. */
|
||
ERROR("persist_apply_pump_run: apply_committed_entry failed for "
|
||
"committed entry lsn=%" PRIu64 " wal_type=%" PRIu32
|
||
" (rc=%d) -- stopping this pass at this position; the "
|
||
"entry stays committed and will be retried, from the same "
|
||
"position, on the next kick; Live Apply Resource Fence "
|
||
"ENGAGED (job-array-single-wal-record Req 9.4)",
|
||
next, wal_type, apply_rc);
|
||
/* job-array-single-wal-record task 4.2 (design.md "Live Apply
|
||
* Resource Fence"): engage the NEW, live, self-healing fence --
|
||
* distinct from recovery_unrecoverable -- so the promotion gate
|
||
* (task 4.3) can refuse this node while it stays alive and
|
||
* keeps retrying. Cleared below, from inside this same walk,
|
||
* the instant a retry of this exact LSN succeeds (Req 9.5). */
|
||
persist_mark_apply_resource_fenced(p, next);
|
||
free(payload);
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
return apply_rc;
|
||
}
|
||
free(payload);
|
||
|
||
atomic_store_explicit(&p->last_applied_lsn, next, memory_order_relaxed);
|
||
applied++;
|
||
|
||
/* job-array-single-wal-record task 4.2 (design.md "Live Apply
|
||
* Resource Fence", Req 9.5): self-heal. Called unconditionally on
|
||
* EVERY successful apply, not just after a known-fenced LSN --
|
||
* persist_clear_apply_resource_fence() is a cheap no-op when nothing
|
||
* is fenced (the overwhelming common case). This is correct and
|
||
* sufficient because the walk above is strictly sequential and
|
||
* returns immediately on any failure (see the halt branch just
|
||
* above): it can never advance PAST a fenced LSN without that exact
|
||
* LSN applying successfully first, so "the next entry to apply
|
||
* successfully after a fence engages" and "a retry of the SAME
|
||
* stuck LSN succeeding" are the same event by construction. No
|
||
* election, promotion check, or external/admin action is involved --
|
||
* the fence clears purely because this walk itself made progress. */
|
||
persist_clear_apply_resource_fence(p);
|
||
|
||
/* raft-apply-decoupling task 6.2 (Req 1.1, 2.1-2.3): consume this
|
||
* LSN's command hand-off, if any (task 3.2's
|
||
* persist_register_pending_apply(), e.g. handle_jobsubmit's
|
||
* commit-outcome continuation) -- strictly AFTER the mutation above
|
||
* and after last_applied_lsn already reflects it, never before or
|
||
* during. A fact entry never registers a hand-off, so
|
||
* persist_take_pending_apply() simply returns 0 for it and this is a
|
||
* correct no-op with no per-entry-type branching needed here.
|
||
* persist_take_pending_apply() unlinks the registration before
|
||
* returning it, so re-applying the same lsn (a future retry/replay
|
||
* landing on this exact walk position again) can never fire the
|
||
* continuation a second time. */
|
||
pending_apply_notify_fn on_applied = NULL;
|
||
void* apply_ctx = NULL;
|
||
|
||
if (persist_take_pending_apply(p, next, &on_applied, &apply_ctx))
|
||
on_applied(apply_ctx, next);
|
||
}
|
||
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
return applied;
|
||
}
|
||
|
||
/* Heap payload for one dispatched apply-pump walk invocation; frees itself
|
||
* after use, mirroring commit.c's commit_resolve_dispatch_t convention. */
|
||
typedef struct {
|
||
persist_t* p;
|
||
} apply_pump_dispatch_t;
|
||
|
||
/* Run walk passes until genuinely caught up, using the same recheck-then-
|
||
* clear idiom commit.c's sweep thread uses (task 1.3) to avoid a lost-wakeup
|
||
* race between "persist_apply_pump_run just decided it is caught up" and "a
|
||
* new commit lands right after". Runs on a pool worker thread (or inline as
|
||
* persist_apply_pump_kick's documented fallback) — never on the calling
|
||
* (potentially replication-ACK/iomgr) thread when dispatched normally.
|
||
*
|
||
* Does NOT immediately retry a failed pass: persist_apply_pump_run() already
|
||
* logs loudly and leaves last_applied_lsn unadvanced past a read failure, so
|
||
* an immediate self-retry here would busy-loop forever on the exact same
|
||
* unresolved head-of-line entry (a real, reproduced hazard — a persist_t
|
||
* whose committed_lsn is ever advanced without a corresponding real WAL
|
||
* entry on disk, e.g. a test fixture exercising commit-engine resolve logic
|
||
* in isolation, spun this into a multi-million-iteration tight loop before
|
||
* this guard existed). A future, genuinely NEW kick — triggered by a later
|
||
* real commit-advance event — retries then, at that event's natural cadence,
|
||
* never in a busy spin. */
|
||
static void apply_pump_drain(persist_t* p)
|
||
{
|
||
for (;;) {
|
||
int rc = persist_apply_pump_run(p);
|
||
|
||
atomic_store_explicit(&p->apply_pump_scheduled, 0, memory_order_release);
|
||
|
||
if (rc < 0)
|
||
return; /* already logged loudly inside persist_apply_pump_run;
|
||
* do not busy-retry the same unresolved entry */
|
||
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t next = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_relaxed) + 1;
|
||
if (next > committed)
|
||
return; /* genuinely caught up: no race window survived open */
|
||
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel,
|
||
memory_order_relaxed))
|
||
return; /* a concurrent kick already reclaimed the flag; it will
|
||
* run its own pass and pick up this same new work */
|
||
/* else: we reclaimed the flag ourselves -- loop and run another pass */
|
||
}
|
||
}
|
||
|
||
static int apply_pump_vtask_entry(vtask_t* task)
|
||
{
|
||
apply_pump_dispatch_t* d = (apply_pump_dispatch_t*)task->data->u.generic;
|
||
|
||
apply_pump_drain(d->p);
|
||
free(d);
|
||
return 0;
|
||
}
|
||
|
||
void persist_apply_pump_kick(persist_t* p, void* pool_arg)
|
||
{
|
||
if (!p || !p->cluster)
|
||
return;
|
||
|
||
/* bug-1416 (raft-apply-decoupling task 7.2 round 2): defer while this
|
||
* node's own boot-time persist_recover() replay+verification pass is
|
||
* still running -- last_applied_lsn is being written UNLOCKED by
|
||
* replay_wal() during that window (single-threaded-recovery
|
||
* assumption), and persist_verify_apply_correctness() is about to
|
||
* compare a live re-read of it (and of the live job table) against a
|
||
* WAL-scan-derived expectation. Letting this walk run concurrently
|
||
* would race BOTH: it could advance last_applied_lsn out from under
|
||
* replay_wal's own plain (non-CAS) store, and it mutates the same job
|
||
* table persist_verify_apply_correctness re-reads as "actual" --
|
||
* producing a false APPLY_DIVERGENT verdict on a perfectly healthy
|
||
* boot (confirmed live). No work is lost: persist_recover() sets this
|
||
* flag back to true and issues exactly one catch-up kick right before
|
||
* it returns, which will walk from last_applied_lsn+1 up through
|
||
* whatever committed_lsn has grown to by then -- a full, ordered
|
||
* drain, not a partial one. Intentionally checked with NO lock: a
|
||
* stale `true` read racing the flag's transition to `false` is
|
||
* harmless (the walk itself still only advances one entry at a time
|
||
* under apply_pump_lock, so at worst this call proceeds a few
|
||
* instructions into a window persist_recover() is about to close, and
|
||
* the deferred re-kick after recover() still runs); a stale `false`
|
||
* merely defers one no-op-cost kick to the next real trigger. */
|
||
if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire))
|
||
return;
|
||
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel,
|
||
memory_order_relaxed))
|
||
return; /* already scheduled/running -- it will pick up any new work */
|
||
|
||
vtaskmgr_t* pool = (vtaskmgr_t*)pool_arg;
|
||
vtask_data_t* data = calloc(1, sizeof(*data));
|
||
vtask_t* task = vtask_alloc();
|
||
apply_pump_dispatch_t* d = calloc(1, sizeof(*d));
|
||
|
||
/* Mirrors commit.c's dispatch_continuation fallback exactly: never leave
|
||
* a scheduled walk silently unrun. pool == NULL is the expected shape of
|
||
* a commit-only / bare-persist-cluster unit fixture that never calls
|
||
* commit_set_pool — a real cluster always wires the pool before any
|
||
* commit activity (cluster_init_phase2), so this fallback path is the
|
||
* normal, deterministic behavior for that class of test, not a rare
|
||
* production condition. */
|
||
if (pool == NULL || data == NULL || task == NULL || d == NULL) {
|
||
free(data);
|
||
free(task);
|
||
free(d);
|
||
apply_pump_drain(p);
|
||
return;
|
||
}
|
||
|
||
d->p = p;
|
||
data->vtype = VTASK_GENERIC;
|
||
data->u.generic = d;
|
||
vtask_init(task, (vtask_func_t)apply_pump_vtask_entry, data);
|
||
|
||
/* VTASK_LIVE_BLOCK is never admission-gated (same delivery guarantee as
|
||
* commit.c's own continuation dispatch) — this cannot fast-reject the
|
||
* way VTASK_REQ_BLOCK could. */
|
||
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
|
||
WARN("persist_apply_pump_kick: vtask enqueue failed -- running "
|
||
"inline (stay alive, never leave a scheduled walk unrun)");
|
||
free(data);
|
||
free(task);
|
||
free(d);
|
||
apply_pump_drain(p);
|
||
}
|
||
}
|
||
|
||
/* persist_apply_pump_run_sync (2026-07-10, chaotic-soak regression fix,
|
||
* ROUND 2): synchronously drain the apply pump on THIS thread, GUARANTEEING
|
||
* last_applied_lsn reaches committed_lsn (as observed at entry) before
|
||
* returning -- regardless of what any concurrent caller is doing.
|
||
*
|
||
* Round 1 of this fix CAS-claimed apply_pump_scheduled and, on losing that
|
||
* claim to a concurrent caller (e.g. a heartbeat-driven kick that fired
|
||
* moments before promotion), just returned -- betting the OTHER caller's
|
||
* async vtask-dispatched drain would finish in time. A second live chaotic
|
||
* soak proved that bet wrong: job loss persisted even with the environmental
|
||
* fdatasync noise (bug-924) independently confirmed and eliminated, so this
|
||
* was a genuine, separate bug in the fix itself, not noise. The caller here
|
||
* (promotion) cannot outsource its guarantee to a drain it doesn't control
|
||
* the completion of.
|
||
*
|
||
* Fix: do NOT use the CAS-coalescing flag as a "someone else has it, I'm
|
||
* done" signal -- call persist_apply_pump_run() directly in a loop until
|
||
* THIS call observes last_applied_lsn caught up to committed_lsn. Multiple
|
||
* concurrent callers (this one and any vtask-dispatched persist_apply_pump_
|
||
* kick) safely interleave via persist_apply_pump_run's own internal
|
||
* apply_pump_lock and apply_committed_entry's per-LSN idempotency -- at
|
||
* worst a redundant no-op pass, never lost or duplicated work. Stops (does
|
||
* not busy-loop forever) on a genuine read failure, exactly mirroring
|
||
* apply_pump_drain's own error handling: persist_apply_pump_run() already
|
||
* logs loudly and leaves last_applied_lsn unadvanced on that path. */
|
||
void persist_apply_pump_run_sync(persist_t* p)
|
||
{
|
||
if (!p || !p->cluster)
|
||
return;
|
||
if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire))
|
||
return;
|
||
|
||
for (;;) {
|
||
uint64_t committed = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t applied = atomic_load_explicit(&p->last_applied_lsn,
|
||
memory_order_relaxed);
|
||
if (applied >= committed)
|
||
return; /* caught up -- by us or a concurrent drain */
|
||
|
||
int rc = persist_apply_pump_run(p);
|
||
if (rc < 0)
|
||
return; /* already logged loudly; do not busy-loop on the same
|
||
* unresolved entry (mirrors apply_pump_drain) */
|
||
/* Loop: re-check in case committed_lsn advanced further mid-walk, or
|
||
* a concurrent drain made partial progress this pass didn't see. */
|
||
}
|
||
}
|
||
|
||
/* Read WAL segment files in LSN order; replay entries with lsn > from_lsn.
|
||
*
|
||
* Segment filename format: seg-<start_lsn_hex16>.wal
|
||
* Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes).
|
||
*
|
||
* CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry
|
||
* in a segment). If prev_crc mismatches, stop at last good entry and log WARN.
|
||
*
|
||
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
|
||
* *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped
|
||
* on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a
|
||
* short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF,
|
||
* pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is
|
||
* about the STRUCTURE of the local log, not about being behind the cluster: a
|
||
* clean log that simply ends below the cluster high-water is still CLEAN.
|
||
*
|
||
* new_job_submits_out (raft-apply-decoupling task 5.1, when non-NULL) is set
|
||
* to the number of WAL_JOB_SUBMIT entries this call actually dispatched via
|
||
* apply_committed_entry() that resulted in a GENUINE new job (that function's
|
||
* return value == 1) — not merely the number of WAL_JOB_SUBMIT entries above
|
||
* `from_lsn` present in the log. This is the log-derived half of the
|
||
* post-replay apply-correctness expectation (see apply_expectation_t).
|
||
*
|
||
* A WAL_JOB_SUBMIT entry can legitimately be duplicated for the same jobid
|
||
* (bug-1419): a client's local commit-timeout retry appends a second entry
|
||
* while the first is still in flight during a quorum-loss window, and the
|
||
* timed-out entry is never retracted, so both can end up committed if the
|
||
* first later achieves majority commit anyway. apply_committed_entry()
|
||
* dedupes this via restore_job()'s -EEXIST and returns 0 (no-op) rather than
|
||
* 1 (created) for the duplicate — counting raw WAL_JOB_SUBMIT occurrences
|
||
* instead would tally that duplicate as a second job and spuriously fail the
|
||
* post-replay comparison against jobmgr's actual growth (APPLY_DIVERGENT).
|
||
*
|
||
* Returns number of entries replayed (>= 0) or negative errno on I/O error. */
|
||
static int replay_wal(persist_t* p, uint64_t from_lsn,
|
||
recover_verdict_t* verdict_out,
|
||
uint64_t* new_job_submits_out)
|
||
{
|
||
/* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_CLEAN;
|
||
if (new_job_submits_out)
|
||
*new_job_submits_out = 0;
|
||
|
||
/* --- Collect and sort segment filenames by start LSN --- */
|
||
DIR* d = opendir(p->wal_path);
|
||
if (!d)
|
||
return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */
|
||
|
||
/* Collect up to 4096 segment names */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0) continue;
|
||
uint64_t seg_start = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue;
|
||
seg_lsns[nseg++] = seg_start;
|
||
}
|
||
closedir(d);
|
||
|
||
if (nseg == 0)
|
||
return 0;
|
||
|
||
/* Simple insertion sort — segment count is tiny in practice */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
/* --- Replay entries from each segment --- */
|
||
int replayed = 0;
|
||
uint32_t last_crc = WAL_CRC_NONE;
|
||
|
||
for (int si = 0; si < nseg; si++) {
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[si]);
|
||
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0) {
|
||
WARN("persist_recover: cannot open segment %s: %s",
|
||
seg_path, strerror(errno));
|
||
continue;
|
||
}
|
||
|
||
uint32_t seg_off = 0;
|
||
|
||
for (;;) {
|
||
/* Read header */
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off);
|
||
if (hn == 0)
|
||
break; /* clean end of segment (CLEAN) */
|
||
if (hn < (ssize_t)sizeof(hdr)) {
|
||
/* A partial header (or read error) is a SHORT READ — the log
|
||
* stops mid-entry, so it cannot be reconstructed contiguously
|
||
* (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */
|
||
if (hn < 0)
|
||
WARN("persist_recover: read error in %s at offset %u: %s",
|
||
seg_path, seg_off, strerror(errno));
|
||
else
|
||
WARN("persist_recover: short header read in %s at offset %u "
|
||
"(got %zd of %zu bytes); stopping replay",
|
||
seg_path, seg_off, hn, sizeof(hdr));
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
break;
|
||
}
|
||
|
||
/* Detect pre-allocated zero region (no valid entry starts with
|
||
* all-zero header: lsn == 0 && type == 0 && payload_len == 0).
|
||
* This is the normal fallocate'd tail after the last valid entry —
|
||
* a clean stop, NOT corruption (stays RECOVER_CLEAN). */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
|
||
/* CRC chain validation: prev_crc must match last_crc in segment */
|
||
if (seg_off == 0) {
|
||
/* First entry in this segment: prev_crc must be WAL_CRC_NONE
|
||
* (or we accept any value if last_crc is also WAL_CRC_NONE,
|
||
* e.g. across a segment boundary after checkpoint trim). */
|
||
if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE &&
|
||
hdr.prev_crc != last_crc) {
|
||
WARN("persist_recover: CRC chain break at start of %s "
|
||
"(prev_crc=%08x expected=%08x); stopping replay",
|
||
seg_path, hdr.prev_crc, last_crc);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
break;
|
||
}
|
||
} else {
|
||
if (hdr.prev_crc != last_crc) {
|
||
WARN("persist_recover: CRC chain break in %s at offset %u "
|
||
"(prev_crc=%08x expected=%08x); stopping replay at last good entry",
|
||
seg_path, seg_off, hdr.prev_crc, last_crc);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* Verify entry CRC */
|
||
uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE);
|
||
|
||
uint8_t* payload = NULL;
|
||
if (hdr.payload_len > 0) {
|
||
payload = malloc(hdr.payload_len);
|
||
if (!payload) {
|
||
close(fd);
|
||
return -ENOMEM;
|
||
}
|
||
ssize_t pn = pread(fd, payload, hdr.payload_len,
|
||
(off_t)(seg_off + sizeof(hdr)));
|
||
if (pn != (ssize_t)hdr.payload_len) {
|
||
WARN("persist_recover: short payload read in %s at offset %u",
|
||
seg_path, seg_off);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
free(payload);
|
||
break;
|
||
}
|
||
expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len);
|
||
}
|
||
|
||
if (hdr.crc != expected_crc) {
|
||
WARN("persist_recover: CRC mismatch in %s at offset %u "
|
||
"(got=%08x expected=%08x); stopping replay",
|
||
seg_path, seg_off, hdr.crc, expected_crc);
|
||
if (verdict_out)
|
||
*verdict_out = RECOVER_TRUNCATED;
|
||
free(payload);
|
||
break;
|
||
}
|
||
|
||
/* Advance segment-local CRC */
|
||
last_crc = hdr.crc;
|
||
|
||
/* Record this entry's term at its LSN for recovery read-back
|
||
* (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical
|
||
* entry — including snapshot-covered ones below from_lsn — so the
|
||
* index spans the full LSN range the reconcile/promotion logic may
|
||
* query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and
|
||
* unknown future types (type > WAL_CHECKPOINT): they are skipped
|
||
* below and are never logical Raft entries, so the live append path
|
||
* does not record them either (kept symmetric). The raw hdr.term is
|
||
* stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM
|
||
* mapping is applied only at read-back. */
|
||
if (hdr.type < WAL_CHECKPOINT)
|
||
record_entry_term(p, hdr.lsn, hdr.term);
|
||
|
||
/* Skip entries already covered by the snapshot */
|
||
if (hdr.lsn <= from_lsn) {
|
||
free(payload);
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
continue;
|
||
}
|
||
|
||
/* Skip WAL_CHECKPOINT markers (local bookkeeping only) */
|
||
if (hdr.type == WAL_CHECKPOINT) {
|
||
free(payload);
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
/* Advance current_lsn past the marker */
|
||
if (hdr.lsn + 1 > p->current_lsn)
|
||
p->current_lsn = hdr.lsn + 1;
|
||
continue;
|
||
}
|
||
|
||
/* Unknown future entry types: log and skip */
|
||
if (hdr.type > WAL_CHECKPOINT) {
|
||
WARN("persist_recover: unknown WAL entry type %u at LSN %lu; skipping",
|
||
hdr.type, (unsigned long)hdr.lsn);
|
||
free(payload);
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
continue;
|
||
}
|
||
|
||
/* Logical entity restore via the shared apply seam
|
||
* (raft-apply-decoupling task 2.3, design.md "Apply Seam"): the
|
||
* SAME apply_committed_entry() dispatch the live commit-advance
|
||
* apply pump will use (task 6.1), so a node's materialized state
|
||
* is the deterministic result of its committed log regardless of
|
||
* whether it got there via restart replay or live apply
|
||
* (Req 3.1-3.4). apply_wal_entry() (this replay loop's previous
|
||
* per-entry dispatch) is NOT retired by this change — it remains
|
||
* the live standby catch-up/replicated-apply dispatch used
|
||
* elsewhere (ha.c); it is simply no longer this loop's call. The
|
||
* CRC chain + current_lsn remain owned by this replay loop
|
||
* (current_lsn advanced below; last_crc at line ~1601) — neither
|
||
* dispatch function ever touched them (bug-778).
|
||
*
|
||
* KNOWN HAZARD (tracked in tasks.md Implementation Notes, task
|
||
* 2.3): this replays the FULL post-checkpoint WAL range on every
|
||
* restart (last_applied_lsn is not persisted, deliberately
|
||
* recomputed here), so apply_committed_entry()'s AFTER_EVENT
|
||
* hooks fire again for entries already applied+notified in a
|
||
* prior process life. Confirmed inert today: scheduler hooks
|
||
* aren't registered until a post-boot network JOIN
|
||
* (on_scheduler_join, sched.c), and apiserver hooks — though
|
||
* registered before this runs — gate their notify on a
|
||
* connected endpoint that is empty at first-boot replay time.
|
||
* MUST be re-verified before tasks 4.1/6.1/6.2/6.3 make this
|
||
* path broadly live. */
|
||
int apply_rc = apply_committed_entry(p, hdr.type, hdr.lsn, payload,
|
||
hdr.payload_len);
|
||
free(payload);
|
||
|
||
/* raft-apply-decoupling task 5.1: track the replay-derived
|
||
* LOGICAL apply cursor separately from current_lsn (the physical
|
||
* WAL position owned by this loop, bug-778) — last_applied_lsn is
|
||
* the "highest entry actually dispatched via
|
||
* apply_committed_entry()" position the post-replay verification
|
||
* compares against. Segments are scanned in ascending LSN order
|
||
* (sorted above) and entries within a segment are sequential, so
|
||
* a plain store (no monotonic max needed) is correct here. */
|
||
atomic_store_explicit(&p->last_applied_lsn, hdr.lsn,
|
||
memory_order_relaxed);
|
||
|
||
/* Log-derived half of the post-replay apply-correctness
|
||
* expectation (Req 8.1): every WAL_JOB_SUBMIT that reaches this
|
||
* point is, by construction, above the loaded snapshot's
|
||
* checkpoint_lsn (the from_lsn skip above already filtered out
|
||
* anything the snapshot covers) — so it names a job that must now
|
||
* exist in the live job table that did not exist before this
|
||
* replay pass began. A duplicate WAL_JOB_SUBMIT for an
|
||
* already-existing jobid (bug-1419: timeout-then-late-commit
|
||
* retry) is a legitimate no-op — apply_committed_entry() returns
|
||
* 0 for it, not 1 — so it must NOT be tallied here or the
|
||
* post-replay comparison against jobmgr's real growth spuriously
|
||
* fails (APPLY_DIVERGENT). */
|
||
if (hdr.type == WAL_JOB_SUBMIT && new_job_submits_out &&
|
||
apply_rc == 1)
|
||
(*new_job_submits_out)++;
|
||
|
||
/* job-array-single-wal-record task 6.0a: the array analog of the
|
||
* WAL_JOB_SUBMIT tally immediately above. apply_committed_entry()'s
|
||
* WAL_JOB_ARRAY_SUBMIT case (restore_job_array()'s return,
|
||
* generalized to N elements) returns apply_rc == the genuine
|
||
* count of NEWLY created elements (1..array_size) on a real
|
||
* apply, or 0 for an idempotent whole-record re-apply (every
|
||
* index already existed -- restore_job_array() returned
|
||
* -EEXIST, which this same dispatch maps to 0, mirroring
|
||
* WAL_JOB_SUBMIT's own dedup return). Tallying apply_rc (not a
|
||
* flat +1) is required here, unlike the single-job case above,
|
||
* because one WAL_JOB_ARRAY_SUBMIT entry can materialize many
|
||
* jobmgr entries at once; guarding on apply_rc > 0 excludes the
|
||
* idempotent 0 case from double-counting a re-apply, exactly as
|
||
* apply_rc == 1 excludes WAL_JOB_SUBMIT's duplicate-jobid case
|
||
* (bug-1419) above. Without this branch, replay_wal() silently
|
||
* undercounts every array record's real N-element jobmgr growth
|
||
* to 0, and persist_verify_apply_correctness() then reports a
|
||
* false APPLY_DIVERGENT verdict that engages the PERMANENT,
|
||
* non-self-healing persist_recover_loud_fence_if_unrecoverable()
|
||
* fence (live-reproduced task 6.1 soak, 2026-07-15). */
|
||
if (hdr.type == WAL_JOB_ARRAY_SUBMIT && new_job_submits_out &&
|
||
apply_rc > 0)
|
||
(*new_job_submits_out) += (uint64_t)apply_rc;
|
||
|
||
/* Advance current_lsn to one past this entry */
|
||
if (hdr.lsn + 1 > p->current_lsn)
|
||
p->current_lsn = hdr.lsn + 1;
|
||
|
||
replayed++;
|
||
seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
|
||
close(fd);
|
||
}
|
||
|
||
/* Update last_crc so the next WAL write chains correctly */
|
||
pthread_mutex_lock(&p->lock);
|
||
p->last_crc = last_crc;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
return replayed;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 5.1: cheap read-lock count of c->jobmgr's live
|
||
* entries, mirroring collect_job_frames' iteration idiom (this file, above)
|
||
* without collecting or filtering anything. Used only to derive the
|
||
* post-replay materialized-entity expectation (never on a hot path). Safe on
|
||
* a NULL cluster or jobmgr (a bare unit-test persist_t) — returns 0. */
|
||
static uint64_t count_jobmgr_entries(cluster_t* c)
|
||
{
|
||
if (!c || !c->jobmgr)
|
||
return 0;
|
||
|
||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||
|
||
iterator_t it;
|
||
int irc = iter_make(&it, c->jobmgr);
|
||
if (irc < 0) {
|
||
unlock(c->jobmgr);
|
||
return 0; /* empty table */
|
||
}
|
||
|
||
uint64_t n = 0;
|
||
for (;;) {
|
||
job_t* j = NULL;
|
||
int nrc = iter_next(&it, (void**)&j);
|
||
if (nrc < 0 || nrc == RTTI_ITER_NO_DATA)
|
||
break;
|
||
if (j == NULL)
|
||
continue;
|
||
n++;
|
||
}
|
||
|
||
iter_destroy(&it);
|
||
unlock(c->jobmgr);
|
||
return n;
|
||
}
|
||
|
||
/* persist_verify_apply_correctness — see persist.h for the full contract
|
||
* doc (raft-apply-decoupling task 5.1, Req 8.1/8.2). Pure comparison: no
|
||
* logging, no fence side effects. */
|
||
apply_verdict_t persist_verify_apply_correctness(
|
||
persist_t* p, const apply_expectation_t* exp,
|
||
uint64_t jobmgr_count_before_replay)
|
||
{
|
||
if (!p || !exp)
|
||
return APPLY_CLEAN; /* defensive: nothing to verify */
|
||
|
||
uint64_t actual_last_applied =
|
||
atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire);
|
||
if (actual_last_applied != exp->expected_last_applied_lsn)
|
||
return APPLY_DIVERGENT;
|
||
|
||
/* Entity-materialization half of the check: skip gracefully when there
|
||
* is no live cluster to query — mirrors apply_committed_entry()'s own
|
||
* "nothing to dispatch into" convention for a bare unit-test persist_t.
|
||
* Only the position check above applies in that case. */
|
||
if (!p->cluster || !p->cluster->jobmgr)
|
||
return APPLY_CLEAN;
|
||
|
||
uint64_t jobmgr_count_after = count_jobmgr_entries(p->cluster);
|
||
if (jobmgr_count_after < jobmgr_count_before_replay)
|
||
return APPLY_DIVERGENT; /* table shrank — impossible on a clean replay */
|
||
|
||
uint64_t actual_new_jobs = jobmgr_count_after - jobmgr_count_before_replay;
|
||
if (actual_new_jobs != exp->expected_new_job_count)
|
||
return APPLY_DIVERGENT;
|
||
|
||
return APPLY_CLEAN;
|
||
}
|
||
|
||
apply_verdict_t persist_apply_verdict(persist_t* p)
|
||
{
|
||
return p ? p->apply_verdict : APPLY_CLEAN;
|
||
}
|
||
|
||
uint64_t persist_last_applied_lsn(persist_t* p)
|
||
{
|
||
return p ? atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire)
|
||
: 0;
|
||
}
|
||
|
||
/* bug-1416 (raft-apply-decoupling task 7.2 round 2) -- see persist.h's
|
||
* doc comment on the recovery_complete field for the full contract. */
|
||
bool persist_recovery_complete(persist_t* p)
|
||
{
|
||
return !p || atomic_load_explicit(&p->recovery_complete,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/* bug-1416 round 2 -- see persist.h's doc comment on this function for the
|
||
* full rationale (closes the gap between persist_init() and the FIRST
|
||
* persist_recover() call, which persist_recover()'s own false/true bracket
|
||
* cannot see). */
|
||
void persist_mark_recovery_pending(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return;
|
||
atomic_store_explicit(&p->recovery_complete, false, memory_order_release);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Public: persist_recover */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
int persist_recover(persist_t* p)
|
||
{
|
||
/* bug-1416 (raft-apply-decoupling task 7.2 round 2): close the boot-time
|
||
* race window for the FULL duration of this call, not just the position
|
||
* check inside persist_verify_apply_correctness -- see the
|
||
* recovery_complete field's doc comment (persist.h) for the complete
|
||
* rationale. Held false until the natural (success) return at the
|
||
* bottom of this function, which restores it and issues one deliberate
|
||
* catch-up kick. The two hard-I/O-error early returns below
|
||
* (load_snapshot / replay_wal failure) deliberately do NOT restore it:
|
||
* those are pre-existing, rare disk-I/O-level failures the caller
|
||
* already treats as fatal to this boot attempt (cluster_init_phase2
|
||
* tears down persist_t on a nonzero return) -- leaving the gate closed
|
||
* is the conservative, fail-static choice (never let the live apply
|
||
* pump or live-replication path mutate state on top of a persist_t
|
||
* whose own recovery never finished), not a new hazard. */
|
||
atomic_store_explicit(&p->recovery_complete, false, memory_order_release);
|
||
|
||
/* raft-apply-decoupling task 7.4 perf-fix (round 2): invalidate the apply
|
||
* pump's segment-lookup cache before replay -- the cached fd/offset are
|
||
* only valid relative to the pre-recovery on-disk state, and replay below
|
||
* can change checkpoint_lsn/current_lsn. Doing this first covers every
|
||
* return path below (including the early I/O-failure returns), so the
|
||
* first post-recovery pump kick always falls back to full segment
|
||
* rediscovery from the correct, recovered position -- never trusting a
|
||
* stale pre-recovery cursor. Wrapped in apply_pump_lock (matching
|
||
* persist_truncate_wal_above's identical invalidation site below) so the
|
||
* cache fields' documented invariant -- "guarded ENTIRELY by
|
||
* apply_pump_lock, never p->lock" (persist.h) -- holds unconditionally
|
||
* here too, rather than resting on the implicit fact that no live pump
|
||
* walk can be racing this boot-time call. No caller of persist_recover()
|
||
* holds apply_pump_lock (its sole production call site, cluster.c boot
|
||
* sequence, runs before the pump is ever kicked), so this cannot
|
||
* deadlock. */
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
if (p->apply_pump_cache_fd >= 0) {
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
|
||
/* --- Step 1: find latest snapshot (highest LSN filename) --- */
|
||
char snap_file[4096] = "";
|
||
uint64_t snap_lsn = 0;
|
||
|
||
DIR* d = opendir(p->snap_path);
|
||
if (d) {
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL) {
|
||
size_t len = strlen(e->d_name);
|
||
if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0)
|
||
continue;
|
||
uint64_t lsn = 0;
|
||
if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1)
|
||
continue;
|
||
if (lsn > snap_lsn) {
|
||
snap_lsn = lsn;
|
||
snprintf(snap_file, sizeof(snap_file),
|
||
"%s/%s", p->snap_path, e->d_name);
|
||
}
|
||
}
|
||
closedir(d);
|
||
}
|
||
|
||
/* --- Step 2: load snapshot (if any) --- */
|
||
if (snap_lsn > 0 && snap_file[0] != '\0') {
|
||
/* Local recovery doesn't need the entities-restored indicator to
|
||
* decide behavior: the WAL replay in Step 3 applies everything
|
||
* above checkpoint_lsn regardless, unchanged by ha-entity-snapshots.
|
||
* It IS consumed for observability parity with the wire receive
|
||
* path (ha_handle_snapshot_chunk, task 6.1) — a rolling-upgrade
|
||
* restart against a pre-feature (legacy, position-only) local
|
||
* checkpoint is an expected rollout state, not an error. */
|
||
bool entities_restored = false;
|
||
int rc = load_snapshot(p, snap_file, snap_lsn, &entities_restored);
|
||
if (rc != 0) {
|
||
ERROR("persist_recover: failed to load snapshot %s: %s",
|
||
snap_file, strerror(-rc));
|
||
return rc;
|
||
}
|
||
/* checkpoint_lsn and current_lsn set by load_snapshot */
|
||
if (entities_restored) {
|
||
INFO("persist_recover: snapshot %s loaded lsn=%" PRIu64
|
||
" with full job/queue entity restoration", snap_file,
|
||
p->checkpoint_lsn);
|
||
} else {
|
||
INFO("persist_recover: snapshot %s lsn=%" PRIu64 " is a legacy "
|
||
"position-only checkpoint (no entity content) — continuing "
|
||
"via normal WAL replay", snap_file, p->checkpoint_lsn);
|
||
}
|
||
}
|
||
|
||
/* raft-apply-decoupling task 5.1 (Req 8.1): seed the replay-derived
|
||
* apply cursor from the loaded baseline (checkpoint_lsn stays 0 when no
|
||
* snapshot was found) so that, on a CLEAN replay with no new entries
|
||
* beyond the snapshot, the cursor already sits at the recovered tip —
|
||
* mirroring how current_lsn is seeded by load_snapshot. Step 3's
|
||
* replay_wal advances it further for every entry actually dispatched
|
||
* above this baseline. Also capture the live job table's size BEFORE
|
||
* replay runs: the post-replay verification below needs this baseline to
|
||
* isolate the jobs replay itself materializes from anything the snapshot
|
||
* already restored. */
|
||
atomic_store_explicit(&p->last_applied_lsn, p->checkpoint_lsn,
|
||
memory_order_release);
|
||
uint64_t jobmgr_count_before_replay = count_jobmgr_entries(p->cluster);
|
||
|
||
/* --- Step 3: replay WAL entries after checkpoint_lsn --- */
|
||
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1):
|
||
* replay distinguishes a clean log end from a corruption-truncated end.
|
||
* We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do
|
||
* NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean
|
||
* replay over a loaded snapshot-sentinel (no entries below the sentinel) is
|
||
* CLEAN — the sentinel established the committed position; replay only adds
|
||
* entries above it. */
|
||
recover_verdict_t verdict = RECOVER_CLEAN;
|
||
uint64_t new_job_submits = 0;
|
||
int replayed = replay_wal(p, p->checkpoint_lsn, &verdict, &new_job_submits);
|
||
if (replayed < 0)
|
||
return replayed;
|
||
|
||
p->recovery_verdict = verdict;
|
||
if (verdict == RECOVER_TRUNCATED)
|
||
WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally "
|
||
"corrupt (CRC break / mismatch / short read); member marked "
|
||
"recovery-truncated (non-promotable until caught up)");
|
||
|
||
/* --- Step 3b: re-derive the durable floor from the recovered WAL tip
|
||
* (ha-commit-durability task 3.4, Req 2.3) ---
|
||
*
|
||
* Publish the member's durable floor (the durable last-log index consumed
|
||
* by the Raft freshness gate, task 4.2) as the highest INTACT replayed LSN.
|
||
* replay_wal advanced current_lsn to one past the last intact entry — incl.
|
||
* stopping at a corruption boundary on TRUNCATED — so current_lsn - 1 is
|
||
* exactly that intact WAL tip, and a snapshot-sentinel load leaves it at the
|
||
* sentinel/replayed tip. We store it into durable_lsn because the recovered
|
||
* on-disk entries are GENUINELY durable (each was CRC-validated and read
|
||
* from stable storage by replay_wal), so the freshness advertisement is now
|
||
* backed by durable state rather than a volatile in-memory position.
|
||
*
|
||
* Crux (Req 2.3 — a truncated member cannot over-advertise): this is an
|
||
* ASSIGN to the recovered tip, NOT a max() with whatever durable_lsn held.
|
||
* A fresh recover starts from durable_lsn=0 (persist_init), so on a
|
||
* truncating restart the floor lands at the LOWER intact tip — never a
|
||
* stale higher in-memory value and never the commit position
|
||
* (committed_lsn). The recovered tip is, by construction, <= current_lsn-1,
|
||
* preserving the durable_lsn <= current_lsn-1 invariant. The post-election
|
||
* safe-commit point continues to come from voter replies, not this floor
|
||
* (CLAUDE.md). */
|
||
uint64_t recovered_tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0;
|
||
atomic_store_explicit(&p->durable_lsn, recovered_tip, memory_order_release);
|
||
|
||
/* raft-apply-decoupling task 5.1 (Req 8.1/8.2): post-replay apply-
|
||
* correctness verification, reported ALONGSIDE the structural
|
||
* recovery_verdict set above — neither implies the other. See
|
||
* persist_verify_apply_correctness's doc comment for exactly what is
|
||
* compared. Routing an APPLY_DIVERGENT verdict into the existing
|
||
* non-promotable loud-fence mechanism is task 5.2; this only classifies
|
||
* and logs.
|
||
*
|
||
* Round-2 fix: the expected position MUST be the LOGICAL apply cursor
|
||
* replay_wal() already maintains in p->last_applied_lsn — NOT
|
||
* recovered_tip (current_lsn - 1, the PHYSICAL WAL-scan tip used above
|
||
* for durable_lsn). current_lsn advances for every WAL entry replay_wal
|
||
* scans, including WAL_CHECKPOINT markers and unknown future types,
|
||
* neither of which is ever routed through apply_committed_entry() (see
|
||
* the skip branches in replay_wal). last_applied_lsn only advances when
|
||
* a real dispatch happens. If the highest-LSN entry above checkpoint_lsn
|
||
* is a trailing WAL_CHECKPOINT marker — the exact state left behind by a
|
||
* crash between persist_write_checkpoint_marker() (step 1, durably
|
||
* fsynced) and write_snapshot() (step 3b, persists the advanced
|
||
* checkpoint_lsn) — recovered_tip over-counts past it while
|
||
* last_applied_lsn correctly does not, and comparing against
|
||
* recovered_tip previously produced a false APPLY_DIVERGENT for a
|
||
* perfectly clean recovery (violates Req 8.2). */
|
||
uint64_t apply_cursor =
|
||
atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire);
|
||
apply_expectation_t apply_exp = {
|
||
.expected_last_applied_lsn = apply_cursor,
|
||
.expected_new_job_count = new_job_submits,
|
||
};
|
||
p->apply_verdict = persist_verify_apply_correctness(
|
||
p, &apply_exp, jobmgr_count_before_replay);
|
||
if (p->apply_verdict == APPLY_DIVERGENT) {
|
||
ERROR("persist_recover: APPLY_DIVERGENT — materialized state does "
|
||
"not match the committed log tip (expected last_applied_lsn=%"
|
||
PRIu64 " new_job_count=%" PRIu64 "); a hidden apply defect is "
|
||
"present (ha-commit-durability recover_verdict=%s)",
|
||
apply_exp.expected_last_applied_lsn,
|
||
apply_exp.expected_new_job_count,
|
||
(verdict == RECOVER_TRUNCATED) ? "RECOVER_TRUNCATED"
|
||
: "RECOVER_CLEAN");
|
||
|
||
/* raft-apply-decoupling task 5.2 (Req 8.3/8.4): route the DIVERGENT
|
||
* verdict through the EXISTING non-promotable loud-fence mechanism
|
||
* (persist_recover_loud_fence_if_unrecoverable + the sticky
|
||
* recovery_unrecoverable flag) instead of a new fence path.
|
||
* apply_divergent=true fences UNCONDITIONALLY (see
|
||
* persist_recover_decision), so the peer/HWM fields below are
|
||
* inert placeholders here -- they are not yet known at this
|
||
* boot-time replay call site (unlike the structural verdict's own
|
||
* fence check, which stays deferred to promotion time in ha.c,
|
||
* where live peer/HWM context exists). verdict/recovered_tip are
|
||
* passed through only for accurate, consistent logging. The member
|
||
* stays ALIVE, non-promotable, and keeps retrying catch-up -- it
|
||
* never exits (controller-never-self-terminate invariant,
|
||
* CLAUDE.md). */
|
||
recover_policy_inputs_t apply_fence_in = {
|
||
.verdict = verdict,
|
||
.durable_below_hwm = false,
|
||
.healthy_peer_reachable = false,
|
||
.catch_up_succeeded = false,
|
||
.durable_floor_lsn = recovered_tip,
|
||
.cluster_hwm_lsn = 0,
|
||
.apply_divergent = true,
|
||
};
|
||
persist_recover_loud_fence_if_unrecoverable(p, &apply_fence_in);
|
||
} else
|
||
INFO("persist_recover: APPLY_CLEAN — materialized state matches the "
|
||
"committed log tip (last_applied_lsn=%" PRIu64 ")",
|
||
apply_cursor);
|
||
|
||
/* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */
|
||
|
||
/* bug-1416: replay + verification is fully complete -- safe to let the
|
||
* live apply pump and the standby live-replication apply path resume
|
||
* mutating state (see the recovery_complete field's doc comment).
|
||
* Restored unconditionally here regardless of the verdict computed
|
||
* above: a genuine APPLY_DIVERGENT already fenced this member via
|
||
* persist_recover_loud_fence_if_unrecoverable (non-promotable, stays
|
||
* alive, keeps retrying) -- that sticky fence and its clearing logic
|
||
* (task 5.3) are untouched by this flag, which governs ONLY whether
|
||
* in-memory apply may proceed, not whether this member may lead. Issue
|
||
* one deliberate catch-up kick for anything that committed locally (or
|
||
* arrived via live replication and was durably persisted but had its
|
||
* entity-apply deferred, see ha_apply_single_entry / ha_apply_term_
|
||
* entry_via_library) while this pass ran -- persist_apply_pump_run's
|
||
* own walk drains fully from last_applied_lsn+1 through the current
|
||
* committed_lsn, so this is a complete, ordered catch-up, not a partial
|
||
* one; a harmless no-op if nothing queued. */
|
||
atomic_store_explicit(&p->recovery_complete, true, memory_order_release);
|
||
/* n_vcpus==0 is this codebase's own established "not yet initialized"
|
||
* sentinel for vtaskmgr_t (mirrors cluster_init_phase2's own
|
||
* `if (c->vtaskmgr.n_vcpus == 0) vtaskmgr_init(...)` guard) -- a real
|
||
* cluster always has n_vcpus > 0 by the time persist_recover() runs
|
||
* (vtaskmgr_init happens earlier in myapp_run()/phase2), while a bare
|
||
* recovery-focused unit-test cluster_t (none of which call
|
||
* vtaskmgr_init, confirmed by inspection) leaves it zeroed. Passing a
|
||
* pointer to an uninitialized vtaskmgr_t into vtask_enq_classified
|
||
* would be unsafe, so fall back to persist_apply_pump_kick's own
|
||
* existing, already-exercised NULL-pool inline path in that case --
|
||
* identical to how every apply-pump test fixture in this spec already
|
||
* exercises the pump. */
|
||
void* pool = (p->cluster && p->cluster->vtaskmgr.n_vcpus > 0)
|
||
? &p->cluster->vtaskmgr
|
||
: NULL;
|
||
persist_apply_pump_kick(p, pool);
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */
|
||
recover_verdict_t persist_recovery_verdict(persist_t* p)
|
||
{
|
||
return p ? p->recovery_verdict : RECOVER_CLEAN;
|
||
}
|
||
|
||
bool persist_recovery_truncated(persist_t* p)
|
||
{
|
||
return p && p->recovery_verdict == RECOVER_TRUNCATED;
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ *
|
||
* Boot-policy loud-fence decision (ha-commit-durability task 3.3, Req 2.4;
|
||
* never-self-exit rework 2026-06-18).
|
||
*
|
||
* Implements the FENCE_UNRECOVERABLE terminal of the Boot Policy Decision Table
|
||
* (design.md). The decision is a PURE function of its inputs; the action (loud
|
||
* operator-visible log + set the sticky non-promotable flag) is separated into
|
||
* persist_recover_loud_fence_if_unrecoverable so the decision can be unit-tested
|
||
* in isolation. A reachable healthy peer NEVER yields FENCE_UNRECOVERABLE
|
||
* (rows 3, 5); it is reserved for a member that cannot recover locally AND
|
||
* cannot catch up (no healthy peer, rows 4, 4b).
|
||
*
|
||
* CRITICAL: this terminal is a LOUD, PERMANENT, NON-promotable fence — the
|
||
* process STAYS ALIVE and keeps retrying catch-up. There is NO abort()/exit():
|
||
* a controller MUST NEVER self-terminate as a designed response to an
|
||
* operational condition (user invariant, 2026-06-18). A self-exited controller
|
||
* needs manual admin recovery and degrades quorum meanwhile; one that stays
|
||
* alive and keeps retrying self-heals when a peer returns (Req 2.6).
|
||
* ------------------------------------------------------------------ */
|
||
|
||
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in)
|
||
{
|
||
if (in == NULL)
|
||
return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */
|
||
|
||
/* raft-apply-decoupling task 5.2 (Req 8.3/8.4): an APPLY_DIVERGENT
|
||
* post-replay verdict is a NEW INPUT to this same decision table, not a
|
||
* new fence mechanism (design.md "Restart Replay & Verification"). It
|
||
* fences UNCONDITIONALLY -- every other field below is irrelevant once
|
||
* this is true. Unlike a structurally truncated/behind log (rows 2/3/5),
|
||
* where a healthy peer's missing entries can repair the gap and legitimately
|
||
* downgrade the outcome to a plain FENCE-then-catch-up, an apply-time
|
||
* defect means the LOCAL apply path itself produced the wrong
|
||
* materialized state from bytes it already had -- streaming more entries
|
||
* from a peer does not prove that path is now trustworthy, so this is
|
||
* never downgraded by peer reachability or catch-up success. */
|
||
if (in->apply_divergent)
|
||
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
|
||
|
||
bool truncated = (in->verdict == RECOVER_TRUNCATED);
|
||
|
||
/* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster
|
||
* high-water mark is a normal promotable member — no fence. */
|
||
if (!truncated && !in->durable_below_hwm)
|
||
return RECOVER_DECISION_ELIGIBLE;
|
||
|
||
/* From here the member is truncated OR behind the HWM — it must NOT lead
|
||
* as-is. A truncated log is by definition not provably at/above the HWM
|
||
* (design.md note), so it is treated as behind for fence purposes. */
|
||
|
||
/* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain
|
||
* eligibility (task 4.3). A reachable peer NEVER yields the unrecoverable
|
||
* fence — even for a truncated log or a legitimate snapshot-sentinel. If
|
||
* catch-up has already succeeded the member is eligible; otherwise it stays
|
||
* fenced (4.3 drives the catch-up + release). */
|
||
if (in->healthy_peer_reachable) {
|
||
if (in->catch_up_succeeded)
|
||
return RECOVER_DECISION_ELIGIBLE;
|
||
return RECOVER_DECISION_FENCE;
|
||
}
|
||
|
||
/* No healthy peer. If catch-up did not reach a committed-consistent state,
|
||
* the member can neither recover locally nor catch up right now — rows 4 &
|
||
* 4b: a LOUD, PERMANENT, NON-promotable fence (NOT process exit). It stays
|
||
* alive and keeps retrying; the fence releases when a peer returns
|
||
* (Req 2.4/2.6). */
|
||
if (!in->catch_up_succeeded)
|
||
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
|
||
|
||
/* No peer but catch-up is reported succeeded (e.g. already at/above after a
|
||
* local catch-up): eligible. */
|
||
return RECOVER_DECISION_ELIGIBLE;
|
||
}
|
||
|
||
bool persist_recovery_unrecoverable(persist_t* p)
|
||
{
|
||
return p && atomic_load_explicit(&p->recovery_unrecoverable,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
void persist_clear_recovery_unrecoverable(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* Clear only on a genuine transition so the loud "fence released" log fires
|
||
* once, not on every catch-up tick. */
|
||
if (atomic_exchange_explicit(&p->recovery_unrecoverable, false,
|
||
memory_order_acq_rel)) {
|
||
INFO("persist_recover: unrecoverable fence RELEASED — member reached a "
|
||
"committed-consistent state via catch-up; promotable again "
|
||
"(ha-commit-durability Req 2.6)");
|
||
}
|
||
}
|
||
|
||
/* job-array-single-wal-record task 4.2 (design.md "Live Apply Resource
|
||
* Fence", Req 9.4/9.5). See the apply_resource_fenced_lsn field's doc
|
||
* comment (persist.h) and this trio's own doc comment (persist.h) for the
|
||
* full rationale and the deliberate distinction from
|
||
* persist_recovery_unrecoverable/persist_clear_recovery_unrecoverable just
|
||
* above -- that boot-only mechanism is NOT reused here on purpose. */
|
||
bool persist_apply_resource_fenced(persist_t* p)
|
||
{
|
||
return p && atomic_load_explicit(&p->apply_resource_fenced_lsn,
|
||
memory_order_acquire) != 0;
|
||
}
|
||
|
||
void persist_mark_apply_resource_fenced(persist_t* p, uint64_t lsn)
|
||
{
|
||
if (!p)
|
||
return;
|
||
atomic_store_explicit(&p->apply_resource_fenced_lsn, lsn,
|
||
memory_order_release);
|
||
}
|
||
|
||
void persist_clear_apply_resource_fence(persist_t* p)
|
||
{
|
||
if (!p)
|
||
return;
|
||
/* Clear only on a genuine transition so the self-heal log fires once per
|
||
* episode, not on every ordinary successful apply (the overwhelming
|
||
* common case: nothing was fenced, so this is a cheap no-op). */
|
||
uint64_t prev = atomic_exchange_explicit(&p->apply_resource_fenced_lsn, 0,
|
||
memory_order_acq_rel);
|
||
if (prev != 0) {
|
||
INFO("persist_apply_pump_run: Live Apply Resource Fence RELEASED — "
|
||
"lsn=%" PRIu64 " applied successfully on retry; promotable "
|
||
"again (job-array-single-wal-record Req 9.5)",
|
||
prev);
|
||
}
|
||
}
|
||
|
||
recover_decision_t
|
||
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
|
||
const recover_policy_inputs_t* in)
|
||
{
|
||
recover_decision_t decision = persist_recover_decision(in);
|
||
|
||
if (decision != RECOVER_DECISION_FENCE_UNRECOVERABLE)
|
||
return decision;
|
||
|
||
/* Loud, operator-visible alert. The two triggers reaching this terminal
|
||
* are NOT logged with the same clause (round-2 fix, raft-apply-
|
||
* decoupling task 5.2 remediation):
|
||
*
|
||
* - apply_divergent: fences UNCONDITIONALLY (persist_recover_decision),
|
||
* evaluated at boot-replay time BEFORE peer/HWM context exists (see
|
||
* the call site in persist_recover()). `healthy_peer_reachable` and
|
||
* `cluster_hwm_lsn` are inert placeholders there, not real facts —
|
||
* asserting "no healthy peer reachable" or a durable_floor_lsn < hwm
|
||
* comparison for this trigger would report unevaluated-or-false
|
||
* information to the operator (round-1 review: the placeholders can
|
||
* make the comparison numerically impossible, e.g. "2 < 0"). This
|
||
* branch names the real reason (apply-correctness divergence) only.
|
||
* - structural (truncated / behind-HWM): reaches this terminal only
|
||
* via the pre-existing "no healthy peer reachable" branch in
|
||
* persist_recover_decision, so the peer-reachability and
|
||
* durable_floor_lsn/hwm comparison ARE guaranteed true here — that
|
||
* clause is unchanged from before this task.
|
||
*/
|
||
if (in->apply_divergent) {
|
||
ERROR("persist_recover: UNRECOVERABLE — post-replay apply-correctness "
|
||
"verification is DIVERGENT (materialized state does not match "
|
||
"the committed log tip, raft-apply-decoupling Req 8.3); a local "
|
||
"apply defect cannot be proven fixed by peer reachability or "
|
||
"catch-up alone. Staying ALIVE and PERMANENTLY non-promotable "
|
||
"(refusing to serve divergent state) and RETRYING catch-up "
|
||
"indefinitely — operator attention is needed to diagnose the "
|
||
"apply path (raft-apply-decoupling Req 8.3/8.4)");
|
||
} else {
|
||
const char* why = (in->verdict == RECOVER_TRUNCATED)
|
||
? "recovery-truncated log (structural corruption)"
|
||
: "log behind the cluster committed high-water mark";
|
||
ERROR("persist_recover: UNRECOVERABLE — %s and no healthy peer reachable to "
|
||
"catch up; truncation boundary durable_floor_lsn=%" PRIu64
|
||
" < cluster committed high-water hwm=%" PRIu64 ". Staying ALIVE and "
|
||
"PERMANENTLY non-promotable (refusing to serve divergent state) and "
|
||
"RETRYING catch-up indefinitely — will self-heal when a healthy peer "
|
||
"returns; operator attention may be needed "
|
||
"(ha-commit-durability Req 2.4)",
|
||
why, in->durable_floor_lsn, in->cluster_hwm_lsn);
|
||
}
|
||
|
||
/* Set the sticky non-promotable flag so the promotion gate refuses
|
||
* leadership as belt-and-suspenders until catch-up clears it (Req 2.4/2.6).
|
||
* p may be NULL for pure-classification callers that only want the loud log. */
|
||
if (p != NULL)
|
||
atomic_store_explicit(&p->recovery_unrecoverable, true,
|
||
memory_order_release);
|
||
|
||
return RECOVER_DECISION_FENCE_UNRECOVERABLE;
|
||
}
|
||
|
||
/* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that
|
||
* entry is already durable on local stable storage (ha-commit-durability task
|
||
* 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the
|
||
* commit engine's durable self-cap (commit.c majority_candidate_locked): a
|
||
* majority ACK no longer implies the local copy is fsync'd, because group-
|
||
* commit (task 2.1) deferred the append fsync to the demand-driven flusher. So
|
||
* advancing committed_lsn here on the ACK alone would reintroduce the bug-661
|
||
* ACK-without-durability hazard — committed could be durable on a single
|
||
* member and lost across failover. We gate the store on written_lsn <=
|
||
* durable_lsn so committed_lsn ⇒ durable-on-this-member on every path.
|
||
* committed_lsn is monotonic, so a max() store; the durable cap can only delay
|
||
* the advance until the flusher syncs the entry (the durable-advance commit
|
||
* wake re-drives the engine path; the legacy path is re-evaluated on the next
|
||
* sync_append). */
|
||
static void sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn)
|
||
{
|
||
uint64_t durable = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
/* DURABILITY PRECONDITION: never advance committed past what is durable
|
||
* locally. The LSN-0 first-entry edge stays valid (0 <= 0). */
|
||
if (written_lsn > durable)
|
||
return;
|
||
uint64_t cur = atomic_load_explicit(&p->committed_lsn,
|
||
memory_order_acquire);
|
||
if (written_lsn <= cur)
|
||
return; /* monotonic: committed_lsn never regresses */
|
||
atomic_store_explicit(&p->committed_lsn, written_lsn, memory_order_release);
|
||
#ifdef XCE_TESTING
|
||
/* INVARIANT (debug guard): committed_lsn must never exceed durable_lsn on
|
||
* any commit advance. A future caller reintroducing ACK-without-durability
|
||
* trips this. */
|
||
assert(atomic_load_explicit(&p->committed_lsn, memory_order_acquire)
|
||
<= atomic_load_explicit(&p->durable_lsn, memory_order_acquire));
|
||
#endif
|
||
}
|
||
|
||
int persist_wal_sync_append(persist_t* p, uint32_t type,
|
||
const void* payload, uint32_t len,
|
||
int* sync_rc_out)
|
||
{
|
||
/* ha-term-safety task 3.1: the primary's WAL_JOB_SUBMIT append is routed
|
||
* through the raft library (ha_primary_log_append → raft_log_append) so the
|
||
* library stamps the term + assigns the LSN (firing on_persist_entry → the
|
||
* WAL write), keeping p->current_lsn co-tracked with s->log_len; the existing
|
||
* majority-commit barrier below (repl_wait_majority_ack) is UNCHANGED and runs
|
||
* after the append. Non-raft road / engine-less tests (library not LEADER)
|
||
* fall back inside ha_primary_log_append to persist_wal_async. Standalone
|
||
* persist (p->cluster == NULL, lib-level tests) keeps the direct append. */
|
||
int rc;
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, type, payload, len, NULL);
|
||
else
|
||
rc = persist_wal_async(p, type, payload, len);
|
||
if (rc != 0) {
|
||
if (sync_rc_out)
|
||
*sync_rc_out = rc;
|
||
return rc;
|
||
}
|
||
|
||
/* Majority-commit replication (Req 5.1–5.3; ha-commit-engine Req 1.1–1.3).
|
||
* persist_wal_async appends to the page cache and signals the demand-driven
|
||
* flusher; it no longer fsyncs inline (group-commit, task 2.1). Durability
|
||
* is therefore established by the flusher advancing durable_lsn, and a
|
||
* commit position advances only once the entry is durable locally — the
|
||
* engine path via the durable self-cap (commit.c), the legacy path below
|
||
* via sync_append_legacy_commit_store's written_lsn <= durable_lsn gate.
|
||
* We use current_lsn - 1 because persist_wal_async already incremented it. */
|
||
uint64_t written_lsn;
|
||
pthread_mutex_lock(&p->lock);
|
||
written_lsn = p->current_lsn - 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
int sync_err;
|
||
/* Engine-wired path (ha-commit-engine task 5.3): delegate the commit
|
||
* decision to the commit engine through the walAck adapter. The engine
|
||
* is the single committed_lsn writer (no store here), and its
|
||
* n_peers<=1 self-quorum replaces the old has_standbys → -ENODEV
|
||
* degraded special case (Req 6.3).
|
||
*
|
||
* LSN-0 edge (ACCEPTED interim behavior — spec Implementation Notes):
|
||
* written_lsn is passed unconditionally, but the FIRST-EVER WAL entry
|
||
* of a fresh cluster is LSN 0 and commit_submit rejects lsn==0, so the
|
||
* adapter's entry_lsn != 0 guard routes that one entry to the legacy
|
||
* branch below. This happens once per cluster lifetime;
|
||
* ha-single-leader-role's promotion no-op entry eliminates it in
|
||
* practice. */
|
||
if (p->repl.ce != NULL && written_lsn != 0) {
|
||
sync_err = repl_wait_majority_ack(&p->repl, written_lsn,
|
||
p->config.sync_timeout_ms,
|
||
(uint8_t)OPCode_walAck);
|
||
if (sync_err != 0)
|
||
CTRL_METRIC_INC(ctrl_repl_degraded_count);
|
||
} else {
|
||
/* Residual non-engine branch: engine-less (ce == NULL) unit-test
|
||
* contexts plus the LSN-0 first entry. The legacy connectivity-
|
||
* based counting was retired by ha-commit-barrier-rollout task 2.2:
|
||
* repl_wait_majority_ack now confirms LSN 0 immediately (documented
|
||
* once-per-lifetime edge) and surfaces engine-less walAck waits as
|
||
* -EINVAL via commit_submit, so the committed_lsn store below only
|
||
* fires for the LSN-0 case (a no-op store of 0). */
|
||
mylock_lock(&p->repl.lock);
|
||
int has_standbys = (p->repl.nstandbys > 0);
|
||
mylock_unlock(&p->repl.lock);
|
||
|
||
if (!has_standbys) {
|
||
/* No standbys registered → degraded mode (no quorum to wait for) */
|
||
sync_err = -ENODEV;
|
||
} else {
|
||
sync_err = repl_wait_majority_ack(&p->repl, written_lsn,
|
||
p->config.sync_timeout_ms,
|
||
(uint8_t)OPCode_walAck);
|
||
if (sync_err == 0)
|
||
/* Durable-gated commit advance (task 2.4): advance committed_lsn
|
||
* only if written_lsn is durable locally — never on the ACK
|
||
* alone (group-commit removed the inline append fsync). */
|
||
sync_append_legacy_commit_store(p, written_lsn);
|
||
else
|
||
CTRL_METRIC_INC(ctrl_repl_degraded_count);
|
||
}
|
||
}
|
||
if (sync_rc_out)
|
||
*sync_rc_out = sync_err;
|
||
/* sync_err < 0 — replication failed.
|
||
* -ENODEV: legacy degraded mode (no standbys, engine-less) — acceptable
|
||
* for forward progress.
|
||
* -ETIMEDOUT: no per-entry majority within the deadline — the write
|
||
* must be rejected to preserve the majority-consistency
|
||
* invariant.
|
||
* -XCE_ERR_NOT_READY: engine admission cap exhausted — retriable
|
||
* rejection.
|
||
* Timeout and admission rejection propagate to the job-submit error
|
||
* path (ha-commit-engine Req 1.1, 4.4). */
|
||
if (sync_err == -ETIMEDOUT || sync_err == -XCE_ERR_NOT_READY)
|
||
return sync_err;
|
||
return 0;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Test seam (task 2.4): invoke the EXACT durable-gated commit store that the
|
||
* legacy sync_append branch uses on a majority ACK, so a test can prove the
|
||
* ACK-without-durability hazard is closed without standing up the replication
|
||
* wire. Pre-fix this branch stored committed_lsn = written_lsn unconditionally
|
||
* (over-advancing past durable_lsn). */
|
||
void persist_test_sync_append_legacy_commit_store(persist_t* p,
|
||
uint64_t written_lsn)
|
||
{
|
||
sync_append_legacy_commit_store(p, written_lsn);
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
void persist_log_entry_metadata(const wal_entry_hdr_t* hdr)
|
||
{
|
||
TRACE("WAL entry: type=%u lsn=%lu term=%lu payload_len=%u crc=%08x",
|
||
hdr->type,
|
||
(unsigned long)hdr->lsn,
|
||
(unsigned long)hdr->term,
|
||
hdr->payload_len,
|
||
hdr->crc);
|
||
}
|
||
|
||
/* ------------------------------------------------------------------ */
|
||
/* Hook functions — AFTER_EVENT handlers that write WAL entries */
|
||
/* ------------------------------------------------------------------ */
|
||
|
||
/* Map HOOK_DETAIL_* values to WAL entry types.
|
||
* Returns 0 for details that have no WAL representation (skip). */
|
||
static uint32_t detail_to_wal_type(int detail)
|
||
{
|
||
switch (detail) {
|
||
case HOOK_DETAIL_JOB_NEW: return WAL_JOB_SUBMIT;
|
||
case HOOK_DETAIL_JOB_START: return WAL_JOB_START;
|
||
case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH;
|
||
case HOOK_DETAIL_JOB_FAIL: return WAL_JOB_FAIL;
|
||
case HOOK_DETAIL_JOB_STOP: return WAL_JOB_STOP;
|
||
case HOOK_DETAIL_JOB_RESUME: return WAL_JOB_RESUME;
|
||
case HOOK_DETAIL_JOB_UPDATE: return WAL_JOB_UPDATE;
|
||
case HOOK_DETAIL_JOB_CKPT: return WAL_JOB_CKPT;
|
||
case HOOK_DETAIL_JOB_RUSAGE_UPDATE: return WAL_JOB_RUSAGE;
|
||
case HOOK_DETAIL_SETPREASON: return WAL_JOB_PREASON;
|
||
case HOOK_DETAIL_CLRPREASON: return WAL_JOB_PREASON;
|
||
case HOOK_DETAIL_QUEUE_UPDATE: return WAL_QUEUE_CTRL;
|
||
case HOOK_DETAIL_WORKER_JOIN: return WAL_WORKER_CTRL;
|
||
case HOOK_DETAIL_WORKER_UPDATE: return WAL_WORKER_CTRL;
|
||
case HOOK_DETAIL_WORKER_DEAD: return WAL_WORKER_CTRL;
|
||
default: return 0; /* unknown — skip */
|
||
}
|
||
}
|
||
|
||
/* Write-path commit classification (ha-commit-barrier-rollout task 1.2).
|
||
* Pure lookup over the VERIFIED table (design.md §Write-Path
|
||
* Classification); `detail` disambiguates WAL_WORKER_CTRL only.
|
||
* Unclassified types fail fast under XCE_TESTING (abort) and fall back to
|
||
* RETAIN in production — never dropped, never falsely confirmed (Req 2.2). */
|
||
wal_commit_mode_t wal_classify(uint32_t wal_type, int detail)
|
||
{
|
||
switch (wal_type) {
|
||
case WAL_JOB_SUBMIT:
|
||
/* client job submit — request-shaped; keeps the in-hook sync path
|
||
* (reference implementation, handle_jobsubmit) */
|
||
return WAL_MODE_WAIT;
|
||
case WAL_JOB_ARRAY_SUBMIT:
|
||
/* client job-array submit (job-array-single-wal-record task 1.1) —
|
||
* request-shaped exactly like WAL_JOB_SUBMIT: one requester awaits
|
||
* the single commit barrier before reply, regardless of array
|
||
* size. */
|
||
return WAL_MODE_WAIT;
|
||
case WAL_QUEUE_CTRL:
|
||
/* client queue control — request-shaped; the handler barrier owes
|
||
* commit completion before the requester reply (tasks 2-3) */
|
||
return WAL_MODE_WAIT;
|
||
case WAL_WORKER_CTRL:
|
||
/* the only detail-disambiguated type: JOIN/UPDATE have a requester
|
||
* awaiting confirmation (WAIT); DEAD is a controller-detected fact.
|
||
* Unexpected details take the fact-safe RETAIN side. */
|
||
if (detail == HOOK_DETAIL_WORKER_JOIN ||
|
||
detail == HOOK_DETAIL_WORKER_UPDATE)
|
||
return WAL_MODE_WAIT;
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_JOB_START:
|
||
case WAL_JOB_FINISH:
|
||
case WAL_JOB_FAIL:
|
||
case WAL_JOB_STOP:
|
||
case WAL_JOB_RESUME:
|
||
case WAL_JOB_UPDATE:
|
||
case WAL_JOB_RUSAGE:
|
||
case WAL_JOB_CKPT:
|
||
case WAL_JOB_PREASON:
|
||
/* worker/scheduler-reported facts — retained until committed.
|
||
* FAIL/UPDATE are dual-origin; their request-shaped side is
|
||
* realized by the handler barrier in do_jobcontrol_op
|
||
* (research.md §2 call-site verification). */
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_NOOP:
|
||
/* internal promotion no-op — replicated like any committed entry,
|
||
* retained until committed (ha-commit-barrier-rollout pre-declared
|
||
* row: promotion | internal | RETAIN; guard forces classification) */
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_CONFIG_MARKER:
|
||
/* config-change marker (ha-term-safety task 4.2): a committable
|
||
* (term,lsn) log entry like a job-state fact — replicated and retained
|
||
* until committed; it commits through the SAME barrier as job state
|
||
* (R7.1/R7.11), so it is RETAIN, NOT a no-op or a LOCAL marker. The
|
||
* requester barrier on a config upgrade is owned by the upgrade path
|
||
* (do_upgrade_config_with_bytes), not the per-entry WAIT hook. */
|
||
return WAL_MODE_RETAIN;
|
||
case WAL_CHECKPOINT:
|
||
/* local segment marker — never replicated, by design */
|
||
return WAL_MODE_LOCAL;
|
||
default:
|
||
ERROR("wal_classify: unclassified WAL type %u (detail %d) — "
|
||
"RETAIN fallback; add it to the classification table",
|
||
wal_type, detail);
|
||
#ifdef XCE_TESTING
|
||
assert(0 && "unclassified WAL type — add it to wal_classify");
|
||
#endif
|
||
return WAL_MODE_RETAIN;
|
||
}
|
||
}
|
||
|
||
/* Commit barrier helper (ha-commit-barrier-rollout task 2.2, design.md
|
||
* §Commit Barrier Helper; Req 1.1, 1.2, 1.3).
|
||
*
|
||
* Handler-callable WAIT completion on the appended watermark: snapshot the
|
||
* last-appended LSN, return 0 when the commit index already covers it,
|
||
* else block in the commit engine until coverage or deadline. The engine
|
||
* result is returned UNMODIFIED (0 / -ETIMEDOUT / -XCE_ERR_NOT_READY /
|
||
* -EINVAL) so callers map it to their own error replies.
|
||
*
|
||
* Lock discipline: p->lock is held only for the current_lsn snapshot and
|
||
* is released BEFORE commit_submit(WAIT) — the engine blocks on ce->lock
|
||
* + condvar, and p->lock must never be held across it (engine
|
||
* lock-ordering rule). */
|
||
int persist_commit_barrier(persist_t* p, uint32_t timeout_ms)
|
||
{
|
||
uint64_t watermark;
|
||
|
||
if (p == NULL)
|
||
return -EINVAL;
|
||
|
||
/* Snapshot the last-appended watermark under p->lock. current_lsn is
|
||
* the NEXT LSN to assign (persist.h invariant: committed_lsn <=
|
||
* current_lsn - 1), so the highest appended entry is current_lsn - 1
|
||
* (resolved contract, spec Implementation Notes 2.1). current_lsn ==
|
||
* 0 means nothing was ever appended — vacuously satisfied. */
|
||
pthread_mutex_lock(&p->lock);
|
||
if (p->current_lsn == 0) {
|
||
pthread_mutex_unlock(&p->lock);
|
||
return 0;
|
||
}
|
||
watermark = p->current_lsn - 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* Already covered — committed_lsn is monotone, so coverage observed
|
||
* here can never regress. This check also absorbs the LSN-0 first
|
||
* entry (watermark 0 vs committed_lsn 0): that once-per-lifetime entry
|
||
* has no quorum contract (ha-commit-engine Implementation Notes) and
|
||
* commit_submit would reject lsn == 0 anyway. */
|
||
if (atomic_load_explicit(&p->committed_lsn, memory_order_acquire) >=
|
||
watermark)
|
||
return 0;
|
||
|
||
/* Engine not wired: legacy/bootstrap contexts have no engine commit
|
||
* contract — there is nothing to wait on, so the barrier is a no-op.
|
||
* Production always wires the engine (cluster_init_phase2). */
|
||
if (p->repl.ce == NULL)
|
||
return 0;
|
||
|
||
/* timeout_ms == 0 borrows the configured sync deadline (Req 1.2). */
|
||
return commit_submit(p->repl.ce, watermark, COMMIT_MODE_WAIT,
|
||
timeout_ms != 0 ? timeout_ms
|
||
: p->config.sync_timeout_ms);
|
||
}
|
||
|
||
/* Classified append at the hook seam (barrier-rollout task 1.2).
|
||
*
|
||
* WAIT (queue/worker control): the append stays async here — the handler
|
||
* barrier (tasks 2-3) owes commit completion before the requester reply.
|
||
* WAL_JOB_SUBMIT never reaches this helper (it keeps the in-hook sync
|
||
* path in persist_job_hook_fun).
|
||
*
|
||
* RETAIN (facts): append + broadcast, then register the appended LSN with
|
||
* the commit engine. commit_submit(COMMIT_MODE_RETAIN) is registration-
|
||
* free bookkeeping (commit.c — returns 0 immediately; commit state is
|
||
* derived from registry watermarks at ACK time), so the call is contract
|
||
* documentation: the entry stays pending purely by lsn > committed_lsn
|
||
* and commits later via covering ACKs. Commit-progress results are NEVER
|
||
* propagated to hook callers — only the local append rc is (Req 3.1).
|
||
*
|
||
* Lock context: persist_wal_async takes/releases p->lock internally; the
|
||
* appended-LSN read below re-takes p->lock briefly (same discipline as
|
||
* persist_wal_sync_append) and releases it BEFORE commit_submit, so no
|
||
* p->lock is ever held across ce->lock (engine lock-ordering rule). */
|
||
static int wal_hook_append_classified(persist_t* p, uint32_t wal_type,
|
||
wal_commit_mode_t mode,
|
||
const void* payload, uint32_t len)
|
||
{
|
||
uint64_t appended_lsn;
|
||
int rc;
|
||
|
||
/* ha-term-safety task 3.1: route the primary's local append through the raft
|
||
* library (ha_primary_log_append → raft_log_append) so the LIBRARY stamps the
|
||
* live election term and assigns the LSN (firing on_persist_entry → the WAL
|
||
* write), keeping p->current_lsn co-tracked with s->log_len. The library is
|
||
* the SINGLE primary append entry — no append bypasses it on the raft road.
|
||
* On the non-raft road (library not LEADER) ha_primary_log_append falls back
|
||
* to persist_wal_async, so engine-less / init-time contexts are unchanged.
|
||
* When p->cluster is NULL (persist used standalone in lib tests) keep the
|
||
* direct append (no cluster → no raft state to drive). */
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, wal_type, payload, len, NULL);
|
||
else
|
||
rc = persist_wal_async(p, wal_type, payload, len);
|
||
if (rc != 0 || mode != WAL_MODE_RETAIN)
|
||
return rc;
|
||
|
||
if (p->repl.ce == NULL)
|
||
return rc; /* engine not wired (bootstrap / engine-less contexts) */
|
||
|
||
/* current_lsn - 1 is the LSN persist_wal_async just assigned (it
|
||
* increments under p->lock). A concurrent append may bump it further;
|
||
* that is benign — contiguous advancement means the higher watermark
|
||
* still covers this entry. LSN-0 edge: commit_submit rejects lsn == 0
|
||
* (first-ever entry of a fresh cluster — accepted interim behavior,
|
||
* spec Implementation Notes). */
|
||
pthread_mutex_lock(&p->lock);
|
||
appended_lsn = p->current_lsn - 1;
|
||
pthread_mutex_unlock(&p->lock);
|
||
if (appended_lsn != 0)
|
||
(void)commit_submit(p->repl.ce, appended_lsn, COMMIT_MODE_RETAIN, 0);
|
||
|
||
return rc; /* local append rc only — facts have no commit timeout */
|
||
}
|
||
|
||
/* Shared step for a single WAL_JOB_SUBMIT entry (raft-apply-decoupling task
|
||
* 3.1): encode `job`, append, and block on the majority-commit barrier.
|
||
* Returns 0 when the entry committed on a majority; a negative rc otherwise
|
||
* (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: admission
|
||
* cap; -ENOMEM: encode failure; or a local-write error). Whenever the LOCAL
|
||
* WAL write succeeded (rc == 0, -ETIMEDOUT, or -XCE_ERR_NOT_READY — the entry
|
||
* was appended locally and the LSN advanced even though replication is
|
||
* degraded), *out_lsn is set to the entry's assigned LSN so a caller can tag
|
||
* it for egress-watermark purposes even on a rejected submission. */
|
||
static int submit_job_wal_entry_and_commit(persist_t* p, const xce_job_t* job,
|
||
uint64_t* out_lsn)
|
||
{
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int rc;
|
||
|
||
if (capnp_encode_job(job, &buf, &sz) != 0)
|
||
return -ENOMEM;
|
||
|
||
/* ha-convergence-recovery (3.3): snapshot the LSN this submit entry
|
||
* will receive BEFORE the append. persist_wal_async assigns
|
||
* hdr.lsn = current_lsn then increments, so the pre-append current_lsn
|
||
* is exactly the written entry's LSN. */
|
||
pthread_mutex_lock(&p->lock);
|
||
uint64_t submit_lsn = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
int sync_rc = 0;
|
||
rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz,
|
||
&sync_rc);
|
||
free(buf);
|
||
|
||
if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY)
|
||
*out_lsn = submit_lsn;
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Explicit controller-side raft step for a job submission (raft-apply-decoupling
|
||
* item #1 / bug-1306): WAL-append each job in `jlist` and block on the
|
||
* majority-commit barrier, returning the commit verdict DIRECTLY to the caller
|
||
* (handle_jobsubmit) instead of through the xce_run_hooks fan-out (which returns
|
||
* only the last hook's rc and so clobbered this -ETIMEDOUT with a later notify
|
||
* hook's 0 — the committed-job-loss root cause).
|
||
*
|
||
* Returns 0 when every job's entry committed on a majority; the FIRST negative
|
||
* rc otherwise (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY:
|
||
* admission cap; -ENOMEM: encode failure; or a local-write error). The caller
|
||
* MUST reject the submission on a negative rc (never reply success) to preserve
|
||
* the Commit Durability invariant. On -ETIMEDOUT / -XCE_ERR_NOT_READY the entry
|
||
* WAS appended locally and the LSN advanced, so j->submit_lsn is still tagged —
|
||
* the entry remains a pending, uncommitted tail a new-term leader may discard. */
|
||
int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist)
|
||
{
|
||
int rc = 0;
|
||
|
||
if (p == NULL || jlist == NULL)
|
||
return -EINVAL;
|
||
|
||
for (int i = 0; i < jlist->count && rc == 0; i++) {
|
||
job_t* j = (job_t*)xlist_get(jlist, i);
|
||
uint64_t lsn = 0;
|
||
|
||
rc = submit_job_wal_entry_and_commit(p, &j->job, &lsn);
|
||
if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY)
|
||
j->submit_lsn = lsn;
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* Sibling of persist_submit_wal_and_commit for a job submission whose job
|
||
* object(s) do not exist yet (raft-apply-decoupling task 3.1 / Req 2.1, 2.3):
|
||
* `payloads` is an xlist_t of xce_job_t* that have never been registered in
|
||
* any job table. On full success (return 0), out_lsns[i] holds payloads[i]'s
|
||
* committed LSN — the caller constructs and tags the job object AFTER this
|
||
* call returns, so a failed/timed-out commit never leaves a job object
|
||
* behind (there is nothing to construct until this succeeds). Returns the
|
||
* first negative rc on failure. */
|
||
int persist_submit_job_payloads_and_commit(persist_t* p, xlist_t* payloads,
|
||
uint64_t* out_lsns)
|
||
{
|
||
int rc = 0;
|
||
|
||
if (p == NULL || payloads == NULL || out_lsns == NULL)
|
||
return -EINVAL;
|
||
|
||
for (int i = 0; i < payloads->count && rc == 0; i++) {
|
||
xce_job_t* j = (xce_job_t*)xlist_get(payloads, i);
|
||
|
||
rc = submit_job_wal_entry_and_commit(p, j, &out_lsns[i]);
|
||
}
|
||
|
||
return rc;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 3.2: append-only counterpart of
|
||
* submit_job_wal_entry_and_commit — writes the entry via the SAME
|
||
* primary-append primitive wal_hook_append_classified already uses for
|
||
* RETAIN-mode facts (ha_primary_log_append / persist_wal_async), WITHOUT
|
||
* waiting on the majority-commit barrier. Deliberately does NOT reuse
|
||
* submit_job_wal_entry_and_commit itself: that helper is shared with
|
||
* persist_submit_wal_and_commit's still-synchronous, still-blocking
|
||
* contract (relied on directly by test_egress_filter.c / test_e2e_hooks.c /
|
||
* test_persist_hooks.c), which this task must not disturb. */
|
||
static int append_job_wal_entry_local(persist_t* p, const xce_job_t* job,
|
||
uint64_t* out_lsn)
|
||
{
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int rc;
|
||
uint64_t lsn;
|
||
|
||
if (capnp_encode_job(job, &buf, &sz) != 0)
|
||
return -ENOMEM;
|
||
|
||
/* Snapshot the LSN this entry will receive BEFORE the append (same
|
||
* convention as submit_job_wal_entry_and_commit / wal_hook_append_
|
||
* classified: persist_wal_async / ha_primary_log_append assigns
|
||
* hdr.lsn = current_lsn then increments). */
|
||
pthread_mutex_lock(&p->lock);
|
||
lsn = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, WAL_JOB_SUBMIT, buf,
|
||
(uint32_t)sz, NULL);
|
||
else
|
||
rc = persist_wal_async(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz);
|
||
free(buf);
|
||
|
||
if (rc == 0)
|
||
*out_lsn = lsn;
|
||
return rc;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 3.2: bridges commit_submit_async's
|
||
* commit_continuation_fn shape (commit_outcome_t + rc) to
|
||
* persist_job_commit_notify_fn's shape (a single rc) so persist.h's public
|
||
* declaration never needs commit.h's commit_outcome_t (see persist.h's
|
||
* doc comment on persist_job_commit_notify_fn). One adapter is heap-
|
||
* allocated per registered lsn and frees itself after its single
|
||
* invocation — safe because commit_resolve guarantees exactly-once
|
||
* delivery. */
|
||
typedef struct {
|
||
persist_job_commit_notify_fn fn;
|
||
void* ctx;
|
||
} job_commit_adapter_ctx_t;
|
||
|
||
static void job_commit_resolve_adapter(void* adapter_ctx, uint64_t lsn,
|
||
commit_outcome_t outcome, int rc)
|
||
{
|
||
job_commit_adapter_ctx_t* a = (job_commit_adapter_ctx_t*)adapter_ctx;
|
||
int final_rc = (outcome == COMMIT_OUTCOME_COMMITTED) ? 0 : rc;
|
||
|
||
a->fn(a->ctx, lsn, final_rc);
|
||
free(a);
|
||
}
|
||
|
||
int persist_submit_job_payloads_async(persist_t* p, xlist_t* payloads,
|
||
uint64_t* out_lsns,
|
||
persist_job_commit_notify_fn fn,
|
||
void* ctx)
|
||
{
|
||
int first_rc = 0;
|
||
|
||
if (p == NULL || payloads == NULL || out_lsns == NULL || fn == NULL)
|
||
return -EINVAL;
|
||
|
||
for (int i = 0; i < payloads->count; i++) {
|
||
xce_job_t* j = (xce_job_t*)xlist_get(payloads, i);
|
||
uint64_t lsn = 0;
|
||
int rc;
|
||
|
||
if (first_rc != 0) {
|
||
/* A prior payload in this batch already failed locally: every
|
||
* remaining payload resolves with that SAME failure now,
|
||
* synchronously — so `fn` is invoked exactly once per payload
|
||
* regardless of where the failure occurred, and the caller
|
||
* never has to separately track "never going to be called"
|
||
* entries. */
|
||
fn(ctx, 0, first_rc);
|
||
continue;
|
||
}
|
||
|
||
rc = append_job_wal_entry_local(p, j, &lsn);
|
||
if (rc != 0) {
|
||
first_rc = rc;
|
||
fn(ctx, 0, rc);
|
||
continue;
|
||
}
|
||
out_lsns[i] = lsn;
|
||
|
||
/* LSN-0 edge (documented once-per-cluster-lifetime interim
|
||
* behavior — ha-commit-engine Implementation Notes, mirrored by
|
||
* repl_wait_majority_ack's walAck branch): the first-ever WAL
|
||
* entry of a fresh cluster is structurally un-commitable and
|
||
* commit_submit_async rejects lsn == 0, so resolve it inline.
|
||
* Engine-not-wired (bootstrap / engine-less contexts) mirrors
|
||
* persist_commit_barrier's own "no engine -> no-op success"
|
||
* convention — there is no async completion authority to
|
||
* register with. */
|
||
if (lsn == 0 || p->repl.ce == NULL) {
|
||
fn(ctx, lsn, 0);
|
||
continue;
|
||
}
|
||
|
||
job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a));
|
||
if (a == NULL) {
|
||
first_rc = -ENOMEM;
|
||
fn(ctx, lsn, -ENOMEM);
|
||
continue;
|
||
}
|
||
a->fn = fn;
|
||
a->ctx = ctx;
|
||
|
||
int reg_rc = commit_submit_async(p->repl.ce, lsn,
|
||
job_commit_resolve_adapter, a);
|
||
if (reg_rc != 0) {
|
||
free(a);
|
||
first_rc = reg_rc;
|
||
ERROR("persist_submit_job_payloads_async: lsn=%lu appended but "
|
||
"commit_submit_async registration failed (rc=%d) -- the "
|
||
"entry remains durable and will still be caught up "
|
||
"eventually via replay, but this caller will not be told "
|
||
"its outcome via this path",
|
||
(unsigned long)lsn, reg_rc);
|
||
fn(ctx, lsn, reg_rc);
|
||
continue;
|
||
}
|
||
}
|
||
|
||
return first_rc;
|
||
}
|
||
|
||
/* job-array-single-wal-record task 2.1: serialize a job-array submission
|
||
* (base jobid + array size + one shared job spec) to a malloc'd Cap'n Proto
|
||
* buffer, reusing the SAME capnp struct (JobSubmit) the client's own submit
|
||
* request uses -- mirrors capnp_encode_job above, which mirrors encode_Job/
|
||
* new_Job/write_Job for a single job, except here via encode_JobSubmit/
|
||
* new_JobSubmit/write_JobSubmit. The encoded payload is O(1) in array_size:
|
||
* arraySize is a fixed-width uint64 field, not a per-element list -- so this
|
||
* function's output size never grows with array_size (Req 7.1). Caller must
|
||
* free(*out) on success. */
|
||
static int capnp_encode_job_array(const xce_job_submit_t* s, uint8_t** out,
|
||
size_t* out_len)
|
||
{
|
||
struct capn c;
|
||
struct capn_segment* cs;
|
||
struct JobSubmit cjs;
|
||
JobSubmit_ptr jp;
|
||
|
||
capn_init_malloc(&c);
|
||
cs = capn_root(&c).seg;
|
||
encode_JobSubmit(cs, &cjs, (xce_job_submit_t*)s);
|
||
jp = new_JobSubmit(cs);
|
||
write_JobSubmit(&cjs, jp);
|
||
capn_setp(capn_root(&c), 0, jp.p);
|
||
|
||
*out_len = capn_size(&c);
|
||
*out = malloc(*out_len);
|
||
if (!*out) { capn_free(&c); return -ENOMEM; }
|
||
capn_write_mem(&c, *out, *out_len, 0);
|
||
capn_free(&c);
|
||
return 0;
|
||
}
|
||
|
||
/* job-array-single-wal-record task 2.1: append-only, single-record
|
||
* counterpart of append_job_wal_entry_local for a WHOLE job array. Encodes
|
||
* ONE JobSubmit-shaped payload (base_jobid, array_size, base_spec->submit)
|
||
* and appends it as exactly one WAL_JOB_ARRAY_SUBMIT entry via the SAME
|
||
* primary-append primitive append_job_wal_entry_local already uses
|
||
* (ha_primary_log_append / persist_wal_async) -- never one entry per array
|
||
* element, regardless of array_size (Req 1.1, 1.2, 1.3). base_jobid/
|
||
* array_size are taken as explicit parameters rather than read from
|
||
* base_spec->jobid/base_spec->array_size, so the caller's own
|
||
* already-range-claimed values are always what gets committed, independent
|
||
* of whatever same-shaped fields the client's own wire request happened to
|
||
* carry. */
|
||
static int append_job_array_wal_entry_local(persist_t* p, uint64_t base_jobid,
|
||
uint64_t array_size,
|
||
const xce_job_submit_t* base_spec,
|
||
uint64_t* out_lsn)
|
||
{
|
||
xce_job_submit_t wire = { .jobid = base_jobid,
|
||
.array_size = array_size,
|
||
.submit = base_spec->submit };
|
||
uint8_t* buf = NULL;
|
||
size_t sz = 0;
|
||
int rc;
|
||
uint64_t lsn;
|
||
|
||
if (capnp_encode_job_array(&wire, &buf, &sz) != 0)
|
||
return -ENOMEM;
|
||
|
||
/* Snapshot the LSN this entry will receive BEFORE the append (same
|
||
* convention as append_job_wal_entry_local / submit_job_wal_entry_and_
|
||
* commit: persist_wal_async / ha_primary_log_append assigns
|
||
* hdr.lsn = current_lsn then increments). */
|
||
pthread_mutex_lock(&p->lock);
|
||
lsn = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
if (p->cluster != NULL)
|
||
rc = ha_primary_log_append(p->cluster, WAL_JOB_ARRAY_SUBMIT, buf,
|
||
(uint32_t)sz, NULL);
|
||
else
|
||
rc = persist_wal_async(p, WAL_JOB_ARRAY_SUBMIT, buf, (uint32_t)sz);
|
||
free(buf);
|
||
|
||
if (rc == 0)
|
||
*out_lsn = lsn;
|
||
return rc;
|
||
}
|
||
|
||
/* job-array-single-wal-record task 2.1 (design.md §WAL Array Submission,
|
||
* Service Interface): encodes and appends ONE array-shaped WAL entry
|
||
* (WAL_JOB_ARRAY_SUBMIT) and registers ONE commit continuation for it,
|
||
* regardless of array_size -- the single-record sibling of
|
||
* persist_submit_job_payloads_async's per-payload loop, specialized to
|
||
* exactly one payload. No job_t/xce_job_t object is constructed here or
|
||
* anywhere on this call path -- array expansion into individual job
|
||
* entries happens later, in the shared apply seam (task 3.x's
|
||
* restore_job_array), never here.
|
||
*
|
||
* Mirrors persist_submit_job_payloads_async's contract exactly:
|
||
* - a local encode/append failure resolves `fn` synchronously with that
|
||
* failure and returns the same negative rc;
|
||
* - the LSN-0 / engine-not-wired edge cases (documented on
|
||
* persist_submit_job_payloads_async above) resolve `fn` inline with
|
||
* rc=0;
|
||
* - otherwise `fn` is registered via commit_submit_async and invoked
|
||
* exactly once, asynchronously, once the entry's commit outcome is
|
||
* known. */
|
||
int persist_submit_job_array_async(persist_t* p, uint64_t base_jobid,
|
||
uint64_t array_size,
|
||
const xce_job_submit_t* base_spec,
|
||
uint64_t* out_lsn,
|
||
persist_job_commit_notify_fn fn, void* ctx)
|
||
{
|
||
uint64_t lsn = 0;
|
||
int rc;
|
||
|
||
if (p == NULL || base_spec == NULL || base_spec->submit == NULL ||
|
||
out_lsn == NULL || fn == NULL || array_size <= 1)
|
||
return -EINVAL;
|
||
|
||
rc = append_job_array_wal_entry_local(p, base_jobid, array_size,
|
||
base_spec, &lsn);
|
||
if (rc != 0) {
|
||
fn(ctx, 0, rc);
|
||
return rc;
|
||
}
|
||
*out_lsn = lsn;
|
||
|
||
/* LSN-0 / engine-not-wired edge (see persist_submit_job_payloads_async's
|
||
* doc comment above for the full rationale -- identical here,
|
||
* specialized to one entry instead of a loop). */
|
||
if (lsn == 0 || p->repl.ce == NULL) {
|
||
fn(ctx, lsn, 0);
|
||
return 0;
|
||
}
|
||
|
||
job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a));
|
||
if (a == NULL) {
|
||
fn(ctx, lsn, -ENOMEM);
|
||
return -ENOMEM;
|
||
}
|
||
a->fn = fn;
|
||
a->ctx = ctx;
|
||
|
||
int reg_rc = commit_submit_async(p->repl.ce, lsn,
|
||
job_commit_resolve_adapter, a);
|
||
if (reg_rc != 0) {
|
||
free(a);
|
||
ERROR("persist_submit_job_array_async: lsn=%lu appended but "
|
||
"commit_submit_async registration failed (rc=%d) -- the "
|
||
"entry remains durable and will still be caught up "
|
||
"eventually via replay, but this caller will not be told "
|
||
"its outcome via this path",
|
||
(unsigned long)lsn, reg_rc);
|
||
fn(ctx, lsn, reg_rc);
|
||
return reg_rc;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
void persist_register_pending_apply(persist_t* p, uint64_t lsn,
|
||
pending_apply_notify_fn on_applied,
|
||
void* ctx)
|
||
{
|
||
pending_command_apply_t* node;
|
||
|
||
if (p == NULL || on_applied == NULL)
|
||
return;
|
||
|
||
/* bug-1415 self-healing (round 2): commit.c's continuation dispatch and
|
||
* the apply pump's walk (persist_apply_pump_run) are two independent
|
||
* vtasks with no completion-order guarantee -- see this function's doc
|
||
* comment in persist.h for the full race. If the walk has already
|
||
* applied `lsn` (last_applied_lsn already at or past it) by the time we
|
||
* get here, linking this registration onto pending_applies would orphan
|
||
* it forever: the walk only ever visits each lsn once, and it already
|
||
* called persist_take_pending_apply() for this lsn and found nothing.
|
||
* Fire the hand-off immediately, synchronously, on THIS thread instead
|
||
* -- never leave a continuation unresolved (controllers never
|
||
* self-terminate; the same "fail loud and complete the work" posture
|
||
* commit.c's dispatch_continuation fallback uses). */
|
||
if (lsn <= persist_last_applied_lsn(p)) {
|
||
on_applied(ctx, lsn);
|
||
return;
|
||
}
|
||
|
||
node = calloc(1, sizeof(*node));
|
||
if (node == NULL) {
|
||
ERROR("persist_register_pending_apply: OOM registering the apply "
|
||
"hand-off for lsn=%lu -- the entry is still durably committed "
|
||
"and will still be applied by a future restart replay, but no "
|
||
"continuation will fire for it via this path",
|
||
(unsigned long)lsn);
|
||
return;
|
||
}
|
||
node->lsn = lsn;
|
||
node->on_applied = on_applied;
|
||
node->ctx = ctx;
|
||
|
||
pthread_mutex_lock(&p->pending_applies_lock);
|
||
node->next = (pending_command_apply_t*)p->pending_applies;
|
||
p->pending_applies = node;
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
}
|
||
|
||
int persist_has_pending_apply(persist_t* p, uint64_t lsn)
|
||
{
|
||
pending_command_apply_t* n;
|
||
int found = 0;
|
||
|
||
if (p == NULL)
|
||
return 0;
|
||
|
||
pthread_mutex_lock(&p->pending_applies_lock);
|
||
for (n = (pending_command_apply_t*)p->pending_applies; n != NULL;
|
||
n = n->next) {
|
||
if (n->lsn == lsn) {
|
||
found = 1;
|
||
break;
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
|
||
return found;
|
||
}
|
||
|
||
int persist_take_pending_apply(persist_t* p, uint64_t lsn,
|
||
pending_apply_notify_fn* out_fn, void** out_ctx)
|
||
{
|
||
pending_command_apply_t* prev = NULL;
|
||
pending_command_apply_t* n;
|
||
int found = 0;
|
||
|
||
if (p == NULL)
|
||
return 0;
|
||
|
||
pthread_mutex_lock(&p->pending_applies_lock);
|
||
for (n = (pending_command_apply_t*)p->pending_applies; n != NULL;
|
||
n = n->next) {
|
||
if (n->lsn == lsn) {
|
||
if (prev == NULL)
|
||
p->pending_applies = n->next;
|
||
else
|
||
prev->next = n->next;
|
||
if (out_fn)
|
||
*out_fn = n->on_applied;
|
||
if (out_ctx)
|
||
*out_ctx = n->ctx;
|
||
found = 1;
|
||
break;
|
||
}
|
||
prev = n;
|
||
}
|
||
pthread_mutex_unlock(&p->pending_applies_lock);
|
||
|
||
free(n); /* NULL when not found -- free(NULL) is a no-op */
|
||
|
||
return found;
|
||
}
|
||
|
||
/* raft-apply-decoupling task 4.1 (Req 5.1): the receipt-time durable-append
|
||
* primitive for a worker-reported job-state fact. Unlike persist_job_hook_fun
|
||
* below (an AFTER_EVENT hook that fires only once a mutation has already
|
||
* happened), this is a direct, hook-fan-out-free call: job.c's fact receipt
|
||
* handlers (record_job_start_fact/record_job_finish_fact/record_job_fail_fact/
|
||
* record_job_postdone_fact) call this (or persist_submit_job_fact_ex below)
|
||
* INSTEAD of mutating and firing AFTER_EVENT, so no notification (scheduler
|
||
* dispatch, slot release, apiserver push) can ever fire for an uncommitted
|
||
* fact (Req 5.3). `state` is always WAL_MODE_RETAIN for every job-state fact
|
||
* type (verified against wal_classify's table) — hardcoded here rather than
|
||
* routed through wal_classify since none of these types are detail-
|
||
* disambiguated the way WAL_WORKER_CTRL is.
|
||
*
|
||
* Thin wrapper over persist_submit_job_fact_ex() with pad = 0 — the "legacy
|
||
* absolute state" discriminator (see wal_types.h's doc comment on
|
||
* wal_jobstate_t._pad). Unchanged contract/signature for every existing
|
||
* caller. */
|
||
int persist_submit_job_fact(persist_t* p, uint32_t wal_type, uint64_t jobid,
|
||
int state)
|
||
{
|
||
return persist_submit_job_fact_ex(p, wal_type, jobid, state, 0);
|
||
}
|
||
|
||
/* raft-apply-decoupling task 6.1 round 2 (Req 3.3/7.1-7.3): full-control
|
||
* sibling of persist_submit_job_fact() above, letting the caller set
|
||
* wal_jobstate_t's `_pad` discriminator explicitly. Used by
|
||
* record_job_fail_fact (WAL_JOB_FAIL_STAGE_FLAG | stage) and
|
||
* record_job_postdone_fact (WAL_JOB_UPDATE_POSTDONE_MARK) to mark a fact
|
||
* whose real mutation must be re-derived at apply time against the job's
|
||
* CURRENT state, rather than trusting a receipt-time snapshot baked into
|
||
* `state` — see wal_types.h's doc comment on wal_jobstate_t._pad for the
|
||
* full rationale. */
|
||
int persist_submit_job_fact_ex(persist_t* p, uint32_t wal_type,
|
||
uint64_t jobid, int state, uint32_t pad)
|
||
{
|
||
wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state,
|
||
._pad = pad };
|
||
|
||
return wal_hook_append_classified(p, wal_type, WAL_MODE_RETAIN, &js,
|
||
sizeof(js));
|
||
}
|
||
|
||
/* P0 payload: just the detail value (4 bytes).
|
||
* Full Cap'n Proto serialization is wired in Task 15. */
|
||
|
||
int persist_job_hook_fun(int type, int situation, int detail,
|
||
void* cls, va_list ap)
|
||
{
|
||
(void)type;
|
||
/* Write to WAL only after the state change is complete.
|
||
* BEFORE_EVENT has a different va_list layout and must be ignored. */
|
||
if (situation != HOOK_AFTER_EVENT)
|
||
return 0;
|
||
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1): this AFTER_EVENT firing
|
||
* originated from apply_committed_entry()'s own post-mutation notify —
|
||
* the entry is already durably committed (that is why it is being
|
||
* applied at all), so appending it again here would duplicate it. See
|
||
* t_apply_seam_notifying's doc comment (above apply_committed_entry) for
|
||
* why this MUST be a call-provenance guard, not a `detail` filter:
|
||
* job_ctrl's mark_job_update/mark_job_fail still fire this exact same
|
||
* hook for their own live, still-legitimate append. */
|
||
if (t_apply_seam_notifying)
|
||
return 0;
|
||
|
||
persist_t* p = cls;
|
||
uint32_t wal_type = detail_to_wal_type(detail);
|
||
wal_commit_mode_t mode;
|
||
|
||
if (wal_type == 0)
|
||
return 0;
|
||
mode = wal_classify(wal_type, detail);
|
||
|
||
if (mode == WAL_MODE_WAIT) {
|
||
/* WAL_JOB_SUBMIT durability (WAL append + majority-commit barrier) is
|
||
* an explicit step invoked directly by handle_jobsubmit —
|
||
* persist_submit_job_payloads_and_commit(), called strictly before
|
||
* any job object is constructed (raft-apply-decoupling task 3.1 /
|
||
* bug-1306) — never from this hook. This hook fires only after that
|
||
* commit has already succeeded, purely as a notification, so there
|
||
* is nothing left for it to do here. */
|
||
return 0;
|
||
}
|
||
|
||
/* All other job events: payload = wal_jobstate_t {jobid, state}.
|
||
* State is read from the job_t after the AFTER_EVENT hook updates it. */
|
||
wal_jobstate_t js = {0};
|
||
switch (detail) {
|
||
case HOOK_DETAIL_JOB_START: {
|
||
/* args: cluster_t*, job_pidinfo_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
(void)va_arg(ap, void*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
case HOOK_DETAIL_JOB_EXIT: {
|
||
/* args: cluster_t*, xce_jobfinish_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
(void)va_arg(ap, void*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
case HOOK_DETAIL_JOB_FAIL: {
|
||
/* args: cluster_t*, job_failinfo_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
(void)va_arg(ap, void*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
default: {
|
||
/* STOP, RESUME, UPDATE, CKPT, PREASON, RUSAGE — args: cluster_t*, job_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
job_t* j = va_arg(ap, job_t*);
|
||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* mode == WAL_MODE_RETAIN — fact-shaped job state transition */
|
||
return wal_hook_append_classified(p, wal_type, mode, &js, sizeof(js));
|
||
}
|
||
|
||
int persist_queue_hook_fun(int type, int situation, int detail,
|
||
void* cls, va_list ap)
|
||
{
|
||
(void)type;
|
||
if (situation != HOOK_AFTER_EVENT)
|
||
return 0;
|
||
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1, queue analog): see
|
||
* persist_job_hook_fun's identical guard above for the full rationale —
|
||
* apply_committed_entry()'s WAL_QUEUE_CTRL branch fires this same
|
||
* AFTER_EVENT to notify, and this hook appends unconditionally
|
||
* (WAL_QUEUE_CTRL is WAL_MODE_WAIT with no early-return here), so
|
||
* without this guard a live-applied queue entry would re-append and
|
||
* re-trigger a nested majority-commit wait every time the pump walks
|
||
* it. */
|
||
if (t_apply_seam_notifying)
|
||
return 0;
|
||
|
||
persist_t* p = cls;
|
||
uint32_t wal_type = detail_to_wal_type(detail);
|
||
wal_commit_mode_t mode;
|
||
|
||
if (wal_type == 0)
|
||
return 0;
|
||
mode = wal_classify(wal_type, detail);
|
||
|
||
/* args: cluster_t*, queue_t* */
|
||
(void)va_arg(ap, cluster_t*);
|
||
queue_t* q = va_arg(ap, queue_t*);
|
||
|
||
/* Payload: [wal_queuehdr_t][queue name NUL-terminated] */
|
||
const char* name = q->queue.queue ? q->queue.queue : "";
|
||
size_t name_len = strlen(name) + 1;
|
||
size_t total = sizeof(wal_queuehdr_t) + name_len;
|
||
uint8_t* pkt = malloc(total);
|
||
if (!pkt) return -ENOMEM;
|
||
|
||
wal_queuehdr_t hdr = { .status = q->queue.status };
|
||
memcpy(pkt, &hdr, sizeof(hdr));
|
||
memcpy(pkt + sizeof(hdr), name, name_len);
|
||
|
||
/* mode == WAL_MODE_WAIT — the append stays async; the handler barrier
|
||
* owes commit completion before the requester reply (task 3) */
|
||
int rc = wal_hook_append_classified(p, wal_type, mode, pkt,
|
||
(uint32_t)total);
|
||
free(pkt);
|
||
return rc;
|
||
}
|
||
|
||
int persist_worker_hook_fun(int type, int situation, int detail,
|
||
void* cls, va_list ap)
|
||
{
|
||
(void)type;
|
||
if (situation != HOOK_AFTER_EVENT)
|
||
return 0;
|
||
|
||
/* raft-apply-decoupling task 6.1 (Hazard 1, worker analog): see
|
||
* persist_job_hook_fun's identical guard above for the full rationale. */
|
||
if (t_apply_seam_notifying)
|
||
return 0;
|
||
|
||
persist_t* p = cls;
|
||
uint32_t wal_type = detail_to_wal_type(detail);
|
||
wal_commit_mode_t mode;
|
||
|
||
if (wal_type == 0)
|
||
return 0;
|
||
mode = wal_classify(wal_type, detail);
|
||
|
||
/* WAL_WORKER_CTRL payload layout: [uint32_t detail][capnp worker OR name] */
|
||
uint32_t disc = (uint32_t)detail;
|
||
uint8_t* buf = NULL;
|
||
size_t capnp_sz = 0;
|
||
int rc = 0;
|
||
|
||
/* All worker detail types: args are cluster_t* then worker_t*.
|
||
* JOIN uses a unified signature with an extra char* name between them;
|
||
* DEAD and UPDATE use (cluster_t*, worker_t*) directly. */
|
||
(void)va_arg(ap, cluster_t*);
|
||
if (detail == HOOK_DETAIL_WORKER_JOIN)
|
||
(void)va_arg(ap, char*); /* skip name — present only for JOIN */
|
||
worker_t* w = va_arg(ap, worker_t*);
|
||
if (!w) return 0; /* guard: failed join produced no worker slot */
|
||
rc = capnp_encode_worker(&w->host, &buf, &capnp_sz);
|
||
if (rc != 0) return rc;
|
||
|
||
/* Build [disc | capnp bytes] in one allocation */
|
||
size_t total = WORKER_WAL_HDR_SIZE + capnp_sz;
|
||
uint8_t* pkt = malloc(total);
|
||
if (!pkt) { free(buf); return -ENOMEM; }
|
||
memcpy(pkt, &disc, WORKER_WAL_HDR_SIZE);
|
||
memcpy(pkt + WORKER_WAL_HDR_SIZE, buf, capnp_sz);
|
||
free(buf);
|
||
|
||
/* JOIN/UPDATE → WAIT (handler barrier owes completion, task 3);
|
||
* DEAD → RETAIN (controller-detected fact, registered with the engine) */
|
||
rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total);
|
||
free(pkt);
|
||
return rc;
|
||
}
|
||
|
||
/* ============================================================
|
||
* GC retention predicates (P0-5a / P0-8 / P0-13)
|
||
*
|
||
* Placed in persist.c so the unit test binary (which links only
|
||
* persist.c from the controller sources) can access them without
|
||
* pulling in RTTI-dependent job.c / worker.c.
|
||
* ============================================================ */
|
||
|
||
bool job_is_terminal(const job_t* j)
|
||
{
|
||
return (j->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0;
|
||
}
|
||
|
||
bool job_should_retain(const job_t* j, const persist_config_t* cfg)
|
||
{
|
||
time_t now = time(NULL);
|
||
|
||
/* TRANSIENT jobs: auto-terminate when TTL elapses after failover.
|
||
* transient_since == 0 means the timestamp was never set — do not GC. */
|
||
if (j->job.state & JOB_STAT_TRANSIENT) {
|
||
if (j->job.transient_since > 0 &&
|
||
now > (time_t)j->job.transient_since + (time_t)cfg->transient_job_timeout_s) {
|
||
return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
/* Non-terminal jobs are always retained */
|
||
if (!job_is_terminal(j)) return true;
|
||
|
||
/* Terminal jobs: retain until finished_retain_s expires */
|
||
return (j->job.finish_time + (time_t)cfg->finished_retain_s >= now);
|
||
}
|
||
|
||
bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg)
|
||
{
|
||
time_t now = time(NULL);
|
||
|
||
/* HIDDEN workers: admin explicitly removed — clean up at next checkpoint */
|
||
if (w->status & XCE_WORKER_HIDDEN) return false;
|
||
|
||
/* UNAVAILABLE workers: retain until worker_retain_s expires.
|
||
* dead_since == 0 means not yet set — do not GC. */
|
||
if (w->status & XCE_WORKER_UNAVAIL) {
|
||
if (w->dead_since > 0 &&
|
||
now > w->dead_since + (time_t)cfg->worker_retain_s) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
return true;
|
||
}
|
||
|
||
/* ============================================================
|
||
* WAL truncation (task 7.5)
|
||
*
|
||
* Remove all WAL entries with LSN > cutoff_lsn.
|
||
* Idempotent: calling twice with same cutoff_lsn returns 0 both times.
|
||
* ============================================================ */
|
||
|
||
/* Clamp durable_lsn down to `tip` when the log tip recedes below it (truncate
|
||
* / divergent-suffix discard). Without this, durable_lsn could exceed
|
||
* current_lsn - 1 after a truncate, violating its invariant and letting the
|
||
* commit majority count self at an LSN whose bytes were just removed. MUST be
|
||
* called with p->lock held (mirrors the committed_lsn clamp in the truncate
|
||
* paths). durable_lsn only ever decreases here — the sole exception to its
|
||
* monotonic-non-decreasing rule (ha-commit-durability 2.1 State Management). */
|
||
static void clamp_durable_lsn_locked(persist_t* p, uint64_t tip)
|
||
{
|
||
uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire);
|
||
if (dur > tip)
|
||
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
|
||
}
|
||
|
||
int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn)
|
||
{
|
||
pthread_mutex_lock(&p->lock);
|
||
uint64_t before = p->current_lsn;
|
||
int rc = truncate_wal_above_locked(p, cutoff_lsn);
|
||
uint64_t after = p->current_lsn;
|
||
pthread_mutex_unlock(&p->lock);
|
||
|
||
/* raft-apply-decoupling task 1.4: fire the truncation wake only when a
|
||
* truncation actually removed entries above cutoff_lsn (current_lsn
|
||
* strictly decreased -- both truncate_wal_above_locked branches set
|
||
* current_lsn = cutoff_lsn + 1 on a real truncation) -- mirrors the
|
||
* durable-advance wake's own before/after guard (see fsync_worker
|
||
* above) so a no-op truncate (nothing above cutoff_lsn, or an I/O
|
||
* error before current_lsn is touched) never spuriously fails a
|
||
* continuation whose entry is still safely in the log. Called AFTER
|
||
* releasing p->lock: the callback (the commit engine) takes only
|
||
* ce->lock, preserving the cycle-free p->lock -> ce->lock edge already
|
||
* established by the durable-advance wake. */
|
||
if (rc == 0 && after < before) {
|
||
/* raft-apply-decoupling task 7.4 perf-fix: a real truncation may
|
||
* ftruncate()/unlink() segment files the apply-pump's segment-lookup
|
||
* cache still points into -- invalidate it so a subsequent
|
||
* apply_pump_read_entry can never read truncated-away data. p->lock
|
||
* is already released above and apply_pump_lock is never held by
|
||
* this call path (persist_apply_pump_run is the only place that
|
||
* holds it, during its own walk), so no lock-order hazard here. */
|
||
pthread_mutex_lock(&p->apply_pump_lock);
|
||
if (p->apply_pump_cache_fd >= 0) {
|
||
close(p->apply_pump_cache_fd);
|
||
p->apply_pump_cache_fd = -1;
|
||
}
|
||
p->apply_pump_cache_next_lsn = 0;
|
||
pthread_mutex_unlock(&p->apply_pump_lock);
|
||
|
||
if (p->on_truncate != NULL) {
|
||
p->on_truncate(p->truncate_cb_ctx, cutoff_lsn);
|
||
}
|
||
}
|
||
return rc;
|
||
}
|
||
|
||
/* persist_committed_lsn_advance_atomic (2026-07-10, controller-invariant-audit
|
||
* finding): race-safe monotonic-max advance of p->committed_lsn via a CAS
|
||
* retry loop. Replaces the naive "load; if target>cur store" pattern, which
|
||
* can lose an update if two writers race: persist_promotion_advance_committed
|
||
* (below) and ha_standby_advance_committed_and_apply (ha.c) can both run
|
||
* during the promotion window before role flips to CTRL_PRIMARY (a
|
||
* still-in-flight WAL catch-up entry processed on another thread at the same
|
||
* instant as this node's own election completing). No lock required -- CAS
|
||
* only succeeds if the value hasn't changed since the read, so a losing
|
||
* racer retries with the freshly-observed value and can never clobber a
|
||
* larger value with a smaller one. */
|
||
uint64_t persist_committed_lsn_advance_atomic(persist_t* p, uint64_t target)
|
||
{
|
||
uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire);
|
||
while (target > cur) {
|
||
if (atomic_compare_exchange_weak_explicit(&p->committed_lsn, &cur, target,
|
||
memory_order_release,
|
||
memory_order_acquire))
|
||
return target;
|
||
/* cur was refreshed to the current value by the failed CAS; retry. */
|
||
}
|
||
return cur;
|
||
}
|
||
|
||
/* persist_promotion_advance_committed (2026-07-10, promotion apply-lag fix):
|
||
* unconditionally advance p->committed_lsn toward `safe_lsn` at promotion
|
||
* time. persist_truncate_wal_above only sets committed_lsn on the branches
|
||
* of truncate_wal_above_locked that perform a REAL truncation (partial
|
||
* truncate, or the full-segment-purge demotion-convergence case); its no-op
|
||
* early return (nothing above cutoff_lsn to remove -- the common case for a
|
||
* newly-elected leader whose own log already covers safe_lsn) leaves
|
||
* committed_lsn untouched at its pre-election value. Without this call,
|
||
* committed_lsn -- and therefore apply, which is commit-gated
|
||
* (raft-apply-decoupling) -- stalls until the normal per-entry ack-driven
|
||
* commit path organically catches it up, which can take seconds while
|
||
* surviving standbys reconnect and re-ACK. During that gap a job already
|
||
* committed on a majority is invisible via expect-job/joblist on the very
|
||
* primary that is authoritative for it (bug-828 class).
|
||
*
|
||
* DESIGN NOTE -- why this function never forces a synchronous flush (round 3,
|
||
* 2026-07-10, controller-invariant-audit): an earlier round of this fix
|
||
* called flusher_flush_once(p) here when durable_lsn lagged safe_lsn, to
|
||
* force this node's own fsync to catch up immediately rather than waiting on
|
||
* fsync_worker's cadence. That was a real bug-1198-CLASS thread-role
|
||
* violation: the real multi-node winning-vote-reply path reaches
|
||
* raft_cb_become_leader (and so this function) via peer_ctrl_classify's
|
||
* VTASK_LIVENESS dispatch (ha.c ~5935-5940) -- a task class the design
|
||
* DOCUMENTS as "never admission-gated" / never expected to block (see the
|
||
* bug-1198 hazard box a few hundred lines above peer_ctrl_classify's
|
||
* definition, which closed the EXACT same class of bug for
|
||
* HA_REPL_SEND_SNAPSHOT's persist_checkpoint). A blocking fdatasync here
|
||
* would reintroduce that closed bug class on the pool's liveness-reserved
|
||
* workers. The visibility gap the force-flush was trying to close is instead
|
||
* closed non-blockingly by ha_start_primary_heartbeat sending its first
|
||
* heartbeat synchronously at promotion (bug-1432) -- that lets a reconnecting
|
||
* standby's real position get learned, and the entry gets committed via the
|
||
* normal ack-driven commit_advance_locked path, within one network RTT
|
||
* instead of waiting a full heartbeat interval. A live chaotic soak (10
|
||
* rounds) confirmed durable-capped-only + immediate-heartbeat together reach
|
||
* committed_job_loss=0 with zero blocking calls added to this function.
|
||
*
|
||
* Durable-capped (never advances committed_lsn past this node's own
|
||
* durable_lsn -- see persistence.md's "acknowledged implies durable on a
|
||
* majority" core invariant) and monotonic + race-safe via
|
||
* persist_committed_lsn_advance_atomic. */
|
||
void persist_promotion_advance_committed(persist_t* p, uint64_t safe_lsn)
|
||
{
|
||
if (!p)
|
||
return;
|
||
uint64_t durable = atomic_load_explicit(&p->durable_lsn,
|
||
memory_order_acquire);
|
||
uint64_t target = safe_lsn < durable ? safe_lsn : durable;
|
||
persist_committed_lsn_advance_atomic(p, target);
|
||
}
|
||
|
||
/* Lock-held core of persist_truncate_wal_above (see the forward declaration and
|
||
* the conflict-guard reuse near persist_wal_write_replicated). Caller MUST hold
|
||
* p->lock; this function never touches the lock so it is reusable from another
|
||
* already-locked path. */
|
||
static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn)
|
||
{
|
||
/* No-op: nothing written yet, or cutoff covers all written entries */
|
||
if (p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn) {
|
||
return 0;
|
||
}
|
||
|
||
/* --- Collect segment filenames and sort by start LSN --- */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
|
||
DIR* d = opendir(p->wal_path);
|
||
if (d) {
|
||
struct dirent* e;
|
||
while ((e = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) {
|
||
if (strncmp(e->d_name, "seg-", 4) != 0)
|
||
continue;
|
||
uint64_t sl = 0;
|
||
if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1)
|
||
continue;
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
}
|
||
|
||
if (nseg == 0) {
|
||
return 0;
|
||
}
|
||
|
||
/* Insertion sort ascending by start LSN */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) {
|
||
seg_lsns[j + 1] = seg_lsns[j];
|
||
j--;
|
||
}
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
/* --- Find segment containing cutoff_lsn --- */
|
||
int cutoff_seg_idx = -1;
|
||
for (int i = 0; i < nseg; i++) {
|
||
if (seg_lsns[i] <= cutoff_lsn)
|
||
cutoff_seg_idx = i;
|
||
else
|
||
break;
|
||
}
|
||
|
||
if (cutoff_seg_idx < 0) {
|
||
/* cutoff_lsn is below the oldest retained segment's start LSN, so EVERY
|
||
* on-disk entry has LSN > cutoff_lsn and must be removed (the function's
|
||
* contract: "remove all entries with LSN > cutoff_lsn"). This is the
|
||
* demotion-convergence case (ha-convergence-recovery Req 1.2): a demoted
|
||
* ex-primary whose committed boundary sits below its entire retained log
|
||
* must discard all of it and collapse the tip to cutoff_lsn + 1. Delete
|
||
* every segment, close the active fd, and reset the in-memory tip. */
|
||
for (int i = 0; i < nseg; i++) {
|
||
char del_path[4096];
|
||
snprintf(del_path, sizeof(del_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[i]);
|
||
unlink(del_path);
|
||
}
|
||
if (p->wal_fd >= 0) {
|
||
fdatasync(p->wal_fd);
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
p->seg_offset = 0;
|
||
p->seg_path[0] = '\0';
|
||
p->current_lsn = cutoff_lsn + 1;
|
||
p->last_crc = WAL_CRC_NONE;
|
||
atomic_store_explicit(&p->committed_lsn, cutoff_lsn,
|
||
memory_order_release);
|
||
clamp_durable_lsn_locked(p, cutoff_lsn);
|
||
/* Keep the per-LSN term index consistent: drop terms for removed LSNs. */
|
||
truncate_term_index_locked(p, cutoff_lsn);
|
||
return 0;
|
||
}
|
||
|
||
/* --- Scan segment to find byte offset of first entry with lsn > cutoff --- */
|
||
char cutoff_seg_path[4096];
|
||
snprintf(cutoff_seg_path, sizeof(cutoff_seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]);
|
||
|
||
int scan_fd = open(cutoff_seg_path, O_RDONLY);
|
||
if (scan_fd < 0) {
|
||
return -errno;
|
||
}
|
||
|
||
uint32_t scan_off = 0;
|
||
uint32_t trunc_off = 0;
|
||
uint32_t new_last_crc = WAL_CRC_NONE;
|
||
bool found_cut = false;
|
||
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t n = pread(scan_fd, &hdr, sizeof(hdr), (off_t)scan_off);
|
||
if (n < (ssize_t)sizeof(hdr))
|
||
break; /* short read or EOF */
|
||
/* Pre-allocated zero region: type==0 is not a valid WAL type (min=1) */
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0)
|
||
break;
|
||
if (hdr.lsn > cutoff_lsn) {
|
||
trunc_off = scan_off;
|
||
found_cut = true;
|
||
break;
|
||
}
|
||
new_last_crc = hdr.crc;
|
||
scan_off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(scan_fd);
|
||
|
||
if (!found_cut)
|
||
trunc_off = scan_off; /* end of valid data in segment */
|
||
|
||
/* --- Close the current wal_fd --- */
|
||
if (p->wal_fd >= 0) {
|
||
fdatasync(p->wal_fd);
|
||
close(p->wal_fd);
|
||
p->wal_fd = -1;
|
||
}
|
||
|
||
/* --- Truncate the cutoff segment at the computed offset --- */
|
||
int new_fd = open(cutoff_seg_path, O_RDWR);
|
||
if (new_fd < 0) {
|
||
return -errno;
|
||
}
|
||
if (ftruncate(new_fd, (off_t)trunc_off) != 0) {
|
||
int saved = errno;
|
||
close(new_fd);
|
||
return -saved;
|
||
}
|
||
fdatasync(new_fd);
|
||
|
||
/* --- Delete all segments after the cutoff segment --- */
|
||
for (int i = cutoff_seg_idx + 1; i < nseg; i++) {
|
||
char del_path[4096];
|
||
snprintf(del_path, sizeof(del_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[i]);
|
||
unlink(del_path);
|
||
}
|
||
|
||
/* --- Update persist_t state under lock --- */
|
||
p->wal_fd = new_fd;
|
||
p->seg_offset = trunc_off;
|
||
snprintf(p->seg_path, sizeof(p->seg_path),
|
||
WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]);
|
||
p->current_lsn = cutoff_lsn + 1;
|
||
p->last_crc = new_last_crc;
|
||
atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release);
|
||
clamp_durable_lsn_locked(p, cutoff_lsn);
|
||
/* Keep the per-LSN term index consistent: drop terms for removed LSNs. */
|
||
truncate_term_index_locked(p, cutoff_lsn);
|
||
|
||
return 0;
|
||
}
|