#define _GNU_SOURCE #include "persist.h" #include "config.h" #include "ctrl_metrics.h" #include "hook.h" #include "xce/cluster_def.h" #include "xce/errnox.h" #include "xce/types.h" #include "controller.capnp.h" #include "xlist.h" #include "cluster.h" #include "commit.h" /* commit_submit RETAIN registration (barrier-rollout 1.2) */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* restore_* functions defined in job.c / worker.c / queue.c */ extern int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload, size_t len, uint64_t entry_lsn); extern int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state); extern int worker_from_capnp(cluster_t* c, const void* capnp_payload, size_t len); extern int restore_queue_state(cluster_t* c, const char* name, uint32_t status); /* restore_host_state defined in worker.c (config-upgrade-snapshot task 3, * Req 1.10/1.11/5.8) -- create-if-missing host admin-state restore, called * from load_snapshot_entities' host-frame loop below (task 4). */ extern int restore_host_state(cluster_t* c, const char* name, uint32_t admin_status); /* raft-apply-decoupling task 3.2: pending command-apply hand-off list node * (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, is * the consumer). Declared here (rather than persist.h) so the list stays * an implementation detail — persist.h exposes only the opaque `void*` * head field plus persist_register_pending_apply/persist_has_pending_apply. * See persist.h's doc comment on persist_register_pending_apply for the * full contract. */ typedef struct pending_command_apply { uint64_t lsn; pending_apply_notify_fn on_applied; void* ctx; struct pending_command_apply* next; } pending_command_apply_t; /* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */ /* Serialize xce_job_t to a malloc'd Cap'n Proto buffer. * Caller must free(*out) on success. */ static int capnp_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Job cj; Job_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Job(cs, &cj, (xce_job_t*)j); jp = new_Job(cs); write_Job(&cj, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize xce_worker_t to a malloc'd Cap'n Proto buffer. */ static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Worker cw; Worker_ptr wp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Worker(cs, &cw, (xce_worker_t*)w); wp = new_Worker(cs); write_Worker(&cw, wp); capn_setp(capn_root(&c), 0, wp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize a host admin-state frame (config-upgrade-snapshot task 2.1, Req * 1.6/1.7/1.9) to a malloc'd Cap'n Proto buffer. Mirrors capnp_encode_job * above exactly -- capnp_host_admin_state_t (typemap.h) is a frame-only * (name, status) pair, never the full xce_worker_t (design.md Design * Invariant 5). Caller must free(*out) on success. */ static int capnp_encode_host_admin_state(const capnp_host_admin_state_t* h, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct HostAdminState ch; HostAdminState_ptr hp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_HostAdminState(cs, &ch, (capnp_host_admin_state_t*)h); hp = new_HostAdminState(cs); write_HostAdminState(&ch, hp); capn_setp(capn_root(&c), 0, hp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize a queue admin-state frame (config-upgrade-snapshot task 2.2, Req * 1.9/5.7) to a malloc'd Cap'n Proto buffer. Mirrors * capnp_encode_host_admin_state above exactly -- capnp_queue_admin_state_t * (typemap.h) is a frame-only (name, status) pair, matching * restore_queue_state's exact (name, status) contract (queue.c:208), same * field content as the prior raw layout, only the wire encoding changes. * Caller must free(*out) on success. */ static int capnp_encode_queue_admin_state(const capnp_queue_admin_state_t* q, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct QueueAdminState cq; QueueAdminState_ptr qp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_QueueAdminState(cs, &cq, (capnp_queue_admin_state_t*)q); qp = new_QueueAdminState(cs); write_QueueAdminState(&cq, qp); capn_setp(capn_root(&c), 0, qp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */ #define WORKER_WAL_HDR_SIZE sizeof(uint32_t) /* ------------------------------------------------------------------ */ /* Constants */ /* ------------------------------------------------------------------ */ #define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal" /* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */ #define NFS_SUPER_MAGIC 0x6969 /* ------------------------------------------------------------------ */ /* Internal helpers */ /* ------------------------------------------------------------------ */ static uint64_t now_us(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return (uint64_t)ts.tv_sec * 1000000ULL + (uint64_t)(ts.tv_nsec / 1000); } /* Recursive mkdir — equivalent to mkdir -p */ static int mkdir_p(const char* path, mode_t mode) { char tmp[4096]; char* p; size_t len; snprintf(tmp, sizeof(tmp), "%s", path); len = strlen(tmp); if (len > 0 && tmp[len - 1] == '/') tmp[len - 1] = '\0'; for (p = tmp + 1; *p; p++) { if (*p == '/') { *p = '\0'; if (mkdir(tmp, mode) != 0 && errno != EEXIST) return -1; *p = '/'; } } if (mkdir(tmp, mode) != 0 && errno != EEXIST) return -1; return 0; } /* Open (and pre-allocate) a new WAL segment named after current_lsn */ static int open_new_segment(persist_t* p) { snprintf(p->seg_path, sizeof(p->seg_path), WAL_SEG_FMT, p->wal_path, p->current_lsn); int fd = open(p->seg_path, O_RDWR | O_CREAT | O_TRUNC, 0600); if (fd < 0) return -errno; /* Pre-allocate segment space to avoid fragmentation and metadata updates * on every write. posix_fallocate is best-effort — ignore errors. */ posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes); p->wal_fd = fd; p->seg_offset = 0; return 0; } /* Close current segment and open a fresh one */ static int rotate_segment(persist_t* p) { if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } return open_new_segment(p); } /* ------------------------------------------------------------------ */ /* Demand-driven group-commit flusher (ha-commit-durability task 2.1) */ /* */ /* Repurposes the old fixed-interval fsync timer into a flusher driven */ /* by the append-vs-synced gap. On each pass it batches one fdatasync */ /* over all entries appended since the last sync, advances durable_lsn */ /* to the synced tip, and wakes the commit evaluation. --wal-sync-ms */ /* now bounds the MAX batching latency (a no-further-append entry is */ /* still synced within the cadence), not a durability gap. */ /* */ /* LOST-WAKEUP / CORRECTNESS REASONING (gating item): */ /* */ /* The "work pending" condition is LEVEL-triggered on two persistent */ /* variables, NOT an edge: pending ⇔ durable_lsn < current_lsn - 1 */ /* (flusher_work_pending below, evaluated under p->lock). Because it */ /* is a comparison of stored state — not a transient flag consumed on */ /* read — a signal that races the flusher's wait can never be lost: */ /* */ /* - persist_wal_async advances current_lsn under p->lock, THEN */ /* signals fsync_cond. If the flusher is mid-flush when the append */ /* lands, current_lsn has already moved past the tip the flusher */ /* captured, so after that flush durable_lsn < current_lsn - 1 */ /* STILL holds; the flusher re-checks the level under p->lock on */ /* its next loop iteration and flushes again (it does NOT wait */ /* first). The missed signal is therefore harmless — the level */ /* itself carries the pending work forward. */ /* - If the append's signal arrives while the flusher sits in */ /* pthread_cond_wait, it wakes it. If it arrives in the tiny window */ /* after the flusher decided "no work" but before it re-enters the */ /* wait, the flusher re-evaluates the level under p->lock before */ /* waiting (we re-check inside the fsync_mutex-held region by */ /* reading current_lsn/durable_lsn under p->lock first), so it */ /* will not block while work is pending. */ /* */ /* "NEVER durable before its fdatasync": the flusher snapshots */ /* tip = current_lsn - 1 AND calls fdatasync WHILE HOLDING p->lock. */ /* persist_wal_async also pwrites under p->lock, so no append can */ /* extend the segment between the tip snapshot and the fdatasync — by */ /* the time fdatasync returns, every byte for LSNs <= tip is on stable */ /* storage. durable_lsn is only then advanced to tip (monotonic max). */ /* Releasing p->lock and broadcasting commit-eval happens after the */ /* advance, so a WAIT caller that observes durable_lsn >= its LSN is */ /* guaranteed the bytes are synced. */ /* */ /* Lock discipline: p->lock and fsync_mutex are NEVER held together */ /* (fsync_mutex guards only the cond wait; the flush section takes */ /* p->lock alone). This keeps fdatasync off any lock the iomgr / */ /* heartbeat path needs and preserves liveness (Req 1.6). */ /* ------------------------------------------------------------------ */ /* Test-only fault-injection probe (ha-commit-durability task 2.1 error path). * Returns true when the test has forced fdatasync to be treated as failed. * Compiled to a constant `false` in production (no -DXCE_TESTING), so the * fail-loud gate below collapses to the pure rc != 0 check with zero overhead * and no dependency on the test-only struct field. */ static inline bool flusher_fsync_forced_fail(persist_t* p) { #ifdef XCE_TESTING return atomic_load_explicit(&p->test_force_fsync_fail, memory_order_acquire) != 0; #else (void)p; return false; #endif } /* Returns true iff there is un-synced appended data (level predicate). * MUST be called with p->lock held. current_lsn - 1 is the appended tip * (current_lsn is the NEXT lsn to assign); LSN 0 reserved, so a fresh * store has current_lsn == 1 → tip 0 → no pending work. */ static bool flusher_work_pending_locked(persist_t* p) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); return dur < tip; } /* Perform one group-commit flush: snapshot the appended tip, fdatasync the * live segment, and advance durable_lsn to the synced tip. Takes p->lock * for the whole snapshot+sync+advance so no concurrent append can extend * the segment between the tip read and the fdatasync. Returns the new * durable_lsn (unchanged if there was nothing to sync). */ static uint64_t flusher_flush_once(persist_t* p) { pthread_mutex_lock(&p->lock); uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (dur >= tip || p->wal_fd < 0) { /* Nothing new to sync (or no segment yet). Still flush the fd for a * clean-shutdown / fallback pass, but do not advance the watermark. * The watermark is already <= tip here, so a failed fallback sync * cannot make it claim more than is durable — but we still surface * the error loudly so a degrading disk is never silent (Req 1.1). */ if (p->wal_fd >= 0) { int rc = fdatasync(p->wal_fd); if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: flusher fallback fdatasync FAILED (errno=%d %s)" " — durable_lsn held at %" PRIu64 "; WAL not advanced", e, strerror(e), dur); } } pthread_mutex_unlock(&p->lock); return dur; } /* fdatasync under p->lock: appends also hold p->lock to pwrite, so the * segment cannot grow between this tip snapshot and the sync. Every byte * for LSNs <= tip is therefore on stable storage when fdatasync returns * SUCCESSFULLY. */ int rc = fdatasync(p->wal_fd); /* FAIL-LOUD GATE (Req 1.1/1.2): advance durable_lsn ONLY on a successful * sync. On EIO/ENOSPC the bytes are NOT on stable storage, so leaving the * watermark unchanged keeps the non-durable LSNs from being counted toward * the commit majority. We do NOT retry into an advanced watermark — the * level predicate stays pending and the next pass retries the sync. */ if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: flusher fdatasync FAILED (errno=%d %s) — durable_lsn " "held at %" PRIu64 " (tip=%" PRIu64 " NOT made durable); commit " "majority will not count these LSNs", e, strerror(e), dur, tip); pthread_mutex_unlock(&p->lock); return dur; } /* Monotonic advance to the synced tip — never past it (the invariant * durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1 * was read under this same lock). One fdatasync advances durable_lsn past * however many LSNs were appended since the last sync (batching). */ if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); #ifdef XCE_TESTING /* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6): * a liveness test asserts this is the dedicated flusher thread, never the * caller/iomgr/heartbeat thread. Written under p->lock; read after the * flusher has caught up, so no extra synchronization is needed. */ p->test_last_fdatasync_thread = pthread_self(); atomic_store_explicit(&p->test_last_fdatasync_thread_set, 1, memory_order_release); #endif pthread_mutex_unlock(&p->lock); return tip; } static void* fsync_worker(void* arg) { persist_t* p = arg; pthread_mutex_lock(&p->fsync_mutex); while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) { /* Re-evaluate the level predicate BEFORE waiting so a signal that * raced in just before we entered the wait is not needed for * correctness — if work is already pending we skip the wait entirely * and flush. (Reads current_lsn/durable_lsn under p->lock.) */ pthread_mutex_lock(&p->lock); bool pending = flusher_work_pending_locked(p); pthread_mutex_unlock(&p->lock); if (!pending) { /* No pending work: wait for an append-signal OR the cadence * deadline (bounded fallback flush so a lone no-further-append * entry is still synced within wal_sync_ms — Req 3.1). */ long ms = (long)p->config.wal_sync_ms; struct timespec dl; clock_gettime(CLOCK_REALTIME, &dl); dl.tv_sec += ms / 1000; dl.tv_nsec += (ms % 1000) * 1000000L; if (dl.tv_nsec >= 1000000000L) { dl.tv_sec += 1; dl.tv_nsec -= 1000000000L; } pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl); if (atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) break; } /* Flush section: never hold fsync_mutex across the fdatasync (the * flush takes p->lock alone). Release fsync_mutex, flush, re-acquire. * After the flush we loop and re-check the level — if a new append * landed during the flush, the level is still pending and we flush * again without needing its signal (lost-wakeup-safe). */ pthread_mutex_unlock(&p->fsync_mutex); uint64_t before = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); uint64_t after = flusher_flush_once(p); /* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced * INSIDE flusher_flush_once (atomic store with release) BEFORE we * return here, and flusher_flush_once has already RELEASED p->lock. * Now — holding NO persist lock — invoke the registered durable-advance * callback so the commit engine re-evaluates its majority and wakes any * COMMIT_MODE_WAIT caller whose entry just became durable locally. * * LOST-WAKEUP SAFETY: the wake fires strictly AFTER the durable_lsn * release-store, and the commit engine's predicate re-reads durable_lsn * via an acquire load under ce->lock — so a WAIT caller either (a) sees * the advanced watermark on its own re-check, or (b) is blocked in * pthread_cond_wait and is woken by this broadcast; it can never both * miss the watermark AND miss the wake. We only fire the callback when * durable_lsn actually advanced (after > before): a no-op flush (or a * failed fdatasync that held the watermark) needs no commit re-eval, * avoiding spurious wakes. * * LOCK ORDERING (cycle-free): the flusher releases p->lock inside * flusher_flush_once, THEN the callback takes ce->lock (commit side). * The commit side only ever takes ce->lock and reads persist atomics — * it never takes p->lock while holding ce->lock. So the only ordering * is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no * deadlock cycle. The callback explicitly MUST NOT take p->lock. */ if (after > before && p->on_durable_advance != NULL) { p->on_durable_advance(p->durable_cb_ctx); } pthread_mutex_lock(&p->fsync_mutex); } pthread_mutex_unlock(&p->fsync_mutex); return NULL; } /* ------------------------------------------------------------------ * * WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) * * ------------------------------------------------------------------ */ /* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN * term index (R1.5). Grows the index on demand; LSNs replay/append contiguously * so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous * LSN (should not happen on the contiguous WAL) is zero-filled by realloc-then- * memset, which read back as "absent" until/unless written. The raw term * (including a legacy 0) is stored verbatim; the legacy-0 -> WAL_LEGACY_TERM * mapping is applied only at READ-BACK (persist_entry_term), keeping "present * legacy-0" distinguishable from "absent". * * Callers: the live append path (holds p->lock) and the single-threaded * recovery path (replay_wal — no other thread touches the index before the node * participates, so no lock is required there). Best-effort: an allocation * failure is logged loudly and leaves the index unextended (read-back then * reports the LSN absent) — it never aborts and never corrupts existing * entries. */ static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term) { if (!p || lsn == 0) return; /* LSN 0 is reserved — never an entry */ if (lsn > p->term_index_cap) { /* Grow geometrically to amortize; never below the requested LSN. */ uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64; if (new_cap < lsn) new_cap = lsn; uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t)); if (!grown) { ERROR("persist: term-index realloc failed (lsn=%" PRIu64 ", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read " "back as absent", lsn, p->term_index_cap, new_cap); return; } /* Zero-fill the newly grown region so any unwritten slot reads absent. */ memset(grown + p->term_index_cap, 0, (new_cap - p->term_index_cap) * sizeof(uint64_t)); p->term_index = grown; p->term_index_cap = new_cap; } p->term_index[lsn - 1] = term; if (lsn > p->term_index_count) p->term_index_count = lsn; } /* entry_term_locked: the lock-free core of persist_entry_term — read the * recorded Raft term for `lsn` from the in-memory term index, applying the * legacy-0 rule (R1.8: a present term-0 entry maps to WAL_LEGACY_TERM; an absent * LSN returns 0). Caller MUST hold p->lock. Used by the append path * (persist_wal_async) to source the AppendEntries prev_log_term while already * under the lock — calling persist_entry_term there would re-acquire p->lock * and deadlock. */ static uint64_t entry_term_locked(const persist_t* p, uint64_t lsn) { if (!p || lsn == 0) return 0; if (lsn <= p->term_index_count && p->term_index != NULL) { uint64_t raw = p->term_index[lsn - 1]; return (raw == 0) ? WAL_LEGACY_TERM : raw; } return 0; } /* Shrink the in-memory per-LSN term index so that every LSN > `keep_lsn` reads * back as absent (ha-term-safety task 2.2). Called from the truncate path after * the on-disk entries above keep_lsn are removed, keeping the term index * consistent with the WAL: a later persist_entry_term on a truncated LSN must * return 0. The backing allocation is left as-is (only the live count moves); * the zeroed-on-grow contract means a later re-append at those LSNs overwrites * cleanly. Caller must hold p->lock. */ static void truncate_term_index_locked(persist_t* p, uint64_t keep_lsn) { if (!p) return; if (keep_lsn < p->term_index_count) p->term_index_count = keep_lsn; } void persist_set_term(persist_t* p, uint64_t term) { if (!p) return; /* p->term is read on the append path under p->lock; take the lock so the * stamped value is consistent with the append that reads it. */ pthread_mutex_lock(&p->lock); p->term = term; pthread_mutex_unlock(&p->lock); } uint64_t persist_entry_term(const persist_t* p, uint64_t lsn) { if (!p || lsn == 0) return 0; /* Take p->lock to read consistently against concurrent record_entry_term * on the append path. Cast away const — the lock is logically a mutable * sub-object of an otherwise-const read. */ persist_t* mp = (persist_t*)p; pthread_mutex_lock(&mp->lock); /* Legacy-zero rule (R1.8) lives in entry_term_locked: a present entry * carrying term 0 maps to the single lowest-known real Raft term, applied * uniformly so two legacy-0 entries at the same LSN match; absent LSNs * return 0. */ uint64_t result = entry_term_locked(mp, lsn); pthread_mutex_unlock(&mp->lock); return result; } uint64_t persist_entry_term_locked(const persist_t* p, uint64_t lsn) { return entry_term_locked(p, lsn); } /* persist_reconcile_checkpoint_term — raft-ha-core-safety task 3.2 (Req * 3.1/5.1), Snapshot Boundary Reconcile. See persist.h for the full contract. * Reuses record_entry_term (same translation unit) to authoritatively * overwrite whatever term (if any) the local term index held at * checkpoint_lsn — this is a genuine reconcile-overwrite, not an idempotent * skip, because a pre-snapshot divergent entry may have left a stale local * term at this exact LSN. Only the checkpoint LSN's own term is reconciled: * post-install Raft semantics mean no AppendEntries prevLogIndex/prevLogTerm * check against this member ever references an LSN below the checkpoint * again (the checkpoint is the new log-start boundary), and the WAL bytes * below it were compacted away, so there is no data to reconstruct a full * per-LSN history from even if it were needed. */ void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn, uint64_t checkpoint_term) { if (!p || checkpoint_lsn == 0) return; pthread_mutex_lock(&p->lock); record_entry_term(p, checkpoint_lsn, checkpoint_term); pthread_mutex_unlock(&p->lock); } /* ------------------------------------------------------------------ */ /* Public API */ /* ------------------------------------------------------------------ */ void persist_set_workdir(persist_t* p, const char* workdir, const char* name) { char effective[4096]; snprintf(effective, sizeof(effective), "%s/%s", workdir, name); snprintf(p->wal_path, sizeof(p->wal_path), "%s/data/wal", effective); snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective); } int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c) { p->cluster = c; p->config = cfg ? *cfg : persist_config_defaults(); p->wal_fd = -1; p->last_replicated_crc = WAL_CRC_NONE; p->last_crc = WAL_CRC_NONE; /* LSN 0 is RESERVED as the "none / empty" sentinel — it is NEVER assigned * to a real WAL entry. A fresh store therefore starts current_lsn at 1, so * the first-ever appended entry (e.g. the promotion WAL_NOOP) is lsn 1 and * lsn 0 unambiguously means "nothing here": * - committed_lsn == 0 → nothing committed (the lsn-1 NOOP is committable * 0 → 1, vs. the old lsn-0 entry which could never advance committed_lsn); * - writable_lsn == 0 → own-term gate off (the gate armed at lsn 1 is * effective, vs. lsn 0 which read as "gate off" on a cold-start leader); * - standby_lsn == 0 → empty standby (the streamer's `hdr.lsn > 0` * filter streams the lsn-1 entry, closing the cold-start WAL hole that * dropped lsn 0 on `0 > 0`). * persist_recover() overwrites current_lsn from the durable snapshot/WAL tip * (load_snapshot / replay_wal), so this fresh default only applies before any * recovery — i.e. a brand-new store with no snapshot and no WAL. */ p->current_lsn = 1; /* Enforce minimum value for max_wal_records to prevent runaway checkpointing */ if (p->config.max_wal_records < 100) { ERROR("persist: max_wal_records=%u too low (minimum 100) — setting to 100", p->config.max_wal_records); p->config.max_wal_records = 100; } /* Enforce minimum value for wal_segment_bytes to prevent a rotation storm * (must comfortably fit at least one max-payload entry: header + up to * MAX_WAL_PAYLOAD_BYTES). */ if (p->config.wal_segment_bytes < 4096) { ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096", p->config.wal_segment_bytes); p->config.wal_segment_bytes = 4096; } /* Initialise atomic checkpoint-trigger counters */ atomic_store(&p->wal_record_count, 0); atomic_store(&p->checkpoint_in_progress, 0); /* Durable watermark starts at 0 ("nothing fdatasync'd"): mirrors the * LSN-0-reserved committed_lsn convention. It advances only on the * flusher thread after a successful fdatasync (ha-commit-durability 2.1). */ atomic_store(&p->durable_lsn, 0); /* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1): * a fresh store has no corruption. persist_recover overwrites this with the * replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */ p->recovery_verdict = RECOVER_CLEAN; /* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh * store is promotable. Set by persist_recover_loud_fence_if_unrecoverable * on an unrecoverable boot; cleared on catch-up (never self-exits). */ atomic_store(&p->recovery_unrecoverable, false); /* Post-replay apply-correctness verdict defaults to CLEAN (raft-apply- * decoupling task 5.1, Req 8.2): a fresh store has nothing to diverge * from. persist_recover overwrites both after every replay. */ p->apply_verdict = APPLY_CLEAN; atomic_store(&p->last_applied_lsn, 0); /* Live Apply Resource Fence starts clear (job-array-single-wal-record * task 4.2, Req 9.4): a fresh store has nothing stuck failing to apply. * Set by persist_mark_apply_resource_fenced on a resource-exhaustion * apply failure; cleared by persist_clear_apply_resource_fence the * instant a retry of the same LSN succeeds -- see persist_t's field doc * comment for why this is deliberately separate from * recovery_unrecoverable just above. */ atomic_store(&p->apply_resource_fenced_lsn, 0); /* bug-1416: defaults to true (nothing to gate) so every fixture that * calls persist_init() but never persist_recover() -- most of this * spec's own test suite -- sees identical, ungated apply-pump/live- * replication behavior to before this flag existed. Only persist_ * recover() ever sets this false, for its own single-threaded * replay+verify duration (see the field's doc comment in persist.h). */ atomic_store(&p->recovery_complete, true); /* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2): * the commit engine registers it in commit_engine_init. Until then every * flusher durable advance is a no-op wake. */ p->on_durable_advance = NULL; p->durable_cb_ctx = NULL; /* Truncation wake hook starts unregistered (raft-apply-decoupling task * 1.4): the commit engine registers it in commit_engine_init. Until * then every truncation is a no-op wake. */ p->on_truncate = NULL; p->truncate_cb_ctx = NULL; /* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries * recorded yet. Grown on demand by record_entry_term during replay/append; * freed in persist_fini. */ p->term_index = NULL; p->term_index_count = 0; p->term_index_cap = 0; /* Pending command-apply hand-off list starts empty (raft-apply- * decoupling task 3.2): no committed-outcome continuations have * registered yet. Freed in persist_fini. */ p->pending_applies = NULL; pthread_mutex_init(&p->pending_applies_lock, NULL); /* Ordered apply pump starts idle (raft-apply-decoupling task 6.1): no * walk scheduled or running yet. Freed/destroyed in persist_fini. */ pthread_mutex_init(&p->apply_pump_lock, NULL); atomic_store(&p->apply_pump_scheduled, 0); /* Apply-pump segment-lookup cache starts invalid (raft-apply-decoupling * task 7.4 perf-fix): no segment/fd cached yet. -1 is always a safe * fallback trigger for apply_pump_read_entry's fast path (see persist.h * doc comment). */ p->apply_pump_cache_fd = -1; p->apply_pump_cache_seg_lsn = 0; p->apply_pump_cache_off = 0; p->apply_pump_cache_next_lsn = 0; /* Create WAL and snapshot directories */ if (mkdir_p(p->wal_path, 0700) != 0) return -errno; if (mkdir_p(p->snap_path, 0700) != 0) return -errno; /* Reject NFS mounts — fsync durability is not guaranteed over NFS */ struct statfs sfs; if (statfs(p->wal_path, &sfs) == 0 && (unsigned long)sfs.f_type == (unsigned long)NFS_SUPER_MAGIC) { ERROR("persist: WAL path %s is on NFS — unsafe for fsync durability", p->wal_path); return -EREMOTE; } pthread_mutex_init(&p->lock, NULL); pthread_mutex_init(&p->checkpoint_mutex, NULL); pthread_cond_init(&p->checkpoint_done, NULL); pthread_mutex_init(&p->fsync_mutex, NULL); pthread_cond_init(&p->fsync_cond, NULL); repl_init(&p->repl); /* P2-4: fan-out replicator lifecycle */ WARN("WAL/snapshot stored unencrypted in %s — " "ensure filesystem is encrypted if job payloads contain sensitive data", p->wal_path); return 0; } int persist_start(persist_t* p) { ctrl_metrics_init(); /* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */ if (p->cluster != NULL) { xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun, p); xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun, p); xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun, p); } atomic_store_explicit(&p->fsync_stop, 0, memory_order_release); int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p); if (rc == 0) atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release); return rc; } void persist_stop(persist_t* p) { if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) { /* Set the stop flag and wake the timer wait under fsync_mutex so the * worker cannot miss the signal between its flag check and the wait. */ pthread_mutex_lock(&p->fsync_mutex); atomic_store_explicit(&p->fsync_stop, 1, memory_order_release); pthread_cond_signal(&p->fsync_cond); pthread_mutex_unlock(&p->fsync_mutex); pthread_join(p->fsync_thread, NULL); atomic_store_explicit(&p->fsync_thread_running, 0, memory_order_release); } pthread_mutex_lock(&p->lock); if (p->wal_fd >= 0) { /* Final synchronous flush-or-exit on stop: fdatasync everything * appended so far and advance durable_lsn to the synced tip so a * post-stop read reflects the durable state (the flusher thread is * already joined and will not advance it again). * * FAIL-LOUD GATE (Req 1.1/1.2): advance the watermark ONLY when the * final sync SUCCEEDS — on EIO/ENOSPC the bytes are not on stable * storage, so durable_lsn must stay where it is and the failure is * surfaced loudly. */ int rc = fdatasync(p->wal_fd); if (rc == 0 && !flusher_fsync_forced_fail(p)) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } else { int e = (rc != 0) ? errno : EIO; ERROR("persist: final fdatasync on stop FAILED (errno=%d %s) — " "durable_lsn left unchanged; appended-but-unsynced entries " "are NOT durable", e, strerror(e)); } close(p->wal_fd); p->wal_fd = -1; } pthread_mutex_unlock(&p->lock); } void persist_fini(persist_t* p) { repl_fini(&p->repl); /* P2-4: fan-out replicator lifecycle */ pthread_mutex_destroy(&p->lock); pthread_mutex_destroy(&p->checkpoint_mutex); pthread_cond_destroy(&p->checkpoint_done); pthread_mutex_destroy(&p->fsync_mutex); pthread_cond_destroy(&p->fsync_cond); /* Free the in-memory per-LSN term index (ha-term-safety 1.3). */ free(p->term_index); p->term_index = NULL; p->term_index_count = 0; p->term_index_cap = 0; /* Free any pending command-apply hand-off entries never drained by a * (not-yet-built) task 6 apply pump (raft-apply-decoupling task 3.2). * Each node's `ctx` is opaque to persist.c (owned by the registering * caller, e.g. job.c's jobsubmit_async_ctx_t) — freeing the node here * does NOT free ctx; a caller shutting down with entries still pending * is responsible for its own ctx lifetime (test harnesses that tear * down mid-flight must account for this). */ pending_command_apply_t* pn = (pending_command_apply_t*)p->pending_applies; while (pn != NULL) { pending_command_apply_t* next = pn->next; free(pn); pn = next; } p->pending_applies = NULL; pthread_mutex_destroy(&p->pending_applies_lock); /* raft-apply-decoupling task 6.1: no walk can still be in flight by the * time persist_fini runs (the caller owns quiescing any vtask pool * first, same precondition every other persist_t resource here already * assumes). */ pthread_mutex_destroy(&p->apply_pump_lock); /* raft-apply-decoupling task 7.4 perf-fix: close any cached segment fd * left open by apply_pump_read_entry. No walk can be in flight here * (same precondition as apply_pump_lock's destroy just above), so no * lock is needed. */ if (p->apply_pump_cache_fd >= 0) { close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } } /* Forward declaration — defined after checkpoint internals */ static void schedule_checkpoint(persist_t* persist); int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len) { /* WAL_CHECKPOINT is a local segment marker, never written via this path */ if (type == WAL_CHECKPOINT) return -EINVAL; pthread_mutex_lock(&p->lock); /* Open first segment lazily */ if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* Build entry header */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = p->current_lsn++; hdr.timestamp = now_us(); hdr.type = type; hdr.payload_len = len; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; /* CRC32 over WAL_HDR_CRC_SIZE bytes of header (excludes crc + prev_crc * fields themselves) then over the payload */ uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (payload && len > 0) c = crc32(c, (const Bytef*)payload, len); hdr.crc = c; /* Write header */ ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } /* Write payload (if any) */ if (payload && len > 0) { ssize_t pw = pwrite(p->wal_fd, payload, len, (off_t)(p->seg_offset + sizeof(hdr))); if (pw != (ssize_t)len) { pthread_mutex_unlock(&p->lock); return -EIO; } } uint32_t entry_size = (uint32_t)(sizeof(hdr) + len); p->seg_offset += entry_size; p->total_wal_bytes += entry_size; p->last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back (ha-term-safety * 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */ record_entry_term(p, hdr.lsn, hdr.term); /* Replicate to standbys — never replicate WAL_CHECKPOINT entries. * last_replicated_crc is distinct from last_crc: * last_crc includes checkpoint markers; last_replicated_crc skips them. * This maintains a continuous CRC chain for standbys (design doc Part 2.3). * Note: WAL_CHECKPOINT is blocked by the assert at function entry, so * every entry that reaches here is a replicable entry. */ p->last_replicated_crc = hdr.crc; /* ha-term-safety task 3.4: source the AppendEntries prev_log_term (term of * the entry at hdr.lsn-1) under p->lock via the lock-free accessor, so the * live broadcast carries the leader's term + prev tuple and the standby * routes it through the library (raft_on_append_entries_request). 0 when * hdr.lsn == 1 (no predecessor). */ uint64_t prev_log_term = entry_term_locked(p, hdr.lsn - 1); repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term); /* Rotate segment when full */ if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } pthread_mutex_unlock(&p->lock); /* Signal the group-commit flusher (ha-commit-durability 2.1): current_lsn * advanced and the entry's bytes are pwritten, so durable_lsn < tip now * holds (the level predicate). The signal is sent AFTER releasing p->lock * and is level-triggered — if it races the flusher and is "lost", the * flusher re-checks durable_lsn < current_lsn-1 under p->lock and flushes * anyway (see fsync_worker). We do NOT fdatasync inline on this path; the * flusher batches the sync across all pending appends. */ pthread_mutex_lock(&p->fsync_mutex); pthread_cond_signal(&p->fsync_cond); pthread_mutex_unlock(&p->fsync_mutex); /* WAL record-count checkpoint trigger: schedule an async checkpoint once * the record count reaches the configured threshold. The fetch_add is * outside the lock — atomic_fetch_add is sufficient for the counter; the * checkpoint itself is guarded by its own CAS inside schedule_checkpoint. */ if (atomic_fetch_add(&p->wal_record_count, 1) + 1 >= p->config.max_wal_records) { schedule_checkpoint(p); } return 0; } /* Lock-held core of persist_truncate_wal_above (defined below): assumes the * caller already holds p->lock and never touches the lock itself. The * replicated-write conflict guard reuses it to repair a divergent entry while * still under p->lock (the follower-AE-conflict truncate contract). */ static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn); /* read_entry_meta_locked: scan the on-disk WAL for the entry at `lsn` and * return its (payload_len, payload-only CRC, raw stored term). The * replicated-write conflict guard uses this to compare a re-streamed entry * against the copy already on disk WITHOUT trusting the LSN alone. pcrc is * computed as crc32(0, payload, len) — the SAME payload-only fingerprint * walcompare/waldump compare across members (NOT the chained header CRC); an * empty (PLEN=0) entry yields len=0, pcrc=0. Reads through the page cache so * a written-but-not-yet-fdatasync'd entry is still visible. Caller MUST hold * p->lock. Returns 0 if the entry is found, -1 if absent or unreadable. */ static int read_entry_meta_locked(persist_t* p, uint64_t lsn, uint32_t* out_len, uint32_t* out_pcrc, uint64_t* out_term) { if (!p || lsn == 0) return -1; DIR* d = opendir(p->wal_path); if (!d) return -1; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return -1; /* Ascending sort by start LSN (segment count is tiny). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } int rc = -1; for (int si = 0; si < nseg && rc != 0; si++) { /* Segments are LSN-ordered; one whose start LSN already exceeds the * target cannot contain it (nor can any later one). */ if (seg_lsns[si] > lsn) break; char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (hn < (ssize_t)sizeof(hdr)) break; /* short read / EOF / pre-allocated zeros */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn == lsn) { uint32_t pc = 0; if (hdr.payload_len > 0) { uint8_t* pl = malloc(hdr.payload_len); if (!pl) break; /* leave rc == -1: unreadable → conservative */ ssize_t pn = pread(fd, pl, hdr.payload_len, (off_t)(off + sizeof(hdr))); if (pn == (ssize_t)hdr.payload_len) pc = crc32(0L, (const Bytef*)pl, hdr.payload_len); free(pl); if (pn != (ssize_t)hdr.payload_len) break; /* unreadable body → conservative */ } if (out_len) *out_len = hdr.payload_len; if (out_pcrc) *out_pcrc = pc; if (out_term) *out_term = hdr.term; rc = 0; break; } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return rc; } int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type, const void* payload, uint32_t len) { if (!p) return -EINVAL; /* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */ if (type == WAL_CHECKPOINT) return -EINVAL; /* Reject oversized payloads BEFORE touching the segment. */ if (len > MAX_WAL_PAYLOAD_BYTES) return -EMSGSIZE; pthread_mutex_lock(&p->lock); /* Tip-repair bookkeeping (set only on the TIP-conflict repair branch below): * the truncate-and-reappend repair transiently lowers committed_lsn to * lsn-1, so we snapshot the true committed boundary here and restore it * EXACTLY after the re-append re-establishes the entry at `lsn` (never let a * same-(term,lsn) payload repair regress the committed boundary — commit * monotonicity / State Machine Safety). */ int repaired_tip = 0; uint64_t repaired_committed_save = 0; /* Payload-aware conflict guard (raft-ha-core-safety task 2.2): this lsn is * already persisted (current_lsn advanced past it). The legacy guard was * (lsn)-only — it no-op'd ANY re-stream, including a corrective full-payload * write over a previously-stored empty (PLEN=0) body, which is exactly the * cross-term catch-up payload-loss signature (a committed LSN that carries * the right (term,lsn) but an empty payload). Compare (payload-crc, len) * against the copy already on disk and act on the difference: * (c) identical payload → idempotent no-op (keep). * (b) incoming EMPTY over a stored entry → never degrade (keep stored); * an empty body is never the durable form of a known entry. * (a) incoming NON-EMPTY whose payload differs from the stored copy at an * existing (term,lsn) → a genuine payload conflict whose handling * depends on WHERE the conflict sits relative to the stored tip: * · AT THE TIP (lsn == current_lsn - 1, nothing stored above) → * REPAIR: truncate just this one entry and re-append the * authoritative payload at the same (term,lsn). Because nothing * is stored above it, no distinct committed entry is destroyed, * and the committed boundary is restored verbatim afterwards. * · BELOW THE TIP (lsn < current_lsn - 1, committed entries stored * ABOVE the conflict) → MUST NOT truncate: dropping [lsn..tip] * would delete committed entries above the conflict AND roll * committed_lsn/durable_lsn backward — a State Machine Safety / * commit-monotonicity violation. Instead keep the stored entry * untouched, raise a loud non-promotable fence, and let catch-up * reconverge. The real source fix (the streamer must stop * shipping an empty body) is task 2.4; the guard's only job here * is to NOT make things worse (never drop a committed entry, * never regress a boundary) and to surface the corruption loudly. * An unreadable stored entry stays a conservative no-op — never truncate a * committed LSN we cannot read back. */ if (lsn + 1 <= p->current_lsn) { uint32_t inc_pcrc = (payload && len > 0) ? crc32(0L, (const Bytef*)payload, len) : 0; uint32_t st_len = 0; uint32_t st_pcrc = 0; uint64_t st_term = 0; int have = read_entry_meta_locked(p, lsn, &st_len, &st_pcrc, &st_term); int payload_matches = (have == 0 && st_len == len && st_pcrc == inc_pcrc); /* (c) identical payload, (b) empty incoming, or an unreadable stored * entry → leave the stored entry untouched. */ if (payload_matches || len == 0 || have != 0) { pthread_mutex_unlock(&p->lock); return 0; } /* (a) non-empty incoming whose payload differs at an existing (term,lsn) * — a genuine payload conflict. Branch on tip vs below-tip. */ uint64_t tip_lsn = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; if (lsn != tip_lsn) { /* BELOW-TIP conflict: committed entries are stored ABOVE `lsn`. * Truncating here would delete them and roll committed_lsn/ * durable_lsn backward (State Machine Safety / commit-monotonicity * violation — bug-class the round-1 review rejected). Do NOT * truncate, do NOT lower any boundary, do NOT degrade the stored * entry. Surface loudly and raise the sticky non-promotable fence * (reuse the recovery_unrecoverable mechanism: loud, stays ALIVE, * non-promotable, self-heals via persist_clear_recovery_unrecoverable * once catch-up reaches a committed-consistent state — never * self-exits, per the controller invariant). The source fix lives * in task 2.4 (the streamer must stop shipping an empty payload). */ ERROR("persist: persist_wal_write_replicated payload CONFLICT BELOW " "TIP at lsn=%" PRIu64 " (stored_term=%" PRIu64 " stored_len=%u stored_pcrc=%08x vs incoming_len=%u " "incoming_pcrc=%08x); current_lsn=%" PRIu64 " committed_lsn=%" PRIu64 " — refusing to truncate committed entries above the " "conflict (would regress the committed boundary). Keeping the " "stored entry, raising the non-promotable fence and staying " "ALIVE; awaiting a correct re-stream (source fix: task 2.4)", lsn, st_term, st_len, st_pcrc, len, inc_pcrc, p->current_lsn, atomic_load_explicit(&p->committed_lsn, memory_order_acquire)); atomic_store_explicit(&p->recovery_unrecoverable, true, memory_order_release); pthread_mutex_unlock(&p->lock); return 0; /* no-op the write: keep stored, never degrade/regress */ } /* TIP conflict: `lsn` is the highest stored entry, so nothing is stored * above it — truncate-and-reappend replaces EXACTLY this one divergent * entry. truncate_wal_above_locked(lsn-1) drops it and sets * current_lsn = lsn, so control falls through to the append below and * re-persists the authoritative entry at this exact LSN. Snapshot the * committed boundary first: the truncate transiently rewrites * committed_lsn to lsn-1, and we restore it verbatim after the * re-append so a same-(term,lsn) payload repair never regresses (nor * spuriously advances) the committed boundary. */ repaired_committed_save = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); int trc = truncate_wal_above_locked(p, lsn - 1); if (trc != 0) { ERROR("persist: persist_wal_write_replicated repair-truncate FAILED " "for lsn=%" PRIu64 " (rc=%d) — divergent entry NOT repaired", lsn, trc); pthread_mutex_unlock(&p->lock); return trc; } repaired_tip = 1; /* fall through to the contiguity guard + append (lsn == current_lsn). */ } /* Contiguity guard (bug-850): REFUSE a forward GAP. current_lsn is the * next-expected LSN, so a replicated entry with lsn > current_lsn would * leave [current_lsn .. lsn-1] missing AND advance current_lsn past that * hole — after which the idempotency guard above silently DROPS the later * in-order catch-up of the prefix (the failover prefix-loss mechanism). * Append only the contiguous next entry (lsn == current_lsn); reject a gap * without writing or acking so the ACK-driven catch-up continuation * re-streams from current_lsn and the log fills in order. This is Raft * log-matching: append entry N only when the log already holds N-1. A * behind standby that keeps receiving live broadcasts simply drops the * out-of-order ones until catch-up backfills the gap (no loss). -EAGAIN is * the "retry via catch-up" signal the apply path treats as a quiet skip. */ if (lsn > p->current_lsn) { pthread_mutex_unlock(&p->lock); return -EAGAIN; } /* Open first segment lazily. open_new_segment names the file from * current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is * 1 (reserve-lsn-0), so the segment is seg-...001.wal. */ if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not * current_lsn++). term isn't streamed, so use our best-effort p->term. * The CRC chain is SELF-computed (the streamed entry carries no crc): * recovery validates an intra-segment chain (prev_crc == last_crc), so the * standby's own writes must chain to each other — cross-node CRC equality * is neither required nor possible. */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = lsn; hdr.timestamp = now_us(); hdr.type = type; hdr.payload_len = len; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (payload && len > 0) c = crc32(c, (const Bytef*)payload, len); hdr.crc = c; /* Write header */ ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } /* Write payload (if any) */ if (payload && len > 0) { ssize_t pw = pwrite(p->wal_fd, payload, len, (off_t)(p->seg_offset + sizeof(hdr))); if (pw != (ssize_t)len) { pthread_mutex_unlock(&p->lock); return -EIO; } } uint32_t entry_size = (uint32_t)(sizeof(hdr) + len); p->seg_offset += entry_size; p->total_wal_bytes += entry_size; p->last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back (ha-term-safety * 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed * envelope does not yet carry the leader's term, so this records the local * best-effort p->term (the design's known hazard closed by task 2.x, which * carries the leader's term on the envelope; this task only provides the * read-back seam over whatever term was persisted). */ record_entry_term(p, hdr.lsn, hdr.term); /* Advance current_lsn to one past the given lsn (max-idempotent — mirrors * apply_wal_entry's monotonic advance). */ if (lsn + 1 > p->current_lsn) p->current_lsn = lsn + 1; /* Durability: fsync the segment BEFORE returning so the caller's ACK * implies the entry is on stable storage (persist-before-ACK). * persist_wal_async defers fsync to the background thread / persist_flush; * here we sync inline because the entry feeds the primary's commit * majority and must survive a standby restart. fdatasync is sufficient * (entry bytes + segment metadata are pre-allocated by open_new_segment). * * Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the * append-path gate at task 2.1): the ACK is sent by the caller ONLY when * this returns success, and the ACK claims the entry is durable. So if * the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must * NOT return success — we log loudly and return -EIO so the caller's * existing error path (ha_apply_single_entry) does NOT apply or ACK. */ int fsrc = fdatasync(p->wal_fd); #ifdef XCE_TESTING atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1, memory_order_relaxed); #endif if (fsrc != 0 || flusher_fsync_forced_fail(p)) { int e = (fsrc != 0) ? errno : EIO; ERROR("persist: persist_wal_write_replicated fdatasync FAILED for " "lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning " "error so the standby does NOT ACK", lsn, e, strerror(e)); pthread_mutex_unlock(&p->lock); return -EIO; } /* bug-828 secondary: the entry is now fdatasync'd durable, so advance * durable_lsn to the synced tip immediately (mirror persist_flush). The * 10ms cadence flusher would otherwise advance it lazily, leaving this * standby's ADVERTISED durable position (persist_durable_floor → * RequestVoteReply.peerLastLogLsn) lagging its own synced tip during the * vote window. A standby that has durably persisted the committed prefix * must advertise it at once so a clean failover's safe-commit-from-votes * computation sees the true majority-held LSN. Monotonic max; preserves * durable_lsn <= current_lsn - 1 (tip read under this same p->lock, after * the current_lsn advance above). Reached ONLY past the fail-loud rc-gate * — never advances on an fdatasync error. */ { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } /* Tip-repair committed-boundary restore (raft-ha-core-safety task 2.2): a * TIP-conflict repair above truncated [lsn..tip] (rewriting committed_lsn to * lsn-1) and has now re-appended + fdatasync'd the authoritative entry at the * SAME (term,lsn). Restore the committed boundary VERBATIM so the repair is * commit-boundary-neutral: a same-(term,lsn) payload correction must never * regress committed_lsn (State Machine Safety / commit monotonicity) nor * spuriously advance it (truncate would otherwise have raised it to lsn-1). * Reached only past the fail-loud fdatasync gate, with current_lsn re-advanced * to lsn+1 and durable_lsn at the synced tip, so the invariant * committed_lsn <= min(current_lsn - 1, durable_lsn) holds (saved value was * itself <= the pre-repair tip == lsn). The whole repair is serialized under * p->lock, so no external observer ever sees the transient lowered value. */ if (repaired_tip) { atomic_store_explicit(&p->committed_lsn, repaired_committed_save, memory_order_release); } /* Rotate segment when full (same threshold as the primary append). */ if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* NOTE: no repl_broadcast — standbys never re-replicate. */ pthread_mutex_unlock(&p->lock); return 0; } int persist_flush(persist_t* p) { pthread_mutex_lock(&p->lock); int rc = (p->wal_fd >= 0) ? fdatasync(p->wal_fd) : 0; /* persist_flush is a synchronous durability point (tests, clean shutdown): * it fdatasync'd everything appended so far, so advance durable_lsn to the * synced tip too. Without this, an explicit-flush-only path (slow flusher * cadence) would leave durable_lsn lagging even though the bytes are on * disk. Monotonic max; preserves durable_lsn <= current_lsn - 1 (tip read * under the same p->lock). Gated on rc == 0 (fail-loud, Req 1.1/1.2): a * failed flush must not advance the watermark. */ if (rc == 0 && !flusher_fsync_forced_fail(p)) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } else if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: persist_flush fdatasync FAILED (errno=%d %s) — " "durable_lsn not advanced; flushed bytes are NOT durable", e, strerror(e)); } pthread_mutex_unlock(&p->lock); return rc; } uint64_t persist_durable_lsn(persist_t* p) { return atomic_load_explicit(&p->durable_lsn, memory_order_acquire); } uint64_t persist_durable_floor(persist_t* p) { /* The durable freshness floor (durable last-log index) IS the durable * watermark: persist_recover re-derives it from the recovered WAL tip * (ha-commit-durability task 3.4), and the flusher advances it on every * successful fdatasync — so a thin wrapper avoids a redundant field. NULL * guard keeps the accessor safe for the election consumers (task 4.2). */ return p ? atomic_load_explicit(&p->durable_lsn, memory_order_acquire) : 0; } void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx) { if (!p) return; /* Set ctx before fn so the flusher (which checks fn != NULL) never sees a * live callback with a stale/uninitialised ctx. Registration happens in * commit_engine_init, before the engine is reachable by any ACK path. */ p->durable_cb_ctx = ctx; p->on_durable_advance = fn; } /* raft-apply-decoupling task 1.4: registration counterpart of * persist_set_durable_cb -- see persist.h for the full contract. */ void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn), void* ctx) { if (!p) return; p->truncate_cb_ctx = ctx; p->on_truncate = fn; } #ifdef XCE_TESTING void persist_test_set_force_fsync_fail(persist_t* p, int fail) { if (p) atomic_store_explicit(&p->test_force_fsync_fail, fail ? 1 : 0, memory_order_release); } void persist_test_set_force_job_encode_fail_at(persist_t* p, int index) { if (p) atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1, memory_order_relaxed); } unsigned persist_test_replicated_fdatasync_count(persist_t* p) { if (!p) return 0; return atomic_load_explicit(&p->test_replicated_fdatasync_count, memory_order_relaxed); } pthread_t persist_test_last_fdatasync_thread(persist_t* p) { /* If no flusher fdatasync has advanced durable_lsn yet, return the * caller's own id so an unprimed read never spuriously looks off-thread. * After a flush, return the recorded flusher thread id (Req 1.6). */ if (!p || atomic_load_explicit(&p->test_last_fdatasync_thread_set, memory_order_acquire) == 0) return pthread_self(); return p->test_last_fdatasync_thread; } /* Force a fresh WAL segment at the current LSN (test seam — see persist.h). * Takes p->lock for the same single-writer serialization the append path uses, * so the segment swap cannot race a concurrent write. */ int persist_test_force_new_segment(persist_t* p) { if (!p) return -EINVAL; pthread_mutex_lock(&p->lock); int rc = rotate_segment(p); pthread_mutex_unlock(&p->lock); return rc; } /* ------------------------------------------------------------------ * * Durability-verification test seam (ha-commit-durability Req 3.2/3.3)* * * * Re-read + CRC-validate the WAL segments from stable storage and * * return the highest fully-persisted LSN — independent of the * * in-memory current_lsn / committed_lsn watermarks. Mirrors the * * segment-scan + CRC-chain validation of replay_wal(), but is strictly * * read-only: it applies no entries, fires no hooks, and mutates no * * persist_t state, so it cannot change the externally observable * * acknowledgment semantics (Req 3.3). * * * * Stops at the first corruption / short read / pre-allocated zero * * region (exactly like recovery), so only entries genuinely on stable * * storage are counted. Returns the highest such LSN, or 0 when nothing * * is durable. * * ------------------------------------------------------------------ */ uint64_t persist_test_durable_lsn_on_disk(persist_t* p) { if (!p) return 0; DIR* d = opendir(p->wal_path); if (!d) return 0; /* no WAL dir → nothing durable */ /* Collect segment start LSNs (same bound as replay_wal). */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue; seg_lsns[nseg++] = seg_start; } closedir(d); if (nseg == 0) return 0; /* Insertion sort by start LSN (segment count is tiny). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } uint64_t highest_durable = 0; uint32_t last_crc = WAL_CRC_NONE; for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t seg_off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); if (hn == 0) break; /* clean end of segment */ if (hn < (ssize_t)sizeof(hdr)) break; /* truncated / pre-allocated zeros */ /* Pre-allocated zero region: no valid entry is all-zero. */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; /* CRC chain validation — identical rule to replay_wal. */ if (seg_off == 0) { if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && hdr.prev_crc != last_crc) break; } else { if (hdr.prev_crc != last_crc) break; } /* Verify entry CRC over header + payload. */ uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); uint8_t* payload = NULL; if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { close(fd); return highest_durable; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(seg_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { free(payload); break; } free(payload); /* Entry is intact and durable on disk. */ last_crc = hdr.crc; if (hdr.lsn > highest_durable) highest_durable = hdr.lsn; seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return highest_durable; } #endif /* XCE_TESTING */ /* ------------------------------------------------------------------ */ /* Checkpoint internals */ /* ------------------------------------------------------------------ */ /* Write a WAL_CHECKPOINT marker to the current segment. * Never calls repl_broadcast() — checkpoint markers are local only. */ static int persist_write_checkpoint_marker(persist_t* p) { pthread_mutex_lock(&p->lock); if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = p->current_lsn++; hdr.timestamp = now_us(); hdr.type = WAL_CHECKPOINT; hdr.payload_len = 0; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); hdr.crc = c; ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } fdatasync(p->wal_fd); p->seg_offset += (uint32_t)sizeof(hdr); p->total_wal_bytes += sizeof(hdr); p->last_crc = hdr.crc; /* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */ /* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS * checkpoint markers (local bookkeeping, not a logical Raft entry), so the * recovered index never holds a checkpoint LSN; the live index must match * (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which * never compares against a checkpoint marker. */ pthread_mutex_unlock(&p->lock); return 0; } /* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2/3.2, Req * 1.1-1.4). Physical layout appended after the existing legacy header * (design.md "Checkpoint File Layout"): * [uint32_t format_version] * [uint32_t job_count] * [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }] * [uint32_t queue_count] * [queue_count * { uint32_t name_len; char name[name_len]; uint32_t status }] * * config-upgrade-snapshot task 1 (Req 1.9, 5.7): bumped 1->2 to reserve the * version for the HostAdminState/QueueAdminState capnp migration -- the * queue-frame layout above (name_len/name/status raw bytes) is the PRIOR * layout this version now hard-rejects; the byte-layout migration itself * (new host section, queue frames re-encoded as capnp) lands in tasks * 2.1/2.2. A binary built against version 2 must never misinterpret a * checkpoint written under version 1's raw queue-frame layout (Design * Invariant 4: fail loud, never silently reinterpret). */ #define SNAPSHOT_FORMAT_VERSION 2u typedef struct { uint8_t* buf; uint32_t len; } snap_job_frame_t; static void snap_free_job_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every retained job into `frames`, one malloc'd * snap_job_frame_t per job, under a point-in-time-consistent walk * (design.md "Existing Architecture Analysis" / Requirement 1.2): the * jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes * concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is * ADDITIONALLY taken nested around just that job's encode call (excludes a * concurrent per-entity writer for exactly the duration of its encode -- * the table lock alone does not do this). No lock is held during the * disk write itself: this function only builds in-memory frames; the * caller writes them to disk after the table walk has fully released both * locks. * * Filters via job_should_retain (Requirement 1.1) so the checkpoint's * notion of "live" matches the existing GC retention policy. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd -- even zero results -- so the caller can always * xlist_fini it) or a negative errno on the first encode failure, in which * case every frame collected so far has already been freed and `frames` * is empty (Requirement 1.3: a mid-walk failure must not leave partial * entity content behind). */ static int collect_job_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; int retained_index = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero jobs is not a failure. */ if (c == NULL || c->jobmgr == NULL) return 0; lock(c->jobmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->jobmgr); if (irc < 0) { unlock(c->jobmgr); return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */ } for (;;) { job_t* j = NULL; int nrc = iter_next(&it, (void**)&j); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (j == NULL) continue; if (!job_should_retain(j, &p->config)) continue; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(j, RTTI_LOCK_READ); #ifdef XCE_TESTING int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at, memory_order_relaxed); if (forced > 0 && (forced - 1) == retained_index) enc_rc = -EIO; /* simulated mid-loop encode failure */ else enc_rc = capnp_encode_job(&j->job, &buf, &sz); #else enc_rc = capnp_encode_job(&j->job, &buf, &sz); #endif unlock(j); retained_index++; if (enc_rc != 0) { rc = enc_rc; break; } snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->jobmgr); if (rc != 0) snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Queue admin-status frame (config-upgrade-snapshot task 2.2, Req 1.9, 5.7). * Migrated from the prior raw {name_len, name, status} layout to a capnp * QueueAdminState payload -- identical {buf,len} shape to snap_host_frame_t * (both carry a capnp payload), same (name, status) field content as * before, restore_queue_state's contract (queue.c:208) unchanged; only the * wire encoding changes (design.md "Snapshot Entity/Config Payload"). */ typedef struct { uint8_t* buf; uint32_t len; } snap_queue_frame_t; static void snap_free_queue_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every queue in `frames`, one malloc'd snap_queue_frame_t * per queue, under the same point-in-time-consistent walk discipline as * collect_job_frames above (design.md "Existing Architecture Analysis" / * Requirement 1.2): the qmgr table's own RTTI_LOCK_READ is held for the * ENTIRE walk, and each queue's own RTTI_LOCK_READ is ADDITIONALLY taken * nested around just that queue's (name, status) read and capnp encode -- * mirroring collect_host_frames' identical lock-then-encode-then-unlock * shape. No lock is held during the disk write itself. * * Unlike collect_job_frames, no retention predicate is applied: a queue's * mere presence in the live qmgr table already means config provisioned * it (design.md "write_snapshot (extended)" Responsibilities), so every * queue found is included. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd, even for zero results, so the caller can always xlist_fini * it) or a negative errno on the first encode failure, in which case every * frame collected so far has already been freed and `frames` is empty * (Requirement 1.3: a mid-walk failure must not leave partial entity * content behind). */ static int collect_queue_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero queues is not a failure. */ if (c == NULL || c->qmgr == NULL) return 0; lock(c->qmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->qmgr); if (irc < 0) { unlock(c->qmgr); return 0; /* nothing to walk -- mirrors collect_queues' empty-table handling */ } for (;;) { queue_t* q = NULL; int nrc = iter_next(&it, (void**)&q); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (q == NULL) continue; capnp_queue_admin_state_t qs; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(q, RTTI_LOCK_READ); qs.name = q->queue.queue; qs.status = q->queue.status; enc_rc = capnp_encode_queue_admin_state(&qs, &buf, &sz); unlock(q); if (enc_rc != 0) { rc = enc_rc; break; } snap_queue_frame_t* fr = (snap_queue_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->qmgr); if (rc != 0) snap_free_queue_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Bits set by admin commands that must survive a checkpoint round-trip. * Mirrors worker.c's and lib/wobj.c's own local XCE_WORKER_ADMIN_MASK * definitions -- both already independently duplicate this exact bit list * (design.md Design Invariant 6 notes this pre-existing duplication); this * is a third, deliberate copy for the same reason (no shared header exists * for it today), not a new layering dependency on worker.c. */ #define PERSIST_WORKER_ADMIN_MASK \ (XCE_WORKER_LOCKEDU | XCE_WORKER_CLOSED | XCE_WORKER_HIDDEN \ | XCE_WORKER_CLOSED_BY_ADMIN) typedef struct { uint8_t* buf; uint32_t len; } snap_host_frame_t; static void snap_free_host_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every host in c->workermgr into `frames`, one malloc'd * snap_host_frame_t (identical {buf,len} shape to snap_job_frame_t -- both * carry a capnp payload) per host, under the same point-in-time-consistent * walk discipline as * collect_job_frames/collect_queue_frames above (design.md "Existing * Architecture Analysis" / Requirement 1.2): the workermgr table's own * RTTI_LOCK_READ is held for the ENTIRE walk, and each worker's own * RTTI_LOCK_READ is ADDITIONALLY taken nested around just that host's * (name, status) read and encode. No lock is held during the disk write * itself. * * Like collect_queue_frames and UNLIKE collect_job_frames, no retention * predicate is applied: every host currently in the live workermgr table * gets exactly one frame (mirrors collect_queue_frames's own "mere presence * already means it's included" precedent above), so restore later sees a * frame for every host this member currently knows about, whether or not * any admin bit happens to be set on it right now. * * Each frame carries ONLY the host's admin-issued status bits * (w->host.status & PERSIST_WORKER_ADMIN_MASK) -- never the raw status * (which also carries dynamic/liveness bits like XCE_WORKER_UNAVAIL) and * never any other xce_worker_t field (attrs, resvals, dead_since, ...): * Requirement 1.7's exclusion, design.md Design Invariant 5 (never inherit * a live struct's dynamic fields just because reusing it is convenient -- * capnp_encode_worker/Worker is deliberately NOT used here). `name` is * read directly off w->host.name -- the same field key_find(c->workermgr, * name) looks up elsewhere in this codebase (join.c:91, worker.c:524,645,751, * lib/wobj.c:212), so a later restore's key_find(name) lookup matches * exactly what was collected here. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd -- even zero results -- so the caller can always * xlist_fini it) or a negative errno on the first encode failure, in which * case every frame collected so far has already been freed and `frames` * is empty (Requirement 1.3: a mid-walk failure must not leave partial * entity content behind). */ static int collect_host_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero hosts is not a failure. */ if (c == NULL || c->workermgr == NULL) return 0; lock(c->workermgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->workermgr); if (irc < 0) { unlock(c->workermgr); return 0; /* nothing to walk -- mirrors collect_job/queue_frames' empty-table handling */ } for (;;) { worker_t* w = NULL; int nrc = iter_next(&it, (void**)&w); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (w == NULL) continue; capnp_host_admin_state_t h; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(w, RTTI_LOCK_READ); h.name = w->host.name; h.status = w->host.status & PERSIST_WORKER_ADMIN_MASK; enc_rc = capnp_encode_host_admin_state(&h, &buf, &sz); unlock(w); if (enc_rc != 0) { rc = enc_rc; break; } snap_host_frame_t* fr = (snap_host_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->workermgr); if (rc != 0) snap_free_host_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Bound on the config bytes this file will embed in a checkpoint -- mirrors * config_repl.h's CONF_MAX_SIZE (1 MiB) exactly. Duplicated as its own local * constant rather than including config_repl.h, per the zero-config_repl.c- * dependency decision documented on read_active_config_bytes() below. */ #define PERSIST_CONFIG_BYTES_MAX_SIZE (1u * 1024u * 1024u) /* read_active_config_bytes (config-upgrade-snapshot task 2.3, Req 2.1, 2.3, * 2.4, 5.1): read the locally-held active configuration's raw bytes straight * off disk, for embedding in every checkpoint's trailing config-bytes * section. * * DELIBERATE DESIGN CHOICE -- read this before changing the call site below: * this function duplicates config_repl.c's `conf_read_current_filename` + * `conf_read_raw` path-resolution-and-read logic INLINE, rather than calling * those functions (config_repl.h is not included by this file at all). This * mirrors scan_latest_committed_config_ref()'s own precedent just above, * which independently decodes WAL_CONFIG_MARKER payloads inline "so persist * stays free of a config_repl layering dependency" instead of adding a * persist.c -> config_repl.c function-call edge -- and design.md's * "Snapshot Entity/Config Payload" + File Structure Plan sections direct the * SAME treatment here verbatim ("no new config_repl.c function is added... * this deliberately mirrors scan_latest_committed_config_ref's own existing * precedent of avoiding a persist.c -> config_repl.c function-call * dependency, rather than introducing the layering violation a new accessor * would create"). * * A future reader who spots ha.c's two `extern int conf_read_raw(...)` call * sites (config push to standbys) and wonders why this file doesn't do the * same: ha.c is the intentional CONSUMER of config-bytes sourcing * (design.md's Architecture boundary map: "config replication (config_repl.c) * owns config-bytes sourcing"), so a ha.c -> config_repl.c call is expected, * in-boundary integration. persist.c owns materialization/on-disk-format and * is deliberately kept free of any dependency on config_repl.c's domain -- * the same boundary this file already draws for the WAL-marker case above. * This is a considered choice, not an oversight: duplicating this ~30-line * read-with-validation routine once was judged cheaper than adding this * file's first cross-domain coupling edge into config_repl.c. * * Returns 0 with *out_bytes (caller must free()) / *out_len set on success. * Returns -ENOENT when no local config bytes exist yet: no cluster/myconf * back-pointer (e.g. a bare unit-test persist_t), no workdir/name configured, * no conf/current pointer file yet (this node has not received its first * config push), or the pointer names a versioned file that is not present. * This is a BENIGN, expected state (early boot / not-yet-provisioned), never * corruption -- callers must treat -ENOENT as "no config yet", not failure. * Any OTHER negative errno (-EINVAL malformed pointer/oversized file, -EIO * read failure, -ENOMEM, -EFBIG) indicates genuine on-disk corruption or a * real I/O failure and must propagate as a hard failure, never be silently * downgraded to an empty section. */ static int read_active_config_bytes(persist_t* p, uint8_t** out_bytes, size_t* out_len) { cluster_t* c = p->cluster; if (c == NULL || c->myconf == NULL || c->myconf->workdir == NULL || c->myconf->name == NULL) return -ENOENT; /* effective_dir = workdir/name -- must match conf_save_config's own * path construction (config_repl.c:781) exactly. */ char conf_dir[4096]; snprintf(conf_dir, sizeof(conf_dir), "%s/%s", c->myconf->workdir, c->myconf->name); /* Step 1: resolve the "current" versioned filename -- mirrors * conf_read_current_filename's validation exactly (non-empty, no path * separator, "cluster.bin." prefix + 12 lowercase hex chars) so a * corrupted pointer file is caught here just as loudly as it would be * there. */ char head_path[4160]; snprintf(head_path, sizeof(head_path), "%s/conf/current", conf_dir); FILE* hf = fopen(head_path, "r"); if (hf == NULL) return -ENOENT; /* no config pushed to this node yet */ struct stat hst; if (fstat(fileno(hf), &hst) == 0 && hst.st_size == 0) { fclose(hf); return -ENOENT; /* interrupted-write pointer file -- treat as absent */ } char fname[256]; if (fgets(fname, (int)sizeof(fname), hf) == NULL) { fclose(hf); return -EIO; } fclose(hf); fname[strcspn(fname, "\n")] = '\0'; if (strchr(fname, '/') != NULL) return -EINVAL; /* corrupted pointer file -- embedded path separator */ if (strncmp(fname, "cluster.bin.", 12) != 0) return -EINVAL; const char* hex = fname + 12; if (strlen(hex) != 12 || strspn(hex, "0123456789abcdef") != 12) return -EINVAL; /* Step 2: read the versioned config file's raw bytes. */ char cfg_path[4416]; snprintf(cfg_path, sizeof(cfg_path), "%s/conf/%s", conf_dir, fname); FILE* cf = fopen(cfg_path, "rb"); if (cf == NULL) return -ENOENT; /* pointer references a file that is not there -- * treat as not-yet-ready rather than corruption */ if (fseek(cf, 0, SEEK_END) != 0) { fclose(cf); return -EIO; } long sz = ftell(cf); if (sz < 0) { fclose(cf); return -EIO; } if (fseek(cf, 0, SEEK_SET) != 0) { fclose(cf); return -EIO; } if (sz == 0) { fclose(cf); return -EIO; /* zero-byte versioned config file: genuine corruption */ } if ((size_t)sz > PERSIST_CONFIG_BYTES_MAX_SIZE) { fclose(cf); return -EFBIG; } uint8_t* buf = (uint8_t*)malloc((size_t)sz); if (buf == NULL) { fclose(cf); return -ENOMEM; } if (fread(buf, 1, (size_t)sz, cf) != (size_t)sz) { free(buf); fclose(cf); return -EIO; } fclose(cf); *out_bytes = buf; *out_len = (size_t)sz; return 0; } /* Write an entity-bearing checkpoint: legacy header (unchanged) + entity * payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any * failure at any step -- collect, encode, write, sync -- leaves NO file at * the final path; the tmp file is always unlinked on error. This is a * retriable-next-checkpoint-cycle failure, never a crash/abort (controller * never self-terminates on an operational condition). */ static int write_snapshot(persist_t* p) { char tmp_path[4096]; char final_path[4096]; snprintf(tmp_path, sizeof(tmp_path), "%s/snap-%016" PRIx64 ".tmp", p->snap_path, p->checkpoint_lsn); snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn); /* Collect + encode entities BEFORE opening the tmp file: a collection * failure (Requirement 1.3) then never creates a tmp file at all. */ xlist_t job_frames; int rc = collect_job_frames(p, &job_frames); if (rc != 0) return rc; xlist_t queue_frames; rc = collect_queue_frames(p, &queue_frames); if (rc != 0) { snap_free_job_frames(&job_frames); return rc; } xlist_t host_frames; rc = collect_host_frames(p, &host_frames); if (rc != 0) { snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); return rc; } int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600); if (fd < 0) { snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); snap_free_host_frames(&host_frames); return -errno; } /* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference * (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1). * The config-ref tail is OPTIONAL on read-back, so an older 16-byte * sentinel still loads (load_snapshot reads the tail best-effort). The * tag is little-endian on this arch, matching the checkpoint_lsn field. */ char header[16 + 1 + 8 + 32]; memset(header, 0, sizeof(header)); memcpy(header, "XCESNAP\0", 8); memcpy(header + 8, &p->checkpoint_lsn, 8); header[16] = (char)(p->cfgref_present ? 1 : 0); memcpy(header + 17, &p->cfgref_tag, 8); memcpy(header + 25, p->cfgref_sha, 32); if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header)) rc = -EIO; /* Entity payload: format_version, then job frames (design.md "Checkpoint * File Layout"). No table/entity lock is held here -- collect_job_frames * already released both before returning. */ if (rc == 0) { uint32_t fmt = SNAPSHOT_FORMAT_VERSION; if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) rc = -EIO; } if (rc == 0) { uint32_t job_count = (uint32_t)job_frames.count; if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < job_frames.count; i++) { snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Queue trailer (config-upgrade-snapshot task 2.2, Req 1.9, 5.7): * capnp QueueAdminState frames, same [payload_len][capnp bytes] shape * as the job/host frame loops -- migrated from the prior raw * {name_len, name, status} layout (design.md "Checkpoint File Layout": * same field content, new wire encoding only). */ if (rc == 0) { uint32_t queue_count = (uint32_t)queue_frames.count; if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < queue_frames.count; i++) { snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(&queue_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Host trailer (config-upgrade-snapshot task 2.1, Req 1.6, 1.7, 1.9): * capnp HostAdminState frames, same [payload_len][capnp bytes] shape as * the job frame loop above (design.md "Checkpoint File Layout": host * frames come after queue frames). */ if (rc == 0) { uint32_t host_count = (uint32_t)host_frames.count; if (write(fd, &host_count, sizeof(host_count)) != (ssize_t)sizeof(host_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < host_frames.count; i++) { snap_host_frame_t* fr = (snap_host_frame_t*)xlist_get(&host_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Config-bytes section (config-upgrade-snapshot task 2.3, Req 2.1, 2.3, * 2.4, 5.1): raw active-configuration bytes, appended LAST. This is a * single flat length-prefixed blob for the whole file -- NOT a * [count][frame...] shape like the job/queue/host sections above, since * there is exactly one active configuration, never a set of them. * Populated on EVERY checkpoint (marker-triggered or periodic alike, * never gated on which triggered this call) so a member joining or * catching up after the marker-triggered upgrade snapshot has rotated * away still gets configuration and materialized state together from * whichever checkpoint it installs (Req 2.4). A pure function of * whatever bytes currently sit at the resolved config path -- no * timestamp, no ordering dependence -- so reading the same unchanged * file twice yields byte-identical output (Req 5.1 determinism). */ uint8_t* cfg_bytes = NULL; size_t cfg_len = 0; if (rc == 0) { int cfg_rc = read_active_config_bytes(p, &cfg_bytes, &cfg_len); if (cfg_rc == -ENOENT) { /* No local config bytes yet -- no cluster/myconf back-pointer * (bare unit-test persist_t) or this node has not received its * first config push. Mirrors the "no live cluster back-pointer" * empty-is-not-failure discipline collect_job/queue/host_frames * already apply above: write an explicit zero-length section * rather than failing the whole checkpoint over a benign, * expected early-boot / not-yet-provisioned state. */ cfg_bytes = NULL; cfg_len = 0; } else if (cfg_rc != 0) { /* Any other error means genuinely corrupt/unreadable on-disk * config state, not benign absence -- fail the checkpoint like * any other collect/restore failure (design.md Error Handling: * "Host placeholder creation failure ... treated as any other * collect/restore failure today -- discard partial state, fail * the checkpoint/restore attempt, retriable on the next * cycle"). */ rc = cfg_rc; } } if (rc == 0) { uint32_t cfg_bytes_len = (uint32_t)cfg_len; if (write(fd, &cfg_bytes_len, sizeof(cfg_bytes_len)) != (ssize_t)sizeof(cfg_bytes_len)) rc = -EIO; } if (rc == 0 && cfg_len > 0 && write(fd, cfg_bytes, cfg_len) != (ssize_t)cfg_len) rc = -EIO; free(cfg_bytes); if (rc == 0 && fdatasync(fd) != 0) rc = -errno; close(fd); snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); snap_free_host_frames(&host_frames); if (rc != 0) { unlink(tmp_path); return rc; } if (rename(tmp_path, final_path) != 0) { int saved = errno; unlink(tmp_path); return -saved; } return 0; } /* scan_latest_committed_config_ref (ha-term-safety task 4.2 remediation r1): * walk the local WAL segments and capture the HIGHEST-LSN WAL_CONFIG_MARKER * whose LSN is <= committed_lsn into p->cfgref_{present,tag,sha}. This is the * SAME committed-prefix marker config_committed_marker derives, cached durably * so a later checkpoint can trim the marker's segment without losing the * derived committed-config reference (compaction safety). The marker payload is * a fixed 40-byte reference (8-byte big-endian version_tag + 32-byte sha256 — * mirrors config_marker_encode; the config BYTES are never in the WAL), decoded * inline here so persist stays free of a config_repl layering dependency. */ static void scan_latest_committed_config_ref(persist_t* p) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); if (committed == 0) return; /* nothing committed → keep any prior cached ref unchanged */ DIR* d = opendir(p->wal_path); if (!d) return; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { uint64_t sl = 0; if (sscanf(e->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1) seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return; for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (n < (ssize_t)sizeof(hdr)) break; if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn > committed) break; /* past committed prefix */ if (hdr.type == WAL_CONFIG_MARKER && hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) { uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN]; ssize_t pr = pread(fd, buf, sizeof(buf), (off_t)(off + sizeof(hdr))); if (pr == (ssize_t)sizeof(buf)) { p->cfgref_tag = ((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) | ((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) | ((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) | ((uint64_t)buf[6] << 8) | ((uint64_t)buf[7]); memcpy(p->cfgref_sha, buf + 8, 32); p->cfgref_present = 1; /* later markers overwrite — latest wins */ } } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } } /* Delete WAL segment files whose start LSN is strictly less than the * active segment's start LSN AND strictly less than the trim floor. * * Bug-1147 (stopgap, raft-drive-async task 3.2): the trim floor is the * minimum of the active-segment LSN, every active standby's last acked LSN, * and every recently-disconnected slot's last_disconnected_lsn (strategy-A * retention discipline). This ensures we NEVER delete a WAL segment that a * live standby still needs for catch-up (the bug-795/828 committed-job-loss * family). It does NOT fix the empty/far-behind join case (cursor=0) — * that requires entity-bearing snapshots (separate spec). * * Lock ordering (verified 2026-06-26): persist_checkpoint takes p->lock only * to snapshot checkpoint_lsn, then RELEASES it before calling * trim_wal_segments — so this function runs with NO p->lock held. * repl_trim_lsn and repl_expire_disconnect_slots take only r->lock * (p->repl.lock); neither ever acquires p->lock or ce->lock, so there is no * p->lock↔repl.lock (or ce->lock↔repl.lock) inversion. */ static void trim_wal_segments(persist_t* p) { /* Extract the start LSN of the currently-open segment from seg_path. * seg_path has the form "/seg-.wal". * If no segment is open yet (wal_fd < 0), skip trimming. */ if (p->wal_fd < 0 || p->seg_path[0] == '\0') return; const char* seg_basename = strrchr(p->seg_path, '/'); if (!seg_basename) return; seg_basename++; /* skip the '/' */ uint64_t active_seg_lsn = 0; if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1) return; /* Expire stale disconnected slots so their last_disconnected_lsn guards * don't hold WAL segments indefinitely (slot guard expires at * last_disconnect_time + snap_interval_s). */ repl_expire_disconnect_slots(&p->repl, time(NULL), p->config.snap_interval_s); /* Compute the trim floor: min(active_seg_lsn, all active standby * acked_lsn, all valid disconnected-slot last_disconnected_lsn). * Any segment with seg_start_lsn BELOW this floor is disposable. */ uint64_t floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s); DIR* d = opendir(p->wal_path); if (!d) return; struct dirent* e; while ((e = readdir(d)) != NULL) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start_lsn = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue; /* Only delete segments below BOTH the active segment AND the * standby-cursor floor (bug-1147 guard). floor <= active_seg_lsn * by construction (repl_trim_lsn caps at checkpoint_lsn=active_seg_lsn), * so this check subsumes the old < active_seg_lsn guard. */ if (seg_start_lsn < floor) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name); unlink(seg_path); } } closedir(d); } /* Snapshot filename comparison helper for qsort — sorts ascending by name * (which sorts ascending by LSN since filenames embed the LSN as hex). */ static int snap_name_cmp(const void* a, const void* b) { return strcmp(*(const char* const*)a, *(const char* const*)b); } /* Keep only the snap_keep_n most recent snapshot files; delete the rest. */ /* True when `lsn` is the checkpoint position an in-flight peer snapshot * transfer currently references (ha-entity-snapshots task 4.1/4.3, Req * 3.1-3.3) — checked directly against snapshot_inflight_lsn[], not gated on * snapshot_inflight[] first: the LSN clears before the flag in every * ha_snapshot_send_dispatch/snapshot_lane_fn path, so a stale flag=1 can * momentarily coexist with an already-cleared LSN=0 (see task 4.1's * Implementation Note). A checkpoint's LSN is never 0, so "nonzero and * matches" is the correct, race-free protection signal on its own. No * cluster (p->cluster == NULL, e.g. standalone lib-level tests) means no HA * layer exists to be mid-transfer — nothing is protected. */ static bool snapshot_lsn_in_flight(persist_t* p, uint64_t lsn) { if (p->cluster == NULL || lsn == 0) return false; for (int i = 0; i < MAX_HA_CANDIDATES; i++) { if (atomic_load_explicit(&p->cluster->ha.snapshot_inflight_lsn[i], memory_order_acquire) == lsn) return true; } return false; } static void rotate_snapshots(persist_t* p) { DIR* d = opendir(p->snap_path); if (!d) return; /* Collect all .capnp snapshot filenames */ char* names[1024]; int n = 0; struct dirent* e; while ((e = readdir(d)) != NULL && n < (int)(sizeof(names) / sizeof(names[0]))) { size_t len = strlen(e->d_name); if (len <= 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue; names[n] = strdup(e->d_name); if (names[n]) n++; } closedir(d); if (n == 0) return; /* Sort ascending — oldest first */ qsort(names, (size_t)n, sizeof(names[0]), snap_name_cmp); /* Delete all but the snap_keep_n newest, skipping any file an in-flight * peer transfer is still reading from (task 4.3) — it is retried on the * next rotation pass once that transfer completes. */ int keep = (int)p->config.snap_keep_n; int delete_up_to = n - keep; /* first index to keep */ for (int i = 0; i < delete_up_to && i < n; i++) { uint64_t lsn = 0; if (sscanf(names[i], "snap-%" SCNx64 ".capnp", &lsn) == 1 && snapshot_lsn_in_flight(p, lsn)) continue; char path[4096]; snprintf(path, sizeof(path), "%s/%s", p->snap_path, names[i]); unlink(path); } for (int i = 0; i < n; i++) free(names[i]); } /* ------------------------------------------------------------------ */ /* Async checkpoint scheduling (schedule_checkpoint + task fn) */ /* ------------------------------------------------------------------ */ /* checkpoint_task_fn: executed by delaymgr worker thread. * * The CAS in schedule_checkpoint already holds the "in_progress" slot (=1). * We must clear it before calling persist_checkpoint(), which has its own * atomic_exchange guard. After persist_checkpoint() returns (it resets the * flag to 0), we also reset wal_record_count so the counter starts fresh. */ static int checkpoint_task_fn(void* arg) { persist_t* persist = (persist_t*)arg; /* Release the "scheduled" slot so persist_checkpoint's guard can acquire */ atomic_store(&persist->checkpoint_in_progress, 0); persist_checkpoint(persist); /* Reset the record counter so the next checkpoint window starts clean. * Done after persist_checkpoint returns — the flag is already 0 by now. */ atomic_store(&persist->wal_record_count, 0); return 0; } /* schedule_checkpoint: idempotent async checkpoint request via delaymgr. * Safe to call from any thread. Uses atomic CAS to ensure at most one * checkpoint is scheduled or running at a time. * * If cluster is NULL (unit tests without a live delaymgr) the function * falls back to a synchronous checkpoint. */ static void schedule_checkpoint(persist_t* persist) { int expected = 0; if (!atomic_compare_exchange_strong(&persist->checkpoint_in_progress, &expected, 1)) return; /* already scheduled or running — idempotency guard */ if (persist->cluster == NULL) { /* Unit-test / no-cluster path: run synchronously and reset. */ atomic_store(&persist->checkpoint_in_progress, 0); persist_checkpoint(persist); atomic_store(&persist->wal_record_count, 0); return; } /* Schedule via delaymgr with 1 s coalescing delay, single execution */ delay_task(&persist->cluster->delaymgr, 1, 1, checkpoint_task_fn, persist); } /* ------------------------------------------------------------------ */ /* Checkpoint core (shared by persist_checkpoint and the marker-apply */ /* trigger, config-upgrade-snapshot task 5) */ /* ------------------------------------------------------------------ */ /* persist_checkpoint_write_locked: pin checkpoint_lsn to the EXPLICIT * `checkpoint_lsn` given (never re-derived internally from * last_applied_lsn), capture the committed-config reference, and serialize * live state to a snapshot file. * * Caller MUST already hold apply_pump_lock (or otherwise guarantee no * concurrent apply-pump walk can be racing this call, e.g. the * single-threaded boot-replay window persist_recover() already documents * elsewhere in this file) -- this function never locks or unlocks * apply_pump_lock itself, so it is safe to call both from a context that * already holds it (the live apply pump) and from a context where no lock * is needed at all (boot replay). * * The explicit-lsn parameter exists because the two callers need two * different values at two different moments: * - persist_checkpoint() (periodic/explicit trigger): last_applied_lsn, * read fresh by the caller immediately before this call. * - the marker-apply trigger (apply_committed_entry's WAL_CONFIG_MARKER * case): the marker's OWN lsn -- last_applied_lsn is NOT yet advanced * to the marker's position at the moment that case runs (both live * apply and replay advance it strictly AFTER dispatching this call), * so reading last_applied_lsn from inside that case would mislabel the * checkpoint with the marker's PREDECESSOR's lsn, not its own. */ static int persist_checkpoint_write_locked(persist_t* p, uint64_t checkpoint_lsn) { p->checkpoint_lsn = checkpoint_lsn; /* Capture the latest committed config-marker reference BEFORE trimming * (ha-term-safety task 4.2 r1): trim_wal_segments (run by both callers, * after this function returns) can unlink the segment holding a * committed WAL_CONFIG_MARKER, so the derived committed config must be * cached durably (into the sentinel) before that happens. */ scan_latest_committed_config_ref(p); /* Serialize in-memory state to a snapshot file (P0: no live kvmgrs — * writes sentinel header + cached config ref). */ return write_snapshot(p); } /* ------------------------------------------------------------------ */ /* Public: persist_checkpoint */ /* ------------------------------------------------------------------ */ int persist_checkpoint(persist_t* p) { /* Guard against concurrent checkpoints */ if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0) return -EBUSY; int rc = 0; /* 1. Write WAL_CHECKPOINT marker (local only — never replicated) */ rc = persist_write_checkpoint_marker(p); if (rc != 0) goto done; /* 2. Record checkpoint_lsn = the APPLIED cursor (last_applied_lsn), not * the physical WAL append tip (current_lsn). write_snapshot() below * serializes live jobmgr state, which only reflects entries the * apply pump has actually dispatched -- NOT every entry durably * appended/committed, since apply is decoupled from commit * (raft-apply-decoupling) and can lag behind current_lsn under load. * Stamping current_lsn here mislabels the snapshot: on restart, * last_applied_lsn is re-seeded FROM checkpoint_lsn and replay * resumes strictly above it, so any committed-but-not-yet-applied * entries between the true apply cursor and current_lsn at * checkpoint time are silently skipped forever, even though jobmgr * never actually contained them (persist.h last_applied_lsn doc). * Held under apply_pump_lock together with 3a/3b so the label and * the jobmgr walk it describes are captured atomically w.r.t. the * live apply pump -- the pump can only ever move last_applied_lsn * and jobmgr forward, never backward, so freezing it here rules out * the label ever claiming more than jobmgr actually holds. */ pthread_mutex_lock(&p->apply_pump_lock); rc = persist_checkpoint_write_locked( p, atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire)); pthread_mutex_unlock(&p->apply_pump_lock); if (rc != 0) goto done; /* 4. Trim old WAL segments before the checkpoint */ trim_wal_segments(p); /* 5. Rotate old snapshots — keep only snap_keep_n most recent */ rotate_snapshots(p); done: atomic_store(&p->checkpoint_in_progress, 0); return rc; } /* persist_checkpoint_marker_triggered (config-upgrade-snapshot task 5, Req * 1.1, 1.5, 1.8, 4.4, 5.2, 5.3): force an immediate checkpoint at exactly * `marker_lsn` -- the configuration-change marker's OWN lsn -- called from * apply_committed_entry's WAL_CONFIG_MARKER case. * * Unlike persist_checkpoint(), this function NEVER locks apply_pump_lock: * it relies entirely on the caller's own locking context, because the two * production call sites of apply_committed_entry() differ in that context * and this function must be correct under BOTH without deadlocking under * either: * - persist_apply_pump_run() (live online-apply, both primary and * standby route through this) holds apply_pump_lock across its ENTIRE * per-LSN dispatch loop, including the call into apply_committed_entry * that reaches this function. Re-locking that same, non-recursive * mutex here would self-deadlock -- so this function must NOT lock it; * it is already held by the caller for the whole duration of this call. * - replay_wal() (boot/restart replay, persist_recover()'s sole caller) * does NOT hold apply_pump_lock during its per-entry dispatch loop -- * by design, since persist_recover() runs before the apply pump is * ever kicked and nothing else can be racing it (see persist_recover's * own doc comment on its brief, unrelated apply_pump_lock use for * cache invalidation). No lock is needed here either, for the same * reason. * Both call sites are internal to persist.c and fully controlled, so this * documented precondition (caller already serializes, one way or another) * is safe to rely on rather than adding a redundant/ambiguous lock here. * * Shares checkpoint_in_progress -- the SAME single-in-flight CAS guard * persist_checkpoint() uses -- so a concurrent WAL-threshold-triggered * periodic checkpoint and this marker-triggered one can never both * proceed at once (Req 5.3). Returns -EBUSY when that guard is already * held by someone else; the caller (apply_committed_entry) treats that as * an expected, self-resolving condition, not a failure (Req 4.4) -- the * in-flight checkpoint (whichever triggered it) still completes and * either it or a later checkpoint picks up this marker's configuration * (every checkpoint embeds the currently active configuration, task 2.3). */ static int persist_checkpoint_marker_triggered(persist_t* p, uint64_t marker_lsn) { if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0) return -EBUSY; int rc = persist_write_checkpoint_marker(p); if (rc == 0) rc = persist_checkpoint_write_locked(p, marker_lsn); if (rc == 0) { trim_wal_segments(p); rotate_snapshots(p); } atomic_store(&p->checkpoint_in_progress, 0); return rc; } /* ------------------------------------------------------------------ */ /* Recovery internals */ /* ------------------------------------------------------------------ */ /* Parse and restore every job frame, then every queue frame, then every host * frame from the entity payload beginning at the current `fd` offset * (design.md "Checkpoint File Layout", SNAPSHOT_FORMAT_VERSION=2): * [job_count][job_count * {payload_len; capnp Job bytes}] * [queue_count][queue_count * {payload_len; capnp QueueAdminState bytes}] * [host_count][host_count * {payload_len; capnp HostAdminState bytes}] * A trailing config-bytes section follows the host section in the file * (config-upgrade-snapshot task 2.3) but is deliberately NOT read here -- * out of this function's scope; any bytes after the host section are simply * left unread, exactly as host-frame bytes were already tolerated as * present-but-unread before this function grew a host section of its own. * * `file_size` is the checkpoint file's total size (from fstat) and * `consumed` is the number of bytes already read from it (legacy header + * format_version) — every length-prefixed frame's length is bounds-checked * against the ACTUAL remaining bytes in the file before being trusted for a * read or allocation, never blindly followed (a corrupt/attacker-controlled * length must fail fast, not drive an oversized read). * * Restores in file order: every job via restore_job, then every queue via * restore_queue_state, then every host via restore_host_state * (config-upgrade-snapshot task 4, Req 1.10 — design.md sequence diagram * and "Referential integrity" note: restore order does not matter * functionally, since re-homing decisions are already baked into each * entity's encoded content at materialization time, never re-derived at * restore time, Design Invariant 2). Returns 0 only if every frame * restored; on the FIRST failure of any kind (short read, oversized * length, decode error, or restore error) returns immediately with a * negative errno — the caller MUST NOT commit any position change on a * non-zero return (Requirement 2.3). * * restore_job's -EEXIST is treated as success, not a failure: it is job.c's * own documented idempotent-replay guard (job already present), needed so a * retried load — after an earlier partial attempt already created some jobs * before failing on a later one — can continue past already-restored entries * rather than aborting on them again. restore_queue_state and * restore_host_state have no such idempotent-duplicate case of their own to * special-case here (re-applying the same status is already a harmless * overwrite/OR, and restore_host_state's create-if-missing placeholder path * is independently idempotent under retry — Requirement 5.8, worker.c), so * any non-zero return from either is a genuine failure. */ static int load_snapshot_entities(int fd, off_t file_size, off_t consumed, persist_t* p, uint64_t stored_lsn) { uint32_t job_count = 0; if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) return -EIO; consumed += (off_t)sizeof(job_count); if (job_count > 0 && (p->cluster == NULL || p->cluster->jobmgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < job_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; int rrc = restore_job(p->cluster, 0, buf, payload_len, stored_lsn); free(buf); if (rrc != 0 && rrc != -EEXIST) return rrc; } uint32_t queue_count = 0; if (read(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count)) return -EIO; consumed += (off_t)sizeof(queue_count); if (queue_count > 0 && (p->cluster == NULL || p->cluster->qmgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < queue_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; /* Decode the capnp QueueAdminState frame (config-upgrade-snapshot * task 4, Req 1.9/5.7 decode side — migrated off the prior raw * {name_len, name, status} layout collect_queue_frames stopped * writing in task 2.2). Mirrors the WAL_JOB_SUBMIT re-decode block * above in this file (capn_init_mem/capn_getp/read_*) and * test_persist_snapshot_entities.c's own * snapent_read_checkpoint_queues reader exactly. */ struct capn c_ctx; capn_init_mem(&c_ctx, buf, payload_len, 0); QueueAdminState_ptr qp; qp.p = capn_getp(capn_root(&c_ctx), 0, 1); struct QueueAdminState cq; memset(&cq, 0, sizeof(cq)); read_QueueAdminState(&cq, qp); capnp_queue_admin_state_t decoded; memset(&decoded, 0, sizeof(decoded)); decode_QueueAdminState(&decoded, &cq); capn_free(&c_ctx); free(buf); int qrc = restore_queue_state(p->cluster, decoded.name, decoded.status); free_QueueAdminState(&decoded); if (qrc != 0) return qrc; } /* Host section (config-upgrade-snapshot task 4, Req 1.10): new in * SNAPSHOT_FORMAT_VERSION=2, added after the queue section (design.md * "Checkpoint File Layout"). Same [payload_len][capnp bytes] per-frame * shape as the job/queue sections above. Every host frame dispatches * to restore_host_state (worker.c, task 3), which is create-if-missing * (a worker_t is join-provisioned, not config-provisioned like a queue * — Design Invariant 1) and idempotent under retry on its own * (Requirement 5.8), so this loop applies the exact same "any failure * returns immediately" discipline as the job/queue loops above without * needing any extra idempotency handling here. */ uint32_t host_count = 0; if (read(fd, &host_count, sizeof(host_count)) != (ssize_t)sizeof(host_count)) return -EIO; consumed += (off_t)sizeof(host_count); if (host_count > 0 && (p->cluster == NULL || p->cluster->workermgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < host_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; /* Decode the capnp HostAdminState frame -- mirrors the queue decode * block immediately above and * test_persist_snapshot_entities.c's snapent_read_checkpoint_hosts * reader exactly. */ struct capn c_ctx; capn_init_mem(&c_ctx, buf, payload_len, 0); HostAdminState_ptr hp; hp.p = capn_getp(capn_root(&c_ctx), 0, 1); struct HostAdminState ch; memset(&ch, 0, sizeof(ch)); read_HostAdminState(&ch, hp); capnp_host_admin_state_t decoded; memset(&decoded, 0, sizeof(decoded)); decode_HostAdminState(&decoded, &ch); capn_free(&c_ctx); free(buf); int hrc = restore_host_state(p->cluster, decoded.name, decoded.status); free_HostAdminState(&decoded); if (hrc != 0) return hrc; } return 0; } /* Load a snapshot: legacy header (magic + checkpoint_lsn + optional * config-ref tail), then — for an entity-bearing checkpoint — every job, * queue, and host frame that follows it (ha-entity-snapshots task 5.2, * config-upgrade-snapshot task 4; Requirements 2.1-2.4, 5.1, 5.2, 1.10). * See design.md "load_snapshot (extended)" and its sequence diagram for the * exact contract this implements. * * Ordering (Requirement 2.1, 2.2, 2.3 — restore FIRST, validate FULLY, THEN * advance position, never the reverse, never partial): the header and any * config-ref tail are parsed into LOCAL variables only. Format is then * detected by whether any bytes remain past the header (design.md's stated * discriminator — not a flag bit). A short file (nothing remains) is a * legacy, pre-feature checkpoint: not an error, so its recorded position is * committed immediately and *out_entities_restored is false. Otherwise every * job, then every queue, then every host frame is restored via * load_snapshot_entities() before ANYTHING is written to * p->checkpoint_lsn/p->current_lsn/p->cfgref_*; on any restore failure this * function returns non-zero with p's fields PROVABLY untouched (the single * commit point below is only ever reached after load_snapshot_entities() * returns 0). *out_entities_restored is only set on a rc==0 return, matching * persist.h's documented contract. * * Returns 0 on success, -EINVAL on bad magic / unrecognized entity format, * -EIO on a short/corrupt read, -ENOMEM on allocation failure, -errno on * open/fstat failure, or a restore_job/restore_queue_state/restore_host_state * error code. */ static int load_snapshot(persist_t* p, const char* path, uint64_t lsn, bool* out_entities_restored) { (void)lsn; /* lsn is encoded in the file itself — we re-read it */ int fd = open(path, O_RDONLY); if (fd < 0) return -errno; struct stat st; if (fstat(fd, &st) != 0) { int saved = errno; close(fd); return -saved; } /* magic(8) + checkpoint_lsn(8) + optional config-ref tail(1+8+32). */ char header[16 + 1 + 8 + 32]; ssize_t n = read(fd, header, sizeof(header)); if (n < 16) { close(fd); return -EIO; } if (memcmp(header, "XCESNAP\0", 8) != 0) { close(fd); return -EINVAL; } /* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch). * Held LOCALLY -- not yet written to p-> -- until the single commit * point below (legacy short-circuit, or after every entity restores). */ uint64_t stored_lsn; memcpy(&stored_lsn, header + 8, 8); /* Optional committed-config reference (ha-term-safety task 4.2 r1): * present only when the sentinel carries the 41-byte tail. Also held * locally until the same commit point. */ int cfgref_present = 0; uint64_t cfgref_tag = 0; uint8_t cfgref_sha[32]; memset(cfgref_sha, 0, sizeof(cfgref_sha)); if (n >= 16 + 1 + 8 + 32 && header[16] != 0) { cfgref_present = 1; memcpy(&cfgref_tag, header + 17, 8); memcpy(cfgref_sha, header + 25, 32); } off_t consumed = (off_t)n; /* Legacy-format detection (Requirement 2.4, 5.1, 5.2): "does more data * exist" is the discriminator (design.md), not a flag bit -- a file * ending exactly at the legacy header is unambiguously a pre-feature * checkpoint. This is an expected rollout shape, not corruption: commit * the position it records (existing, preserved legacy behavior) and * report entities_restored=false so the caller falls back to normal * catch-up rather than treating this checkpoint as fully complete. */ if (st.st_size <= consumed) { close(fd); p->checkpoint_lsn = stored_lsn; p->current_lsn = stored_lsn; if (cfgref_present) { p->cfgref_present = 1; p->cfgref_tag = cfgref_tag; memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha)); } if (out_entities_restored) *out_entities_restored = false; return 0; } /* Entity-bearing checkpoint: format_version, then job frames, then queue * frames (design.md "Checkpoint File Layout"). */ uint32_t fmt = 0; if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) { close(fd); return -EIO; } consumed += (off_t)sizeof(fmt); /* An unrecognized format_version is corruption or a future layout this * binary cannot read -- fail loud rather than silently treating it as * legacy (task 5.1 review note: a version mismatch must never be masked * as an expected legacy shape, or real corruption goes undetected). */ if (fmt != SNAPSHOT_FORMAT_VERSION) { close(fd); return -EINVAL; } int rc = load_snapshot_entities(fd, st.st_size, consumed, p, stored_lsn); close(fd); if (rc != 0) return rc; /* Requirement 2.3: p->*_lsn / p->cfgref_* untouched */ /* Single commit point (Requirement 2.1, 2.2): reached ONLY after every * job and queue frame restored successfully above. */ p->checkpoint_lsn = stored_lsn; p->current_lsn = stored_lsn; if (cfgref_present) { p->cfgref_present = 1; p->cfgref_tag = cfgref_tag; memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha)); } if (out_entities_restored) *out_entities_restored = true; return 0; } /* Public wrapper for load_snapshot — exposed via persist.h for ha.c (Task * 3.5c). load_snapshot() above now owns the full restore-before-advance * implementation (ha-entity-snapshots task 5.2): this wrapper is a thin * pass-through, since load_snapshot() already gates *out_entities_restored * and the p->checkpoint_lsn/current_lsn commit on entity-restore success * (Requirements 2.1-2.4, 5.1, 5.2). */ int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn, bool *out_entities_restored) { return load_snapshot(p, path, lsn, out_entities_restored); } /* P2-2: Standby WAL replay — apply a single replicated entry to in-memory state. * * No hooks fire on standbys; the hook system is empty until promotion. * * Returns 0 on success (including skip for checkpoint / unknown types), * negative errno on error (NULL args, oversized payload). */ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn, const void* payload, uint32_t payload_len) { if (!p) return -EINVAL; /* Reject oversized payloads BEFORE any allocation */ if (payload_len > MAX_WAL_PAYLOAD_BYTES) { ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u", payload_len, MAX_WAL_PAYLOAD_BYTES); return -EMSGSIZE; } /* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain * (p->last_crc), and current_lsn are owned exclusively by the physical * layer — persist_wal_write_replicated on the live replicated path, and * replay_wal/persist_recover on recovery. This function MUST NOT touch * them. (bug-778: it previously set p->last_crc from a synthetic crc=0 * header, breaking the on-disk CRC chain and cold-restart recovery.) */ /* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it * via replication. Skip silently. */ if (type == WAL_CHECKPOINT) return 0; /* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward * compatibility — an older controller can boot on WAL written by a newer * one without hard-failing on unrecognised entry types. */ if (type > WAL_ENTRY_TYPE_MAX) { WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u" " (lsn=%" PRIu64 ") — skipping for forward compatibility", type, WAL_ENTRY_TYPE_MAX, lsn); return 0; } /* Advance the current_lsn high-water (monotonic, idempotent). This is a * write-cursor high-water that is safe to bump from multiple paths. The * CRC chain link (p->last_crc) is deliberately NOT touched here — it has a * single owner: the physical writer (persist_wal_write_replicated) and * replay (persist_recover). (bug-778) */ pthread_mutex_lock(&p->lock); if (lsn + 1 > p->current_lsn) p->current_lsn = lsn + 1; pthread_mutex_unlock(&p->lock); if (!p->cluster) return 0; /* unit tests without a full cluster — skip entity restore */ switch (type) { case WAL_JOB_SUBMIT: /* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN * through so the restored job is tagged for the egress watermark * (ha-convergence-recovery 3.3). */ restore_job(p->cluster, 0, payload, payload_len, lsn); break; case WAL_JOB_ARRAY_SUBMIT: /* job-array-single-wal-record task 3.3 (design.md "Array Apply * Expansion"): the array-record analog of WAL_JOB_SUBMIT's * restore_job() call directly above, dispatched from this SEPARATE, * still-live standby catch-up / replicated-apply seam so it can * never diverge from apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT * case (Req 3.1, 3.2, 4.1). Mirrors the single-job case's exact * minimalism: no xlist_t, no hook fire — this dispatcher is * deliberately hook-free (standby catch-up does not notify the * local scheduler/API server; that is the primary's job, done via * apply_committed_entry()). out_new_jobs is not needed here, so * NULL is passed (restore_job_array()'s out_new_jobs parameter is * optional — see test_rja_out_new_jobs_is_optional, * test_job_array_apply.c). The return value is intentionally * discarded, matching every other case in this switch today (task * 4.1 — Live Apply Resource Fence — is what later makes this * function's callers respect a negative rc; not this task's * boundary). */ restore_job_array(p->cluster, payload, payload_len, lsn, NULL); break; case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: case WAL_JOB_RUSAGE: /* Payload: wal_jobstate_t {jobid, state} */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; restore_job_state(p->cluster, js->jobid, js->state); } break; case WAL_WORKER_CTRL: /* Payload: [uint32_t detail][capnp worker] */ if (payload_len > WORKER_WAL_HDR_SIZE) worker_from_capnp(p->cluster, payload, payload_len); break; case WAL_QUEUE_CTRL: /* Payload: [wal_queuehdr_t][name NUL-terminated] */ if (payload_len > sizeof(wal_queuehdr_t)) { const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload; const char* name = (const char*)(payload + sizeof(wal_queuehdr_t)); restore_queue_state(p->cluster, name, qh->status); } break; case WAL_NOOP: /* Internal promotion no-op: empty payload, no entity restore, no * hooks. current_lsn is advanced by the physical layer (writer / * replay), not here — recognised explicitly rather than via default. */ break; case WAL_CONFIG_MARKER: /* Config-change marker (ha-term-safety task 4.2): a {version_tag, * sha256} reference, NEVER the config bytes (R7.4). It carries NO * entity state, so replay performs no restore here — the committed * configuration is DERIVED from the latest committed marker on demand * (config_committed_marker), and the bytes are obtained + hash-verified * out-of-band before activation (task 4.3). Recognised explicitly so * the per-LSN term index (built above this switch's caller) records its * term and on_get_entry_term / (term,lsn) reconcile treat it as an * ordinary log entry. */ break; default: break; } return 0; } /* raft-apply-decoupling task 2.2 — see persist.h for the full contract doc * (routing table, the job-state-family hook-mapping deviation, and what this * function deliberately does NOT own). Not yet wired into any live call * site — replay (2.3) and the live apply pump (6.1) will call this later. */ /* raft-apply-decoupling task 6.1 (Hazard 1 fix): thread-local "this call * stack is inside apply_committed_entry's own post-mutation notification" * guard. persist_job_hook_fun / persist_queue_hook_fun / persist_worker_hook_fun * are AFTER_EVENT subscribers that (for RETAIN/fact-shaped entry types) * durably append a NEW WAL entry every time they fire — the correct behavior * for their ORIGINAL, still-live purpose (job_ctrl's mark_job_update/ * mark_job_fail synchronous command path; the legacy queue/worker control * command handlers) which mutate live state and rely on this exact hook * firing to durably record that mutation. * * apply_committed_entry() now ALSO fires these same AFTER_EVENT hooks (job * facts always as the generic HOOK_DETAIL_JOB_UPDATE, per task 2.2's * documented wire-format deviation; queue/worker control at their native * detail) to deliver its own post-mutation NOTIFICATION — but that entry is * by definition already durably committed (it just got applied FROM the * WAL), so re-appending it would silently duplicate the entry on every live * apply. This is fixable, correctly, ONLY by provenance (which caller fired * the hook), NOT by hook `detail` value: HOOK_DETAIL_JOB_UPDATE in * particular is genuinely dual-origin (job_ctrl's SIGCONT/SIGSTOP path uses * the identical detail for its own, still-legitimate live append) — an * unconditional detail-based filter would silently break job_ctrl's suspend/ * resume durability instead of fixing anything. A shared persist_t-wide flag * would ALSO be wrong: it would race across threads (e.g. job_ctrl mutating * a DIFFERENT job on the iomgr thread while this thread's apply walk is * mid-hook for an unrelated fact) and could wrongly suppress or wrongly * permit an append depending on scheduling. A thread-local flag, scoped * narrowly around each xce_run_hooks() call this function makes, is race-free * by construction: it is only ever observed by hook subscribers invoked * SYNCHRONOUSLY on this exact call stack, from this exact function. */ static _Thread_local bool t_apply_seam_notifying = false; int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, const void* payload, uint32_t len) { if (!p) return -EINVAL; if (!p->cluster) return 0; /* unit tests without a full cluster — nothing to dispatch into */ cluster_t* c = p->cluster; switch (wal_type) { case WAL_JOB_SUBMIT: { /* Return value distinguishes a genuine creation from an idempotent * no-op (bug-1419): 1 = restore_job() actually constructed a new * job, 0 = the jobid already existed (duplicate WAL entry — see * below), negative = a real error. replay_wal()'s new-job tally * relies on this distinction; the live apply pump discards it. * * A duplicate WAL_JOB_SUBMIT for the same jobid is an EXPECTED, * harmless occurrence, not a bug: a client's local commit-timeout * retry can append a second entry for the same jobid while the * first is still in flight during a quorum-loss window; the WAL * entry that timed out is never retracted, so if it later achieves * majority commit anyway, both entries end up committed. restore_job * ()'s own -EEXIST here is what makes that safe to replay twice. */ int rc = restore_job(c, 0, payload, len, lsn); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) return rc; struct capn jc_ctx; capn_init_mem(&jc_ctx, (const uint8_t*)payload, len, 0); Job_ptr jp; jp.p = capn_getp(capn_root(&jc_ctx), 0, 1); struct Job cj; read_Job(&cj, jp); uint64_t jobid = cj.jobid; capn_free(&jc_ctx); job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)jobid, (void**)&j); unlock(c->jobmgr); if (!j) return 1; /* defensive: should not happen after a successful restore */ void* jptr = j; xlist_t jlist = { .count = 1, .caps = 1, .delta = 1, .data = &jptr }; t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist); t_apply_seam_notifying = false; return 1; } case WAL_JOB_ARRAY_SUBMIT: { /* job-array-single-wal-record task 3.2 (design.md "Array Apply * Expansion"): the array-record analog of WAL_JOB_SUBMIT's * restore_job() above, dispatched from this SAME shared seam so live * apply, boot replay (both callers of this function), and (task * 3.3) standby catch-up can never diverge in how they expand a * committed array record (Req 3.1, 3.2). * * restore_job_array()'s return generalizes restore_job()'s * three-way contract to N elements: a positive count = the number * of genuinely NEW elements this call constructed (this is exactly, * and only, what the hook fire below carries — Req 3.4); -EEXIST = * every index in the record already existed (idempotent * whole-record no-op, Req 3.3) — no hook; a negative errno = a * genuine construction/resource-exhaustion failure (Req 9.4) — * propagated so persist_apply_pump_run() (task 4.1) can halt and * fence on it rather than silently skipping it. **Task 7.1 * (bug-1443) update**: a negative return no longer unconditionally * means "no hook" — restore_job_array()'s bulk-insert-phase residual * gap can leave a genuinely, permanently inserted prefix in * out_new_jobs even on failure (job.c's doc comment, "Residual * atomicity gap"); see below. */ xlist_t out_new_jobs; memset(&out_new_jobs, 0, sizeof(out_new_jobs)); int rc = restore_job_array(c, payload, len, lsn, &out_new_jobs); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) { /* job-array-single-wal-record task 7.1 (bug-1443 partial-prefix * fix): out_new_jobs is untouched (count == 0, from the memset * above) for a reservation or scratch-build-phase failure — * nothing was ever inserted, so no hook fires, unchanged from * before this task. But for the narrower bulk-insert-phase * residual gap, out_new_jobs now carries a non-empty, genuinely * and permanently inserted prefix (job.c) that must still get * its one HOOK_DETAIL_JOB_NEW fire — a job that is really in * c->jobmgr must never be silently un-notified. The overall * apply is still, correctly, reported as failed via the * negative rc returned below, so persist_apply_pump_run() halts * the LSN and the Live Apply Resource Fence engages for the * un-constructed remainder (Req 2.3, 9.4, 9.6). */ if (out_new_jobs.count > 0) { t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &out_new_jobs); t_apply_seam_notifying = false; } xlist_fini(&out_new_jobs); return rc; } /* Exactly ONE fire carrying every newly created element in one * list — never once per element (Req 3.4). out_new_jobs is already * the precise newly-created set restore_job_array() built; no * re-derivation via key_find needed. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &out_new_jobs); t_apply_seam_notifying = false; /* Only the list's own backing array is released here — ownership of * every job_t* it holds already transferred to c->jobmgr inside * restore_job_array()'s bulk-insert pass, mirroring * test_job_array_apply.c's own xlist_fini(&out) usage after a * successful call. */ xlist_fini(&out_new_jobs); return rc; } case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_RUSAGE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: { /* Every job-state fact type in this group shares one wire shape — * wal_jobstate_t {jobid, state} — and therefore one guarded core. * Their payloads are deterministic/state-independent (a START is * always JOB_STAT_RUN, a FINISH's outcome depends only on * exit_code), so an absolute `state` baked at receipt time never * goes stale — unlike WAL_JOB_FAIL/WAL_JOB_UPDATE below, which have * their own dedicated cases. See persist.h's doc comment on * apply_committed_entry for the full routing table and why this * does not dispatch to the richer type-specific apply_job_{start, * finish}_transition() cores (their parameter structs are not * present on this wire format). */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = apply_job_update_transition(j, (int)js->state); unlock(j); if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate — no hook */ /* Hazard 1 guard (see t_apply_seam_notifying's doc comment above): * HOOK_DETAIL_JOB_UPDATE is dual-origin — job_ctrl's mark_job_update * (SIGCONT/SIGSTOP) fires the identical detail for its own live, * still-legitimate append via persist_job_hook_fun. Without this * guard, EVERY fact this apply pump applies would re-append itself * as a duplicate, mis-typed WAL_JOB_UPDATE entry the moment the live * pump (task 6.1) starts calling this function. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); t_apply_seam_notifying = false; return 0; } case WAL_JOB_FAIL: { /* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3) fix for a job-state * corruption bug: unlike the deterministic group above, a FAIL's * resulting state depends on the job's CURRENT state (PERR merges * onto whatever terminal state already holds). Dispatch on _pad * (see wal_types.h's doc comment on wal_jobstate_t._pad): * - WAL_JOB_FAIL_STAGE_FLAG set: a deferred fail-fact * (record_job_fail_fact, job.c) — reconstruct a minimal * job_failinfo_t from {jobid, stage} and call the REAL * apply_job_fail_transition() against the job's ACTUAL CURRENT * state at apply time, never a receipt-time snapshot. * - _pad == 0: the legacy live path (job_ctrl's mark_job_fail) — * `state` is already the correct, current, absolute value, * computed synchronously under the same lock as the mutation * (no staleness window) — an absolute overwrite is safe and * unchanged. */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result; if (js->_pad & WAL_JOB_FAIL_STAGE_FLAG) { job_failinfo_t failinfo = { .jobid = js->jobid, .stage = js->_pad & ~WAL_JOB_FAIL_STAGE_FLAG, .code = 0, .detail = NULL, }; result = apply_job_fail_transition(j, &failinfo); } else { result = apply_job_update_transition(j, (int)js->state); } unlock(j); if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate — no hook */ /* Hazard 1 guard — see the identical guard above. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); t_apply_seam_notifying = false; return 0; } case WAL_JOB_UPDATE: { /* task 6.1 round 2 (Req 3.3/7.1-7.3, 11.3): same class of fix as * WAL_JOB_FAIL above. Dispatch on _pad: * - WAL_JOB_UPDATE_POSTDONE_MARK: a deferred postdone fact * (record_job_postdone_fact, job.c) — call * apply_job_postdone_transition(), which re-derives the * postdone merge (conditional EXIT backstop + OR JOB_STAT_PDONE) * fresh against the job's ACTUAL CURRENT state at apply time. * - _pad == 0: the legacy live path (job_ctrl's mark_job_update, * SIGCONT/SIGSTOP) — `state` is already correct/absolute, * computed synchronously under the same lock as the mutation — * an absolute overwrite is safe and unchanged. */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = (js->_pad == WAL_JOB_UPDATE_POSTDONE_MARK) ? apply_job_postdone_transition(j) : apply_job_update_transition(j, (int)js->state); unlock(j); if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate — no hook */ /* Hazard 1 guard — see the identical guard above. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); t_apply_seam_notifying = false; return 0; } case WAL_QUEUE_CTRL: { if (len <= sizeof(wal_queuehdr_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload; const char* name = (const char*)((const uint8_t*)payload + sizeof(wal_queuehdr_t)); /* restore_queue_state's -ENOENT ("queue not yet built from config") * and -EINVAL (bad name) are both benign skips here, matching * apply_wal_entry's existing behavior of never fencing on this * entry type's restore outcome. */ if (restore_queue_state(c, name, qh->status) < 0) return 0; queue_t* q = NULL; lock(c->qmgr, RTTI_LOCK_READ); key_find(c->qmgr, (void*)name, (void**)&q); unlock(c->qmgr); if (!q) return 0; /* Same double-append class as the job-fact guard above: the legacy * queue-control command handler still mutates then relies on this * exact AFTER_EVENT firing to durably append via * persist_queue_hook_fun (WAL_QUEUE_CTRL is unconditionally * appended on AFTER_EVENT there, not gated on WAL_MODE_WAIT like the * job-submit case is) — guard so this live-pump re-notify does not * re-append an already-committed entry. */ t_apply_seam_notifying = true; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, HOOK_AFTER_EVENT, HOOK_DETAIL_QUEUE_UPDATE, c, q); t_apply_seam_notifying = false; return 0; } case WAL_WORKER_CTRL: { if (len <= WORKER_WAL_HDR_SIZE) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ uint32_t hook_detail; memcpy(&hook_detail, payload, sizeof(hook_detail)); int rc = worker_from_capnp(c, payload, len); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) return rc; /* Recover the worker's name via a cheap raw capnp field read * (read_Worker's capn_get_text is a borrowed, NUL-terminated pointer * into the decode buffer — no allocation, unlike decode_Worker) so * the just-restored worker can be looked up for the notification * hook without changing worker_from_capnp's own contract. */ const uint8_t* wpayload = (const uint8_t*)payload + WORKER_WAL_HDR_SIZE; size_t wlen = len - WORKER_WAL_HDR_SIZE; struct capn wc_ctx; capn_init_mem(&wc_ctx, wpayload, wlen, 0); Worker_ptr wp; wp.p = capn_getp(capn_root(&wc_ctx), 0, 1); struct Worker cw; read_Worker(&cw, wp); worker_t* w = NULL; lock(c->workermgr, RTTI_LOCK_READ); key_find(c->workermgr, (void*)cw.name.str, (void**)&w); unlock(c->workermgr); capn_free(&wc_ctx); if (!w) return 0; /* defensive: should not happen after a successful restore */ /* Same double-append class as the job-fact/queue guards above: * persist_worker_hook_fun unconditionally appends on AFTER_EVENT for * every worker detail (no WAL_MODE_WAIT gate), and the legacy worker * join/update/dead command paths still rely on that firing for their * own live durability — guard the whole re-notify switch so this * live-pump call never re-appends an already-committed entry. */ t_apply_seam_notifying = true; switch (hook_detail) { case HOOK_DETAIL_WORKER_JOIN: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c, w->host.name, w); break; case HOOK_DETAIL_WORKER_UPDATE: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_UPDATE, c, w); break; case HOOK_DETAIL_WORKER_DEAD: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w); break; default: break; /* unrecognized discriminator — mutation already applied, * notification skipped rather than guessed at */ } t_apply_seam_notifying = false; return 0; } case WAL_CONFIG_MARKER: { /* config-upgrade-snapshot task 5 (Req 1.1, 1.5, 1.8): force an * immediate, deterministic checkpoint at exactly this marker's own * (term, lsn) boundary, materializing the upgrade snapshot the * moment every member applies the marker -- primary online-apply, * standby online-apply, and boot/restart replay alike, since this * one unified apply seam is exactly what all three route through * (design.md "Architecture" / "Key simplification found during * design": no new leader-specific "resume" logic is needed either, * since a newly-elected leader satisfies this the same way, by * continuing to apply its own committed log). * * Deliberately does NOT touch entity interpretation/re-homing * (lost_and_found etc.) -- Req 1.8 requires apply and activation to * stay separate steps; this case's only effect is materialization, * identical in kind to any other checkpoint. * * persist_checkpoint_marker_triggered() (not a bare * persist_checkpoint() call) is required here to avoid two real * hazards a naive direct call would hit: * 1. Lock reentrancy: persist_apply_pump_run() (the live pump's * call site for online-apply, both primary and standby) holds * apply_pump_lock across this entire dispatch call; * persist_checkpoint()'s own body re-acquires that same, * non-recursive mutex, which would self-deadlock. * persist_checkpoint_marker_triggered() never locks * apply_pump_lock itself -- see its own doc comment for why * that is safe under both production call sites. * 2. Stale checkpoint_lsn: both live-apply and replay advance * last_applied_lsn AFTER this dispatch call returns, never * before (persist_apply_pump_run / replay_wal), so reading * last_applied_lsn from inside this case would capture the * PRECEDING entry's lsn, not the marker's own, mislabeling the * checkpoint. `lsn` (this function's own parameter) is * already the marker's exact position, so it is passed * through explicitly instead of re-derived. */ int rc = persist_checkpoint_marker_triggered(p, lsn); if (rc == -EBUSY) { /* Expected, self-resolving (Req 4.4/5.3): a concurrent * WAL-threshold-triggered or earlier marker-triggered * checkpoint already holds the shared single-in-flight guard * (checkpoint_in_progress). That in-flight checkpoint * completes on its own; this marker's own configuration is * still picked up by it or a later checkpoint (every * checkpoint embeds the currently active configuration, task * 2.3) -- not an error, no retry loop here. */ TRACE("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64 " checkpoint already in progress -- skipping, " "self-resolving", lsn); return 0; } if (rc != 0) { /* A genuine checkpoint failure (I/O error, etc.) -- propagate * so the caller's existing halt-and-retry-from-same-position * handling engages (persist_apply_pump_run / replay_wal), * exactly mirroring every other negative-return case in this * dispatch. last_applied_lsn is NOT advanced past this marker, * so the next kick/retry re-attempts the checkpoint fresh * (Req 5.2 crash-safety: resume materialization from the * marker). */ ERROR("apply_committed_entry: WAL_CONFIG_MARKER lsn=%" PRIu64 " forced checkpoint failed (rc=%d)", lsn, rc); return rc; } return 0; } default: /* WAL_NOOP / WAL_CHECKPOINT / unknown — no entity effect at this * seam, no hook (unchanged from apply_wal_entry's existing * treatment of these types). WAL_CONFIG_MARKER has its own explicit * case above (config-upgrade-snapshot task 5) and no longer falls * through to here. */ return 0; } } /* ============================================================ * Ordered apply pump (raft-apply-decoupling task 6.1) * ============================================================ */ /* Locate and read back the WAL entry at exactly `lsn` from the on-disk * segment files, for the live apply pump to hand to apply_committed_entry() * (mirrors read_entry_meta_locked's segment-scan idiom above, but returns * the entry's type and a malloc'd copy of its payload instead of just * metadata, and re-verifies the entry CRC — this is live-path plumbing, not * a metadata probe). * * Does not take p->lock: every LSN this is ever called with is already * majority-committed (durable and replicated), strictly below the physical * append tip a concurrent persist_wal_async could be extending — so this * never races the single-writer append path for the bytes it actually * reads. * * Returns 0 and populates *out_type/*out_payload (malloc'd if payload_len > * 0, caller frees; NULL when payload_len is 0)/*out_len on success. * Returns -ENOENT if no segment contains this LSN (should not happen for a * genuinely committed entry) or a negative errno on a read/CRC failure. */ static int apply_pump_read_entry(persist_t* p, uint64_t lsn, uint32_t* out_type, uint8_t** out_payload, uint32_t* out_len) { if (!p || lsn == 0 || !out_type || !out_payload || !out_len) return -EINVAL; /* Fast path (raft-apply-decoupling task 7.4 perf-fix): the sole caller * (persist_apply_pump_run) calls this once per LSN in strict ascending * order with no gaps in the common case, and holds apply_pump_lock * across the entire walk -- so the cache fields below need no lock of * their own here (see persist.h doc comment). A cache hit reads the next * header directly at the remembered offset on the already-open fd, * giving amortized O(1) per entry instead of a fresh * opendir/readdir/segment-sort plus a scan-from-zero on every single * call (the O(n^2) regression this fixes). ANY miss below just drops the * cache and falls through to the unchanged slow path -- correctness * never depends on the cache being right, only performance does. */ if (p->apply_pump_cache_fd >= 0 && p->apply_pump_cache_next_lsn == lsn) { wal_entry_hdr_t hdr; ssize_t hn = pread(p->apply_pump_cache_fd, &hdr, sizeof(hdr), (off_t)p->apply_pump_cache_off); bool hit = (hn == (ssize_t)sizeof(hdr)) && (hdr.lsn == lsn); uint8_t* payload = NULL; if (hit && hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { hit = false; } else { ssize_t pn = pread(p->apply_pump_cache_fd, payload, hdr.payload_len, (off_t)(p->apply_pump_cache_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); payload = NULL; hit = false; } } } if (hit) { uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (hdr.payload_len > 0) expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); if (hdr.crc != expected_crc) { free(payload); payload = NULL; hit = false; } } if (hit) { *out_type = hdr.type; *out_payload = payload; *out_len = hdr.payload_len; p->apply_pump_cache_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); p->apply_pump_cache_next_lsn = lsn + 1; return 0; } /* Miss (short read/EOF meaning the entry rolled into the next * segment, header mismatch, CRC failure, whatever): don't try to be * clever about advancing to the next segment inline. Close and fall * through to the EXISTING slow path unchanged -- it already * correctly finds the right segment from scratch. */ close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } DIR* d = opendir(p->wal_path); if (!d) return -ENOENT; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return -ENOENT; /* Ascending sort by start LSN (segment count is tiny in practice). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } int rc = -ENOENT; for (int si = 0; si < nseg; si++) { /* Segments are LSN-ordered; one whose start LSN already exceeds the * target cannot contain it, nor can any later one. */ if (seg_lsns[si] > lsn) break; char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (hn < (ssize_t)sizeof(hdr)) break; /* short read / EOF / pre-allocated zero tail */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn == lsn) { uint8_t* payload = NULL; uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { rc = -ENOMEM; break; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); rc = -EIO; break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { free(payload); rc = -EIO; break; } *out_type = hdr.type; *out_payload = payload; *out_len = hdr.payload_len; rc = 0; /* raft-apply-decoupling task 7.4 perf-fix: populate the * segment-lookup cache with this fd positioned just past the * entry we just returned, BEFORE closing it below, so the * next expected LSN (lsn+1) hits the fast path above instead * of re-scanning from segment start next time. Every OTHER * branch (error paths, not-found-in-this-segment * continuation) closes fd exactly as before -- caching only * ever happens on this success branch. */ p->apply_pump_cache_fd = fd; p->apply_pump_cache_seg_lsn = seg_lsns[si]; p->apply_pump_cache_off = off + (uint32_t)(sizeof(hdr) + hdr.payload_len); p->apply_pump_cache_next_lsn = lsn + 1; fd = -1; /* ownership transferred to the cache -- the * close(fd) below becomes a guarded no-op. */ break; } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } if (fd >= 0) close(fd); if (rc == 0 || rc == -ENOMEM || rc == -EIO) break; /* found it, or hit a genuine error inside the segment * that should contain it */ } return rc; } int persist_apply_pump_run(persist_t* p) { if (!p) return -EINVAL; pthread_mutex_lock(&p->apply_pump_lock); int applied = 0; for (;;) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); uint64_t next = atomic_load_explicit(&p->last_applied_lsn, memory_order_relaxed) + 1; if (next > committed) break; /* caught up as of this instant */ uint32_t wal_type = 0; uint8_t* payload = NULL; uint32_t len = 0; int rc = apply_pump_read_entry(p, next, &wal_type, &payload, &len); if (rc < 0) { ERROR("persist_apply_pump_run: failed to read committed entry " "lsn=%" PRIu64 " (rc=%d) -- stopping this pass; the entry " "stays committed and will be retried, from the same " "position, on the next kick", next, rc); pthread_mutex_unlock(&p->apply_pump_lock); return rc; } /* Strict order, no skipping or reordering: this is the single * dispatch call every newly committed entry passes through here, * regardless of its commit mode (WAIT command or RETAIN fact) — the * loop only ever advances last_applied_lsn one entry at a time, in * ascending order, so a burst of commands and facts that committed * out of their original submission order still applies in strict * commit (LSN) order (Req 3.3, 7.3). */ int apply_rc = apply_committed_entry(p, wal_type, next, payload, len); if (apply_rc < 0) { /* job-array-single-wal-record task 4.1 (design.md "Live Apply * Resource Fence"): this return value was previously discarded * -- (void)apply_committed_entry(...) -- with last_applied_lsn * advancing unconditionally, silently and permanently losing * any apply failure. A negative rc here is never a content- * derived failure (Req 9.1-9.3's pre-commit validation already * made those impossible post-commit); it is a node-local * resource-exhaustion failure (e.g. OOM mid array expansion, * Req 9.4) that must not be skipped. Mirror the * apply_pump_read_entry failure branch immediately above: stop * the walk exactly at this LSN, do not advance * last_applied_lsn, and do not take this LSN's pending-apply * hand-off (persist_take_pending_apply/on_applied) -- the entry * never actually applied, so that hand-off must not fire for * it. The entry stays committed and will be retried, from the * same position, on the next kick. This is a GENERAL fix: every * WAL type's apply outcome is now respected, not just * WAL_JOB_ARRAY_SUBMIT's. Under NORMAL operation every other * existing type's apply case returns 0 or positive, so this * branch is a behavior no-op for them in practice; the * fact-type cases' pre-existing "-ENOENT: referenced job not * found" branch (persist.h's dispatch table) is one other real, * pre-existing source of a negative return here, and now also * correctly halts the walk here instead of being silently * discarded, exactly as this fix intends. */ ERROR("persist_apply_pump_run: apply_committed_entry failed for " "committed entry lsn=%" PRIu64 " wal_type=%" PRIu32 " (rc=%d) -- stopping this pass at this position; the " "entry stays committed and will be retried, from the same " "position, on the next kick; Live Apply Resource Fence " "ENGAGED (job-array-single-wal-record Req 9.4)", next, wal_type, apply_rc); /* job-array-single-wal-record task 4.2 (design.md "Live Apply * Resource Fence"): engage the NEW, live, self-healing fence -- * distinct from recovery_unrecoverable -- so the promotion gate * (task 4.3) can refuse this node while it stays alive and * keeps retrying. Cleared below, from inside this same walk, * the instant a retry of this exact LSN succeeds (Req 9.5). */ persist_mark_apply_resource_fenced(p, next); free(payload); pthread_mutex_unlock(&p->apply_pump_lock); return apply_rc; } free(payload); atomic_store_explicit(&p->last_applied_lsn, next, memory_order_relaxed); applied++; /* job-array-single-wal-record task 4.2 (design.md "Live Apply * Resource Fence", Req 9.5): self-heal. Called unconditionally on * EVERY successful apply, not just after a known-fenced LSN -- * persist_clear_apply_resource_fence() is a cheap no-op when nothing * is fenced (the overwhelming common case). This is correct and * sufficient because the walk above is strictly sequential and * returns immediately on any failure (see the halt branch just * above): it can never advance PAST a fenced LSN without that exact * LSN applying successfully first, so "the next entry to apply * successfully after a fence engages" and "a retry of the SAME * stuck LSN succeeding" are the same event by construction. No * election, promotion check, or external/admin action is involved -- * the fence clears purely because this walk itself made progress. */ persist_clear_apply_resource_fence(p); /* raft-apply-decoupling task 6.2 (Req 1.1, 2.1-2.3): consume this * LSN's command hand-off, if any (task 3.2's * persist_register_pending_apply(), e.g. handle_jobsubmit's * commit-outcome continuation) -- strictly AFTER the mutation above * and after last_applied_lsn already reflects it, never before or * during. A fact entry never registers a hand-off, so * persist_take_pending_apply() simply returns 0 for it and this is a * correct no-op with no per-entry-type branching needed here. * persist_take_pending_apply() unlinks the registration before * returning it, so re-applying the same lsn (a future retry/replay * landing on this exact walk position again) can never fire the * continuation a second time. */ pending_apply_notify_fn on_applied = NULL; void* apply_ctx = NULL; if (persist_take_pending_apply(p, next, &on_applied, &apply_ctx)) on_applied(apply_ctx, next); } pthread_mutex_unlock(&p->apply_pump_lock); return applied; } /* Heap payload for one dispatched apply-pump walk invocation; frees itself * after use, mirroring commit.c's commit_resolve_dispatch_t convention. */ typedef struct { persist_t* p; } apply_pump_dispatch_t; /* Run walk passes until genuinely caught up, using the same recheck-then- * clear idiom commit.c's sweep thread uses (task 1.3) to avoid a lost-wakeup * race between "persist_apply_pump_run just decided it is caught up" and "a * new commit lands right after". Runs on a pool worker thread (or inline as * persist_apply_pump_kick's documented fallback) — never on the calling * (potentially replication-ACK/iomgr) thread when dispatched normally. * * Does NOT immediately retry a failed pass: persist_apply_pump_run() already * logs loudly and leaves last_applied_lsn unadvanced past a read failure, so * an immediate self-retry here would busy-loop forever on the exact same * unresolved head-of-line entry (a real, reproduced hazard — a persist_t * whose committed_lsn is ever advanced without a corresponding real WAL * entry on disk, e.g. a test fixture exercising commit-engine resolve logic * in isolation, spun this into a multi-million-iteration tight loop before * this guard existed). A future, genuinely NEW kick — triggered by a later * real commit-advance event — retries then, at that event's natural cadence, * never in a busy spin. */ static void apply_pump_drain(persist_t* p) { for (;;) { int rc = persist_apply_pump_run(p); atomic_store_explicit(&p->apply_pump_scheduled, 0, memory_order_release); if (rc < 0) return; /* already logged loudly inside persist_apply_pump_run; * do not busy-retry the same unresolved entry */ uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); uint64_t next = atomic_load_explicit(&p->last_applied_lsn, memory_order_relaxed) + 1; if (next > committed) return; /* genuinely caught up: no race window survived open */ int expected = 0; if (!atomic_compare_exchange_strong_explicit( &p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel, memory_order_relaxed)) return; /* a concurrent kick already reclaimed the flag; it will * run its own pass and pick up this same new work */ /* else: we reclaimed the flag ourselves -- loop and run another pass */ } } static int apply_pump_vtask_entry(vtask_t* task) { apply_pump_dispatch_t* d = (apply_pump_dispatch_t*)task->data->u.generic; apply_pump_drain(d->p); free(d); return 0; } void persist_apply_pump_kick(persist_t* p, void* pool_arg) { if (!p || !p->cluster) return; /* bug-1416 (raft-apply-decoupling task 7.2 round 2): defer while this * node's own boot-time persist_recover() replay+verification pass is * still running -- last_applied_lsn is being written UNLOCKED by * replay_wal() during that window (single-threaded-recovery * assumption), and persist_verify_apply_correctness() is about to * compare a live re-read of it (and of the live job table) against a * WAL-scan-derived expectation. Letting this walk run concurrently * would race BOTH: it could advance last_applied_lsn out from under * replay_wal's own plain (non-CAS) store, and it mutates the same job * table persist_verify_apply_correctness re-reads as "actual" -- * producing a false APPLY_DIVERGENT verdict on a perfectly healthy * boot (confirmed live). No work is lost: persist_recover() sets this * flag back to true and issues exactly one catch-up kick right before * it returns, which will walk from last_applied_lsn+1 up through * whatever committed_lsn has grown to by then -- a full, ordered * drain, not a partial one. Intentionally checked with NO lock: a * stale `true` read racing the flag's transition to `false` is * harmless (the walk itself still only advances one entry at a time * under apply_pump_lock, so at worst this call proceeds a few * instructions into a window persist_recover() is about to close, and * the deferred re-kick after recover() still runs); a stale `false` * merely defers one no-op-cost kick to the next real trigger. */ if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire)) return; int expected = 0; if (!atomic_compare_exchange_strong_explicit( &p->apply_pump_scheduled, &expected, 1, memory_order_acq_rel, memory_order_relaxed)) return; /* already scheduled/running -- it will pick up any new work */ vtaskmgr_t* pool = (vtaskmgr_t*)pool_arg; vtask_data_t* data = calloc(1, sizeof(*data)); vtask_t* task = vtask_alloc(); apply_pump_dispatch_t* d = calloc(1, sizeof(*d)); /* Mirrors commit.c's dispatch_continuation fallback exactly: never leave * a scheduled walk silently unrun. pool == NULL is the expected shape of * a commit-only / bare-persist-cluster unit fixture that never calls * commit_set_pool — a real cluster always wires the pool before any * commit activity (cluster_init_phase2), so this fallback path is the * normal, deterministic behavior for that class of test, not a rare * production condition. */ if (pool == NULL || data == NULL || task == NULL || d == NULL) { free(data); free(task); free(d); apply_pump_drain(p); return; } d->p = p; data->vtype = VTASK_GENERIC; data->u.generic = d; vtask_init(task, (vtask_func_t)apply_pump_vtask_entry, data); /* VTASK_LIVE_BLOCK is never admission-gated (same delivery guarantee as * commit.c's own continuation dispatch) — this cannot fast-reject the * way VTASK_REQ_BLOCK could. */ if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { WARN("persist_apply_pump_kick: vtask enqueue failed -- running " "inline (stay alive, never leave a scheduled walk unrun)"); free(data); free(task); free(d); apply_pump_drain(p); } } /* persist_apply_pump_run_sync (2026-07-10, chaotic-soak regression fix, * ROUND 2): synchronously drain the apply pump on THIS thread, GUARANTEEING * last_applied_lsn reaches committed_lsn (as observed at entry) before * returning -- regardless of what any concurrent caller is doing. * * Round 1 of this fix CAS-claimed apply_pump_scheduled and, on losing that * claim to a concurrent caller (e.g. a heartbeat-driven kick that fired * moments before promotion), just returned -- betting the OTHER caller's * async vtask-dispatched drain would finish in time. A second live chaotic * soak proved that bet wrong: job loss persisted even with the environmental * fdatasync noise (bug-924) independently confirmed and eliminated, so this * was a genuine, separate bug in the fix itself, not noise. The caller here * (promotion) cannot outsource its guarantee to a drain it doesn't control * the completion of. * * Fix: do NOT use the CAS-coalescing flag as a "someone else has it, I'm * done" signal -- call persist_apply_pump_run() directly in a loop until * THIS call observes last_applied_lsn caught up to committed_lsn. Multiple * concurrent callers (this one and any vtask-dispatched persist_apply_pump_ * kick) safely interleave via persist_apply_pump_run's own internal * apply_pump_lock and apply_committed_entry's per-LSN idempotency -- at * worst a redundant no-op pass, never lost or duplicated work. Stops (does * not busy-loop forever) on a genuine read failure, exactly mirroring * apply_pump_drain's own error handling: persist_apply_pump_run() already * logs loudly and leaves last_applied_lsn unadvanced on that path. */ void persist_apply_pump_run_sync(persist_t* p) { if (!p || !p->cluster) return; if (!atomic_load_explicit(&p->recovery_complete, memory_order_acquire)) return; for (;;) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); uint64_t applied = atomic_load_explicit(&p->last_applied_lsn, memory_order_relaxed); if (applied >= committed) return; /* caught up -- by us or a concurrent drain */ int rc = persist_apply_pump_run(p); if (rc < 0) return; /* already logged loudly; do not busy-loop on the same * unresolved entry (mirrors apply_pump_drain) */ /* Loop: re-check in case committed_lsn advanced further mid-walk, or * a concurrent drain made partial progress this pass didn't see. */ } } /* Read WAL segment files in LSN order; replay entries with lsn > from_lsn. * * Segment filename format: seg-.wal * Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes). * * CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry * in a segment). If prev_crc mismatches, stop at last good entry and log WARN. * * Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1): * *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped * on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a * short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF, * pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is * about the STRUCTURE of the local log, not about being behind the cluster: a * clean log that simply ends below the cluster high-water is still CLEAN. * * new_job_submits_out (raft-apply-decoupling task 5.1, when non-NULL) is set * to the number of WAL_JOB_SUBMIT entries this call actually dispatched via * apply_committed_entry() that resulted in a GENUINE new job (that function's * return value == 1) — not merely the number of WAL_JOB_SUBMIT entries above * `from_lsn` present in the log. This is the log-derived half of the * post-replay apply-correctness expectation (see apply_expectation_t). * * A WAL_JOB_SUBMIT entry can legitimately be duplicated for the same jobid * (bug-1419): a client's local commit-timeout retry appends a second entry * while the first is still in flight during a quorum-loss window, and the * timed-out entry is never retracted, so both can end up committed if the * first later achieves majority commit anyway. apply_committed_entry() * dedupes this via restore_job()'s -EEXIST and returns 0 (no-op) rather than * 1 (created) for the duplicate — counting raw WAL_JOB_SUBMIT occurrences * instead would tally that duplicate as a second job and spuriously fail the * post-replay comparison against jobmgr's actual growth (APPLY_DIVERGENT). * * Returns number of entries replayed (>= 0) or negative errno on I/O error. */ static int replay_wal(persist_t* p, uint64_t from_lsn, recover_verdict_t* verdict_out, uint64_t* new_job_submits_out) { /* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */ if (verdict_out) *verdict_out = RECOVER_CLEAN; if (new_job_submits_out) *new_job_submits_out = 0; /* --- Collect and sort segment filenames by start LSN --- */ DIR* d = opendir(p->wal_path); if (!d) return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */ /* Collect up to 4096 segment names */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue; seg_lsns[nseg++] = seg_start; } closedir(d); if (nseg == 0) return 0; /* Simple insertion sort — segment count is tiny in practice */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } /* --- Replay entries from each segment --- */ int replayed = 0; uint32_t last_crc = WAL_CRC_NONE; /* job-array-single-wal-record task 7.2 (design.md "Live Apply Resource * Fence", task 4.2's self-flagged gap): once a negative * apply_committed_entry() return is hit during this replay pass, every * SUBSEQUENT entry is skipped for LOGICAL apply purposes -- mirroring * persist_apply_pump_run()'s halt-and-fence branch, which never * dispatches past a failed LSN either. This preserves the strict * sequential-no-skip apply invariant (task 4.1): a later entry may be * causally dependent on the one that never actually applied (e.g. a * job-state fact for an array element that was never constructed). * The PHYSICAL scan (CRC chain, current_lsn, seg_off) below is * deliberately NOT gated on this flag -- see the apply_rc < 0 branch's * doc comment for why current_lsn/last_crc must stay accurate for the * entire on-disk log regardless of the logical apply outcome. */ bool apply_halted = false; for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) { WARN("persist_recover: cannot open segment %s: %s", seg_path, strerror(errno)); continue; } uint32_t seg_off = 0; for (;;) { /* Read header */ wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); if (hn == 0) break; /* clean end of segment (CLEAN) */ if (hn < (ssize_t)sizeof(hdr)) { /* A partial header (or read error) is a SHORT READ — the log * stops mid-entry, so it cannot be reconstructed contiguously * (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */ if (hn < 0) WARN("persist_recover: read error in %s at offset %u: %s", seg_path, seg_off, strerror(errno)); else WARN("persist_recover: short header read in %s at offset %u " "(got %zd of %zu bytes); stopping replay", seg_path, seg_off, hn, sizeof(hdr)); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } /* Detect pre-allocated zero region (no valid entry starts with * all-zero header: lsn == 0 && type == 0 && payload_len == 0). * This is the normal fallocate'd tail after the last valid entry — * a clean stop, NOT corruption (stays RECOVER_CLEAN). */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; /* CRC chain validation: prev_crc must match last_crc in segment */ if (seg_off == 0) { /* First entry in this segment: prev_crc must be WAL_CRC_NONE * (or we accept any value if last_crc is also WAL_CRC_NONE, * e.g. across a segment boundary after checkpoint trim). */ if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && hdr.prev_crc != last_crc) { WARN("persist_recover: CRC chain break at start of %s " "(prev_crc=%08x expected=%08x); stopping replay", seg_path, hdr.prev_crc, last_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } } else { if (hdr.prev_crc != last_crc) { WARN("persist_recover: CRC chain break in %s at offset %u " "(prev_crc=%08x expected=%08x); stopping replay at last good entry", seg_path, seg_off, hdr.prev_crc, last_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } } /* Verify entry CRC */ uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); uint8_t* payload = NULL; if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { close(fd); return -ENOMEM; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(seg_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { WARN("persist_recover: short payload read in %s at offset %u", seg_path, seg_off); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; free(payload); break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { WARN("persist_recover: CRC mismatch in %s at offset %u " "(got=%08x expected=%08x); stopping replay", seg_path, seg_off, hdr.crc, expected_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; free(payload); break; } /* Advance segment-local CRC */ last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back * (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical * entry — including snapshot-covered ones below from_lsn — so the * index spans the full LSN range the reconcile/promotion logic may * query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and * unknown future types (type > WAL_CHECKPOINT): they are skipped * below and are never logical Raft entries, so the live append path * does not record them either (kept symmetric). The raw hdr.term is * stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM * mapping is applied only at read-back. */ if (hdr.type < WAL_CHECKPOINT) record_entry_term(p, hdr.lsn, hdr.term); /* Skip entries already covered by the snapshot */ if (hdr.lsn <= from_lsn) { free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); continue; } /* Skip WAL_CHECKPOINT markers (local bookkeeping only) */ if (hdr.type == WAL_CHECKPOINT) { free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); /* Advance current_lsn past the marker */ if (hdr.lsn + 1 > p->current_lsn) p->current_lsn = hdr.lsn + 1; continue; } /* Unknown future entry types: log and skip */ if (hdr.type > WAL_CHECKPOINT) { WARN("persist_recover: unknown WAL entry type %u at LSN %lu; skipping", hdr.type, (unsigned long)hdr.lsn); free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); continue; } /* Logical entity restore via the shared apply seam * (raft-apply-decoupling task 2.3, design.md "Apply Seam"): the * SAME apply_committed_entry() dispatch the live commit-advance * apply pump will use (task 6.1), so a node's materialized state * is the deterministic result of its committed log regardless of * whether it got there via restart replay or live apply * (Req 3.1-3.4). apply_wal_entry() (this replay loop's previous * per-entry dispatch) is NOT retired by this change — it remains * the live standby catch-up/replicated-apply dispatch used * elsewhere (ha.c); it is simply no longer this loop's call. The * CRC chain + current_lsn remain owned by this replay loop * (current_lsn advanced below; last_crc at line ~1601) — neither * dispatch function ever touched them (bug-778). * * KNOWN HAZARD (tracked in tasks.md Implementation Notes, task * 2.3): this replays the FULL post-checkpoint WAL range on every * restart (last_applied_lsn is not persisted, deliberately * recomputed here), so apply_committed_entry()'s AFTER_EVENT * hooks fire again for entries already applied+notified in a * prior process life. Confirmed inert today: scheduler hooks * aren't registered until a post-boot network JOIN * (on_scheduler_join, sched.c), and apiserver hooks — though * registered before this runs — gate their notify on a * connected endpoint that is empty at first-boot replay time. * MUST be re-verified before tasks 4.1/6.1/6.2/6.3 make this * path broadly live. */ int apply_rc = 0; if (!apply_halted) { apply_rc = apply_committed_entry(p, hdr.type, hdr.lsn, payload, hdr.payload_len); } free(payload); if (apply_halted) { /* job-array-single-wal-record task 7.2: an earlier entry in * THIS replay pass already fenced (see the apply_rc < 0 * branch below) -- this entry and every later one is * skipped for logical apply, exactly like the * WAL_CHECKPOINT/unknown-type skip branches above. Physical * bookkeeping (current_lsn/seg_off) still runs * unconditionally below. */ } else if (apply_rc < 0) { /* job-array-single-wal-record task 7.2 (design.md "Live * Apply Resource Fence", task 4.2's self-flagged residual * gap): mirror persist_apply_pump_run()'s halt-and-fence * branch exactly. This is never a content-derived failure * (Req 9.1-9.3's pre-commit validation already made those * impossible post-commit) -- it is the SAME node-local * resource-exhaustion class (Req 9.4) the live pump already * guards against, just hit during boot/restart replay * instead of live apply. Do NOT advance last_applied_lsn * past this LSN (skip that store entirely -- last_applied_lsn * stays parked at this entry's predecessor) and do NOT tally * this entry's job count; engage the SAME * persist_mark_apply_resource_fenced() primitive the live * path uses -- reused verbatim, no parallel mechanism. This * closes task 4.2's documented residual gap: without this, a * node that crashes while fenced and re-hits the identical * resource-exhaustion condition on restart would silently * discard the failure here (as this branch previously did, * unconditionally advancing last_applied_lsn regardless of * apply_rc's sign) and boot up looking fully promotable * while actually divergent/behind. * * Engaging synchronously HERE, inside this boot-time call * (persist_recover(), called synchronously from * cluster_init_phase2() before it returns), matters: * persist_apply_pump_kick()'s post-recovery catch-up kick * dispatches ASYNCHRONOUSLY onto a pool worker vtask (or * only runs inline as its own documented fallback) -- relying * on that kick alone to eventually re-fail and set the fence * would leave a real window, between persist_recover() * returning and that async retry actually running, where a * promotion-gate check could observe "not fenced" and * wrongly treat this node as eligible. * * KNOWN BOUNDARY (task 7.2 adversarial review, NOT fixed by * this task -- ha.c change required, out of this function's * scope): this fence, once engaged here, protects every * SUBSEQUENT promotion evaluation in this process's life -- * a later election's raft_cb_become_leader() gate (ha.c:1337 * reads persist_apply_resource_fenced(), inside the * eligibility block at ha.c:1303-1408) and the periodic * re-alert timer (promotion_fence_timeout_fn, ha.c ~876) * both correctly observe it. It does NOT protect this SAME * boot's own first-ever promotion evaluation: on a node's * first-ever election evaluation in its process's life (i.e. * BEFORE cluster_init_phase2() has ever run), ha.c's * raft_cb_become_leader() runs its eligibility gate at * ha.c:1303-1408 strictly BEFORE the is_first_boot block at * ha.c:1427-1439 that calls cluster_init_phase2() -- which is * what calls persist_recover()/replay_wal() (cluster.c:1482) * and is what would engage this fence for the first time. * There is no re-check of persist_apply_resource_fenced() * after cluster_init_phase2() returns; the code proceeds * straight to CTRL_PRIMARY. So a fence engaged by THIS SAME * first replay is invisible to the one promotion check * gating that specific boot -- exactly the "all nodes * restart simultaneously" race this whole task exists to * guard against. Closing this requires an ha.c change (a * re-check of persist_apply_resource_fenced() after * cluster_init_phase2() returns, or reordering recovery * ahead of the promotion decision), which is out of this * spec's current scope -- see tasks.md's task 7.2 * Implementation Notes for the full finding. */ ERROR("replay_wal: apply_committed_entry failed for " "committed entry lsn=%" PRIu64 " wal_type=%" PRIu32 " (rc=%d) during boot/restart replay -- halting this " "replay's LOGICAL apply at this position (the entry " "stays committed and will be retried by the live apply " "pump's post-recovery kick, from the same position); " "Live Apply Resource Fence ENGAGED " "(job-array-single-wal-record Req 9.4/9.5)", hdr.lsn, hdr.type, apply_rc); persist_mark_apply_resource_fenced(p, hdr.lsn); apply_halted = true; } else { /* raft-apply-decoupling task 5.1: track the replay-derived * LOGICAL apply cursor separately from current_lsn (the * physical WAL position owned by this loop, bug-778) -- * last_applied_lsn is the "highest entry actually dispatched * via apply_committed_entry()" position the post-replay * verification compares against. Segments are scanned in * ascending LSN order (sorted above) and entries within a * segment are sequential, so a plain store (no monotonic max * needed) is correct here. */ atomic_store_explicit(&p->last_applied_lsn, hdr.lsn, memory_order_relaxed); /* Log-derived half of the post-replay apply-correctness * expectation (Req 8.1): every WAL_JOB_SUBMIT that reaches * this point is, by construction, above the loaded * snapshot's checkpoint_lsn (the from_lsn skip above already * filtered out anything the snapshot covers) — so it names a * job that must now exist in the live job table that did not * exist before this replay pass began. A duplicate * WAL_JOB_SUBMIT for an already-existing jobid (bug-1419: * timeout-then-late-commit retry) is a legitimate no-op — * apply_committed_entry() returns 0 for it, not 1 — so it * must NOT be tallied here or the post-replay comparison * against jobmgr's real growth spuriously fails * (APPLY_DIVERGENT). */ if (hdr.type == WAL_JOB_SUBMIT && new_job_submits_out && apply_rc == 1) (*new_job_submits_out)++; /* job-array-single-wal-record task 6.0a: the array analog of * the WAL_JOB_SUBMIT tally immediately above. * apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case * (restore_job_array()'s return, generalized to N elements) * returns apply_rc == the genuine count of NEWLY created * elements (1..array_size) on a real apply, or 0 for an * idempotent whole-record re-apply (every index already * existed -- restore_job_array() returned -EEXIST, which * this same dispatch maps to 0, mirroring WAL_JOB_SUBMIT's * own dedup return). Tallying apply_rc (not a flat +1) is * required here, unlike the single-job case above, because * one WAL_JOB_ARRAY_SUBMIT entry can materialize many jobmgr * entries at once; guarding on apply_rc > 0 excludes the * idempotent 0 case from double-counting a re-apply, exactly * as apply_rc == 1 excludes WAL_JOB_SUBMIT's duplicate-jobid * case (bug-1419) above. Without this branch, replay_wal() * silently undercounts every array record's real N-element * jobmgr growth to 0, and persist_verify_apply_correctness() * then reports a false APPLY_DIVERGENT verdict that engages * the PERMANENT, non-self-healing * persist_recover_loud_fence_if_unrecoverable() fence * (live-reproduced task 6.1 soak, 2026-07-15). */ if (hdr.type == WAL_JOB_ARRAY_SUBMIT && new_job_submits_out && apply_rc > 0) (*new_job_submits_out) += (uint64_t)apply_rc; replayed++; } /* Advance current_lsn to one past this entry -- unconditional, * regardless of the LOGICAL apply outcome above (mirrors the * WAL_CHECKPOINT/unknown-type skip branches earlier in this * loop, which also advance current_lsn without ever dispatching * through apply_committed_entry()). job-array-single-wal-record * task 7.2: the physical WAL scan must account for every * on-disk entry so current_lsn/last_crc stay correct for any * future append this process makes (as a promoted primary, or * as a replication standby receiving new entries) -- halting * this advance too (treating a fenced entry as physically * absent) would understate the true on-disk tail and risk a * later append colliding with/overwriting already-durable * entries beyond the fenced position (persist_recover()'s own * "Round-2 fix" comment above, for the durable_lsn/apply_cursor * split, already establishes that current_lsn and * last_applied_lsn are expected to diverge -- this is the same * principle, applied to a halted apply instead of a skipped * checkpoint marker). */ if (hdr.lsn + 1 > p->current_lsn) p->current_lsn = hdr.lsn + 1; seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } /* Update last_crc so the next WAL write chains correctly */ pthread_mutex_lock(&p->lock); p->last_crc = last_crc; pthread_mutex_unlock(&p->lock); return replayed; } /* raft-apply-decoupling task 5.1: cheap read-lock count of c->jobmgr's live * entries, mirroring collect_job_frames' iteration idiom (this file, above) * without collecting or filtering anything. Used only to derive the * post-replay materialized-entity expectation (never on a hot path). Safe on * a NULL cluster or jobmgr (a bare unit-test persist_t) — returns 0. */ static uint64_t count_jobmgr_entries(cluster_t* c) { if (!c || !c->jobmgr) return 0; lock(c->jobmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->jobmgr); if (irc < 0) { unlock(c->jobmgr); return 0; /* empty table */ } uint64_t n = 0; for (;;) { job_t* j = NULL; int nrc = iter_next(&it, (void**)&j); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (j == NULL) continue; n++; } iter_destroy(&it); unlock(c->jobmgr); return n; } /* persist_verify_apply_correctness — see persist.h for the full contract * doc (raft-apply-decoupling task 5.1, Req 8.1/8.2). Pure comparison: no * logging, no fence side effects. */ apply_verdict_t persist_verify_apply_correctness( persist_t* p, const apply_expectation_t* exp, uint64_t jobmgr_count_before_replay) { if (!p || !exp) return APPLY_CLEAN; /* defensive: nothing to verify */ uint64_t actual_last_applied = atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire); if (actual_last_applied != exp->expected_last_applied_lsn) return APPLY_DIVERGENT; /* Entity-materialization half of the check: skip gracefully when there * is no live cluster to query — mirrors apply_committed_entry()'s own * "nothing to dispatch into" convention for a bare unit-test persist_t. * Only the position check above applies in that case. */ if (!p->cluster || !p->cluster->jobmgr) return APPLY_CLEAN; uint64_t jobmgr_count_after = count_jobmgr_entries(p->cluster); if (jobmgr_count_after < jobmgr_count_before_replay) return APPLY_DIVERGENT; /* table shrank — impossible on a clean replay */ uint64_t actual_new_jobs = jobmgr_count_after - jobmgr_count_before_replay; if (actual_new_jobs != exp->expected_new_job_count) return APPLY_DIVERGENT; return APPLY_CLEAN; } apply_verdict_t persist_apply_verdict(persist_t* p) { return p ? p->apply_verdict : APPLY_CLEAN; } uint64_t persist_last_applied_lsn(persist_t* p) { return p ? atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire) : 0; } /* bug-1416 (raft-apply-decoupling task 7.2 round 2) -- see persist.h's * doc comment on the recovery_complete field for the full contract. */ bool persist_recovery_complete(persist_t* p) { return !p || atomic_load_explicit(&p->recovery_complete, memory_order_acquire); } /* bug-1416 round 2 -- see persist.h's doc comment on this function for the * full rationale (closes the gap between persist_init() and the FIRST * persist_recover() call, which persist_recover()'s own false/true bracket * cannot see). */ void persist_mark_recovery_pending(persist_t* p) { if (!p) return; atomic_store_explicit(&p->recovery_complete, false, memory_order_release); } /* ------------------------------------------------------------------ */ /* Public: persist_recover */ /* ------------------------------------------------------------------ */ int persist_recover(persist_t* p) { /* bug-1416 (raft-apply-decoupling task 7.2 round 2): close the boot-time * race window for the FULL duration of this call, not just the position * check inside persist_verify_apply_correctness -- see the * recovery_complete field's doc comment (persist.h) for the complete * rationale. Held false until the natural (success) return at the * bottom of this function, which restores it and issues one deliberate * catch-up kick. The two hard-I/O-error early returns below * (load_snapshot / replay_wal failure) deliberately do NOT restore it: * those are pre-existing, rare disk-I/O-level failures the caller * already treats as fatal to this boot attempt (cluster_init_phase2 * tears down persist_t on a nonzero return) -- leaving the gate closed * is the conservative, fail-static choice (never let the live apply * pump or live-replication path mutate state on top of a persist_t * whose own recovery never finished), not a new hazard. */ atomic_store_explicit(&p->recovery_complete, false, memory_order_release); /* raft-apply-decoupling task 7.4 perf-fix (round 2): invalidate the apply * pump's segment-lookup cache before replay -- the cached fd/offset are * only valid relative to the pre-recovery on-disk state, and replay below * can change checkpoint_lsn/current_lsn. Doing this first covers every * return path below (including the early I/O-failure returns), so the * first post-recovery pump kick always falls back to full segment * rediscovery from the correct, recovered position -- never trusting a * stale pre-recovery cursor. Wrapped in apply_pump_lock (matching * persist_truncate_wal_above's identical invalidation site below) so the * cache fields' documented invariant -- "guarded ENTIRELY by * apply_pump_lock, never p->lock" (persist.h) -- holds unconditionally * here too, rather than resting on the implicit fact that no live pump * walk can be racing this boot-time call. No caller of persist_recover() * holds apply_pump_lock (its sole production call site, cluster.c boot * sequence, runs before the pump is ever kicked), so this cannot * deadlock. */ pthread_mutex_lock(&p->apply_pump_lock); if (p->apply_pump_cache_fd >= 0) { close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } pthread_mutex_unlock(&p->apply_pump_lock); /* --- Step 1: find latest snapshot (highest LSN filename) --- */ char snap_file[4096] = ""; uint64_t snap_lsn = 0; DIR* d = opendir(p->snap_path); if (d) { struct dirent* e; while ((e = readdir(d)) != NULL) { size_t len = strlen(e->d_name); if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue; uint64_t lsn = 0; if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1) continue; if (lsn > snap_lsn) { snap_lsn = lsn; snprintf(snap_file, sizeof(snap_file), "%s/%s", p->snap_path, e->d_name); } } closedir(d); } /* --- Step 2: load snapshot (if any) --- */ if (snap_lsn > 0 && snap_file[0] != '\0') { /* Local recovery doesn't need the entities-restored indicator to * decide behavior: the WAL replay in Step 3 applies everything * above checkpoint_lsn regardless, unchanged by ha-entity-snapshots. * It IS consumed for observability parity with the wire receive * path (ha_handle_snapshot_chunk, task 6.1) — a rolling-upgrade * restart against a pre-feature (legacy, position-only) local * checkpoint is an expected rollout state, not an error. */ bool entities_restored = false; int rc = load_snapshot(p, snap_file, snap_lsn, &entities_restored); if (rc != 0) { ERROR("persist_recover: failed to load snapshot %s: %s", snap_file, strerror(-rc)); return rc; } /* checkpoint_lsn and current_lsn set by load_snapshot */ if (entities_restored) { INFO("persist_recover: snapshot %s loaded lsn=%" PRIu64 " with full job/queue entity restoration", snap_file, p->checkpoint_lsn); } else { INFO("persist_recover: snapshot %s lsn=%" PRIu64 " is a legacy " "position-only checkpoint (no entity content) — continuing " "via normal WAL replay", snap_file, p->checkpoint_lsn); } } /* raft-apply-decoupling task 5.1 (Req 8.1): seed the replay-derived * apply cursor from the loaded baseline (checkpoint_lsn stays 0 when no * snapshot was found) so that, on a CLEAN replay with no new entries * beyond the snapshot, the cursor already sits at the recovered tip — * mirroring how current_lsn is seeded by load_snapshot. Step 3's * replay_wal advances it further for every entry actually dispatched * above this baseline. Also capture the live job table's size BEFORE * replay runs: the post-replay verification below needs this baseline to * isolate the jobs replay itself materializes from anything the snapshot * already restored. */ atomic_store_explicit(&p->last_applied_lsn, p->checkpoint_lsn, memory_order_release); uint64_t jobmgr_count_before_replay = count_jobmgr_entries(p->cluster); /* --- Step 3: replay WAL entries after checkpoint_lsn --- */ /* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1): * replay distinguishes a clean log end from a corruption-truncated end. * We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do * NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean * replay over a loaded snapshot-sentinel (no entries below the sentinel) is * CLEAN — the sentinel established the committed position; replay only adds * entries above it. */ recover_verdict_t verdict = RECOVER_CLEAN; uint64_t new_job_submits = 0; int replayed = replay_wal(p, p->checkpoint_lsn, &verdict, &new_job_submits); if (replayed < 0) return replayed; p->recovery_verdict = verdict; if (verdict == RECOVER_TRUNCATED) WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally " "corrupt (CRC break / mismatch / short read); member marked " "recovery-truncated (non-promotable until caught up)"); /* --- Step 3b: re-derive the durable floor from the recovered WAL tip * (ha-commit-durability task 3.4, Req 2.3) --- * * Publish the member's durable floor (the durable last-log index consumed * by the Raft freshness gate, task 4.2) as the highest INTACT replayed LSN. * replay_wal advanced current_lsn to one past the last intact entry — incl. * stopping at a corruption boundary on TRUNCATED — so current_lsn - 1 is * exactly that intact WAL tip, and a snapshot-sentinel load leaves it at the * sentinel/replayed tip. We store it into durable_lsn because the recovered * on-disk entries are GENUINELY durable (each was CRC-validated and read * from stable storage by replay_wal), so the freshness advertisement is now * backed by durable state rather than a volatile in-memory position. * * Crux (Req 2.3 — a truncated member cannot over-advertise): this is an * ASSIGN to the recovered tip, NOT a max() with whatever durable_lsn held. * A fresh recover starts from durable_lsn=0 (persist_init), so on a * truncating restart the floor lands at the LOWER intact tip — never a * stale higher in-memory value and never the commit position * (committed_lsn). The recovered tip is, by construction, <= current_lsn-1, * preserving the durable_lsn <= current_lsn-1 invariant. The post-election * safe-commit point continues to come from voter replies, not this floor * (CLAUDE.md). */ uint64_t recovered_tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; atomic_store_explicit(&p->durable_lsn, recovered_tip, memory_order_release); /* raft-apply-decoupling task 5.1 (Req 8.1/8.2): post-replay apply- * correctness verification, reported ALONGSIDE the structural * recovery_verdict set above — neither implies the other. See * persist_verify_apply_correctness's doc comment for exactly what is * compared. Routing an APPLY_DIVERGENT verdict into the existing * non-promotable loud-fence mechanism is task 5.2; this only classifies * and logs. * * Round-2 fix: the expected position MUST be the LOGICAL apply cursor * replay_wal() already maintains in p->last_applied_lsn — NOT * recovered_tip (current_lsn - 1, the PHYSICAL WAL-scan tip used above * for durable_lsn). current_lsn advances for every WAL entry replay_wal * scans, including WAL_CHECKPOINT markers and unknown future types, * neither of which is ever routed through apply_committed_entry() (see * the skip branches in replay_wal). last_applied_lsn only advances when * a real dispatch happens. If the highest-LSN entry above checkpoint_lsn * is a trailing WAL_CHECKPOINT marker — the exact state left behind by a * crash between persist_write_checkpoint_marker() (step 1, durably * fsynced) and write_snapshot() (step 3b, persists the advanced * checkpoint_lsn) — recovered_tip over-counts past it while * last_applied_lsn correctly does not, and comparing against * recovered_tip previously produced a false APPLY_DIVERGENT for a * perfectly clean recovery (violates Req 8.2). */ uint64_t apply_cursor = atomic_load_explicit(&p->last_applied_lsn, memory_order_acquire); apply_expectation_t apply_exp = { .expected_last_applied_lsn = apply_cursor, .expected_new_job_count = new_job_submits, }; p->apply_verdict = persist_verify_apply_correctness( p, &apply_exp, jobmgr_count_before_replay); if (p->apply_verdict == APPLY_DIVERGENT) { ERROR("persist_recover: APPLY_DIVERGENT — materialized state does " "not match the committed log tip (expected last_applied_lsn=%" PRIu64 " new_job_count=%" PRIu64 "); a hidden apply defect is " "present (ha-commit-durability recover_verdict=%s)", apply_exp.expected_last_applied_lsn, apply_exp.expected_new_job_count, (verdict == RECOVER_TRUNCATED) ? "RECOVER_TRUNCATED" : "RECOVER_CLEAN"); /* raft-apply-decoupling task 5.2 (Req 8.3/8.4): route the DIVERGENT * verdict through the EXISTING non-promotable loud-fence mechanism * (persist_recover_loud_fence_if_unrecoverable + the sticky * recovery_unrecoverable flag) instead of a new fence path. * apply_divergent=true fences UNCONDITIONALLY (see * persist_recover_decision), so the peer/HWM fields below are * inert placeholders here -- they are not yet known at this * boot-time replay call site (unlike the structural verdict's own * fence check, which stays deferred to promotion time in ha.c, * where live peer/HWM context exists). verdict/recovered_tip are * passed through only for accurate, consistent logging. The member * stays ALIVE, non-promotable, and keeps retrying catch-up -- it * never exits (controller-never-self-terminate invariant, * CLAUDE.md). */ recover_policy_inputs_t apply_fence_in = { .verdict = verdict, .durable_below_hwm = false, .healthy_peer_reachable = false, .catch_up_succeeded = false, .durable_floor_lsn = recovered_tip, .cluster_hwm_lsn = 0, .apply_divergent = true, }; persist_recover_loud_fence_if_unrecoverable(p, &apply_fence_in); } else INFO("persist_recover: APPLY_CLEAN — materialized state matches the " "committed log tip (last_applied_lsn=%" PRIu64 ")", apply_cursor); /* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */ /* bug-1416: replay + verification is fully complete -- safe to let the * live apply pump and the standby live-replication apply path resume * mutating state (see the recovery_complete field's doc comment). * Restored unconditionally here regardless of the verdict computed * above: a genuine APPLY_DIVERGENT already fenced this member via * persist_recover_loud_fence_if_unrecoverable (non-promotable, stays * alive, keeps retrying) -- that sticky fence and its clearing logic * (task 5.3) are untouched by this flag, which governs ONLY whether * in-memory apply may proceed, not whether this member may lead. Issue * one deliberate catch-up kick for anything that committed locally (or * arrived via live replication and was durably persisted but had its * entity-apply deferred, see ha_apply_single_entry / ha_apply_term_ * entry_via_library) while this pass ran -- persist_apply_pump_run's * own walk drains fully from last_applied_lsn+1 through the current * committed_lsn, so this is a complete, ordered catch-up, not a partial * one; a harmless no-op if nothing queued. */ atomic_store_explicit(&p->recovery_complete, true, memory_order_release); /* n_vcpus==0 is this codebase's own established "not yet initialized" * sentinel for vtaskmgr_t (mirrors cluster_init_phase2's own * `if (c->vtaskmgr.n_vcpus == 0) vtaskmgr_init(...)` guard) -- a real * cluster always has n_vcpus > 0 by the time persist_recover() runs * (vtaskmgr_init happens earlier in myapp_run()/phase2), while a bare * recovery-focused unit-test cluster_t (none of which call * vtaskmgr_init, confirmed by inspection) leaves it zeroed. Passing a * pointer to an uninitialized vtaskmgr_t into vtask_enq_classified * would be unsafe, so fall back to persist_apply_pump_kick's own * existing, already-exercised NULL-pool inline path in that case -- * identical to how every apply-pump test fixture in this spec already * exercises the pump. */ void* pool = (p->cluster && p->cluster->vtaskmgr.n_vcpus > 0) ? &p->cluster->vtaskmgr : NULL; persist_apply_pump_kick(p, pool); return 0; } /* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */ recover_verdict_t persist_recovery_verdict(persist_t* p) { return p ? p->recovery_verdict : RECOVER_CLEAN; } bool persist_recovery_truncated(persist_t* p) { return p && p->recovery_verdict == RECOVER_TRUNCATED; } /* ------------------------------------------------------------------ * * Boot-policy loud-fence decision (ha-commit-durability task 3.3, Req 2.4; * never-self-exit rework 2026-06-18). * * Implements the FENCE_UNRECOVERABLE terminal of the Boot Policy Decision Table * (design.md). The decision is a PURE function of its inputs; the action (loud * operator-visible log + set the sticky non-promotable flag) is separated into * persist_recover_loud_fence_if_unrecoverable so the decision can be unit-tested * in isolation. A reachable healthy peer NEVER yields FENCE_UNRECOVERABLE * (rows 3, 5); it is reserved for a member that cannot recover locally AND * cannot catch up (no healthy peer, rows 4, 4b). * * CRITICAL: this terminal is a LOUD, PERMANENT, NON-promotable fence — the * process STAYS ALIVE and keeps retrying catch-up. There is NO abort()/exit(): * a controller MUST NEVER self-terminate as a designed response to an * operational condition (user invariant, 2026-06-18). A self-exited controller * needs manual admin recovery and degrades quorum meanwhile; one that stays * alive and keeps retrying self-heals when a peer returns (Req 2.6). * ------------------------------------------------------------------ */ recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in) { if (in == NULL) return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */ /* raft-apply-decoupling task 5.2 (Req 8.3/8.4): an APPLY_DIVERGENT * post-replay verdict is a NEW INPUT to this same decision table, not a * new fence mechanism (design.md "Restart Replay & Verification"). It * fences UNCONDITIONALLY -- every other field below is irrelevant once * this is true. Unlike a structurally truncated/behind log (rows 2/3/5), * where a healthy peer's missing entries can repair the gap and legitimately * downgrade the outcome to a plain FENCE-then-catch-up, an apply-time * defect means the LOCAL apply path itself produced the wrong * materialized state from bytes it already had -- streaming more entries * from a peer does not prove that path is now trustworthy, so this is * never downgraded by peer reachability or catch-up success. */ if (in->apply_divergent) return RECOVER_DECISION_FENCE_UNRECOVERABLE; bool truncated = (in->verdict == RECOVER_TRUNCATED); /* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster * high-water mark is a normal promotable member — no fence. */ if (!truncated && !in->durable_below_hwm) return RECOVER_DECISION_ELIGIBLE; /* From here the member is truncated OR behind the HWM — it must NOT lead * as-is. A truncated log is by definition not provably at/above the HWM * (design.md note), so it is treated as behind for fence purposes. */ /* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain * eligibility (task 4.3). A reachable peer NEVER yields the unrecoverable * fence — even for a truncated log or a legitimate snapshot-sentinel. If * catch-up has already succeeded the member is eligible; otherwise it stays * fenced (4.3 drives the catch-up + release). */ if (in->healthy_peer_reachable) { if (in->catch_up_succeeded) return RECOVER_DECISION_ELIGIBLE; return RECOVER_DECISION_FENCE; } /* No healthy peer. If catch-up did not reach a committed-consistent state, * the member can neither recover locally nor catch up right now — rows 4 & * 4b: a LOUD, PERMANENT, NON-promotable fence (NOT process exit). It stays * alive and keeps retrying; the fence releases when a peer returns * (Req 2.4/2.6). */ if (!in->catch_up_succeeded) return RECOVER_DECISION_FENCE_UNRECOVERABLE; /* No peer but catch-up is reported succeeded (e.g. already at/above after a * local catch-up): eligible. */ return RECOVER_DECISION_ELIGIBLE; } bool persist_recovery_unrecoverable(persist_t* p) { return p && atomic_load_explicit(&p->recovery_unrecoverable, memory_order_acquire); } void persist_clear_recovery_unrecoverable(persist_t* p) { if (!p) return; /* Clear only on a genuine transition so the loud "fence released" log fires * once, not on every catch-up tick. */ if (atomic_exchange_explicit(&p->recovery_unrecoverable, false, memory_order_acq_rel)) { INFO("persist_recover: unrecoverable fence RELEASED — member reached a " "committed-consistent state via catch-up; promotable again " "(ha-commit-durability Req 2.6)"); } } /* job-array-single-wal-record task 4.2 (design.md "Live Apply Resource * Fence", Req 9.4/9.5). See the apply_resource_fenced_lsn field's doc * comment (persist.h) and this trio's own doc comment (persist.h) for the * full rationale and the deliberate distinction from * persist_recovery_unrecoverable/persist_clear_recovery_unrecoverable just * above -- that boot-only mechanism is NOT reused here on purpose. */ bool persist_apply_resource_fenced(persist_t* p) { return p && atomic_load_explicit(&p->apply_resource_fenced_lsn, memory_order_acquire) != 0; } void persist_mark_apply_resource_fenced(persist_t* p, uint64_t lsn) { if (!p) return; atomic_store_explicit(&p->apply_resource_fenced_lsn, lsn, memory_order_release); } void persist_clear_apply_resource_fence(persist_t* p) { if (!p) return; /* Clear only on a genuine transition so the self-heal log fires once per * episode, not on every ordinary successful apply (the overwhelming * common case: nothing was fenced, so this is a cheap no-op). */ uint64_t prev = atomic_exchange_explicit(&p->apply_resource_fenced_lsn, 0, memory_order_acq_rel); if (prev != 0) { INFO("persist_apply_pump_run: Live Apply Resource Fence RELEASED — " "lsn=%" PRIu64 " applied successfully on retry; promotable " "again (job-array-single-wal-record Req 9.5)", prev); } } recover_decision_t persist_recover_loud_fence_if_unrecoverable(persist_t* p, const recover_policy_inputs_t* in) { recover_decision_t decision = persist_recover_decision(in); if (decision != RECOVER_DECISION_FENCE_UNRECOVERABLE) return decision; /* Loud, operator-visible alert. The two triggers reaching this terminal * are NOT logged with the same clause (round-2 fix, raft-apply- * decoupling task 5.2 remediation): * * - apply_divergent: fences UNCONDITIONALLY (persist_recover_decision), * evaluated at boot-replay time BEFORE peer/HWM context exists (see * the call site in persist_recover()). `healthy_peer_reachable` and * `cluster_hwm_lsn` are inert placeholders there, not real facts — * asserting "no healthy peer reachable" or a durable_floor_lsn < hwm * comparison for this trigger would report unevaluated-or-false * information to the operator (round-1 review: the placeholders can * make the comparison numerically impossible, e.g. "2 < 0"). This * branch names the real reason (apply-correctness divergence) only. * - structural (truncated / behind-HWM): reaches this terminal only * via the pre-existing "no healthy peer reachable" branch in * persist_recover_decision, so the peer-reachability and * durable_floor_lsn/hwm comparison ARE guaranteed true here — that * clause is unchanged from before this task. */ if (in->apply_divergent) { ERROR("persist_recover: UNRECOVERABLE — post-replay apply-correctness " "verification is DIVERGENT (materialized state does not match " "the committed log tip, raft-apply-decoupling Req 8.3); a local " "apply defect cannot be proven fixed by peer reachability or " "catch-up alone. Staying ALIVE and PERMANENTLY non-promotable " "(refusing to serve divergent state) and RETRYING catch-up " "indefinitely — operator attention is needed to diagnose the " "apply path (raft-apply-decoupling Req 8.3/8.4)"); } else { const char* why = (in->verdict == RECOVER_TRUNCATED) ? "recovery-truncated log (structural corruption)" : "log behind the cluster committed high-water mark"; ERROR("persist_recover: UNRECOVERABLE — %s and no healthy peer reachable to " "catch up; truncation boundary durable_floor_lsn=%" PRIu64 " < cluster committed high-water hwm=%" PRIu64 ". Staying ALIVE and " "PERMANENTLY non-promotable (refusing to serve divergent state) and " "RETRYING catch-up indefinitely — will self-heal when a healthy peer " "returns; operator attention may be needed " "(ha-commit-durability Req 2.4)", why, in->durable_floor_lsn, in->cluster_hwm_lsn); } /* Set the sticky non-promotable flag so the promotion gate refuses * leadership as belt-and-suspenders until catch-up clears it (Req 2.4/2.6). * p may be NULL for pure-classification callers that only want the loud log. */ if (p != NULL) atomic_store_explicit(&p->recovery_unrecoverable, true, memory_order_release); return RECOVER_DECISION_FENCE_UNRECOVERABLE; } /* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that * entry is already durable on local stable storage (ha-commit-durability task * 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the * commit engine's durable self-cap (commit.c majority_candidate_locked): a * majority ACK no longer implies the local copy is fsync'd, because group- * commit (task 2.1) deferred the append fsync to the demand-driven flusher. So * advancing committed_lsn here on the ACK alone would reintroduce the bug-661 * ACK-without-durability hazard — committed could be durable on a single * member and lost across failover. We gate the store on written_lsn <= * durable_lsn so committed_lsn ⇒ durable-on-this-member on every path. * committed_lsn is monotonic, so a max() store; the durable cap can only delay * the advance until the flusher syncs the entry (the durable-advance commit * wake re-drives the engine path; the legacy path is re-evaluated on the next * sync_append). */ static void sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn) { uint64_t durable = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); /* DURABILITY PRECONDITION: never advance committed past what is durable * locally. The LSN-0 first-entry edge stays valid (0 <= 0). */ if (written_lsn > durable) return; uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); if (written_lsn <= cur) return; /* monotonic: committed_lsn never regresses */ atomic_store_explicit(&p->committed_lsn, written_lsn, memory_order_release); #ifdef XCE_TESTING /* INVARIANT (debug guard): committed_lsn must never exceed durable_lsn on * any commit advance. A future caller reintroducing ACK-without-durability * trips this. */ assert(atomic_load_explicit(&p->committed_lsn, memory_order_acquire) <= atomic_load_explicit(&p->durable_lsn, memory_order_acquire)); #endif } int persist_wal_sync_append(persist_t* p, uint32_t type, const void* payload, uint32_t len, int* sync_rc_out) { /* ha-term-safety task 3.1: the primary's WAL_JOB_SUBMIT append is routed * through the raft library (ha_primary_log_append → raft_log_append) so the * library stamps the term + assigns the LSN (firing on_persist_entry → the * WAL write), keeping p->current_lsn co-tracked with s->log_len; the existing * majority-commit barrier below (repl_wait_majority_ack) is UNCHANGED and runs * after the append. Non-raft road / engine-less tests (library not LEADER) * fall back inside ha_primary_log_append to persist_wal_async. Standalone * persist (p->cluster == NULL, lib-level tests) keeps the direct append. */ int rc; if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, type, payload, len, NULL); else rc = persist_wal_async(p, type, payload, len); if (rc != 0) { if (sync_rc_out) *sync_rc_out = rc; return rc; } /* Majority-commit replication (Req 5.1–5.3; ha-commit-engine Req 1.1–1.3). * persist_wal_async appends to the page cache and signals the demand-driven * flusher; it no longer fsyncs inline (group-commit, task 2.1). Durability * is therefore established by the flusher advancing durable_lsn, and a * commit position advances only once the entry is durable locally — the * engine path via the durable self-cap (commit.c), the legacy path below * via sync_append_legacy_commit_store's written_lsn <= durable_lsn gate. * We use current_lsn - 1 because persist_wal_async already incremented it. */ uint64_t written_lsn; pthread_mutex_lock(&p->lock); written_lsn = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); int sync_err; /* Engine-wired path (ha-commit-engine task 5.3): delegate the commit * decision to the commit engine through the walAck adapter. The engine * is the single committed_lsn writer (no store here), and its * n_peers<=1 self-quorum replaces the old has_standbys → -ENODEV * degraded special case (Req 6.3). * * LSN-0 edge (ACCEPTED interim behavior — spec Implementation Notes): * written_lsn is passed unconditionally, but the FIRST-EVER WAL entry * of a fresh cluster is LSN 0 and commit_submit rejects lsn==0, so the * adapter's entry_lsn != 0 guard routes that one entry to the legacy * branch below. This happens once per cluster lifetime; * ha-single-leader-role's promotion no-op entry eliminates it in * practice. */ if (p->repl.ce != NULL && written_lsn != 0) { sync_err = repl_wait_majority_ack(&p->repl, written_lsn, p->config.sync_timeout_ms, (uint8_t)OPCode_walAck); if (sync_err != 0) CTRL_METRIC_INC(ctrl_repl_degraded_count); } else { /* Residual non-engine branch: engine-less (ce == NULL) unit-test * contexts plus the LSN-0 first entry. The legacy connectivity- * based counting was retired by ha-commit-barrier-rollout task 2.2: * repl_wait_majority_ack now confirms LSN 0 immediately (documented * once-per-lifetime edge) and surfaces engine-less walAck waits as * -EINVAL via commit_submit, so the committed_lsn store below only * fires for the LSN-0 case (a no-op store of 0). */ mylock_lock(&p->repl.lock); int has_standbys = (p->repl.nstandbys > 0); mylock_unlock(&p->repl.lock); if (!has_standbys) { /* No standbys registered → degraded mode (no quorum to wait for) */ sync_err = -ENODEV; } else { sync_err = repl_wait_majority_ack(&p->repl, written_lsn, p->config.sync_timeout_ms, (uint8_t)OPCode_walAck); if (sync_err == 0) /* Durable-gated commit advance (task 2.4): advance committed_lsn * only if written_lsn is durable locally — never on the ACK * alone (group-commit removed the inline append fsync). */ sync_append_legacy_commit_store(p, written_lsn); else CTRL_METRIC_INC(ctrl_repl_degraded_count); } } if (sync_rc_out) *sync_rc_out = sync_err; /* sync_err < 0 — replication failed. * -ENODEV: legacy degraded mode (no standbys, engine-less) — acceptable * for forward progress. * -ETIMEDOUT: no per-entry majority within the deadline — the write * must be rejected to preserve the majority-consistency * invariant. * -XCE_ERR_NOT_READY: engine admission cap exhausted — retriable * rejection. * Timeout and admission rejection propagate to the job-submit error * path (ha-commit-engine Req 1.1, 4.4). */ if (sync_err == -ETIMEDOUT || sync_err == -XCE_ERR_NOT_READY) return sync_err; return 0; } #ifdef XCE_TESTING /* Test seam (task 2.4): invoke the EXACT durable-gated commit store that the * legacy sync_append branch uses on a majority ACK, so a test can prove the * ACK-without-durability hazard is closed without standing up the replication * wire. Pre-fix this branch stored committed_lsn = written_lsn unconditionally * (over-advancing past durable_lsn). */ void persist_test_sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn) { sync_append_legacy_commit_store(p, written_lsn); } #endif /* XCE_TESTING */ void persist_log_entry_metadata(const wal_entry_hdr_t* hdr) { TRACE("WAL entry: type=%u lsn=%lu term=%lu payload_len=%u crc=%08x", hdr->type, (unsigned long)hdr->lsn, (unsigned long)hdr->term, hdr->payload_len, hdr->crc); } /* ------------------------------------------------------------------ */ /* Hook functions — AFTER_EVENT handlers that write WAL entries */ /* ------------------------------------------------------------------ */ /* Map HOOK_DETAIL_* values to WAL entry types. * Returns 0 for details that have no WAL representation (skip). */ static uint32_t detail_to_wal_type(int detail) { switch (detail) { case HOOK_DETAIL_JOB_NEW: return WAL_JOB_SUBMIT; case HOOK_DETAIL_JOB_START: return WAL_JOB_START; case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH; case HOOK_DETAIL_JOB_FAIL: return WAL_JOB_FAIL; case HOOK_DETAIL_JOB_STOP: return WAL_JOB_STOP; case HOOK_DETAIL_JOB_RESUME: return WAL_JOB_RESUME; case HOOK_DETAIL_JOB_UPDATE: return WAL_JOB_UPDATE; case HOOK_DETAIL_JOB_CKPT: return WAL_JOB_CKPT; case HOOK_DETAIL_JOB_RUSAGE_UPDATE: return WAL_JOB_RUSAGE; case HOOK_DETAIL_SETPREASON: return WAL_JOB_PREASON; case HOOK_DETAIL_CLRPREASON: return WAL_JOB_PREASON; case HOOK_DETAIL_QUEUE_UPDATE: return WAL_QUEUE_CTRL; case HOOK_DETAIL_WORKER_JOIN: return WAL_WORKER_CTRL; case HOOK_DETAIL_WORKER_UPDATE: return WAL_WORKER_CTRL; case HOOK_DETAIL_WORKER_DEAD: return WAL_WORKER_CTRL; default: return 0; /* unknown — skip */ } } /* Write-path commit classification (ha-commit-barrier-rollout task 1.2). * Pure lookup over the VERIFIED table (design.md §Write-Path * Classification); `detail` disambiguates WAL_WORKER_CTRL only. * Unclassified types fail fast under XCE_TESTING (abort) and fall back to * RETAIN in production — never dropped, never falsely confirmed (Req 2.2). */ wal_commit_mode_t wal_classify(uint32_t wal_type, int detail) { switch (wal_type) { case WAL_JOB_SUBMIT: /* client job submit — request-shaped; keeps the in-hook sync path * (reference implementation, handle_jobsubmit) */ return WAL_MODE_WAIT; case WAL_JOB_ARRAY_SUBMIT: /* client job-array submit (job-array-single-wal-record task 1.1) — * request-shaped exactly like WAL_JOB_SUBMIT: one requester awaits * the single commit barrier before reply, regardless of array * size. */ return WAL_MODE_WAIT; case WAL_QUEUE_CTRL: /* client queue control — request-shaped; the handler barrier owes * commit completion before the requester reply (tasks 2-3) */ return WAL_MODE_WAIT; case WAL_WORKER_CTRL: /* the only detail-disambiguated type: JOIN/UPDATE have a requester * awaiting confirmation (WAIT); DEAD is a controller-detected fact. * Unexpected details take the fact-safe RETAIN side. */ if (detail == HOOK_DETAIL_WORKER_JOIN || detail == HOOK_DETAIL_WORKER_UPDATE) return WAL_MODE_WAIT; return WAL_MODE_RETAIN; case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: case WAL_JOB_RUSAGE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: /* worker/scheduler-reported facts — retained until committed. * FAIL/UPDATE are dual-origin; their request-shaped side is * realized by the handler barrier in do_jobcontrol_op * (research.md §2 call-site verification). */ return WAL_MODE_RETAIN; case WAL_NOOP: /* internal promotion no-op — replicated like any committed entry, * retained until committed (ha-commit-barrier-rollout pre-declared * row: promotion | internal | RETAIN; guard forces classification) */ return WAL_MODE_RETAIN; case WAL_CONFIG_MARKER: /* config-change marker (ha-term-safety task 4.2): a committable * (term,lsn) log entry like a job-state fact — replicated and retained * until committed; it commits through the SAME barrier as job state * (R7.1/R7.11), so it is RETAIN, NOT a no-op or a LOCAL marker. The * requester barrier on a config upgrade is owned by the upgrade path * (do_upgrade_config_with_bytes), not the per-entry WAIT hook. */ return WAL_MODE_RETAIN; case WAL_CHECKPOINT: /* local segment marker — never replicated, by design */ return WAL_MODE_LOCAL; default: ERROR("wal_classify: unclassified WAL type %u (detail %d) — " "RETAIN fallback; add it to the classification table", wal_type, detail); #ifdef XCE_TESTING assert(0 && "unclassified WAL type — add it to wal_classify"); #endif return WAL_MODE_RETAIN; } } /* Commit barrier helper (ha-commit-barrier-rollout task 2.2, design.md * §Commit Barrier Helper; Req 1.1, 1.2, 1.3). * * Handler-callable WAIT completion on the appended watermark: snapshot the * last-appended LSN, return 0 when the commit index already covers it, * else block in the commit engine until coverage or deadline. The engine * result is returned UNMODIFIED (0 / -ETIMEDOUT / -XCE_ERR_NOT_READY / * -EINVAL) so callers map it to their own error replies. * * Lock discipline: p->lock is held only for the current_lsn snapshot and * is released BEFORE commit_submit(WAIT) — the engine blocks on ce->lock * + condvar, and p->lock must never be held across it (engine * lock-ordering rule). */ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms) { uint64_t watermark; if (p == NULL) return -EINVAL; /* Snapshot the last-appended watermark under p->lock. current_lsn is * the NEXT LSN to assign (persist.h invariant: committed_lsn <= * current_lsn - 1), so the highest appended entry is current_lsn - 1 * (resolved contract, spec Implementation Notes 2.1). current_lsn == * 0 means nothing was ever appended — vacuously satisfied. */ pthread_mutex_lock(&p->lock); if (p->current_lsn == 0) { pthread_mutex_unlock(&p->lock); return 0; } watermark = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); /* Already covered — committed_lsn is monotone, so coverage observed * here can never regress. This check also absorbs the LSN-0 first * entry (watermark 0 vs committed_lsn 0): that once-per-lifetime entry * has no quorum contract (ha-commit-engine Implementation Notes) and * commit_submit would reject lsn == 0 anyway. */ if (atomic_load_explicit(&p->committed_lsn, memory_order_acquire) >= watermark) return 0; /* Engine not wired: legacy/bootstrap contexts have no engine commit * contract — there is nothing to wait on, so the barrier is a no-op. * Production always wires the engine (cluster_init_phase2). */ if (p->repl.ce == NULL) return 0; /* timeout_ms == 0 borrows the configured sync deadline (Req 1.2). */ return commit_submit(p->repl.ce, watermark, COMMIT_MODE_WAIT, timeout_ms != 0 ? timeout_ms : p->config.sync_timeout_ms); } /* Classified append at the hook seam (barrier-rollout task 1.2). * * WAIT (queue/worker control): the append stays async here — the handler * barrier (tasks 2-3) owes commit completion before the requester reply. * WAL_JOB_SUBMIT never reaches this helper (it keeps the in-hook sync * path in persist_job_hook_fun). * * RETAIN (facts): append + broadcast, then register the appended LSN with * the commit engine. commit_submit(COMMIT_MODE_RETAIN) is registration- * free bookkeeping (commit.c — returns 0 immediately; commit state is * derived from registry watermarks at ACK time), so the call is contract * documentation: the entry stays pending purely by lsn > committed_lsn * and commits later via covering ACKs. Commit-progress results are NEVER * propagated to hook callers — only the local append rc is (Req 3.1). * * Lock context: persist_wal_async takes/releases p->lock internally; the * appended-LSN read below re-takes p->lock briefly (same discipline as * persist_wal_sync_append) and releases it BEFORE commit_submit, so no * p->lock is ever held across ce->lock (engine lock-ordering rule). */ static int wal_hook_append_classified(persist_t* p, uint32_t wal_type, wal_commit_mode_t mode, const void* payload, uint32_t len) { uint64_t appended_lsn; int rc; /* ha-term-safety task 3.1: route the primary's local append through the raft * library (ha_primary_log_append → raft_log_append) so the LIBRARY stamps the * live election term and assigns the LSN (firing on_persist_entry → the WAL * write), keeping p->current_lsn co-tracked with s->log_len. The library is * the SINGLE primary append entry — no append bypasses it on the raft road. * On the non-raft road (library not LEADER) ha_primary_log_append falls back * to persist_wal_async, so engine-less / init-time contexts are unchanged. * When p->cluster is NULL (persist used standalone in lib tests) keep the * direct append (no cluster → no raft state to drive). */ if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, wal_type, payload, len, NULL); else rc = persist_wal_async(p, wal_type, payload, len); if (rc != 0 || mode != WAL_MODE_RETAIN) return rc; if (p->repl.ce == NULL) return rc; /* engine not wired (bootstrap / engine-less contexts) */ /* current_lsn - 1 is the LSN persist_wal_async just assigned (it * increments under p->lock). A concurrent append may bump it further; * that is benign — contiguous advancement means the higher watermark * still covers this entry. LSN-0 edge: commit_submit rejects lsn == 0 * (first-ever entry of a fresh cluster — accepted interim behavior, * spec Implementation Notes). */ pthread_mutex_lock(&p->lock); appended_lsn = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); if (appended_lsn != 0) (void)commit_submit(p->repl.ce, appended_lsn, COMMIT_MODE_RETAIN, 0); return rc; /* local append rc only — facts have no commit timeout */ } /* Shared step for a single WAL_JOB_SUBMIT entry (raft-apply-decoupling task * 3.1): encode `job`, append, and block on the majority-commit barrier. * Returns 0 when the entry committed on a majority; a negative rc otherwise * (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: admission * cap; -ENOMEM: encode failure; or a local-write error). Whenever the LOCAL * WAL write succeeded (rc == 0, -ETIMEDOUT, or -XCE_ERR_NOT_READY — the entry * was appended locally and the LSN advanced even though replication is * degraded), *out_lsn is set to the entry's assigned LSN so a caller can tag * it for egress-watermark purposes even on a rejected submission. */ static int submit_job_wal_entry_and_commit(persist_t* p, const xce_job_t* job, uint64_t* out_lsn) { uint8_t* buf = NULL; size_t sz = 0; int rc; if (capnp_encode_job(job, &buf, &sz) != 0) return -ENOMEM; /* ha-convergence-recovery (3.3): snapshot the LSN this submit entry * will receive BEFORE the append. persist_wal_async assigns * hdr.lsn = current_lsn then increments, so the pre-append current_lsn * is exactly the written entry's LSN. */ pthread_mutex_lock(&p->lock); uint64_t submit_lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); int sync_rc = 0; rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz, &sync_rc); free(buf); if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY) *out_lsn = submit_lsn; return rc; } /* Explicit controller-side raft step for a job submission (raft-apply-decoupling * item #1 / bug-1306): WAL-append each job in `jlist` and block on the * majority-commit barrier, returning the commit verdict DIRECTLY to the caller * (handle_jobsubmit) instead of through the xce_run_hooks fan-out (which returns * only the last hook's rc and so clobbered this -ETIMEDOUT with a later notify * hook's 0 — the committed-job-loss root cause). * * Returns 0 when every job's entry committed on a majority; the FIRST negative * rc otherwise (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: * admission cap; -ENOMEM: encode failure; or a local-write error). The caller * MUST reject the submission on a negative rc (never reply success) to preserve * the Commit Durability invariant. On -ETIMEDOUT / -XCE_ERR_NOT_READY the entry * WAS appended locally and the LSN advanced, so j->submit_lsn is still tagged — * the entry remains a pending, uncommitted tail a new-term leader may discard. */ int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist) { int rc = 0; if (p == NULL || jlist == NULL) return -EINVAL; for (int i = 0; i < jlist->count && rc == 0; i++) { job_t* j = (job_t*)xlist_get(jlist, i); uint64_t lsn = 0; rc = submit_job_wal_entry_and_commit(p, &j->job, &lsn); if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY) j->submit_lsn = lsn; } return rc; } /* Sibling of persist_submit_wal_and_commit for a job submission whose job * object(s) do not exist yet (raft-apply-decoupling task 3.1 / Req 2.1, 2.3): * `payloads` is an xlist_t of xce_job_t* that have never been registered in * any job table. On full success (return 0), out_lsns[i] holds payloads[i]'s * committed LSN — the caller constructs and tags the job object AFTER this * call returns, so a failed/timed-out commit never leaves a job object * behind (there is nothing to construct until this succeeds). Returns the * first negative rc on failure. */ int persist_submit_job_payloads_and_commit(persist_t* p, xlist_t* payloads, uint64_t* out_lsns) { int rc = 0; if (p == NULL || payloads == NULL || out_lsns == NULL) return -EINVAL; for (int i = 0; i < payloads->count && rc == 0; i++) { xce_job_t* j = (xce_job_t*)xlist_get(payloads, i); rc = submit_job_wal_entry_and_commit(p, j, &out_lsns[i]); } return rc; } /* raft-apply-decoupling task 3.2: append-only counterpart of * submit_job_wal_entry_and_commit — writes the entry via the SAME * primary-append primitive wal_hook_append_classified already uses for * RETAIN-mode facts (ha_primary_log_append / persist_wal_async), WITHOUT * waiting on the majority-commit barrier. Deliberately does NOT reuse * submit_job_wal_entry_and_commit itself: that helper is shared with * persist_submit_wal_and_commit's still-synchronous, still-blocking * contract (relied on directly by test_egress_filter.c / test_e2e_hooks.c / * test_persist_hooks.c), which this task must not disturb. */ static int append_job_wal_entry_local(persist_t* p, const xce_job_t* job, uint64_t* out_lsn) { uint8_t* buf = NULL; size_t sz = 0; int rc; uint64_t lsn; if (capnp_encode_job(job, &buf, &sz) != 0) return -ENOMEM; /* Snapshot the LSN this entry will receive BEFORE the append (same * convention as submit_job_wal_entry_and_commit / wal_hook_append_ * classified: persist_wal_async / ha_primary_log_append assigns * hdr.lsn = current_lsn then increments). */ pthread_mutex_lock(&p->lock); lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, WAL_JOB_SUBMIT, buf, (uint32_t)sz, NULL); else rc = persist_wal_async(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz); free(buf); if (rc == 0) *out_lsn = lsn; return rc; } /* raft-apply-decoupling task 3.2: bridges commit_submit_async's * commit_continuation_fn shape (commit_outcome_t + rc) to * persist_job_commit_notify_fn's shape (a single rc) so persist.h's public * declaration never needs commit.h's commit_outcome_t (see persist.h's * doc comment on persist_job_commit_notify_fn). One adapter is heap- * allocated per registered lsn and frees itself after its single * invocation — safe because commit_resolve guarantees exactly-once * delivery. */ typedef struct { persist_job_commit_notify_fn fn; void* ctx; } job_commit_adapter_ctx_t; static void job_commit_resolve_adapter(void* adapter_ctx, uint64_t lsn, commit_outcome_t outcome, int rc) { job_commit_adapter_ctx_t* a = (job_commit_adapter_ctx_t*)adapter_ctx; int final_rc = (outcome == COMMIT_OUTCOME_COMMITTED) ? 0 : rc; a->fn(a->ctx, lsn, final_rc); free(a); } int persist_submit_job_payloads_async(persist_t* p, xlist_t* payloads, uint64_t* out_lsns, persist_job_commit_notify_fn fn, void* ctx) { int first_rc = 0; if (p == NULL || payloads == NULL || out_lsns == NULL || fn == NULL) return -EINVAL; for (int i = 0; i < payloads->count; i++) { xce_job_t* j = (xce_job_t*)xlist_get(payloads, i); uint64_t lsn = 0; int rc; if (first_rc != 0) { /* A prior payload in this batch already failed locally: every * remaining payload resolves with that SAME failure now, * synchronously — so `fn` is invoked exactly once per payload * regardless of where the failure occurred, and the caller * never has to separately track "never going to be called" * entries. */ fn(ctx, 0, first_rc); continue; } rc = append_job_wal_entry_local(p, j, &lsn); if (rc != 0) { first_rc = rc; fn(ctx, 0, rc); continue; } out_lsns[i] = lsn; /* LSN-0 edge (documented once-per-cluster-lifetime interim * behavior — ha-commit-engine Implementation Notes, mirrored by * repl_wait_majority_ack's walAck branch): the first-ever WAL * entry of a fresh cluster is structurally un-commitable and * commit_submit_async rejects lsn == 0, so resolve it inline. * Engine-not-wired (bootstrap / engine-less contexts) mirrors * persist_commit_barrier's own "no engine -> no-op success" * convention — there is no async completion authority to * register with. */ if (lsn == 0 || p->repl.ce == NULL) { fn(ctx, lsn, 0); continue; } job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a)); if (a == NULL) { first_rc = -ENOMEM; fn(ctx, lsn, -ENOMEM); continue; } a->fn = fn; a->ctx = ctx; int reg_rc = commit_submit_async(p->repl.ce, lsn, job_commit_resolve_adapter, a); if (reg_rc != 0) { free(a); first_rc = reg_rc; ERROR("persist_submit_job_payloads_async: lsn=%lu appended but " "commit_submit_async registration failed (rc=%d) -- the " "entry remains durable and will still be caught up " "eventually via replay, but this caller will not be told " "its outcome via this path", (unsigned long)lsn, reg_rc); fn(ctx, lsn, reg_rc); continue; } } return first_rc; } /* job-array-single-wal-record task 2.1: serialize a job-array submission * (base jobid + array size + one shared job spec) to a malloc'd Cap'n Proto * buffer, reusing the SAME capnp struct (JobSubmit) the client's own submit * request uses -- mirrors capnp_encode_job above, which mirrors encode_Job/ * new_Job/write_Job for a single job, except here via encode_JobSubmit/ * new_JobSubmit/write_JobSubmit. The encoded payload is O(1) in array_size: * arraySize is a fixed-width uint64 field, not a per-element list -- so this * function's output size never grows with array_size (Req 7.1). Caller must * free(*out) on success. */ static int capnp_encode_job_array(const xce_job_submit_t* s, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct JobSubmit cjs; JobSubmit_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_JobSubmit(cs, &cjs, (xce_job_submit_t*)s); jp = new_JobSubmit(cs); write_JobSubmit(&cjs, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* job-array-single-wal-record task 2.1: append-only, single-record * counterpart of append_job_wal_entry_local for a WHOLE job array. Encodes * ONE JobSubmit-shaped payload (base_jobid, array_size, base_spec->submit) * and appends it as exactly one WAL_JOB_ARRAY_SUBMIT entry via the SAME * primary-append primitive append_job_wal_entry_local already uses * (ha_primary_log_append / persist_wal_async) -- never one entry per array * element, regardless of array_size (Req 1.1, 1.2, 1.3). base_jobid/ * array_size are taken as explicit parameters rather than read from * base_spec->jobid/base_spec->array_size, so the caller's own * already-range-claimed values are always what gets committed, independent * of whatever same-shaped fields the client's own wire request happened to * carry. */ static int append_job_array_wal_entry_local(persist_t* p, uint64_t base_jobid, uint64_t array_size, const xce_job_submit_t* base_spec, uint64_t* out_lsn) { xce_job_submit_t wire = { .jobid = base_jobid, .array_size = array_size, .submit = base_spec->submit }; uint8_t* buf = NULL; size_t sz = 0; int rc; uint64_t lsn; if (capnp_encode_job_array(&wire, &buf, &sz) != 0) return -ENOMEM; /* Snapshot the LSN this entry will receive BEFORE the append (same * convention as append_job_wal_entry_local / submit_job_wal_entry_and_ * commit: persist_wal_async / ha_primary_log_append assigns * hdr.lsn = current_lsn then increments). */ pthread_mutex_lock(&p->lock); lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, WAL_JOB_ARRAY_SUBMIT, buf, (uint32_t)sz, NULL); else rc = persist_wal_async(p, WAL_JOB_ARRAY_SUBMIT, buf, (uint32_t)sz); free(buf); if (rc == 0) *out_lsn = lsn; return rc; } /* job-array-single-wal-record task 2.1 (design.md §WAL Array Submission, * Service Interface): encodes and appends ONE array-shaped WAL entry * (WAL_JOB_ARRAY_SUBMIT) and registers ONE commit continuation for it, * regardless of array_size -- the single-record sibling of * persist_submit_job_payloads_async's per-payload loop, specialized to * exactly one payload. No job_t/xce_job_t object is constructed here or * anywhere on this call path -- array expansion into individual job * entries happens later, in the shared apply seam (task 3.x's * restore_job_array), never here. * * Mirrors persist_submit_job_payloads_async's contract exactly: * - a local encode/append failure resolves `fn` synchronously with that * failure and returns the same negative rc; * - the LSN-0 / engine-not-wired edge cases (documented on * persist_submit_job_payloads_async above) resolve `fn` inline with * rc=0; * - otherwise `fn` is registered via commit_submit_async and invoked * exactly once, asynchronously, once the entry's commit outcome is * known. */ int persist_submit_job_array_async(persist_t* p, uint64_t base_jobid, uint64_t array_size, const xce_job_submit_t* base_spec, uint64_t* out_lsn, persist_job_commit_notify_fn fn, void* ctx) { uint64_t lsn = 0; int rc; if (p == NULL || base_spec == NULL || base_spec->submit == NULL || out_lsn == NULL || fn == NULL || array_size <= 1) return -EINVAL; rc = append_job_array_wal_entry_local(p, base_jobid, array_size, base_spec, &lsn); if (rc != 0) { fn(ctx, 0, rc); return rc; } *out_lsn = lsn; /* LSN-0 / engine-not-wired edge (see persist_submit_job_payloads_async's * doc comment above for the full rationale -- identical here, * specialized to one entry instead of a loop). */ if (lsn == 0 || p->repl.ce == NULL) { fn(ctx, lsn, 0); return 0; } job_commit_adapter_ctx_t* a = calloc(1, sizeof(*a)); if (a == NULL) { fn(ctx, lsn, -ENOMEM); return -ENOMEM; } a->fn = fn; a->ctx = ctx; int reg_rc = commit_submit_async(p->repl.ce, lsn, job_commit_resolve_adapter, a); if (reg_rc != 0) { free(a); ERROR("persist_submit_job_array_async: lsn=%lu appended but " "commit_submit_async registration failed (rc=%d) -- the " "entry remains durable and will still be caught up " "eventually via replay, but this caller will not be told " "its outcome via this path", (unsigned long)lsn, reg_rc); fn(ctx, lsn, reg_rc); return reg_rc; } return 0; } void persist_register_pending_apply(persist_t* p, uint64_t lsn, pending_apply_notify_fn on_applied, void* ctx) { pending_command_apply_t* node; if (p == NULL || on_applied == NULL) return; /* bug-1415 self-healing (round 2): commit.c's continuation dispatch and * the apply pump's walk (persist_apply_pump_run) are two independent * vtasks with no completion-order guarantee -- see this function's doc * comment in persist.h for the full race. If the walk has already * applied `lsn` (last_applied_lsn already at or past it) by the time we * get here, linking this registration onto pending_applies would orphan * it forever: the walk only ever visits each lsn once, and it already * called persist_take_pending_apply() for this lsn and found nothing. * Fire the hand-off immediately, synchronously, on THIS thread instead * -- never leave a continuation unresolved (controllers never * self-terminate; the same "fail loud and complete the work" posture * commit.c's dispatch_continuation fallback uses). */ if (lsn <= persist_last_applied_lsn(p)) { on_applied(ctx, lsn); return; } node = calloc(1, sizeof(*node)); if (node == NULL) { ERROR("persist_register_pending_apply: OOM registering the apply " "hand-off for lsn=%lu -- the entry is still durably committed " "and will still be applied by a future restart replay, but no " "continuation will fire for it via this path", (unsigned long)lsn); return; } node->lsn = lsn; node->on_applied = on_applied; node->ctx = ctx; pthread_mutex_lock(&p->pending_applies_lock); node->next = (pending_command_apply_t*)p->pending_applies; p->pending_applies = node; pthread_mutex_unlock(&p->pending_applies_lock); } int persist_has_pending_apply(persist_t* p, uint64_t lsn) { pending_command_apply_t* n; int found = 0; if (p == NULL) return 0; pthread_mutex_lock(&p->pending_applies_lock); for (n = (pending_command_apply_t*)p->pending_applies; n != NULL; n = n->next) { if (n->lsn == lsn) { found = 1; break; } } pthread_mutex_unlock(&p->pending_applies_lock); return found; } int persist_take_pending_apply(persist_t* p, uint64_t lsn, pending_apply_notify_fn* out_fn, void** out_ctx) { pending_command_apply_t* prev = NULL; pending_command_apply_t* n; int found = 0; if (p == NULL) return 0; pthread_mutex_lock(&p->pending_applies_lock); for (n = (pending_command_apply_t*)p->pending_applies; n != NULL; n = n->next) { if (n->lsn == lsn) { if (prev == NULL) p->pending_applies = n->next; else prev->next = n->next; if (out_fn) *out_fn = n->on_applied; if (out_ctx) *out_ctx = n->ctx; found = 1; break; } prev = n; } pthread_mutex_unlock(&p->pending_applies_lock); free(n); /* NULL when not found -- free(NULL) is a no-op */ return found; } /* raft-apply-decoupling task 4.1 (Req 5.1): the receipt-time durable-append * primitive for a worker-reported job-state fact. Unlike persist_job_hook_fun * below (an AFTER_EVENT hook that fires only once a mutation has already * happened), this is a direct, hook-fan-out-free call: job.c's fact receipt * handlers (record_job_start_fact/record_job_finish_fact/record_job_fail_fact/ * record_job_postdone_fact) call this (or persist_submit_job_fact_ex below) * INSTEAD of mutating and firing AFTER_EVENT, so no notification (scheduler * dispatch, slot release, apiserver push) can ever fire for an uncommitted * fact (Req 5.3). `state` is always WAL_MODE_RETAIN for every job-state fact * type (verified against wal_classify's table) — hardcoded here rather than * routed through wal_classify since none of these types are detail- * disambiguated the way WAL_WORKER_CTRL is. * * Thin wrapper over persist_submit_job_fact_ex() with pad = 0 — the "legacy * absolute state" discriminator (see wal_types.h's doc comment on * wal_jobstate_t._pad). Unchanged contract/signature for every existing * caller. */ int persist_submit_job_fact(persist_t* p, uint32_t wal_type, uint64_t jobid, int state) { return persist_submit_job_fact_ex(p, wal_type, jobid, state, 0); } /* raft-apply-decoupling task 6.1 round 2 (Req 3.3/7.1-7.3): full-control * sibling of persist_submit_job_fact() above, letting the caller set * wal_jobstate_t's `_pad` discriminator explicitly. Used by * record_job_fail_fact (WAL_JOB_FAIL_STAGE_FLAG | stage) and * record_job_postdone_fact (WAL_JOB_UPDATE_POSTDONE_MARK) to mark a fact * whose real mutation must be re-derived at apply time against the job's * CURRENT state, rather than trusting a receipt-time snapshot baked into * `state` — see wal_types.h's doc comment on wal_jobstate_t._pad for the * full rationale. */ int persist_submit_job_fact_ex(persist_t* p, uint32_t wal_type, uint64_t jobid, int state, uint32_t pad) { wal_jobstate_t js = { .jobid = jobid, .state = (uint32_t)state, ._pad = pad }; return wal_hook_append_classified(p, wal_type, WAL_MODE_RETAIN, &js, sizeof(js)); } /* P0 payload: just the detail value (4 bytes). * Full Cap'n Proto serialization is wired in Task 15. */ int persist_job_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; /* Write to WAL only after the state change is complete. * BEFORE_EVENT has a different va_list layout and must be ignored. */ if (situation != HOOK_AFTER_EVENT) return 0; /* raft-apply-decoupling task 6.1 (Hazard 1): this AFTER_EVENT firing * originated from apply_committed_entry()'s own post-mutation notify — * the entry is already durably committed (that is why it is being * applied at all), so appending it again here would duplicate it. See * t_apply_seam_notifying's doc comment (above apply_committed_entry) for * why this MUST be a call-provenance guard, not a `detail` filter: * job_ctrl's mark_job_update/mark_job_fail still fire this exact same * hook for their own live, still-legitimate append. */ if (t_apply_seam_notifying) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); if (mode == WAL_MODE_WAIT) { /* WAL_JOB_SUBMIT durability (WAL append + majority-commit barrier) is * an explicit step invoked directly by handle_jobsubmit — * persist_submit_job_payloads_and_commit(), called strictly before * any job object is constructed (raft-apply-decoupling task 3.1 / * bug-1306) — never from this hook. This hook fires only after that * commit has already succeeded, purely as a notification, so there * is nothing left for it to do here. */ return 0; } /* All other job events: payload = wal_jobstate_t {jobid, state}. * State is read from the job_t after the AFTER_EVENT hook updates it. */ wal_jobstate_t js = {0}; switch (detail) { case HOOK_DETAIL_JOB_START: { /* args: cluster_t*, job_pidinfo_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_EXIT: { /* args: cluster_t*, xce_jobfinish_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_FAIL: { /* args: cluster_t*, job_failinfo_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } default: { /* STOP, RESUME, UPDATE, CKPT, PREASON, RUSAGE — args: cluster_t*, job_t* */ (void)va_arg(ap, cluster_t*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } } /* mode == WAL_MODE_RETAIN — fact-shaped job state transition */ return wal_hook_append_classified(p, wal_type, mode, &js, sizeof(js)); } int persist_queue_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; if (situation != HOOK_AFTER_EVENT) return 0; /* raft-apply-decoupling task 6.1 (Hazard 1, queue analog): see * persist_job_hook_fun's identical guard above for the full rationale — * apply_committed_entry()'s WAL_QUEUE_CTRL branch fires this same * AFTER_EVENT to notify, and this hook appends unconditionally * (WAL_QUEUE_CTRL is WAL_MODE_WAIT with no early-return here), so * without this guard a live-applied queue entry would re-append and * re-trigger a nested majority-commit wait every time the pump walks * it. */ if (t_apply_seam_notifying) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); /* args: cluster_t*, queue_t* */ (void)va_arg(ap, cluster_t*); queue_t* q = va_arg(ap, queue_t*); /* Payload: [wal_queuehdr_t][queue name NUL-terminated] */ const char* name = q->queue.queue ? q->queue.queue : ""; size_t name_len = strlen(name) + 1; size_t total = sizeof(wal_queuehdr_t) + name_len; uint8_t* pkt = malloc(total); if (!pkt) return -ENOMEM; wal_queuehdr_t hdr = { .status = q->queue.status }; memcpy(pkt, &hdr, sizeof(hdr)); memcpy(pkt + sizeof(hdr), name, name_len); /* mode == WAL_MODE_WAIT — the append stays async; the handler barrier * owes commit completion before the requester reply (task 3) */ int rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total); free(pkt); return rc; } int persist_worker_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; if (situation != HOOK_AFTER_EVENT) return 0; /* raft-apply-decoupling task 6.1 (Hazard 1, worker analog): see * persist_job_hook_fun's identical guard above for the full rationale. */ if (t_apply_seam_notifying) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); /* WAL_WORKER_CTRL payload layout: [uint32_t detail][capnp worker OR name] */ uint32_t disc = (uint32_t)detail; uint8_t* buf = NULL; size_t capnp_sz = 0; int rc = 0; /* All worker detail types: args are cluster_t* then worker_t*. * JOIN uses a unified signature with an extra char* name between them; * DEAD and UPDATE use (cluster_t*, worker_t*) directly. */ (void)va_arg(ap, cluster_t*); if (detail == HOOK_DETAIL_WORKER_JOIN) (void)va_arg(ap, char*); /* skip name — present only for JOIN */ worker_t* w = va_arg(ap, worker_t*); if (!w) return 0; /* guard: failed join produced no worker slot */ rc = capnp_encode_worker(&w->host, &buf, &capnp_sz); if (rc != 0) return rc; /* Build [disc | capnp bytes] in one allocation */ size_t total = WORKER_WAL_HDR_SIZE + capnp_sz; uint8_t* pkt = malloc(total); if (!pkt) { free(buf); return -ENOMEM; } memcpy(pkt, &disc, WORKER_WAL_HDR_SIZE); memcpy(pkt + WORKER_WAL_HDR_SIZE, buf, capnp_sz); free(buf); /* JOIN/UPDATE → WAIT (handler barrier owes completion, task 3); * DEAD → RETAIN (controller-detected fact, registered with the engine) */ rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total); free(pkt); return rc; } /* ============================================================ * GC retention predicates (P0-5a / P0-8 / P0-13) * * Placed in persist.c so the unit test binary (which links only * persist.c from the controller sources) can access them without * pulling in RTTI-dependent job.c / worker.c. * ============================================================ */ bool job_is_terminal(const job_t* j) { return (j->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0; } bool job_should_retain(const job_t* j, const persist_config_t* cfg) { time_t now = time(NULL); /* TRANSIENT jobs: auto-terminate when TTL elapses after failover. * transient_since == 0 means the timestamp was never set — do not GC. */ if (j->job.state & JOB_STAT_TRANSIENT) { if (j->job.transient_since > 0 && now > (time_t)j->job.transient_since + (time_t)cfg->transient_job_timeout_s) { return false; } return true; } /* Non-terminal jobs are always retained */ if (!job_is_terminal(j)) return true; /* Terminal jobs: retain until finished_retain_s expires */ return (j->job.finish_time + (time_t)cfg->finished_retain_s >= now); } bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg) { time_t now = time(NULL); /* HIDDEN workers: admin explicitly removed — clean up at next checkpoint */ if (w->status & XCE_WORKER_HIDDEN) return false; /* UNAVAILABLE workers: retain until worker_retain_s expires. * dead_since == 0 means not yet set — do not GC. */ if (w->status & XCE_WORKER_UNAVAIL) { if (w->dead_since > 0 && now > w->dead_since + (time_t)cfg->worker_retain_s) { return false; } } return true; } /* ============================================================ * WAL truncation (task 7.5) * * Remove all WAL entries with LSN > cutoff_lsn. * Idempotent: calling twice with same cutoff_lsn returns 0 both times. * ============================================================ */ /* Clamp durable_lsn down to `tip` when the log tip recedes below it (truncate * / divergent-suffix discard). Without this, durable_lsn could exceed * current_lsn - 1 after a truncate, violating its invariant and letting the * commit majority count self at an LSN whose bytes were just removed. MUST be * called with p->lock held (mirrors the committed_lsn clamp in the truncate * paths). durable_lsn only ever decreases here — the sole exception to its * monotonic-non-decreasing rule (ha-commit-durability 2.1 State Management). */ static void clamp_durable_lsn_locked(persist_t* p, uint64_t tip) { uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (dur > tip) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn) { pthread_mutex_lock(&p->lock); uint64_t before = p->current_lsn; int rc = truncate_wal_above_locked(p, cutoff_lsn); uint64_t after = p->current_lsn; pthread_mutex_unlock(&p->lock); /* raft-apply-decoupling task 1.4: fire the truncation wake only when a * truncation actually removed entries above cutoff_lsn (current_lsn * strictly decreased -- both truncate_wal_above_locked branches set * current_lsn = cutoff_lsn + 1 on a real truncation) -- mirrors the * durable-advance wake's own before/after guard (see fsync_worker * above) so a no-op truncate (nothing above cutoff_lsn, or an I/O * error before current_lsn is touched) never spuriously fails a * continuation whose entry is still safely in the log. Called AFTER * releasing p->lock: the callback (the commit engine) takes only * ce->lock, preserving the cycle-free p->lock -> ce->lock edge already * established by the durable-advance wake. */ if (rc == 0 && after < before) { /* raft-apply-decoupling task 7.4 perf-fix: a real truncation may * ftruncate()/unlink() segment files the apply-pump's segment-lookup * cache still points into -- invalidate it so a subsequent * apply_pump_read_entry can never read truncated-away data. p->lock * is already released above and apply_pump_lock is never held by * this call path (persist_apply_pump_run is the only place that * holds it, during its own walk), so no lock-order hazard here. */ pthread_mutex_lock(&p->apply_pump_lock); if (p->apply_pump_cache_fd >= 0) { close(p->apply_pump_cache_fd); p->apply_pump_cache_fd = -1; } p->apply_pump_cache_next_lsn = 0; pthread_mutex_unlock(&p->apply_pump_lock); if (p->on_truncate != NULL) { p->on_truncate(p->truncate_cb_ctx, cutoff_lsn); } } return rc; } /* persist_committed_lsn_advance_atomic (2026-07-10, controller-invariant-audit * finding): race-safe monotonic-max advance of p->committed_lsn via a CAS * retry loop. Replaces the naive "load; if target>cur store" pattern, which * can lose an update if two writers race: persist_promotion_advance_committed * (below) and ha_standby_advance_committed_and_apply (ha.c) can both run * during the promotion window before role flips to CTRL_PRIMARY (a * still-in-flight WAL catch-up entry processed on another thread at the same * instant as this node's own election completing). No lock required -- CAS * only succeeds if the value hasn't changed since the read, so a losing * racer retries with the freshly-observed value and can never clobber a * larger value with a smaller one. */ uint64_t persist_committed_lsn_advance_atomic(persist_t* p, uint64_t target) { uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); while (target > cur) { if (atomic_compare_exchange_weak_explicit(&p->committed_lsn, &cur, target, memory_order_release, memory_order_acquire)) return target; /* cur was refreshed to the current value by the failed CAS; retry. */ } return cur; } /* persist_promotion_advance_committed (2026-07-10, promotion apply-lag fix): * unconditionally advance p->committed_lsn toward `safe_lsn` at promotion * time. persist_truncate_wal_above only sets committed_lsn on the branches * of truncate_wal_above_locked that perform a REAL truncation (partial * truncate, or the full-segment-purge demotion-convergence case); its no-op * early return (nothing above cutoff_lsn to remove -- the common case for a * newly-elected leader whose own log already covers safe_lsn) leaves * committed_lsn untouched at its pre-election value. Without this call, * committed_lsn -- and therefore apply, which is commit-gated * (raft-apply-decoupling) -- stalls until the normal per-entry ack-driven * commit path organically catches it up, which can take seconds while * surviving standbys reconnect and re-ACK. During that gap a job already * committed on a majority is invisible via expect-job/joblist on the very * primary that is authoritative for it (bug-828 class). * * DESIGN NOTE -- why this function never forces a synchronous flush (round 3, * 2026-07-10, controller-invariant-audit): an earlier round of this fix * called flusher_flush_once(p) here when durable_lsn lagged safe_lsn, to * force this node's own fsync to catch up immediately rather than waiting on * fsync_worker's cadence. That was a real bug-1198-CLASS thread-role * violation: the real multi-node winning-vote-reply path reaches * raft_cb_become_leader (and so this function) via peer_ctrl_classify's * VTASK_LIVENESS dispatch (ha.c ~5935-5940) -- a task class the design * DOCUMENTS as "never admission-gated" / never expected to block (see the * bug-1198 hazard box a few hundred lines above peer_ctrl_classify's * definition, which closed the EXACT same class of bug for * HA_REPL_SEND_SNAPSHOT's persist_checkpoint). A blocking fdatasync here * would reintroduce that closed bug class on the pool's liveness-reserved * workers. The visibility gap the force-flush was trying to close is instead * closed non-blockingly by ha_start_primary_heartbeat sending its first * heartbeat synchronously at promotion (bug-1432) -- that lets a reconnecting * standby's real position get learned, and the entry gets committed via the * normal ack-driven commit_advance_locked path, within one network RTT * instead of waiting a full heartbeat interval. A live chaotic soak (10 * rounds) confirmed durable-capped-only + immediate-heartbeat together reach * committed_job_loss=0 with zero blocking calls added to this function. * * Durable-capped (never advances committed_lsn past this node's own * durable_lsn -- see persistence.md's "acknowledged implies durable on a * majority" core invariant) and monotonic + race-safe via * persist_committed_lsn_advance_atomic. */ void persist_promotion_advance_committed(persist_t* p, uint64_t safe_lsn) { if (!p) return; uint64_t durable = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); uint64_t target = safe_lsn < durable ? safe_lsn : durable; persist_committed_lsn_advance_atomic(p, target); } /* Lock-held core of persist_truncate_wal_above (see the forward declaration and * the conflict-guard reuse near persist_wal_write_replicated). Caller MUST hold * p->lock; this function never touches the lock so it is reusable from another * already-locked path. */ static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn) { /* No-op: nothing written yet, or cutoff covers all written entries */ if (p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn) { return 0; } /* --- Collect segment filenames and sort by start LSN --- */ uint64_t seg_lsns[4096]; int nseg = 0; DIR* d = opendir(p->wal_path); if (d) { struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); } if (nseg == 0) { return 0; } /* Insertion sort ascending by start LSN */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } /* --- Find segment containing cutoff_lsn --- */ int cutoff_seg_idx = -1; for (int i = 0; i < nseg; i++) { if (seg_lsns[i] <= cutoff_lsn) cutoff_seg_idx = i; else break; } if (cutoff_seg_idx < 0) { /* cutoff_lsn is below the oldest retained segment's start LSN, so EVERY * on-disk entry has LSN > cutoff_lsn and must be removed (the function's * contract: "remove all entries with LSN > cutoff_lsn"). This is the * demotion-convergence case (ha-convergence-recovery Req 1.2): a demoted * ex-primary whose committed boundary sits below its entire retained log * must discard all of it and collapse the tip to cutoff_lsn + 1. Delete * every segment, close the active fd, and reset the in-memory tip. */ for (int i = 0; i < nseg; i++) { char del_path[4096]; snprintf(del_path, sizeof(del_path), WAL_SEG_FMT, p->wal_path, seg_lsns[i]); unlink(del_path); } if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } p->seg_offset = 0; p->seg_path[0] = '\0'; p->current_lsn = cutoff_lsn + 1; p->last_crc = WAL_CRC_NONE; atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release); clamp_durable_lsn_locked(p, cutoff_lsn); /* Keep the per-LSN term index consistent: drop terms for removed LSNs. */ truncate_term_index_locked(p, cutoff_lsn); return 0; } /* --- Scan segment to find byte offset of first entry with lsn > cutoff --- */ char cutoff_seg_path[4096]; snprintf(cutoff_seg_path, sizeof(cutoff_seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]); int scan_fd = open(cutoff_seg_path, O_RDONLY); if (scan_fd < 0) { return -errno; } uint32_t scan_off = 0; uint32_t trunc_off = 0; uint32_t new_last_crc = WAL_CRC_NONE; bool found_cut = false; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(scan_fd, &hdr, sizeof(hdr), (off_t)scan_off); if (n < (ssize_t)sizeof(hdr)) break; /* short read or EOF */ /* Pre-allocated zero region: type==0 is not a valid WAL type (min=1) */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn > cutoff_lsn) { trunc_off = scan_off; found_cut = true; break; } new_last_crc = hdr.crc; scan_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(scan_fd); if (!found_cut) trunc_off = scan_off; /* end of valid data in segment */ /* --- Close the current wal_fd --- */ if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } /* --- Truncate the cutoff segment at the computed offset --- */ int new_fd = open(cutoff_seg_path, O_RDWR); if (new_fd < 0) { return -errno; } if (ftruncate(new_fd, (off_t)trunc_off) != 0) { int saved = errno; close(new_fd); return -saved; } fdatasync(new_fd); /* --- Delete all segments after the cutoff segment --- */ for (int i = cutoff_seg_idx + 1; i < nseg; i++) { char del_path[4096]; snprintf(del_path, sizeof(del_path), WAL_SEG_FMT, p->wal_path, seg_lsns[i]); unlink(del_path); } /* --- Update persist_t state under lock --- */ p->wal_fd = new_fd; p->seg_offset = trunc_off; snprintf(p->seg_path, sizeof(p->seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]); p->current_lsn = cutoff_lsn + 1; p->last_crc = new_last_crc; atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release); clamp_durable_lsn_locked(p, cutoff_lsn); /* Keep the per-LSN term index consistent: drop terms for removed LSNs. */ truncate_term_index_locked(p, cutoff_lsn); return 0; }