#define _GNU_SOURCE #include "persist.h" #include "config.h" #include "ctrl_metrics.h" #include "hook.h" #include "xce/cluster_def.h" #include "xce/errnox.h" #include "xce/types.h" #include "controller.capnp.h" #include "xlist.h" #include "cluster.h" #include "commit.h" /* commit_submit RETAIN registration (barrier-rollout 1.2) */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* restore_* functions defined in job.c / worker.c / queue.c */ extern int restore_job(cluster_t* c, uint64_t jobid, const void* capnp_payload, size_t len, uint64_t entry_lsn); extern int restore_job_state(cluster_t* c, uint64_t jobid, uint32_t state); extern int worker_from_capnp(cluster_t* c, const void* capnp_payload, size_t len); extern int restore_queue_state(cluster_t* c, const char* name, uint32_t status); /* ---- Cap'n Proto serialization helpers (WAL payload encoding) ---------- */ /* Serialize xce_job_t to a malloc'd Cap'n Proto buffer. * Caller must free(*out) on success. */ static int capnp_encode_job(const xce_job_t* j, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Job cj; Job_ptr jp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Job(cs, &cj, (xce_job_t*)j); jp = new_Job(cs); write_Job(&cj, jp); capn_setp(capn_root(&c), 0, jp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* Serialize xce_worker_t to a malloc'd Cap'n Proto buffer. */ static int capnp_encode_worker(const xce_worker_t* w, uint8_t** out, size_t* out_len) { struct capn c; struct capn_segment* cs; struct Worker cw; Worker_ptr wp; capn_init_malloc(&c); cs = capn_root(&c).seg; encode_Worker(cs, &cw, (xce_worker_t*)w); wp = new_Worker(cs); write_Worker(&cw, wp); capn_setp(capn_root(&c), 0, wp.p); *out_len = capn_size(&c); *out = malloc(*out_len); if (!*out) { capn_free(&c); return -ENOMEM; } capn_write_mem(&c, *out, *out_len, 0); capn_free(&c); return 0; } /* WAL_WORKER_CTRL payload: [uint32_t detail][capnp bytes...] or [uint32_t detail][name\0] */ #define WORKER_WAL_HDR_SIZE sizeof(uint32_t) /* ------------------------------------------------------------------ */ /* Constants */ /* ------------------------------------------------------------------ */ #define WAL_SEG_FMT "%s/seg-%016" PRIx64 ".wal" /* NFS_SUPER_MAGIC — reject WAL on NFS mounts (unreliable fsync) */ #define NFS_SUPER_MAGIC 0x6969 /* ------------------------------------------------------------------ */ /* Internal helpers */ /* ------------------------------------------------------------------ */ static uint64_t now_us(void) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); return (uint64_t)ts.tv_sec * 1000000ULL + (uint64_t)(ts.tv_nsec / 1000); } /* Recursive mkdir — equivalent to mkdir -p */ static int mkdir_p(const char* path, mode_t mode) { char tmp[4096]; char* p; size_t len; snprintf(tmp, sizeof(tmp), "%s", path); len = strlen(tmp); if (len > 0 && tmp[len - 1] == '/') tmp[len - 1] = '\0'; for (p = tmp + 1; *p; p++) { if (*p == '/') { *p = '\0'; if (mkdir(tmp, mode) != 0 && errno != EEXIST) return -1; *p = '/'; } } if (mkdir(tmp, mode) != 0 && errno != EEXIST) return -1; return 0; } /* Open (and pre-allocate) a new WAL segment named after current_lsn */ static int open_new_segment(persist_t* p) { snprintf(p->seg_path, sizeof(p->seg_path), WAL_SEG_FMT, p->wal_path, p->current_lsn); int fd = open(p->seg_path, O_RDWR | O_CREAT | O_TRUNC, 0600); if (fd < 0) return -errno; /* Pre-allocate segment space to avoid fragmentation and metadata updates * on every write. posix_fallocate is best-effort — ignore errors. */ posix_fallocate(fd, 0, (off_t)p->config.wal_segment_bytes); p->wal_fd = fd; p->seg_offset = 0; return 0; } /* Close current segment and open a fresh one */ static int rotate_segment(persist_t* p) { if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } return open_new_segment(p); } /* ------------------------------------------------------------------ */ /* Demand-driven group-commit flusher (ha-commit-durability task 2.1) */ /* */ /* Repurposes the old fixed-interval fsync timer into a flusher driven */ /* by the append-vs-synced gap. On each pass it batches one fdatasync */ /* over all entries appended since the last sync, advances durable_lsn */ /* to the synced tip, and wakes the commit evaluation. --wal-sync-ms */ /* now bounds the MAX batching latency (a no-further-append entry is */ /* still synced within the cadence), not a durability gap. */ /* */ /* LOST-WAKEUP / CORRECTNESS REASONING (gating item): */ /* */ /* The "work pending" condition is LEVEL-triggered on two persistent */ /* variables, NOT an edge: pending ⇔ durable_lsn < current_lsn - 1 */ /* (flusher_work_pending below, evaluated under p->lock). Because it */ /* is a comparison of stored state — not a transient flag consumed on */ /* read — a signal that races the flusher's wait can never be lost: */ /* */ /* - persist_wal_async advances current_lsn under p->lock, THEN */ /* signals fsync_cond. If the flusher is mid-flush when the append */ /* lands, current_lsn has already moved past the tip the flusher */ /* captured, so after that flush durable_lsn < current_lsn - 1 */ /* STILL holds; the flusher re-checks the level under p->lock on */ /* its next loop iteration and flushes again (it does NOT wait */ /* first). The missed signal is therefore harmless — the level */ /* itself carries the pending work forward. */ /* - If the append's signal arrives while the flusher sits in */ /* pthread_cond_wait, it wakes it. If it arrives in the tiny window */ /* after the flusher decided "no work" but before it re-enters the */ /* wait, the flusher re-evaluates the level under p->lock before */ /* waiting (we re-check inside the fsync_mutex-held region by */ /* reading current_lsn/durable_lsn under p->lock first), so it */ /* will not block while work is pending. */ /* */ /* "NEVER durable before its fdatasync": the flusher snapshots */ /* tip = current_lsn - 1 AND calls fdatasync WHILE HOLDING p->lock. */ /* persist_wal_async also pwrites under p->lock, so no append can */ /* extend the segment between the tip snapshot and the fdatasync — by */ /* the time fdatasync returns, every byte for LSNs <= tip is on stable */ /* storage. durable_lsn is only then advanced to tip (monotonic max). */ /* Releasing p->lock and broadcasting commit-eval happens after the */ /* advance, so a WAIT caller that observes durable_lsn >= its LSN is */ /* guaranteed the bytes are synced. */ /* */ /* Lock discipline: p->lock and fsync_mutex are NEVER held together */ /* (fsync_mutex guards only the cond wait; the flush section takes */ /* p->lock alone). This keeps fdatasync off any lock the iomgr / */ /* heartbeat path needs and preserves liveness (Req 1.6). */ /* ------------------------------------------------------------------ */ /* Test-only fault-injection probe (ha-commit-durability task 2.1 error path). * Returns true when the test has forced fdatasync to be treated as failed. * Compiled to a constant `false` in production (no -DXCE_TESTING), so the * fail-loud gate below collapses to the pure rc != 0 check with zero overhead * and no dependency on the test-only struct field. */ static inline bool flusher_fsync_forced_fail(persist_t* p) { #ifdef XCE_TESTING return atomic_load_explicit(&p->test_force_fsync_fail, memory_order_acquire) != 0; #else (void)p; return false; #endif } /* Returns true iff there is un-synced appended data (level predicate). * MUST be called with p->lock held. current_lsn - 1 is the appended tip * (current_lsn is the NEXT lsn to assign); LSN 0 reserved, so a fresh * store has current_lsn == 1 → tip 0 → no pending work. */ static bool flusher_work_pending_locked(persist_t* p) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); return dur < tip; } /* Perform one group-commit flush: snapshot the appended tip, fdatasync the * live segment, and advance durable_lsn to the synced tip. Takes p->lock * for the whole snapshot+sync+advance so no concurrent append can extend * the segment between the tip read and the fdatasync. Returns the new * durable_lsn (unchanged if there was nothing to sync). */ static uint64_t flusher_flush_once(persist_t* p) { pthread_mutex_lock(&p->lock); uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (dur >= tip || p->wal_fd < 0) { /* Nothing new to sync (or no segment yet). Still flush the fd for a * clean-shutdown / fallback pass, but do not advance the watermark. * The watermark is already <= tip here, so a failed fallback sync * cannot make it claim more than is durable — but we still surface * the error loudly so a degrading disk is never silent (Req 1.1). */ if (p->wal_fd >= 0) { int rc = fdatasync(p->wal_fd); if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: flusher fallback fdatasync FAILED (errno=%d %s)" " — durable_lsn held at %" PRIu64 "; WAL not advanced", e, strerror(e), dur); } } pthread_mutex_unlock(&p->lock); return dur; } /* fdatasync under p->lock: appends also hold p->lock to pwrite, so the * segment cannot grow between this tip snapshot and the sync. Every byte * for LSNs <= tip is therefore on stable storage when fdatasync returns * SUCCESSFULLY. */ int rc = fdatasync(p->wal_fd); /* FAIL-LOUD GATE (Req 1.1/1.2): advance durable_lsn ONLY on a successful * sync. On EIO/ENOSPC the bytes are NOT on stable storage, so leaving the * watermark unchanged keeps the non-durable LSNs from being counted toward * the commit majority. We do NOT retry into an advanced watermark — the * level predicate stays pending and the next pass retries the sync. */ if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: flusher fdatasync FAILED (errno=%d %s) — durable_lsn " "held at %" PRIu64 " (tip=%" PRIu64 " NOT made durable); commit " "majority will not count these LSNs", e, strerror(e), dur, tip); pthread_mutex_unlock(&p->lock); return dur; } /* Monotonic advance to the synced tip — never past it (the invariant * durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1 * was read under this same lock). One fdatasync advances durable_lsn past * however many LSNs were appended since the last sync (batching). */ if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); #ifdef XCE_TESTING /* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6): * a liveness test asserts this is the dedicated flusher thread, never the * caller/iomgr/heartbeat thread. Written under p->lock; read after the * flusher has caught up, so no extra synchronization is needed. */ p->test_last_fdatasync_thread = pthread_self(); atomic_store_explicit(&p->test_last_fdatasync_thread_set, 1, memory_order_release); #endif pthread_mutex_unlock(&p->lock); return tip; } static void* fsync_worker(void* arg) { persist_t* p = arg; pthread_mutex_lock(&p->fsync_mutex); while (!atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) { /* Re-evaluate the level predicate BEFORE waiting so a signal that * raced in just before we entered the wait is not needed for * correctness — if work is already pending we skip the wait entirely * and flush. (Reads current_lsn/durable_lsn under p->lock.) */ pthread_mutex_lock(&p->lock); bool pending = flusher_work_pending_locked(p); pthread_mutex_unlock(&p->lock); if (!pending) { /* No pending work: wait for an append-signal OR the cadence * deadline (bounded fallback flush so a lone no-further-append * entry is still synced within wal_sync_ms — Req 3.1). */ long ms = (long)p->config.wal_sync_ms; struct timespec dl; clock_gettime(CLOCK_REALTIME, &dl); dl.tv_sec += ms / 1000; dl.tv_nsec += (ms % 1000) * 1000000L; if (dl.tv_nsec >= 1000000000L) { dl.tv_sec += 1; dl.tv_nsec -= 1000000000L; } pthread_cond_timedwait(&p->fsync_cond, &p->fsync_mutex, &dl); if (atomic_load_explicit(&p->fsync_stop, memory_order_acquire)) break; } /* Flush section: never hold fsync_mutex across the fdatasync (the * flush takes p->lock alone). Release fsync_mutex, flush, re-acquire. * After the flush we loop and re-check the level — if a new append * landed during the flush, the level is still pending and we flush * again without needing its signal (lost-wakeup-safe). */ pthread_mutex_unlock(&p->fsync_mutex); uint64_t before = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); uint64_t after = flusher_flush_once(p); /* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced * INSIDE flusher_flush_once (atomic store with release) BEFORE we * return here, and flusher_flush_once has already RELEASED p->lock. * Now — holding NO persist lock — invoke the registered durable-advance * callback so the commit engine re-evaluates its majority and wakes any * COMMIT_MODE_WAIT caller whose entry just became durable locally. * * LOST-WAKEUP SAFETY: the wake fires strictly AFTER the durable_lsn * release-store, and the commit engine's predicate re-reads durable_lsn * via an acquire load under ce->lock — so a WAIT caller either (a) sees * the advanced watermark on its own re-check, or (b) is blocked in * pthread_cond_wait and is woken by this broadcast; it can never both * miss the watermark AND miss the wake. We only fire the callback when * durable_lsn actually advanced (after > before): a no-op flush (or a * failed fdatasync that held the watermark) needs no commit re-eval, * avoiding spurious wakes. * * LOCK ORDERING (cycle-free): the flusher releases p->lock inside * flusher_flush_once, THEN the callback takes ce->lock (commit side). * The commit side only ever takes ce->lock and reads persist atomics — * it never takes p->lock while holding ce->lock. So the only ordering * is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no * deadlock cycle. The callback explicitly MUST NOT take p->lock. */ if (after > before && p->on_durable_advance != NULL) { p->on_durable_advance(p->durable_cb_ctx); } pthread_mutex_lock(&p->fsync_mutex); } pthread_mutex_unlock(&p->fsync_mutex); return NULL; } /* ------------------------------------------------------------------ * * WAL term adapter (ha-term-safety task 1.3, R1.1/R1.5/R1.6/R1.7/R1.8) * * ------------------------------------------------------------------ */ /* Record an entry's RAW hdr.term at its 1-based LSN in the in-memory per-LSN * term index (R1.5). Grows the index on demand; LSNs replay/append contiguously * so the array is dense (slot i == LSN i+1). Any gap left by a non-contiguous * LSN (should not happen on the contiguous WAL) is zero-filled by realloc-then- * memset, which read back as "absent" until/unless written. The raw term * (including a legacy 0) is stored verbatim; the legacy-0 -> WAL_LEGACY_TERM * mapping is applied only at READ-BACK (persist_entry_term), keeping "present * legacy-0" distinguishable from "absent". * * Callers: the live append path (holds p->lock) and the single-threaded * recovery path (replay_wal — no other thread touches the index before the node * participates, so no lock is required there). Best-effort: an allocation * failure is logged loudly and leaves the index unextended (read-back then * reports the LSN absent) — it never aborts and never corrupts existing * entries. */ static void record_entry_term(persist_t* p, uint64_t lsn, uint64_t term) { if (!p || lsn == 0) return; /* LSN 0 is reserved — never an entry */ if (lsn > p->term_index_cap) { /* Grow geometrically to amortize; never below the requested LSN. */ uint64_t new_cap = p->term_index_cap ? p->term_index_cap * 2 : 64; if (new_cap < lsn) new_cap = lsn; uint64_t* grown = realloc(p->term_index, new_cap * sizeof(uint64_t)); if (!grown) { ERROR("persist: term-index realloc failed (lsn=%" PRIu64 ", cap=%" PRIu64 "->%" PRIu64 "); term for this LSN will read " "back as absent", lsn, p->term_index_cap, new_cap); return; } /* Zero-fill the newly grown region so any unwritten slot reads absent. */ memset(grown + p->term_index_cap, 0, (new_cap - p->term_index_cap) * sizeof(uint64_t)); p->term_index = grown; p->term_index_cap = new_cap; } p->term_index[lsn - 1] = term; if (lsn > p->term_index_count) p->term_index_count = lsn; } /* entry_term_locked: the lock-free core of persist_entry_term — read the * recorded Raft term for `lsn` from the in-memory term index, applying the * legacy-0 rule (R1.8: a present term-0 entry maps to WAL_LEGACY_TERM; an absent * LSN returns 0). Caller MUST hold p->lock. Used by the append path * (persist_wal_async) to source the AppendEntries prev_log_term while already * under the lock — calling persist_entry_term there would re-acquire p->lock * and deadlock. */ static uint64_t entry_term_locked(const persist_t* p, uint64_t lsn) { if (!p || lsn == 0) return 0; if (lsn <= p->term_index_count && p->term_index != NULL) { uint64_t raw = p->term_index[lsn - 1]; return (raw == 0) ? WAL_LEGACY_TERM : raw; } return 0; } /* Shrink the in-memory per-LSN term index so that every LSN > `keep_lsn` reads * back as absent (ha-term-safety task 2.2). Called from the truncate path after * the on-disk entries above keep_lsn are removed, keeping the term index * consistent with the WAL: a later persist_entry_term on a truncated LSN must * return 0. The backing allocation is left as-is (only the live count moves); * the zeroed-on-grow contract means a later re-append at those LSNs overwrites * cleanly. Caller must hold p->lock. */ static void truncate_term_index_locked(persist_t* p, uint64_t keep_lsn) { if (!p) return; if (keep_lsn < p->term_index_count) p->term_index_count = keep_lsn; } void persist_set_term(persist_t* p, uint64_t term) { if (!p) return; /* p->term is read on the append path under p->lock; take the lock so the * stamped value is consistent with the append that reads it. */ pthread_mutex_lock(&p->lock); p->term = term; pthread_mutex_unlock(&p->lock); } uint64_t persist_entry_term(const persist_t* p, uint64_t lsn) { if (!p || lsn == 0) return 0; /* Take p->lock to read consistently against concurrent record_entry_term * on the append path. Cast away const — the lock is logically a mutable * sub-object of an otherwise-const read. */ persist_t* mp = (persist_t*)p; pthread_mutex_lock(&mp->lock); /* Legacy-zero rule (R1.8) lives in entry_term_locked: a present entry * carrying term 0 maps to the single lowest-known real Raft term, applied * uniformly so two legacy-0 entries at the same LSN match; absent LSNs * return 0. */ uint64_t result = entry_term_locked(mp, lsn); pthread_mutex_unlock(&mp->lock); return result; } /* persist_reconcile_checkpoint_term — raft-ha-core-safety task 3.2 (Req * 3.1/5.1), Snapshot Boundary Reconcile. See persist.h for the full contract. * Reuses record_entry_term (same translation unit) to authoritatively * overwrite whatever term (if any) the local term index held at * checkpoint_lsn — this is a genuine reconcile-overwrite, not an idempotent * skip, because a pre-snapshot divergent entry may have left a stale local * term at this exact LSN. Only the checkpoint LSN's own term is reconciled: * post-install Raft semantics mean no AppendEntries prevLogIndex/prevLogTerm * check against this member ever references an LSN below the checkpoint * again (the checkpoint is the new log-start boundary), and the WAL bytes * below it were compacted away, so there is no data to reconstruct a full * per-LSN history from even if it were needed. */ void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn, uint64_t checkpoint_term) { if (!p || checkpoint_lsn == 0) return; pthread_mutex_lock(&p->lock); record_entry_term(p, checkpoint_lsn, checkpoint_term); pthread_mutex_unlock(&p->lock); } /* ------------------------------------------------------------------ */ /* Public API */ /* ------------------------------------------------------------------ */ void persist_set_workdir(persist_t* p, const char* workdir, const char* name) { char effective[4096]; snprintf(effective, sizeof(effective), "%s/%s", workdir, name); snprintf(p->wal_path, sizeof(p->wal_path), "%s/data/wal", effective); snprintf(p->snap_path, sizeof(p->snap_path), "%s/data/snap", effective); } int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c) { p->cluster = c; p->config = cfg ? *cfg : persist_config_defaults(); p->wal_fd = -1; p->last_replicated_crc = WAL_CRC_NONE; p->last_crc = WAL_CRC_NONE; /* LSN 0 is RESERVED as the "none / empty" sentinel — it is NEVER assigned * to a real WAL entry. A fresh store therefore starts current_lsn at 1, so * the first-ever appended entry (e.g. the promotion WAL_NOOP) is lsn 1 and * lsn 0 unambiguously means "nothing here": * - committed_lsn == 0 → nothing committed (the lsn-1 NOOP is committable * 0 → 1, vs. the old lsn-0 entry which could never advance committed_lsn); * - writable_lsn == 0 → own-term gate off (the gate armed at lsn 1 is * effective, vs. lsn 0 which read as "gate off" on a cold-start leader); * - standby_lsn == 0 → empty standby (the streamer's `hdr.lsn > 0` * filter streams the lsn-1 entry, closing the cold-start WAL hole that * dropped lsn 0 on `0 > 0`). * persist_recover() overwrites current_lsn from the durable snapshot/WAL tip * (load_snapshot / replay_wal), so this fresh default only applies before any * recovery — i.e. a brand-new store with no snapshot and no WAL. */ p->current_lsn = 1; /* Enforce minimum value for max_wal_records to prevent runaway checkpointing */ if (p->config.max_wal_records < 100) { ERROR("persist: max_wal_records=%u too low (minimum 100) — setting to 100", p->config.max_wal_records); p->config.max_wal_records = 100; } /* Enforce minimum value for wal_segment_bytes to prevent a rotation storm * (must comfortably fit at least one max-payload entry: header + up to * MAX_WAL_PAYLOAD_BYTES). */ if (p->config.wal_segment_bytes < 4096) { ERROR("persist: wal_segment_bytes=%u too low (minimum 4096) — setting to 4096", p->config.wal_segment_bytes); p->config.wal_segment_bytes = 4096; } /* Initialise atomic checkpoint-trigger counters */ atomic_store(&p->wal_record_count, 0); atomic_store(&p->checkpoint_in_progress, 0); /* Durable watermark starts at 0 ("nothing fdatasync'd"): mirrors the * LSN-0-reserved committed_lsn convention. It advances only on the * flusher thread after a successful fdatasync (ha-commit-durability 2.1). */ atomic_store(&p->durable_lsn, 0); /* Recovery-integrity verdict defaults to CLEAN (ha-commit-durability 3.1): * a fresh store has no corruption. persist_recover overwrites this with the * replay outcome (RECOVER_TRUNCATED only on a structural log corruption). */ p->recovery_verdict = RECOVER_CLEAN; /* Sticky "unrecoverable, loudly fenced" flag starts clear (Req 2.4): a fresh * store is promotable. Set by persist_recover_loud_fence_if_unrecoverable * on an unrecoverable boot; cleared on catch-up (never self-exits). */ atomic_store(&p->recovery_unrecoverable, false); /* Durable-advance wake hook starts unregistered (ha-commit-durability 2.2): * the commit engine registers it in commit_engine_init. Until then every * flusher durable advance is a no-op wake. */ p->on_durable_advance = NULL; p->durable_cb_ctx = NULL; /* Truncation wake hook starts unregistered (raft-apply-decoupling task * 1.4): the commit engine registers it in commit_engine_init. Until * then every truncation is a no-op wake. */ p->on_truncate = NULL; p->truncate_cb_ctx = NULL; /* Per-LSN term index starts empty (ha-term-safety 1.3, R1.5): no entries * recorded yet. Grown on demand by record_entry_term during replay/append; * freed in persist_fini. */ p->term_index = NULL; p->term_index_count = 0; p->term_index_cap = 0; /* Create WAL and snapshot directories */ if (mkdir_p(p->wal_path, 0700) != 0) return -errno; if (mkdir_p(p->snap_path, 0700) != 0) return -errno; /* Reject NFS mounts — fsync durability is not guaranteed over NFS */ struct statfs sfs; if (statfs(p->wal_path, &sfs) == 0 && (unsigned long)sfs.f_type == (unsigned long)NFS_SUPER_MAGIC) { ERROR("persist: WAL path %s is on NFS — unsafe for fsync durability", p->wal_path); return -EREMOTE; } pthread_mutex_init(&p->lock, NULL); pthread_mutex_init(&p->checkpoint_mutex, NULL); pthread_cond_init(&p->checkpoint_done, NULL); pthread_mutex_init(&p->fsync_mutex, NULL); pthread_cond_init(&p->fsync_cond, NULL); repl_init(&p->repl); /* P2-4: fan-out replicator lifecycle */ WARN("WAL/snapshot stored unencrypted in %s — " "ensure filesystem is encrypted if job payloads contain sensitive data", p->wal_path); return 0; } int persist_start(persist_t* p) { ctrl_metrics_init(); /* Register AFTER_EVENT persistence hooks if we have a cluster back-pointer */ if (p->cluster != NULL) { xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun, p); xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun, p); xce_add_hook(&p->cluster->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun, p); } atomic_store_explicit(&p->fsync_stop, 0, memory_order_release); int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p); if (rc == 0) atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release); return rc; } void persist_stop(persist_t* p) { if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) { /* Set the stop flag and wake the timer wait under fsync_mutex so the * worker cannot miss the signal between its flag check and the wait. */ pthread_mutex_lock(&p->fsync_mutex); atomic_store_explicit(&p->fsync_stop, 1, memory_order_release); pthread_cond_signal(&p->fsync_cond); pthread_mutex_unlock(&p->fsync_mutex); pthread_join(p->fsync_thread, NULL); atomic_store_explicit(&p->fsync_thread_running, 0, memory_order_release); } pthread_mutex_lock(&p->lock); if (p->wal_fd >= 0) { /* Final synchronous flush-or-exit on stop: fdatasync everything * appended so far and advance durable_lsn to the synced tip so a * post-stop read reflects the durable state (the flusher thread is * already joined and will not advance it again). * * FAIL-LOUD GATE (Req 1.1/1.2): advance the watermark ONLY when the * final sync SUCCEEDS — on EIO/ENOSPC the bytes are not on stable * storage, so durable_lsn must stay where it is and the failure is * surfaced loudly. */ int rc = fdatasync(p->wal_fd); if (rc == 0 && !flusher_fsync_forced_fail(p)) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } else { int e = (rc != 0) ? errno : EIO; ERROR("persist: final fdatasync on stop FAILED (errno=%d %s) — " "durable_lsn left unchanged; appended-but-unsynced entries " "are NOT durable", e, strerror(e)); } close(p->wal_fd); p->wal_fd = -1; } pthread_mutex_unlock(&p->lock); } void persist_fini(persist_t* p) { repl_fini(&p->repl); /* P2-4: fan-out replicator lifecycle */ pthread_mutex_destroy(&p->lock); pthread_mutex_destroy(&p->checkpoint_mutex); pthread_cond_destroy(&p->checkpoint_done); pthread_mutex_destroy(&p->fsync_mutex); pthread_cond_destroy(&p->fsync_cond); /* Free the in-memory per-LSN term index (ha-term-safety 1.3). */ free(p->term_index); p->term_index = NULL; p->term_index_count = 0; p->term_index_cap = 0; } /* Forward declaration — defined after checkpoint internals */ static void schedule_checkpoint(persist_t* persist); int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len) { /* WAL_CHECKPOINT is a local segment marker, never written via this path */ if (type == WAL_CHECKPOINT) return -EINVAL; pthread_mutex_lock(&p->lock); /* Open first segment lazily */ if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* Build entry header */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = p->current_lsn++; hdr.timestamp = now_us(); hdr.type = type; hdr.payload_len = len; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; /* CRC32 over WAL_HDR_CRC_SIZE bytes of header (excludes crc + prev_crc * fields themselves) then over the payload */ uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (payload && len > 0) c = crc32(c, (const Bytef*)payload, len); hdr.crc = c; /* Write header */ ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } /* Write payload (if any) */ if (payload && len > 0) { ssize_t pw = pwrite(p->wal_fd, payload, len, (off_t)(p->seg_offset + sizeof(hdr))); if (pw != (ssize_t)len) { pthread_mutex_unlock(&p->lock); return -EIO; } } uint32_t entry_size = (uint32_t)(sizeof(hdr) + len); p->seg_offset += entry_size; p->total_wal_bytes += entry_size; p->last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back (ha-term-safety * 1.3, R1.5). Under p->lock; stamps the same hdr.term written to disk. */ record_entry_term(p, hdr.lsn, hdr.term); /* Replicate to standbys — never replicate WAL_CHECKPOINT entries. * last_replicated_crc is distinct from last_crc: * last_crc includes checkpoint markers; last_replicated_crc skips them. * This maintains a continuous CRC chain for standbys (design doc Part 2.3). * Note: WAL_CHECKPOINT is blocked by the assert at function entry, so * every entry that reaches here is a replicable entry. */ p->last_replicated_crc = hdr.crc; /* ha-term-safety task 3.4: source the AppendEntries prev_log_term (term of * the entry at hdr.lsn-1) under p->lock via the lock-free accessor, so the * live broadcast carries the leader's term + prev tuple and the standby * routes it through the library (raft_on_append_entries_request). 0 when * hdr.lsn == 1 (no predecessor). */ uint64_t prev_log_term = entry_term_locked(p, hdr.lsn - 1); repl_broadcast(&p->repl, &hdr, payload, len, prev_log_term); /* Rotate segment when full */ if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } pthread_mutex_unlock(&p->lock); /* Signal the group-commit flusher (ha-commit-durability 2.1): current_lsn * advanced and the entry's bytes are pwritten, so durable_lsn < tip now * holds (the level predicate). The signal is sent AFTER releasing p->lock * and is level-triggered — if it races the flusher and is "lost", the * flusher re-checks durable_lsn < current_lsn-1 under p->lock and flushes * anyway (see fsync_worker). We do NOT fdatasync inline on this path; the * flusher batches the sync across all pending appends. */ pthread_mutex_lock(&p->fsync_mutex); pthread_cond_signal(&p->fsync_cond); pthread_mutex_unlock(&p->fsync_mutex); /* WAL record-count checkpoint trigger: schedule an async checkpoint once * the record count reaches the configured threshold. The fetch_add is * outside the lock — atomic_fetch_add is sufficient for the counter; the * checkpoint itself is guarded by its own CAS inside schedule_checkpoint. */ if (atomic_fetch_add(&p->wal_record_count, 1) + 1 >= p->config.max_wal_records) { schedule_checkpoint(p); } return 0; } /* Lock-held core of persist_truncate_wal_above (defined below): assumes the * caller already holds p->lock and never touches the lock itself. The * replicated-write conflict guard reuses it to repair a divergent entry while * still under p->lock (the follower-AE-conflict truncate contract). */ static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn); /* read_entry_meta_locked: scan the on-disk WAL for the entry at `lsn` and * return its (payload_len, payload-only CRC, raw stored term). The * replicated-write conflict guard uses this to compare a re-streamed entry * against the copy already on disk WITHOUT trusting the LSN alone. pcrc is * computed as crc32(0, payload, len) — the SAME payload-only fingerprint * walcompare/waldump compare across members (NOT the chained header CRC); an * empty (PLEN=0) entry yields len=0, pcrc=0. Reads through the page cache so * a written-but-not-yet-fdatasync'd entry is still visible. Caller MUST hold * p->lock. Returns 0 if the entry is found, -1 if absent or unreadable. */ static int read_entry_meta_locked(persist_t* p, uint64_t lsn, uint32_t* out_len, uint32_t* out_pcrc, uint64_t* out_term) { if (!p || lsn == 0) return -1; DIR* d = opendir(p->wal_path); if (!d) return -1; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return -1; /* Ascending sort by start LSN (segment count is tiny). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } int rc = -1; for (int si = 0; si < nseg && rc != 0; si++) { /* Segments are LSN-ordered; one whose start LSN already exceeds the * target cannot contain it (nor can any later one). */ if (seg_lsns[si] > lsn) break; char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (hn < (ssize_t)sizeof(hdr)) break; /* short read / EOF / pre-allocated zeros */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn == lsn) { uint32_t pc = 0; if (hdr.payload_len > 0) { uint8_t* pl = malloc(hdr.payload_len); if (!pl) break; /* leave rc == -1: unreadable → conservative */ ssize_t pn = pread(fd, pl, hdr.payload_len, (off_t)(off + sizeof(hdr))); if (pn == (ssize_t)hdr.payload_len) pc = crc32(0L, (const Bytef*)pl, hdr.payload_len); free(pl); if (pn != (ssize_t)hdr.payload_len) break; /* unreadable body → conservative */ } if (out_len) *out_len = hdr.payload_len; if (out_pcrc) *out_pcrc = pc; if (out_term) *out_term = hdr.term; rc = 0; break; } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return rc; } int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type, const void* payload, uint32_t len) { if (!p) return -EINVAL; /* WAL_CHECKPOINT is a local-only marker, never streamed to standbys. */ if (type == WAL_CHECKPOINT) return -EINVAL; /* Reject oversized payloads BEFORE touching the segment. */ if (len > MAX_WAL_PAYLOAD_BYTES) return -EMSGSIZE; pthread_mutex_lock(&p->lock); /* Tip-repair bookkeeping (set only on the TIP-conflict repair branch below): * the truncate-and-reappend repair transiently lowers committed_lsn to * lsn-1, so we snapshot the true committed boundary here and restore it * EXACTLY after the re-append re-establishes the entry at `lsn` (never let a * same-(term,lsn) payload repair regress the committed boundary — commit * monotonicity / State Machine Safety). */ int repaired_tip = 0; uint64_t repaired_committed_save = 0; /* Payload-aware conflict guard (raft-ha-core-safety task 2.2): this lsn is * already persisted (current_lsn advanced past it). The legacy guard was * (lsn)-only — it no-op'd ANY re-stream, including a corrective full-payload * write over a previously-stored empty (PLEN=0) body, which is exactly the * cross-term catch-up payload-loss signature (a committed LSN that carries * the right (term,lsn) but an empty payload). Compare (payload-crc, len) * against the copy already on disk and act on the difference: * (c) identical payload → idempotent no-op (keep). * (b) incoming EMPTY over a stored entry → never degrade (keep stored); * an empty body is never the durable form of a known entry. * (a) incoming NON-EMPTY whose payload differs from the stored copy at an * existing (term,lsn) → a genuine payload conflict whose handling * depends on WHERE the conflict sits relative to the stored tip: * · AT THE TIP (lsn == current_lsn - 1, nothing stored above) → * REPAIR: truncate just this one entry and re-append the * authoritative payload at the same (term,lsn). Because nothing * is stored above it, no distinct committed entry is destroyed, * and the committed boundary is restored verbatim afterwards. * · BELOW THE TIP (lsn < current_lsn - 1, committed entries stored * ABOVE the conflict) → MUST NOT truncate: dropping [lsn..tip] * would delete committed entries above the conflict AND roll * committed_lsn/durable_lsn backward — a State Machine Safety / * commit-monotonicity violation. Instead keep the stored entry * untouched, raise a loud non-promotable fence, and let catch-up * reconverge. The real source fix (the streamer must stop * shipping an empty body) is task 2.4; the guard's only job here * is to NOT make things worse (never drop a committed entry, * never regress a boundary) and to surface the corruption loudly. * An unreadable stored entry stays a conservative no-op — never truncate a * committed LSN we cannot read back. */ if (lsn + 1 <= p->current_lsn) { uint32_t inc_pcrc = (payload && len > 0) ? crc32(0L, (const Bytef*)payload, len) : 0; uint32_t st_len = 0; uint32_t st_pcrc = 0; uint64_t st_term = 0; int have = read_entry_meta_locked(p, lsn, &st_len, &st_pcrc, &st_term); int payload_matches = (have == 0 && st_len == len && st_pcrc == inc_pcrc); /* (c) identical payload, (b) empty incoming, or an unreadable stored * entry → leave the stored entry untouched. */ if (payload_matches || len == 0 || have != 0) { pthread_mutex_unlock(&p->lock); return 0; } /* (a) non-empty incoming whose payload differs at an existing (term,lsn) * — a genuine payload conflict. Branch on tip vs below-tip. */ uint64_t tip_lsn = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; if (lsn != tip_lsn) { /* BELOW-TIP conflict: committed entries are stored ABOVE `lsn`. * Truncating here would delete them and roll committed_lsn/ * durable_lsn backward (State Machine Safety / commit-monotonicity * violation — bug-class the round-1 review rejected). Do NOT * truncate, do NOT lower any boundary, do NOT degrade the stored * entry. Surface loudly and raise the sticky non-promotable fence * (reuse the recovery_unrecoverable mechanism: loud, stays ALIVE, * non-promotable, self-heals via persist_clear_recovery_unrecoverable * once catch-up reaches a committed-consistent state — never * self-exits, per the controller invariant). The source fix lives * in task 2.4 (the streamer must stop shipping an empty payload). */ ERROR("persist: persist_wal_write_replicated payload CONFLICT BELOW " "TIP at lsn=%" PRIu64 " (stored_term=%" PRIu64 " stored_len=%u stored_pcrc=%08x vs incoming_len=%u " "incoming_pcrc=%08x); current_lsn=%" PRIu64 " committed_lsn=%" PRIu64 " — refusing to truncate committed entries above the " "conflict (would regress the committed boundary). Keeping the " "stored entry, raising the non-promotable fence and staying " "ALIVE; awaiting a correct re-stream (source fix: task 2.4)", lsn, st_term, st_len, st_pcrc, len, inc_pcrc, p->current_lsn, atomic_load_explicit(&p->committed_lsn, memory_order_acquire)); atomic_store_explicit(&p->recovery_unrecoverable, true, memory_order_release); pthread_mutex_unlock(&p->lock); return 0; /* no-op the write: keep stored, never degrade/regress */ } /* TIP conflict: `lsn` is the highest stored entry, so nothing is stored * above it — truncate-and-reappend replaces EXACTLY this one divergent * entry. truncate_wal_above_locked(lsn-1) drops it and sets * current_lsn = lsn, so control falls through to the append below and * re-persists the authoritative entry at this exact LSN. Snapshot the * committed boundary first: the truncate transiently rewrites * committed_lsn to lsn-1, and we restore it verbatim after the * re-append so a same-(term,lsn) payload repair never regresses (nor * spuriously advances) the committed boundary. */ repaired_committed_save = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); int trc = truncate_wal_above_locked(p, lsn - 1); if (trc != 0) { ERROR("persist: persist_wal_write_replicated repair-truncate FAILED " "for lsn=%" PRIu64 " (rc=%d) — divergent entry NOT repaired", lsn, trc); pthread_mutex_unlock(&p->lock); return trc; } repaired_tip = 1; /* fall through to the contiguity guard + append (lsn == current_lsn). */ } /* Contiguity guard (bug-850): REFUSE a forward GAP. current_lsn is the * next-expected LSN, so a replicated entry with lsn > current_lsn would * leave [current_lsn .. lsn-1] missing AND advance current_lsn past that * hole — after which the idempotency guard above silently DROPS the later * in-order catch-up of the prefix (the failover prefix-loss mechanism). * Append only the contiguous next entry (lsn == current_lsn); reject a gap * without writing or acking so the ACK-driven catch-up continuation * re-streams from current_lsn and the log fills in order. This is Raft * log-matching: append entry N only when the log already holds N-1. A * behind standby that keeps receiving live broadcasts simply drops the * out-of-order ones until catch-up backfills the gap (no loss). -EAGAIN is * the "retry via catch-up" signal the apply path treats as a quiet skip. */ if (lsn > p->current_lsn) { pthread_mutex_unlock(&p->lock); return -EAGAIN; } /* Open first segment lazily. open_new_segment names the file from * current_lsn — for a fresh standby receiving lsn 1 first, current_lsn is * 1 (reserve-lsn-0), so the segment is seg-...001.wal. */ if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* Build entry header. Unlike persist_wal_async, lsn is GIVEN (not * current_lsn++). term isn't streamed, so use our best-effort p->term. * The CRC chain is SELF-computed (the streamed entry carries no crc): * recovery validates an intra-segment chain (prev_crc == last_crc), so the * standby's own writes must chain to each other — cross-node CRC equality * is neither required nor possible. */ wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = lsn; hdr.timestamp = now_us(); hdr.type = type; hdr.payload_len = len; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); if (payload && len > 0) c = crc32(c, (const Bytef*)payload, len); hdr.crc = c; /* Write header */ ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } /* Write payload (if any) */ if (payload && len > 0) { ssize_t pw = pwrite(p->wal_fd, payload, len, (off_t)(p->seg_offset + sizeof(hdr))); if (pw != (ssize_t)len) { pthread_mutex_unlock(&p->lock); return -EIO; } } uint32_t entry_size = (uint32_t)(sizeof(hdr) + len); p->seg_offset += entry_size; p->total_wal_bytes += entry_size; p->last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back (ha-term-safety * 1.3, R1.5). Stamps the same hdr.term written to disk. NOTE: the streamed * envelope does not yet carry the leader's term, so this records the local * best-effort p->term (the design's known hazard closed by task 2.x, which * carries the leader's term on the envelope; this task only provides the * read-back seam over whatever term was persisted). */ record_entry_term(p, hdr.lsn, hdr.term); /* Advance current_lsn to one past the given lsn (max-idempotent — mirrors * apply_wal_entry's monotonic advance). */ if (lsn + 1 > p->current_lsn) p->current_lsn = lsn + 1; /* Durability: fsync the segment BEFORE returning so the caller's ACK * implies the entry is on stable storage (persist-before-ACK). * persist_wal_async defers fsync to the background thread / persist_flush; * here we sync inline because the entry feeds the primary's commit * majority and must survive a standby restart. fdatasync is sufficient * (entry bytes + segment metadata are pre-allocated by open_new_segment). * * Fail-loud rc-gate (ha-commit-durability Req 1.3, symmetric with the * append-path gate at task 2.1): the ACK is sent by the caller ONLY when * this returns success, and the ACK claims the entry is durable. So if * the fdatasync FAILS (EIO/ENOSPC, or the forced-fail test seam), we must * NOT return success — we log loudly and return -EIO so the caller's * existing error path (ha_apply_single_entry) does NOT apply or ACK. */ int fsrc = fdatasync(p->wal_fd); #ifdef XCE_TESTING atomic_fetch_add_explicit(&p->test_replicated_fdatasync_count, 1, memory_order_relaxed); #endif if (fsrc != 0 || flusher_fsync_forced_fail(p)) { int e = (fsrc != 0) ? errno : EIO; ERROR("persist: persist_wal_write_replicated fdatasync FAILED for " "lsn=%" PRIu64 " (errno=%d %s) — entry NOT durable; returning " "error so the standby does NOT ACK", lsn, e, strerror(e)); pthread_mutex_unlock(&p->lock); return -EIO; } /* bug-828 secondary: the entry is now fdatasync'd durable, so advance * durable_lsn to the synced tip immediately (mirror persist_flush). The * 10ms cadence flusher would otherwise advance it lazily, leaving this * standby's ADVERTISED durable position (persist_durable_floor → * RequestVoteReply.peerLastLogLsn) lagging its own synced tip during the * vote window. A standby that has durably persisted the committed prefix * must advertise it at once so a clean failover's safe-commit-from-votes * computation sees the true majority-held LSN. Monotonic max; preserves * durable_lsn <= current_lsn - 1 (tip read under this same p->lock, after * the current_lsn advance above). Reached ONLY past the fail-loud rc-gate * — never advances on an fdatasync error. */ { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } /* Tip-repair committed-boundary restore (raft-ha-core-safety task 2.2): a * TIP-conflict repair above truncated [lsn..tip] (rewriting committed_lsn to * lsn-1) and has now re-appended + fdatasync'd the authoritative entry at the * SAME (term,lsn). Restore the committed boundary VERBATIM so the repair is * commit-boundary-neutral: a same-(term,lsn) payload correction must never * regress committed_lsn (State Machine Safety / commit monotonicity) nor * spuriously advance it (truncate would otherwise have raised it to lsn-1). * Reached only past the fail-loud fdatasync gate, with current_lsn re-advanced * to lsn+1 and durable_lsn at the synced tip, so the invariant * committed_lsn <= min(current_lsn - 1, durable_lsn) holds (saved value was * itself <= the pre-repair tip == lsn). The whole repair is serialized under * p->lock, so no external observer ever sees the transient lowered value. */ if (repaired_tip) { atomic_store_explicit(&p->committed_lsn, repaired_committed_save, memory_order_release); } /* Rotate segment when full (same threshold as the primary append). */ if (p->seg_offset >= p->config.wal_segment_bytes) { if (rotate_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } /* NOTE: no repl_broadcast — standbys never re-replicate. */ pthread_mutex_unlock(&p->lock); return 0; } int persist_flush(persist_t* p) { pthread_mutex_lock(&p->lock); int rc = (p->wal_fd >= 0) ? fdatasync(p->wal_fd) : 0; /* persist_flush is a synchronous durability point (tests, clean shutdown): * it fdatasync'd everything appended so far, so advance durable_lsn to the * synced tip too. Without this, an explicit-flush-only path (slow flusher * cadence) would leave durable_lsn lagging even though the bytes are on * disk. Monotonic max; preserves durable_lsn <= current_lsn - 1 (tip read * under the same p->lock). Gated on rc == 0 (fail-loud, Req 1.1/1.2): a * failed flush must not advance the watermark. */ if (rc == 0 && !flusher_fsync_forced_fail(p)) { uint64_t tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (tip > dur) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } else if (rc != 0 || flusher_fsync_forced_fail(p)) { int e = (rc != 0) ? errno : EIO; ERROR("persist: persist_flush fdatasync FAILED (errno=%d %s) — " "durable_lsn not advanced; flushed bytes are NOT durable", e, strerror(e)); } pthread_mutex_unlock(&p->lock); return rc; } uint64_t persist_durable_lsn(persist_t* p) { return atomic_load_explicit(&p->durable_lsn, memory_order_acquire); } uint64_t persist_durable_floor(persist_t* p) { /* The durable freshness floor (durable last-log index) IS the durable * watermark: persist_recover re-derives it from the recovered WAL tip * (ha-commit-durability task 3.4), and the flusher advances it on every * successful fdatasync — so a thin wrapper avoids a redundant field. NULL * guard keeps the accessor safe for the election consumers (task 4.2). */ return p ? atomic_load_explicit(&p->durable_lsn, memory_order_acquire) : 0; } void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx) { if (!p) return; /* Set ctx before fn so the flusher (which checks fn != NULL) never sees a * live callback with a stale/uninitialised ctx. Registration happens in * commit_engine_init, before the engine is reachable by any ACK path. */ p->durable_cb_ctx = ctx; p->on_durable_advance = fn; } /* raft-apply-decoupling task 1.4: registration counterpart of * persist_set_durable_cb -- see persist.h for the full contract. */ void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn), void* ctx) { if (!p) return; p->truncate_cb_ctx = ctx; p->on_truncate = fn; } #ifdef XCE_TESTING void persist_test_set_force_fsync_fail(persist_t* p, int fail) { if (p) atomic_store_explicit(&p->test_force_fsync_fail, fail ? 1 : 0, memory_order_release); } void persist_test_set_force_job_encode_fail_at(persist_t* p, int index) { if (p) atomic_store_explicit(&p->test_force_job_encode_fail_at, index + 1, memory_order_relaxed); } unsigned persist_test_replicated_fdatasync_count(persist_t* p) { if (!p) return 0; return atomic_load_explicit(&p->test_replicated_fdatasync_count, memory_order_relaxed); } pthread_t persist_test_last_fdatasync_thread(persist_t* p) { /* If no flusher fdatasync has advanced durable_lsn yet, return the * caller's own id so an unprimed read never spuriously looks off-thread. * After a flush, return the recorded flusher thread id (Req 1.6). */ if (!p || atomic_load_explicit(&p->test_last_fdatasync_thread_set, memory_order_acquire) == 0) return pthread_self(); return p->test_last_fdatasync_thread; } /* Force a fresh WAL segment at the current LSN (test seam — see persist.h). * Takes p->lock for the same single-writer serialization the append path uses, * so the segment swap cannot race a concurrent write. */ int persist_test_force_new_segment(persist_t* p) { if (!p) return -EINVAL; pthread_mutex_lock(&p->lock); int rc = rotate_segment(p); pthread_mutex_unlock(&p->lock); return rc; } /* ------------------------------------------------------------------ * * Durability-verification test seam (ha-commit-durability Req 3.2/3.3)* * * * Re-read + CRC-validate the WAL segments from stable storage and * * return the highest fully-persisted LSN — independent of the * * in-memory current_lsn / committed_lsn watermarks. Mirrors the * * segment-scan + CRC-chain validation of replay_wal(), but is strictly * * read-only: it applies no entries, fires no hooks, and mutates no * * persist_t state, so it cannot change the externally observable * * acknowledgment semantics (Req 3.3). * * * * Stops at the first corruption / short read / pre-allocated zero * * region (exactly like recovery), so only entries genuinely on stable * * storage are counted. Returns the highest such LSN, or 0 when nothing * * is durable. * * ------------------------------------------------------------------ */ uint64_t persist_test_durable_lsn_on_disk(persist_t* p) { if (!p) return 0; DIR* d = opendir(p->wal_path); if (!d) return 0; /* no WAL dir → nothing durable */ /* Collect segment start LSNs (same bound as replay_wal). */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue; seg_lsns[nseg++] = seg_start; } closedir(d); if (nseg == 0) return 0; /* Insertion sort by start LSN (segment count is tiny). */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } uint64_t highest_durable = 0; uint32_t last_crc = WAL_CRC_NONE; for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t seg_off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); if (hn == 0) break; /* clean end of segment */ if (hn < (ssize_t)sizeof(hdr)) break; /* truncated / pre-allocated zeros */ /* Pre-allocated zero region: no valid entry is all-zero. */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; /* CRC chain validation — identical rule to replay_wal. */ if (seg_off == 0) { if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && hdr.prev_crc != last_crc) break; } else { if (hdr.prev_crc != last_crc) break; } /* Verify entry CRC over header + payload. */ uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); uint8_t* payload = NULL; if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { close(fd); return highest_durable; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(seg_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { free(payload); break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { free(payload); break; } free(payload); /* Entry is intact and durable on disk. */ last_crc = hdr.crc; if (hdr.lsn > highest_durable) highest_durable = hdr.lsn; seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return highest_durable; } #endif /* XCE_TESTING */ /* ------------------------------------------------------------------ */ /* Checkpoint internals */ /* ------------------------------------------------------------------ */ /* Write a WAL_CHECKPOINT marker to the current segment. * Never calls repl_broadcast() — checkpoint markers are local only. */ static int persist_write_checkpoint_marker(persist_t* p) { pthread_mutex_lock(&p->lock); if (p->wal_fd < 0) { if (open_new_segment(p) != 0) { pthread_mutex_unlock(&p->lock); return -EIO; } } wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.term = p->term; hdr.lsn = p->current_lsn++; hdr.timestamp = now_us(); hdr.type = WAL_CHECKPOINT; hdr.payload_len = 0; hdr.prev_crc = (p->seg_offset == 0) ? WAL_CRC_NONE : p->last_crc; uint32_t c = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); hdr.crc = c; ssize_t hw = pwrite(p->wal_fd, &hdr, sizeof(hdr), (off_t)p->seg_offset); if (hw != (ssize_t)sizeof(hdr)) { pthread_mutex_unlock(&p->lock); return -EIO; } fdatasync(p->wal_fd); p->seg_offset += (uint32_t)sizeof(hdr); p->total_wal_bytes += sizeof(hdr); p->last_crc = hdr.crc; /* NOTE: last_replicated_crc is NOT updated — WAL_CHECKPOINT is never replicated */ /* NOTE: the term index is NOT updated for WAL_CHECKPOINT — replay_wal SKIPS * checkpoint markers (local bookkeeping, not a logical Raft entry), so the * recovered index never holds a checkpoint LSN; the live index must match * (ha-term-safety 1.3). persist_entry_term is for (term,lsn) reconcile, which * never compares against a checkpoint marker. */ pthread_mutex_unlock(&p->lock); return 0; } /* Entity-bearing checkpoint payload (ha-entity-snapshots task 2.2/3.2, Req * 1.1-1.4). Physical layout appended after the existing legacy header * (design.md "Checkpoint File Layout"): * [uint32_t format_version] * [uint32_t job_count] * [job_count * { uint32_t payload_len; uint8_t capnp_Job_bytes[payload_len] }] * [uint32_t queue_count] * [queue_count * { uint32_t name_len; char name[name_len]; uint32_t status }] */ #define SNAPSHOT_FORMAT_VERSION 1u typedef struct { uint8_t* buf; uint32_t len; } snap_job_frame_t; static void snap_free_job_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(frames, i); if (fr) { free(fr->buf); free(fr); } } xlist_fini(frames); } /* Collect + encode every retained job into `frames`, one malloc'd * snap_job_frame_t per job, under a point-in-time-consistent walk * (design.md "Existing Architecture Analysis" / Requirement 1.2): the * jobmgr table's own RTTI_LOCK_READ is held for the ENTIRE walk (excludes * concurrent insert/delete), and each retained job's own RTTI_LOCK_READ is * ADDITIONALLY taken nested around just that job's encode call (excludes a * concurrent per-entity writer for exactly the duration of its encode -- * the table lock alone does not do this). No lock is held during the * disk write itself: this function only builds in-memory frames; the * caller writes them to disk after the table walk has fully released both * locks. * * Filters via job_should_retain (Requirement 1.1) so the checkpoint's * notion of "live" matches the existing GC retention policy. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd -- even zero results -- so the caller can always * xlist_fini it) or a negative errno on the first encode failure, in which * case every frame collected so far has already been freed and `frames` * is empty (Requirement 1.3: a mid-walk failure must not leave partial * entity content behind). */ static int collect_job_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; int retained_index = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero jobs is not a failure. */ if (c == NULL || c->jobmgr == NULL) return 0; lock(c->jobmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->jobmgr); if (irc < 0) { unlock(c->jobmgr); return 0; /* nothing to walk -- mirrors collect_jobs' empty-table handling */ } for (;;) { job_t* j = NULL; int nrc = iter_next(&it, (void**)&j); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (j == NULL) continue; if (!job_should_retain(j, &p->config)) continue; uint8_t* buf = NULL; size_t sz = 0; int enc_rc; lock(j, RTTI_LOCK_READ); #ifdef XCE_TESTING int forced = atomic_load_explicit(&p->test_force_job_encode_fail_at, memory_order_relaxed); if (forced > 0 && (forced - 1) == retained_index) enc_rc = -EIO; /* simulated mid-loop encode failure */ else enc_rc = capnp_encode_job(&j->job, &buf, &sz); #else enc_rc = capnp_encode_job(&j->job, &buf, &sz); #endif unlock(j); retained_index++; if (enc_rc != 0) { rc = enc_rc; break; } snap_job_frame_t* fr = (snap_job_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(buf); rc = -ENOMEM; break; } fr->buf = buf; fr->len = (uint32_t)sz; if (xlist_store(frames, fr) < 0) { free(buf); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->jobmgr); if (rc != 0) snap_free_job_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Queue admin-status frame (ha-entity-snapshots task 3.2, Req 1.1). Unlike * jobs, a queue's on-disk frame carries no capnp payload at all -- it is a * plain (name, status) pair, the entirety of restore_queue_state's contract * (queue.c:208). `name` is a malloc'd, NUL-terminated copy; the on-disk * frame itself carries no NUL (design.md "Checkpoint File Layout": * `{ uint32_t name_len; char name[name_len]; uint32_t status }`). */ typedef struct { char* name; uint32_t status; } snap_queue_frame_t; static void snap_free_queue_frames(xlist_t* frames) { for (int i = 0; i < frames->count; i++) { snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(frames, i); if (fr) { free(fr->name); free(fr); } } xlist_fini(frames); } /* Collect every queue in `frames`, one malloc'd snap_queue_frame_t per * queue, under the same point-in-time-consistent walk discipline as * collect_job_frames above (design.md "Existing Architecture Analysis" / * Requirement 1.2): the qmgr table's own RTTI_LOCK_READ is held for the * ENTIRE walk, and each queue's own RTTI_LOCK_READ is ADDITIONALLY taken * nested around just that queue's (name, status) read -- mirroring * restore_queue_state's own per-entity lock pattern on the load side. No * lock is held during the disk write itself. * * Unlike collect_job_frames, no retention predicate is applied: a queue's * mere presence in the live qmgr table already means config provisioned * it (design.md "write_snapshot (extended)" Responsibilities), so every * queue found is included. * * Returns 0 on success (frames populated; `frames` is always left * xlist_init'd, even for zero results, so the caller can always xlist_fini * it) or a negative errno on the first failure (name allocation), in which * case every frame collected so far has already been freed and `frames` * is empty (Requirement 1.3: a mid-walk failure must not leave partial * entity content behind). */ static int collect_queue_frames(persist_t* p, xlist_t* frames) { cluster_t* c = p->cluster; int rc = 0; xlist_init(frames, 64); /* No live cluster back-pointer (e.g. a bare unit-test persist_t) -- * zero queues is not a failure. */ if (c == NULL || c->qmgr == NULL) return 0; lock(c->qmgr, RTTI_LOCK_READ); iterator_t it; int irc = iter_make(&it, c->qmgr); if (irc < 0) { unlock(c->qmgr); return 0; /* nothing to walk -- mirrors collect_queues' empty-table handling */ } for (;;) { queue_t* q = NULL; int nrc = iter_next(&it, (void**)&q); if (nrc < 0 || nrc == RTTI_ITER_NO_DATA) break; if (q == NULL) continue; char* name; uint32_t status; lock(q, RTTI_LOCK_READ); name = q->queue.queue ? strdup(q->queue.queue) : NULL; status = q->queue.status; unlock(q); if (name == NULL) { rc = -ENOMEM; break; } snap_queue_frame_t* fr = (snap_queue_frame_t*)malloc(sizeof(*fr)); if (fr == NULL) { free(name); rc = -ENOMEM; break; } fr->name = name; fr->status = status; if (xlist_store(frames, fr) < 0) { free(name); free(fr); rc = -ENOMEM; break; } } iter_destroy(&it); unlock(c->qmgr); if (rc != 0) snap_free_queue_frames(frames); /* Requirement 1.3: discard, not partial */ return rc; } /* Write an entity-bearing checkpoint: legacy header (unchanged) + entity * payload (ha-entity-snapshots task 2.2/3.2, Req 1.1-1.4, 3.1-3.3). Any * failure at any step -- collect, encode, write, sync -- leaves NO file at * the final path; the tmp file is always unlinked on error. This is a * retriable-next-checkpoint-cycle failure, never a crash/abort (controller * never self-terminates on an operational condition). */ static int write_snapshot(persist_t* p) { char tmp_path[4096]; char final_path[4096]; snprintf(tmp_path, sizeof(tmp_path), "%s/snap-%016" PRIx64 ".tmp", p->snap_path, p->checkpoint_lsn); snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".capnp", p->snap_path, p->checkpoint_lsn); /* Collect + encode entities BEFORE opening the tmp file: a collection * failure (Requirement 1.3) then never creates a tmp file at all. */ xlist_t job_frames; int rc = collect_job_frames(p, &job_frames); if (rc != 0) return rc; xlist_t queue_frames; rc = collect_queue_frames(p, &queue_frames); if (rc != 0) { snap_free_job_frames(&job_frames); return rc; } int fd = open(tmp_path, O_CREAT | O_WRONLY | O_TRUNC, 0600); if (fd < 0) { snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); return -errno; } /* Sentinel: magic (8) + checkpoint_lsn (8) + committed-config reference * (1 present + 8 tag + 32 sha = 41 bytes; ha-term-safety task 4.2 r1). * The config-ref tail is OPTIONAL on read-back, so an older 16-byte * sentinel still loads (load_snapshot reads the tail best-effort). The * tag is little-endian on this arch, matching the checkpoint_lsn field. */ char header[16 + 1 + 8 + 32]; memset(header, 0, sizeof(header)); memcpy(header, "XCESNAP\0", 8); memcpy(header + 8, &p->checkpoint_lsn, 8); header[16] = (char)(p->cfgref_present ? 1 : 0); memcpy(header + 17, &p->cfgref_tag, 8); memcpy(header + 25, p->cfgref_sha, 32); if (write(fd, header, sizeof(header)) != (ssize_t)sizeof(header)) rc = -EIO; /* Entity payload: format_version, then job frames (design.md "Checkpoint * File Layout"). No table/entity lock is held here -- collect_job_frames * already released both before returning. */ if (rc == 0) { uint32_t fmt = SNAPSHOT_FORMAT_VERSION; if (write(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) rc = -EIO; } if (rc == 0) { uint32_t job_count = (uint32_t)job_frames.count; if (write(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < job_frames.count; i++) { snap_job_frame_t* fr = (snap_job_frame_t*)xlist_get(&job_frames, i); if (write(fd, &fr->len, sizeof(fr->len)) != (ssize_t)sizeof(fr->len)) { rc = -EIO; break; } if (write(fd, fr->buf, fr->len) != (ssize_t)fr->len) { rc = -EIO; break; } } /* Queue trailer (ha-entity-snapshots task 3.2, Req 1.1, 1.2): plain * (name, status) frames, no capnp -- matches restore_queue_state's * exact contract (design.md "Checkpoint File Layout"). */ if (rc == 0) { uint32_t queue_count = (uint32_t)queue_frames.count; if (write(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count)) rc = -EIO; } for (int i = 0; rc == 0 && i < queue_frames.count; i++) { snap_queue_frame_t* fr = (snap_queue_frame_t*)xlist_get(&queue_frames, i); uint32_t name_len = (uint32_t)strlen(fr->name); if (write(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len)) { rc = -EIO; break; } if (name_len > 0 && write(fd, fr->name, name_len) != (ssize_t)name_len) { rc = -EIO; break; } if (write(fd, &fr->status, sizeof(fr->status)) != (ssize_t)sizeof(fr->status)) { rc = -EIO; break; } } if (rc == 0 && fdatasync(fd) != 0) rc = -errno; close(fd); snap_free_job_frames(&job_frames); snap_free_queue_frames(&queue_frames); if (rc != 0) { unlink(tmp_path); return rc; } if (rename(tmp_path, final_path) != 0) { int saved = errno; unlink(tmp_path); return -saved; } return 0; } /* scan_latest_committed_config_ref (ha-term-safety task 4.2 remediation r1): * walk the local WAL segments and capture the HIGHEST-LSN WAL_CONFIG_MARKER * whose LSN is <= committed_lsn into p->cfgref_{present,tag,sha}. This is the * SAME committed-prefix marker config_committed_marker derives, cached durably * so a later checkpoint can trim the marker's segment without losing the * derived committed-config reference (compaction safety). The marker payload is * a fixed 40-byte reference (8-byte big-endian version_tag + 32-byte sha256 — * mirrors config_marker_encode; the config BYTES are never in the WAL), decoded * inline here so persist stays free of a config_repl layering dependency. */ static void scan_latest_committed_config_ref(persist_t* p) { uint64_t committed = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); if (committed == 0) return; /* nothing committed → keep any prior cached ref unchanged */ DIR* d = opendir(p->wal_path); if (!d) return; uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { uint64_t sl = 0; if (sscanf(e->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1) seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return; for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (n < (ssize_t)sizeof(hdr)) break; if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn > committed) break; /* past committed prefix */ if (hdr.type == WAL_CONFIG_MARKER && hdr.payload_len >= WAL_CONFIG_MARKER_PAYLOAD_LEN) { uint8_t buf[WAL_CONFIG_MARKER_PAYLOAD_LEN]; ssize_t pr = pread(fd, buf, sizeof(buf), (off_t)(off + sizeof(hdr))); if (pr == (ssize_t)sizeof(buf)) { p->cfgref_tag = ((uint64_t)buf[0] << 56) | ((uint64_t)buf[1] << 48) | ((uint64_t)buf[2] << 40) | ((uint64_t)buf[3] << 32) | ((uint64_t)buf[4] << 24) | ((uint64_t)buf[5] << 16) | ((uint64_t)buf[6] << 8) | ((uint64_t)buf[7]); memcpy(p->cfgref_sha, buf + 8, 32); p->cfgref_present = 1; /* later markers overwrite — latest wins */ } } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } } /* Delete WAL segment files whose start LSN is strictly less than the * active segment's start LSN AND strictly less than the trim floor. * * Bug-1147 (stopgap, raft-drive-async task 3.2): the trim floor is the * minimum of the active-segment LSN, every active standby's last acked LSN, * and every recently-disconnected slot's last_disconnected_lsn (strategy-A * retention discipline). This ensures we NEVER delete a WAL segment that a * live standby still needs for catch-up (the bug-795/828 committed-job-loss * family). It does NOT fix the empty/far-behind join case (cursor=0) — * that requires entity-bearing snapshots (separate spec). * * Lock ordering (verified 2026-06-26): persist_checkpoint takes p->lock only * to snapshot checkpoint_lsn, then RELEASES it before calling * trim_wal_segments — so this function runs with NO p->lock held. * repl_trim_lsn and repl_expire_disconnect_slots take only r->lock * (p->repl.lock); neither ever acquires p->lock or ce->lock, so there is no * p->lock↔repl.lock (or ce->lock↔repl.lock) inversion. */ static void trim_wal_segments(persist_t* p) { /* Extract the start LSN of the currently-open segment from seg_path. * seg_path has the form "/seg-.wal". * If no segment is open yet (wal_fd < 0), skip trimming. */ if (p->wal_fd < 0 || p->seg_path[0] == '\0') return; const char* seg_basename = strrchr(p->seg_path, '/'); if (!seg_basename) return; seg_basename++; /* skip the '/' */ uint64_t active_seg_lsn = 0; if (sscanf(seg_basename, "seg-%" SCNx64 ".wal", &active_seg_lsn) != 1) return; /* Expire stale disconnected slots so their last_disconnected_lsn guards * don't hold WAL segments indefinitely (slot guard expires at * last_disconnect_time + snap_interval_s). */ repl_expire_disconnect_slots(&p->repl, time(NULL), p->config.snap_interval_s); /* Compute the trim floor: min(active_seg_lsn, all active standby * acked_lsn, all valid disconnected-slot last_disconnected_lsn). * Any segment with seg_start_lsn BELOW this floor is disposable. */ uint64_t floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s); DIR* d = opendir(p->wal_path); if (!d) return; struct dirent* e; while ((e = readdir(d)) != NULL) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start_lsn = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start_lsn) != 1) continue; /* Only delete segments below BOTH the active segment AND the * standby-cursor floor (bug-1147 guard). floor <= active_seg_lsn * by construction (repl_trim_lsn caps at checkpoint_lsn=active_seg_lsn), * so this check subsumes the old < active_seg_lsn guard. */ if (seg_start_lsn < floor) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), "%s/%s", p->wal_path, e->d_name); unlink(seg_path); } } closedir(d); } /* Snapshot filename comparison helper for qsort — sorts ascending by name * (which sorts ascending by LSN since filenames embed the LSN as hex). */ static int snap_name_cmp(const void* a, const void* b) { return strcmp(*(const char* const*)a, *(const char* const*)b); } /* Keep only the snap_keep_n most recent snapshot files; delete the rest. */ /* True when `lsn` is the checkpoint position an in-flight peer snapshot * transfer currently references (ha-entity-snapshots task 4.1/4.3, Req * 3.1-3.3) — checked directly against snapshot_inflight_lsn[], not gated on * snapshot_inflight[] first: the LSN clears before the flag in every * ha_snapshot_send_dispatch/snapshot_lane_fn path, so a stale flag=1 can * momentarily coexist with an already-cleared LSN=0 (see task 4.1's * Implementation Note). A checkpoint's LSN is never 0, so "nonzero and * matches" is the correct, race-free protection signal on its own. No * cluster (p->cluster == NULL, e.g. standalone lib-level tests) means no HA * layer exists to be mid-transfer — nothing is protected. */ static bool snapshot_lsn_in_flight(persist_t* p, uint64_t lsn) { if (p->cluster == NULL || lsn == 0) return false; for (int i = 0; i < MAX_HA_CANDIDATES; i++) { if (atomic_load_explicit(&p->cluster->ha.snapshot_inflight_lsn[i], memory_order_acquire) == lsn) return true; } return false; } static void rotate_snapshots(persist_t* p) { DIR* d = opendir(p->snap_path); if (!d) return; /* Collect all .capnp snapshot filenames */ char* names[1024]; int n = 0; struct dirent* e; while ((e = readdir(d)) != NULL && n < (int)(sizeof(names) / sizeof(names[0]))) { size_t len = strlen(e->d_name); if (len <= 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue; names[n] = strdup(e->d_name); if (names[n]) n++; } closedir(d); if (n == 0) return; /* Sort ascending — oldest first */ qsort(names, (size_t)n, sizeof(names[0]), snap_name_cmp); /* Delete all but the snap_keep_n newest, skipping any file an in-flight * peer transfer is still reading from (task 4.3) — it is retried on the * next rotation pass once that transfer completes. */ int keep = (int)p->config.snap_keep_n; int delete_up_to = n - keep; /* first index to keep */ for (int i = 0; i < delete_up_to && i < n; i++) { uint64_t lsn = 0; if (sscanf(names[i], "snap-%" SCNx64 ".capnp", &lsn) == 1 && snapshot_lsn_in_flight(p, lsn)) continue; char path[4096]; snprintf(path, sizeof(path), "%s/%s", p->snap_path, names[i]); unlink(path); } for (int i = 0; i < n; i++) free(names[i]); } /* ------------------------------------------------------------------ */ /* Async checkpoint scheduling (schedule_checkpoint + task fn) */ /* ------------------------------------------------------------------ */ /* checkpoint_task_fn: executed by delaymgr worker thread. * * The CAS in schedule_checkpoint already holds the "in_progress" slot (=1). * We must clear it before calling persist_checkpoint(), which has its own * atomic_exchange guard. After persist_checkpoint() returns (it resets the * flag to 0), we also reset wal_record_count so the counter starts fresh. */ static int checkpoint_task_fn(void* arg) { persist_t* persist = (persist_t*)arg; /* Release the "scheduled" slot so persist_checkpoint's guard can acquire */ atomic_store(&persist->checkpoint_in_progress, 0); persist_checkpoint(persist); /* Reset the record counter so the next checkpoint window starts clean. * Done after persist_checkpoint returns — the flag is already 0 by now. */ atomic_store(&persist->wal_record_count, 0); return 0; } /* schedule_checkpoint: idempotent async checkpoint request via delaymgr. * Safe to call from any thread. Uses atomic CAS to ensure at most one * checkpoint is scheduled or running at a time. * * If cluster is NULL (unit tests without a live delaymgr) the function * falls back to a synchronous checkpoint. */ static void schedule_checkpoint(persist_t* persist) { int expected = 0; if (!atomic_compare_exchange_strong(&persist->checkpoint_in_progress, &expected, 1)) return; /* already scheduled or running — idempotency guard */ if (persist->cluster == NULL) { /* Unit-test / no-cluster path: run synchronously and reset. */ atomic_store(&persist->checkpoint_in_progress, 0); persist_checkpoint(persist); atomic_store(&persist->wal_record_count, 0); return; } /* Schedule via delaymgr with 1 s coalescing delay, single execution */ delay_task(&persist->cluster->delaymgr, 1, 1, checkpoint_task_fn, persist); } /* ------------------------------------------------------------------ */ /* Public: persist_checkpoint */ /* ------------------------------------------------------------------ */ int persist_checkpoint(persist_t* p) { /* Guard against concurrent checkpoints */ if (atomic_exchange(&p->checkpoint_in_progress, 1) != 0) return -EBUSY; int rc = 0; /* 1. Write WAL_CHECKPOINT marker (local only — never replicated) */ rc = persist_write_checkpoint_marker(p); if (rc != 0) goto done; /* 2. Record checkpoint_lsn = current position after the marker */ pthread_mutex_lock(&p->lock); p->checkpoint_lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); /* 3a. Capture the latest committed config-marker reference BEFORE trimming * (ha-term-safety task 4.2 r1): the trim below can unlink the segment * holding a committed WAL_CONFIG_MARKER, so the derived committed config * must be cached durably (into the sentinel) before that happens. */ scan_latest_committed_config_ref(p); /* 3b. Serialize in-memory state to a snapshot file * (P0: no live kvmgrs — writes sentinel header + cached config ref) */ rc = write_snapshot(p); if (rc != 0) goto done; /* 4. Trim old WAL segments before the checkpoint */ trim_wal_segments(p); /* 5. Rotate old snapshots — keep only snap_keep_n most recent */ rotate_snapshots(p); done: atomic_store(&p->checkpoint_in_progress, 0); return rc; } /* ------------------------------------------------------------------ */ /* Recovery internals */ /* ------------------------------------------------------------------ */ /* Parse and restore every job frame then every queue frame from the entity * payload beginning at the current `fd` offset (design.md "Checkpoint File * Layout"): [job_count][job_count * {payload_len; capnp Job bytes}] * [queue_count][queue_count * {name_len; name; status}]. * * `file_size` is the checkpoint file's total size (from fstat) and * `consumed` is the number of bytes already read from it (legacy header + * format_version) — every length-prefixed frame's length is bounds-checked * against the ACTUAL remaining bytes in the file before being trusted for a * read or allocation, never blindly followed (a corrupt/attacker-controlled * length must fail fast, not drive an oversized read). * * Restores in file order: every job via restore_job, then every queue via * restore_queue_state (design.md sequence diagram). Returns 0 only if every * frame restored; on the FIRST failure of any kind (short read, oversized * length, decode error, or restore error) returns immediately with a * negative errno — the caller MUST NOT commit any position change on a * non-zero return (Requirement 2.3). * * restore_job's -EEXIST is treated as success, not a failure: it is job.c's * own documented idempotent-replay guard (job already present), needed so a * retried load — after an earlier partial attempt already created some jobs * before failing on a later one — can continue past already-restored entries * rather than aborting on them again. restore_queue_state has no such * idempotent-duplicate case (re-applying the same status is already a * harmless overwrite), so any non-zero return from it is a genuine failure. */ static int load_snapshot_entities(int fd, off_t file_size, off_t consumed, persist_t* p, uint64_t stored_lsn) { uint32_t job_count = 0; if (read(fd, &job_count, sizeof(job_count)) != (ssize_t)sizeof(job_count)) return -EIO; consumed += (off_t)sizeof(job_count); if (job_count > 0 && (p->cluster == NULL || p->cluster->jobmgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < job_count; i++) { uint32_t payload_len = 0; if (read(fd, &payload_len, sizeof(payload_len)) != (ssize_t)sizeof(payload_len)) return -EIO; consumed += (off_t)sizeof(payload_len); if (payload_len == 0 || (off_t)payload_len > file_size - consumed) return -EIO; uint8_t* buf = (uint8_t*)malloc(payload_len); if (buf == NULL) return -ENOMEM; if (read(fd, buf, payload_len) != (ssize_t)payload_len) { free(buf); return -EIO; } consumed += (off_t)payload_len; int rrc = restore_job(p->cluster, 0, buf, payload_len, stored_lsn); free(buf); if (rrc != 0 && rrc != -EEXIST) return rrc; } uint32_t queue_count = 0; if (read(fd, &queue_count, sizeof(queue_count)) != (ssize_t)sizeof(queue_count)) return -EIO; consumed += (off_t)sizeof(queue_count); if (queue_count > 0 && (p->cluster == NULL || p->cluster->qmgr == NULL)) return -EINVAL; /* entities present but nowhere to restore them into */ for (uint32_t i = 0; i < queue_count; i++) { uint32_t name_len = 0; if (read(fd, &name_len, sizeof(name_len)) != (ssize_t)sizeof(name_len)) return -EIO; consumed += (off_t)sizeof(name_len); if (name_len == 0 || (off_t)name_len > file_size - consumed) return -EIO; char* name = (char*)malloc((size_t)name_len + 1); if (name == NULL) return -ENOMEM; if (read(fd, name, name_len) != (ssize_t)name_len) { free(name); return -EIO; } name[name_len] = '\0'; consumed += (off_t)name_len; uint32_t status = 0; if (read(fd, &status, sizeof(status)) != (ssize_t)sizeof(status)) { free(name); return -EIO; } consumed += (off_t)sizeof(status); int qrc = restore_queue_state(p->cluster, name, status); free(name); if (qrc != 0) return qrc; } return 0; } /* Load a snapshot: legacy header (magic + checkpoint_lsn + optional * config-ref tail), then — for an entity-bearing checkpoint — every job and * queue frame that follows it (ha-entity-snapshots task 5.2, Requirements * 2.1-2.4, 5.1, 5.2). See design.md "load_snapshot (extended)" and its * sequence diagram for the exact contract this implements. * * Ordering (Requirement 2.1, 2.2, 2.3 — restore FIRST, validate FULLY, THEN * advance position, never the reverse, never partial): the header and any * config-ref tail are parsed into LOCAL variables only. Format is then * detected by whether any bytes remain past the header (design.md's stated * discriminator — not a flag bit). A short file (nothing remains) is a * legacy, pre-feature checkpoint: not an error, so its recorded position is * committed immediately and *out_entities_restored is false. Otherwise every * job then every queue frame is restored via load_snapshot_entities() before * ANYTHING is written to p->checkpoint_lsn/p->current_lsn/p->cfgref_*; on * any restore failure this function returns non-zero with p's fields * PROVABLY untouched (the single commit point below is only ever reached * after load_snapshot_entities() returns 0). *out_entities_restored is only * set on a rc==0 return, matching persist.h's documented contract. * * Returns 0 on success, -EINVAL on bad magic / unrecognized entity format, * -EIO on a short/corrupt read, -ENOMEM on allocation failure, -errno on * open/fstat failure, or a restore_job/restore_queue_state error code. */ static int load_snapshot(persist_t* p, const char* path, uint64_t lsn, bool* out_entities_restored) { (void)lsn; /* lsn is encoded in the file itself — we re-read it */ int fd = open(path, O_RDONLY); if (fd < 0) return -errno; struct stat st; if (fstat(fd, &st) != 0) { int saved = errno; close(fd); return -saved; } /* magic(8) + checkpoint_lsn(8) + optional config-ref tail(1+8+32). */ char header[16 + 1 + 8 + 32]; ssize_t n = read(fd, header, sizeof(header)); if (n < 16) { close(fd); return -EIO; } if (memcmp(header, "XCESNAP\0", 8) != 0) { close(fd); return -EINVAL; } /* Extract checkpoint_lsn from bytes 8-15 (little-endian on this arch). * Held LOCALLY -- not yet written to p-> -- until the single commit * point below (legacy short-circuit, or after every entity restores). */ uint64_t stored_lsn; memcpy(&stored_lsn, header + 8, 8); /* Optional committed-config reference (ha-term-safety task 4.2 r1): * present only when the sentinel carries the 41-byte tail. Also held * locally until the same commit point. */ int cfgref_present = 0; uint64_t cfgref_tag = 0; uint8_t cfgref_sha[32]; memset(cfgref_sha, 0, sizeof(cfgref_sha)); if (n >= 16 + 1 + 8 + 32 && header[16] != 0) { cfgref_present = 1; memcpy(&cfgref_tag, header + 17, 8); memcpy(cfgref_sha, header + 25, 32); } off_t consumed = (off_t)n; /* Legacy-format detection (Requirement 2.4, 5.1, 5.2): "does more data * exist" is the discriminator (design.md), not a flag bit -- a file * ending exactly at the legacy header is unambiguously a pre-feature * checkpoint. This is an expected rollout shape, not corruption: commit * the position it records (existing, preserved legacy behavior) and * report entities_restored=false so the caller falls back to normal * catch-up rather than treating this checkpoint as fully complete. */ if (st.st_size <= consumed) { close(fd); p->checkpoint_lsn = stored_lsn; p->current_lsn = stored_lsn; if (cfgref_present) { p->cfgref_present = 1; p->cfgref_tag = cfgref_tag; memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha)); } if (out_entities_restored) *out_entities_restored = false; return 0; } /* Entity-bearing checkpoint: format_version, then job frames, then queue * frames (design.md "Checkpoint File Layout"). */ uint32_t fmt = 0; if (read(fd, &fmt, sizeof(fmt)) != (ssize_t)sizeof(fmt)) { close(fd); return -EIO; } consumed += (off_t)sizeof(fmt); /* An unrecognized format_version is corruption or a future layout this * binary cannot read -- fail loud rather than silently treating it as * legacy (task 5.1 review note: a version mismatch must never be masked * as an expected legacy shape, or real corruption goes undetected). */ if (fmt != SNAPSHOT_FORMAT_VERSION) { close(fd); return -EINVAL; } int rc = load_snapshot_entities(fd, st.st_size, consumed, p, stored_lsn); close(fd); if (rc != 0) return rc; /* Requirement 2.3: p->*_lsn / p->cfgref_* untouched */ /* Single commit point (Requirement 2.1, 2.2): reached ONLY after every * job and queue frame restored successfully above. */ p->checkpoint_lsn = stored_lsn; p->current_lsn = stored_lsn; if (cfgref_present) { p->cfgref_present = 1; p->cfgref_tag = cfgref_tag; memcpy(p->cfgref_sha, cfgref_sha, sizeof(p->cfgref_sha)); } if (out_entities_restored) *out_entities_restored = true; return 0; } /* Public wrapper for load_snapshot — exposed via persist.h for ha.c (Task * 3.5c). load_snapshot() above now owns the full restore-before-advance * implementation (ha-entity-snapshots task 5.2): this wrapper is a thin * pass-through, since load_snapshot() already gates *out_entities_restored * and the p->checkpoint_lsn/current_lsn commit on entity-restore success * (Requirements 2.1-2.4, 5.1, 5.2). */ int persist_load_snapshot(persist_t *p, const char *path, uint64_t lsn, bool *out_entities_restored) { return load_snapshot(p, path, lsn, out_entities_restored); } /* P2-2: Standby WAL replay — apply a single replicated entry to in-memory state. * * No hooks fire on standbys; the hook system is empty until promotion. * * Returns 0 on success (including skip for checkpoint / unknown types), * negative errno on error (NULL args, oversized payload). */ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn, const void* payload, uint32_t payload_len) { if (!p) return -EINVAL; /* Reject oversized payloads BEFORE any allocation */ if (payload_len > MAX_WAL_PAYLOAD_BYTES) { ERROR("apply_wal_entry: payload_len %u exceeds MAX_WAL_PAYLOAD_BYTES %u", payload_len, MAX_WAL_PAYLOAD_BYTES); return -EMSGSIZE; } /* LOGICAL entity restore ONLY. The on-disk WAL header, the CRC chain * (p->last_crc), and current_lsn are owned exclusively by the physical * layer — persist_wal_write_replicated on the live replicated path, and * replay_wal/persist_recover on recovery. This function MUST NOT touch * them. (bug-778: it previously set p->last_crc from a synthetic crc=0 * header, breaking the on-disk CRC chain and cold-restart recovery.) */ /* WAL_CHECKPOINT (100) is a local-only marker — standbys never receive it * via replication. Skip silently. */ if (type == WAL_CHECKPOINT) return 0; /* Unknown types above WAL_ENTRY_TYPE_MAX: log WARN and skip for forward * compatibility — an older controller can boot on WAL written by a newer * one without hard-failing on unrecognised entry types. */ if (type > WAL_ENTRY_TYPE_MAX) { WARN("apply_wal_entry: unknown type %u > WAL_ENTRY_TYPE_MAX %u" " (lsn=%" PRIu64 ") — skipping for forward compatibility", type, WAL_ENTRY_TYPE_MAX, lsn); return 0; } /* Advance the current_lsn high-water (monotonic, idempotent). This is a * write-cursor high-water that is safe to bump from multiple paths. The * CRC chain link (p->last_crc) is deliberately NOT touched here — it has a * single owner: the physical writer (persist_wal_write_replicated) and * replay (persist_recover). (bug-778) */ pthread_mutex_lock(&p->lock); if (lsn + 1 > p->current_lsn) p->current_lsn = lsn + 1; pthread_mutex_unlock(&p->lock); if (!p->cluster) return 0; /* unit tests without a full cluster — skip entity restore */ switch (type) { case WAL_JOB_SUBMIT: /* Payload: Cap'n Proto serialized xce_job_t. Thread the entry LSN * through so the restored job is tagged for the egress watermark * (ha-convergence-recovery 3.3). */ restore_job(p->cluster, 0, payload, payload_len, lsn); break; case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: case WAL_JOB_RUSAGE: /* Payload: wal_jobstate_t {jobid, state} */ if (payload_len >= sizeof(wal_jobstate_t)) { const wal_jobstate_t* js = (const wal_jobstate_t*)payload; restore_job_state(p->cluster, js->jobid, js->state); } break; case WAL_WORKER_CTRL: /* Payload: [uint32_t detail][capnp worker] */ if (payload_len > WORKER_WAL_HDR_SIZE) worker_from_capnp(p->cluster, payload, payload_len); break; case WAL_QUEUE_CTRL: /* Payload: [wal_queuehdr_t][name NUL-terminated] */ if (payload_len > sizeof(wal_queuehdr_t)) { const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload; const char* name = (const char*)(payload + sizeof(wal_queuehdr_t)); restore_queue_state(p->cluster, name, qh->status); } break; case WAL_NOOP: /* Internal promotion no-op: empty payload, no entity restore, no * hooks. current_lsn is advanced by the physical layer (writer / * replay), not here — recognised explicitly rather than via default. */ break; case WAL_CONFIG_MARKER: /* Config-change marker (ha-term-safety task 4.2): a {version_tag, * sha256} reference, NEVER the config bytes (R7.4). It carries NO * entity state, so replay performs no restore here — the committed * configuration is DERIVED from the latest committed marker on demand * (config_committed_marker), and the bytes are obtained + hash-verified * out-of-band before activation (task 4.3). Recognised explicitly so * the per-LSN term index (built above this switch's caller) records its * term and on_get_entry_term / (term,lsn) reconcile treat it as an * ordinary log entry. */ break; default: break; } return 0; } /* raft-apply-decoupling task 2.2 — see persist.h for the full contract doc * (routing table, the job-state-family hook-mapping deviation, and what this * function deliberately does NOT own). Not yet wired into any live call * site — replay (2.3) and the live apply pump (6.1) will call this later. */ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, const void* payload, uint32_t len) { if (!p) return -EINVAL; if (!p->cluster) return 0; /* unit tests without a full cluster — nothing to dispatch into */ cluster_t* c = p->cluster; switch (wal_type) { case WAL_JOB_SUBMIT: { int rc = restore_job(c, 0, payload, len, lsn); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) return rc; /* Recover the jobid via a cheap raw capnp field read (read_Job only * reads the fixed uint64 field — no allocation, unlike decode_Job) * so the just-constructed job can be looked up for the notification * hook without changing restore_job's own contract. */ struct capn jc_ctx; capn_init_mem(&jc_ctx, (const uint8_t*)payload, len, 0); Job_ptr jp; jp.p = capn_getp(capn_root(&jc_ctx), 0, 1); struct Job cj; read_Job(&cj, jp); uint64_t jobid = cj.jobid; capn_free(&jc_ctx); job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)jobid, (void**)&j); unlock(c->jobmgr); if (!j) return 0; /* defensive: should not happen after a successful restore */ void* jptr = j; xlist_t jlist = { .count = 1, .caps = 1, .delta = 1, .data = &jptr }; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_NEW, c, &jlist); return 0; } case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: case WAL_JOB_RUSAGE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: { /* Every job-state fact type shares one wire shape — wal_jobstate_t * {jobid, state} — and therefore one guarded core. See persist.h's * doc comment on apply_committed_entry for why this does not * dispatch to the type-specific apply_job_{start,finish,fail}_ * transition() cores (their richer parameter structs are not * present on this wire format). */ if (len < sizeof(wal_jobstate_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_jobstate_t* js = (const wal_jobstate_t*)payload; job_t* j = NULL; lock(c->jobmgr, RTTI_LOCK_READ); key_find(c->jobmgr, (void*)js->jobid, (void**)&j); unlock(c->jobmgr); if (!j) return -ENOENT; lock(j, RTTI_LOCK_WRITE); job_transition_result_t result = apply_job_update_transition(j, (int)js->state); unlock(j); if (result != JOB_TRANSITION_APPLIED) return 0; /* rejected-backward / no-op-duplicate — no hook */ xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_UPDATE, c, j); return 0; } case WAL_QUEUE_CTRL: { if (len <= sizeof(wal_queuehdr_t)) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ const wal_queuehdr_t* qh = (const wal_queuehdr_t*)payload; const char* name = (const char*)((const uint8_t*)payload + sizeof(wal_queuehdr_t)); /* restore_queue_state's -ENOENT ("queue not yet built from config") * and -EINVAL (bad name) are both benign skips here, matching * apply_wal_entry's existing behavior of never fencing on this * entry type's restore outcome. */ if (restore_queue_state(c, name, qh->status) < 0) return 0; queue_t* q = NULL; lock(c->qmgr, RTTI_LOCK_READ); key_find(c->qmgr, (void*)name, (void**)&q); unlock(c->qmgr); if (!q) return 0; xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, HOOK_AFTER_EVENT, HOOK_DETAIL_QUEUE_UPDATE, c, q); return 0; } case WAL_WORKER_CTRL: { if (len <= WORKER_WAL_HDR_SIZE) return 0; /* undersized payload — skip, mirrors apply_wal_entry */ uint32_t hook_detail; memcpy(&hook_detail, payload, sizeof(hook_detail)); int rc = worker_from_capnp(c, payload, len); if (rc == -EEXIST) return 0; /* idempotent re-apply (replay / resync) — no hook */ if (rc < 0) return rc; /* Recover the worker's name via a cheap raw capnp field read * (read_Worker's capn_get_text is a borrowed, NUL-terminated pointer * into the decode buffer — no allocation, unlike decode_Worker) so * the just-restored worker can be looked up for the notification * hook without changing worker_from_capnp's own contract. */ const uint8_t* wpayload = (const uint8_t*)payload + WORKER_WAL_HDR_SIZE; size_t wlen = len - WORKER_WAL_HDR_SIZE; struct capn wc_ctx; capn_init_mem(&wc_ctx, wpayload, wlen, 0); Worker_ptr wp; wp.p = capn_getp(capn_root(&wc_ctx), 0, 1); struct Worker cw; read_Worker(&cw, wp); worker_t* w = NULL; lock(c->workermgr, RTTI_LOCK_READ); key_find(c->workermgr, (void*)cw.name.str, (void**)&w); unlock(c->workermgr); capn_free(&wc_ctx); if (!w) return 0; /* defensive: should not happen after a successful restore */ switch (hook_detail) { case HOOK_DETAIL_WORKER_JOIN: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_JOIN, c, w->host.name, w); break; case HOOK_DETAIL_WORKER_UPDATE: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_UPDATE, c, w); break; case HOOK_DETAIL_WORKER_DEAD: xce_run_hooks(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, HOOK_AFTER_EVENT, HOOK_DETAIL_WORKER_DEAD, c, w); break; default: break; /* unrecognized discriminator — mutation already applied, * notification skipped rather than guessed at */ } return 0; } default: /* WAL_NOOP / WAL_CONFIG_MARKER / WAL_CHECKPOINT / unknown — no * entity effect at this seam, no hook (unchanged from * apply_wal_entry's existing treatment of these types). */ return 0; } } /* Read WAL segment files in LSN order; replay entries with lsn > from_lsn. * * Segment filename format: seg-.wal * Entry format: wal_entry_hdr_t (40 bytes) + payload (hdr.payload_len bytes). * * CRC chain: hdr.prev_crc must match last_crc (WAL_CRC_NONE for first entry * in a segment). If prev_crc mismatches, stop at last good entry and log WARN. * * Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1): * *verdict_out (when non-NULL) is set to RECOVER_TRUNCATED iff replay stopped * on a STRUCTURAL corruption — a CRC chain break, a CRC mismatch, or a * short/partial entry read — and RECOVER_CLEAN otherwise (clean EOF, * pre-allocated-zero tail, empty WAL, or all entries replayed). The verdict is * about the STRUCTURE of the local log, not about being behind the cluster: a * clean log that simply ends below the cluster high-water is still CLEAN. * * Returns number of entries replayed (>= 0) or negative errno on I/O error. */ static int replay_wal(persist_t* p, uint64_t from_lsn, recover_verdict_t* verdict_out) { /* Default to CLEAN; only a structural corruption flips it to TRUNCATED. */ if (verdict_out) *verdict_out = RECOVER_CLEAN; /* --- Collect and sort segment filenames by start LSN --- */ DIR* d = opendir(p->wal_path); if (!d) return 0; /* no WAL dir → first boot, nothing to replay (CLEAN) */ /* Collect up to 4096 segment names */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns)/sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t seg_start = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &seg_start) != 1) continue; seg_lsns[nseg++] = seg_start; } closedir(d); if (nseg == 0) return 0; /* Simple insertion sort — segment count is tiny in practice */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } /* --- Replay entries from each segment --- */ int replayed = 0; uint32_t last_crc = WAL_CRC_NONE; for (int si = 0; si < nseg; si++) { char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) { WARN("persist_recover: cannot open segment %s: %s", seg_path, strerror(errno)); continue; } uint32_t seg_off = 0; for (;;) { /* Read header */ wal_entry_hdr_t hdr; ssize_t hn = pread(fd, &hdr, sizeof(hdr), (off_t)seg_off); if (hn == 0) break; /* clean end of segment (CLEAN) */ if (hn < (ssize_t)sizeof(hdr)) { /* A partial header (or read error) is a SHORT READ — the log * stops mid-entry, so it cannot be reconstructed contiguously * (RECOVER_TRUNCATED, Req 2.1). Distinct from clean EOF above. */ if (hn < 0) WARN("persist_recover: read error in %s at offset %u: %s", seg_path, seg_off, strerror(errno)); else WARN("persist_recover: short header read in %s at offset %u " "(got %zd of %zu bytes); stopping replay", seg_path, seg_off, hn, sizeof(hdr)); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } /* Detect pre-allocated zero region (no valid entry starts with * all-zero header: lsn == 0 && type == 0 && payload_len == 0). * This is the normal fallocate'd tail after the last valid entry — * a clean stop, NOT corruption (stays RECOVER_CLEAN). */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; /* CRC chain validation: prev_crc must match last_crc in segment */ if (seg_off == 0) { /* First entry in this segment: prev_crc must be WAL_CRC_NONE * (or we accept any value if last_crc is also WAL_CRC_NONE, * e.g. across a segment boundary after checkpoint trim). */ if (hdr.prev_crc != WAL_CRC_NONE && last_crc != WAL_CRC_NONE && hdr.prev_crc != last_crc) { WARN("persist_recover: CRC chain break at start of %s " "(prev_crc=%08x expected=%08x); stopping replay", seg_path, hdr.prev_crc, last_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } } else { if (hdr.prev_crc != last_crc) { WARN("persist_recover: CRC chain break in %s at offset %u " "(prev_crc=%08x expected=%08x); stopping replay at last good entry", seg_path, seg_off, hdr.prev_crc, last_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; break; } } /* Verify entry CRC */ uint32_t expected_crc = crc32(0L, (const Bytef*)&hdr, WAL_HDR_CRC_SIZE); uint8_t* payload = NULL; if (hdr.payload_len > 0) { payload = malloc(hdr.payload_len); if (!payload) { close(fd); return -ENOMEM; } ssize_t pn = pread(fd, payload, hdr.payload_len, (off_t)(seg_off + sizeof(hdr))); if (pn != (ssize_t)hdr.payload_len) { WARN("persist_recover: short payload read in %s at offset %u", seg_path, seg_off); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; free(payload); break; } expected_crc = crc32(expected_crc, (const Bytef*)payload, hdr.payload_len); } if (hdr.crc != expected_crc) { WARN("persist_recover: CRC mismatch in %s at offset %u " "(got=%08x expected=%08x); stopping replay", seg_path, seg_off, hdr.crc, expected_crc); if (verdict_out) *verdict_out = RECOVER_TRUNCATED; free(payload); break; } /* Advance segment-local CRC */ last_crc = hdr.crc; /* Record this entry's term at its LSN for recovery read-back * (ha-term-safety 1.3, R1.5). Record EVERY CRC-validated logical * entry — including snapshot-covered ones below from_lsn — so the * index spans the full LSN range the reconcile/promotion logic may * query. EXCLUDE WAL_CHECKPOINT markers (type == WAL_CHECKPOINT) and * unknown future types (type > WAL_CHECKPOINT): they are skipped * below and are never logical Raft entries, so the live append path * does not record them either (kept symmetric). The raw hdr.term is * stored verbatim (legacy 0 included); the legacy-0 -> WAL_LEGACY_TERM * mapping is applied only at read-back. */ if (hdr.type < WAL_CHECKPOINT) record_entry_term(p, hdr.lsn, hdr.term); /* Skip entries already covered by the snapshot */ if (hdr.lsn <= from_lsn) { free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); continue; } /* Skip WAL_CHECKPOINT markers (local bookkeeping only) */ if (hdr.type == WAL_CHECKPOINT) { free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); /* Advance current_lsn past the marker */ if (hdr.lsn + 1 > p->current_lsn) p->current_lsn = hdr.lsn + 1; continue; } /* Unknown future entry types: log and skip */ if (hdr.type > WAL_CHECKPOINT) { WARN("persist_recover: unknown WAL entry type %u at LSN %lu; skipping", hdr.type, (unsigned long)hdr.lsn); free(payload); seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); continue; } /* Logical entity restore via the shared apply seam * (raft-apply-decoupling task 2.3, design.md "Apply Seam"): the * SAME apply_committed_entry() dispatch the live commit-advance * apply pump will use (task 6.1), so a node's materialized state * is the deterministic result of its committed log regardless of * whether it got there via restart replay or live apply * (Req 3.1-3.4). apply_wal_entry() (this replay loop's previous * per-entry dispatch) is NOT retired by this change — it remains * the live standby catch-up/replicated-apply dispatch used * elsewhere (ha.c); it is simply no longer this loop's call. The * CRC chain + current_lsn remain owned by this replay loop * (current_lsn advanced below; last_crc at line ~1601) — neither * dispatch function ever touched them (bug-778). * * KNOWN HAZARD (tracked in tasks.md Implementation Notes, task * 2.3): this replays the FULL post-checkpoint WAL range on every * restart (last_applied_lsn is not persisted, deliberately * recomputed here), so apply_committed_entry()'s AFTER_EVENT * hooks fire again for entries already applied+notified in a * prior process life. Confirmed inert today: scheduler hooks * aren't registered until a post-boot network JOIN * (on_scheduler_join, sched.c), and apiserver hooks — though * registered before this runs — gate their notify on a * connected endpoint that is empty at first-boot replay time. * MUST be re-verified before tasks 4.1/6.1/6.2/6.3 make this * path broadly live. */ apply_committed_entry(p, hdr.type, hdr.lsn, payload, hdr.payload_len); free(payload); /* Advance current_lsn to one past this entry */ if (hdr.lsn + 1 > p->current_lsn) p->current_lsn = hdr.lsn + 1; replayed++; seg_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } /* Update last_crc so the next WAL write chains correctly */ pthread_mutex_lock(&p->lock); p->last_crc = last_crc; pthread_mutex_unlock(&p->lock); return replayed; } /* ------------------------------------------------------------------ */ /* Public: persist_recover */ /* ------------------------------------------------------------------ */ int persist_recover(persist_t* p) { /* --- Step 1: find latest snapshot (highest LSN filename) --- */ char snap_file[4096] = ""; uint64_t snap_lsn = 0; DIR* d = opendir(p->snap_path); if (d) { struct dirent* e; while ((e = readdir(d)) != NULL) { size_t len = strlen(e->d_name); if (len < 6 || strcmp(e->d_name + len - 6, ".capnp") != 0) continue; uint64_t lsn = 0; if (sscanf(e->d_name, "snap-%" SCNx64 ".capnp", &lsn) != 1) continue; if (lsn > snap_lsn) { snap_lsn = lsn; snprintf(snap_file, sizeof(snap_file), "%s/%s", p->snap_path, e->d_name); } } closedir(d); } /* --- Step 2: load snapshot (if any) --- */ if (snap_lsn > 0 && snap_file[0] != '\0') { /* Local recovery doesn't need the entities-restored indicator to * decide behavior: the WAL replay in Step 3 applies everything * above checkpoint_lsn regardless, unchanged by ha-entity-snapshots. * It IS consumed for observability parity with the wire receive * path (ha_handle_snapshot_chunk, task 6.1) — a rolling-upgrade * restart against a pre-feature (legacy, position-only) local * checkpoint is an expected rollout state, not an error. */ bool entities_restored = false; int rc = load_snapshot(p, snap_file, snap_lsn, &entities_restored); if (rc != 0) { ERROR("persist_recover: failed to load snapshot %s: %s", snap_file, strerror(-rc)); return rc; } /* checkpoint_lsn and current_lsn set by load_snapshot */ if (entities_restored) { INFO("persist_recover: snapshot %s loaded lsn=%" PRIu64 " with full job/queue entity restoration", snap_file, p->checkpoint_lsn); } else { INFO("persist_recover: snapshot %s lsn=%" PRIu64 " is a legacy " "position-only checkpoint (no entity content) — continuing " "via normal WAL replay", snap_file, p->checkpoint_lsn); } } /* --- Step 3: replay WAL entries after checkpoint_lsn --- */ /* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1): * replay distinguishes a clean log end from a corruption-truncated end. * We CLASSIFY + record the verdict here; we do NOT FATAL (task 3.3) and do * NOT fence promotion (task 4.3 consumes p->recovery_verdict). A clean * replay over a loaded snapshot-sentinel (no entries below the sentinel) is * CLEAN — the sentinel established the committed position; replay only adds * entries above it. */ recover_verdict_t verdict = RECOVER_CLEAN; int replayed = replay_wal(p, p->checkpoint_lsn, &verdict); if (replayed < 0) return replayed; p->recovery_verdict = verdict; if (verdict == RECOVER_TRUNCATED) WARN("persist_recover: RECOVER_TRUNCATED — local WAL is structurally " "corrupt (CRC break / mismatch / short read); member marked " "recovery-truncated (non-promotable until caught up)"); /* --- Step 3b: re-derive the durable floor from the recovered WAL tip * (ha-commit-durability task 3.4, Req 2.3) --- * * Publish the member's durable floor (the durable last-log index consumed * by the Raft freshness gate, task 4.2) as the highest INTACT replayed LSN. * replay_wal advanced current_lsn to one past the last intact entry — incl. * stopping at a corruption boundary on TRUNCATED — so current_lsn - 1 is * exactly that intact WAL tip, and a snapshot-sentinel load leaves it at the * sentinel/replayed tip. We store it into durable_lsn because the recovered * on-disk entries are GENUINELY durable (each was CRC-validated and read * from stable storage by replay_wal), so the freshness advertisement is now * backed by durable state rather than a volatile in-memory position. * * Crux (Req 2.3 — a truncated member cannot over-advertise): this is an * ASSIGN to the recovered tip, NOT a max() with whatever durable_lsn held. * A fresh recover starts from durable_lsn=0 (persist_init), so on a * truncating restart the floor lands at the LOWER intact tip — never a * stale higher in-memory value and never the commit position * (committed_lsn). The recovered tip is, by construction, <= current_lsn-1, * preserving the durable_lsn <= current_lsn-1 invariant. The post-election * safe-commit point continues to come from voter replies, not this floor * (CLAUDE.md). */ uint64_t recovered_tip = (p->current_lsn > 0) ? p->current_lsn - 1 : 0; atomic_store_explicit(&p->durable_lsn, recovered_tip, memory_order_release); /* --- Step 4: retention pass (TTL check) — no-op in P0 (no entities) --- */ return 0; } /* Recovery-integrity accessors (ha-commit-durability task 3.1, Req 2.1). */ recover_verdict_t persist_recovery_verdict(persist_t* p) { return p ? p->recovery_verdict : RECOVER_CLEAN; } bool persist_recovery_truncated(persist_t* p) { return p && p->recovery_verdict == RECOVER_TRUNCATED; } /* ------------------------------------------------------------------ * * Boot-policy loud-fence decision (ha-commit-durability task 3.3, Req 2.4; * never-self-exit rework 2026-06-18). * * Implements the FENCE_UNRECOVERABLE terminal of the Boot Policy Decision Table * (design.md). The decision is a PURE function of its inputs; the action (loud * operator-visible log + set the sticky non-promotable flag) is separated into * persist_recover_loud_fence_if_unrecoverable so the decision can be unit-tested * in isolation. A reachable healthy peer NEVER yields FENCE_UNRECOVERABLE * (rows 3, 5); it is reserved for a member that cannot recover locally AND * cannot catch up (no healthy peer, rows 4, 4b). * * CRITICAL: this terminal is a LOUD, PERMANENT, NON-promotable fence — the * process STAYS ALIVE and keeps retrying catch-up. There is NO abort()/exit(): * a controller MUST NEVER self-terminate as a designed response to an * operational condition (user invariant, 2026-06-18). A self-exited controller * needs manual admin recovery and degrades quorum meanwhile; one that stays * alive and keeps retrying self-heals when a peer returns (Req 2.6). * ------------------------------------------------------------------ */ recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in) { if (in == NULL) return RECOVER_DECISION_ELIGIBLE; /* defensive: nothing to fence */ bool truncated = (in->verdict == RECOVER_TRUNCATED); /* Rows 1 & 6: a structurally CLEAN log that is NOT behind the cluster * high-water mark is a normal promotable member — no fence. */ if (!truncated && !in->durable_below_hwm) return RECOVER_DECISION_ELIGIBLE; /* From here the member is truncated OR behind the HWM — it must NOT lead * as-is. A truncated log is by definition not provably at/above the HWM * (design.md note), so it is treated as behind for fence purposes. */ /* Rows 2, 3, 5: a healthy peer is reachable. Catch up from it and regain * eligibility (task 4.3). A reachable peer NEVER yields the unrecoverable * fence — even for a truncated log or a legitimate snapshot-sentinel. If * catch-up has already succeeded the member is eligible; otherwise it stays * fenced (4.3 drives the catch-up + release). */ if (in->healthy_peer_reachable) { if (in->catch_up_succeeded) return RECOVER_DECISION_ELIGIBLE; return RECOVER_DECISION_FENCE; } /* No healthy peer. If catch-up did not reach a committed-consistent state, * the member can neither recover locally nor catch up right now — rows 4 & * 4b: a LOUD, PERMANENT, NON-promotable fence (NOT process exit). It stays * alive and keeps retrying; the fence releases when a peer returns * (Req 2.4/2.6). */ if (!in->catch_up_succeeded) return RECOVER_DECISION_FENCE_UNRECOVERABLE; /* No peer but catch-up is reported succeeded (e.g. already at/above after a * local catch-up): eligible. */ return RECOVER_DECISION_ELIGIBLE; } bool persist_recovery_unrecoverable(persist_t* p) { return p && atomic_load_explicit(&p->recovery_unrecoverable, memory_order_acquire); } void persist_clear_recovery_unrecoverable(persist_t* p) { if (!p) return; /* Clear only on a genuine transition so the loud "fence released" log fires * once, not on every catch-up tick. */ if (atomic_exchange_explicit(&p->recovery_unrecoverable, false, memory_order_acq_rel)) { INFO("persist_recover: unrecoverable fence RELEASED — member reached a " "committed-consistent state via catch-up; promotable again " "(ha-commit-durability Req 2.6)"); } } recover_decision_t persist_recover_loud_fence_if_unrecoverable(persist_t* p, const recover_policy_inputs_t* in) { recover_decision_t decision = persist_recover_decision(in); if (decision != RECOVER_DECISION_FENCE_UNRECOVERABLE) return decision; /* Loud, operator-visible alert: log the truncation boundary * (durable_floor_lsn) and the cluster high-water mark so an operator sees * exactly where the local log stops short of the cluster's committed state. * The member STAYS ALIVE, stays non-promotable, and keeps retrying catch-up * (Req 2.4) — it NEVER self-exits. */ const char* why = (in->verdict == RECOVER_TRUNCATED) ? "recovery-truncated log (structural corruption)" : "log behind the cluster committed high-water mark"; ERROR("persist_recover: UNRECOVERABLE — %s and no healthy peer reachable to " "catch up; truncation boundary durable_floor_lsn=%" PRIu64 " < cluster committed high-water hwm=%" PRIu64 ". Staying ALIVE and " "PERMANENTLY non-promotable (refusing to serve divergent state) and " "RETRYING catch-up indefinitely — will self-heal when a healthy peer " "returns; operator attention may be needed " "(ha-commit-durability Req 2.4)", why, in->durable_floor_lsn, in->cluster_hwm_lsn); /* Set the sticky non-promotable flag so the promotion gate refuses * leadership as belt-and-suspenders until catch-up clears it (Req 2.4/2.6). * p may be NULL for pure-classification callers that only want the loud log. */ if (p != NULL) atomic_store_explicit(&p->recovery_unrecoverable, true, memory_order_release); return RECOVER_DECISION_FENCE_UNRECOVERABLE; } /* Advance committed_lsn to `written_lsn` on a majority ACK — but ONLY if that * entry is already durable on local stable storage (ha-commit-durability task * 2.4, Req 1.1/1.2). This is the legacy (engine-less / LSN-0) sibling of the * commit engine's durable self-cap (commit.c majority_candidate_locked): a * majority ACK no longer implies the local copy is fsync'd, because group- * commit (task 2.1) deferred the append fsync to the demand-driven flusher. So * advancing committed_lsn here on the ACK alone would reintroduce the bug-661 * ACK-without-durability hazard — committed could be durable on a single * member and lost across failover. We gate the store on written_lsn <= * durable_lsn so committed_lsn ⇒ durable-on-this-member on every path. * committed_lsn is monotonic, so a max() store; the durable cap can only delay * the advance until the flusher syncs the entry (the durable-advance commit * wake re-drives the engine path; the legacy path is re-evaluated on the next * sync_append). */ static void sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn) { uint64_t durable = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); /* DURABILITY PRECONDITION: never advance committed past what is durable * locally. The LSN-0 first-entry edge stays valid (0 <= 0). */ if (written_lsn > durable) return; uint64_t cur = atomic_load_explicit(&p->committed_lsn, memory_order_acquire); if (written_lsn <= cur) return; /* monotonic: committed_lsn never regresses */ atomic_store_explicit(&p->committed_lsn, written_lsn, memory_order_release); #ifdef XCE_TESTING /* INVARIANT (debug guard): committed_lsn must never exceed durable_lsn on * any commit advance. A future caller reintroducing ACK-without-durability * trips this. */ assert(atomic_load_explicit(&p->committed_lsn, memory_order_acquire) <= atomic_load_explicit(&p->durable_lsn, memory_order_acquire)); #endif } int persist_wal_sync_append(persist_t* p, uint32_t type, const void* payload, uint32_t len, int* sync_rc_out) { /* ha-term-safety task 3.1: the primary's WAL_JOB_SUBMIT append is routed * through the raft library (ha_primary_log_append → raft_log_append) so the * library stamps the term + assigns the LSN (firing on_persist_entry → the * WAL write), keeping p->current_lsn co-tracked with s->log_len; the existing * majority-commit barrier below (repl_wait_majority_ack) is UNCHANGED and runs * after the append. Non-raft road / engine-less tests (library not LEADER) * fall back inside ha_primary_log_append to persist_wal_async. Standalone * persist (p->cluster == NULL, lib-level tests) keeps the direct append. */ int rc; if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, type, payload, len, NULL); else rc = persist_wal_async(p, type, payload, len); if (rc != 0) { if (sync_rc_out) *sync_rc_out = rc; return rc; } /* Majority-commit replication (Req 5.1–5.3; ha-commit-engine Req 1.1–1.3). * persist_wal_async appends to the page cache and signals the demand-driven * flusher; it no longer fsyncs inline (group-commit, task 2.1). Durability * is therefore established by the flusher advancing durable_lsn, and a * commit position advances only once the entry is durable locally — the * engine path via the durable self-cap (commit.c), the legacy path below * via sync_append_legacy_commit_store's written_lsn <= durable_lsn gate. * We use current_lsn - 1 because persist_wal_async already incremented it. */ uint64_t written_lsn; pthread_mutex_lock(&p->lock); written_lsn = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); int sync_err; /* Engine-wired path (ha-commit-engine task 5.3): delegate the commit * decision to the commit engine through the walAck adapter. The engine * is the single committed_lsn writer (no store here), and its * n_peers<=1 self-quorum replaces the old has_standbys → -ENODEV * degraded special case (Req 6.3). * * LSN-0 edge (ACCEPTED interim behavior — spec Implementation Notes): * written_lsn is passed unconditionally, but the FIRST-EVER WAL entry * of a fresh cluster is LSN 0 and commit_submit rejects lsn==0, so the * adapter's entry_lsn != 0 guard routes that one entry to the legacy * branch below. This happens once per cluster lifetime; * ha-single-leader-role's promotion no-op entry eliminates it in * practice. */ if (p->repl.ce != NULL && written_lsn != 0) { sync_err = repl_wait_majority_ack(&p->repl, written_lsn, p->config.sync_timeout_ms, (uint8_t)OPCode_walAck); if (sync_err != 0) CTRL_METRIC_INC(ctrl_repl_degraded_count); } else { /* Residual non-engine branch: engine-less (ce == NULL) unit-test * contexts plus the LSN-0 first entry. The legacy connectivity- * based counting was retired by ha-commit-barrier-rollout task 2.2: * repl_wait_majority_ack now confirms LSN 0 immediately (documented * once-per-lifetime edge) and surfaces engine-less walAck waits as * -EINVAL via commit_submit, so the committed_lsn store below only * fires for the LSN-0 case (a no-op store of 0). */ mylock_lock(&p->repl.lock); int has_standbys = (p->repl.nstandbys > 0); mylock_unlock(&p->repl.lock); if (!has_standbys) { /* No standbys registered → degraded mode (no quorum to wait for) */ sync_err = -ENODEV; } else { sync_err = repl_wait_majority_ack(&p->repl, written_lsn, p->config.sync_timeout_ms, (uint8_t)OPCode_walAck); if (sync_err == 0) /* Durable-gated commit advance (task 2.4): advance committed_lsn * only if written_lsn is durable locally — never on the ACK * alone (group-commit removed the inline append fsync). */ sync_append_legacy_commit_store(p, written_lsn); else CTRL_METRIC_INC(ctrl_repl_degraded_count); } } if (sync_rc_out) *sync_rc_out = sync_err; /* sync_err < 0 — replication failed. * -ENODEV: legacy degraded mode (no standbys, engine-less) — acceptable * for forward progress. * -ETIMEDOUT: no per-entry majority within the deadline — the write * must be rejected to preserve the majority-consistency * invariant. * -XCE_ERR_NOT_READY: engine admission cap exhausted — retriable * rejection. * Timeout and admission rejection propagate to the job-submit error * path (ha-commit-engine Req 1.1, 4.4). */ if (sync_err == -ETIMEDOUT || sync_err == -XCE_ERR_NOT_READY) return sync_err; return 0; } #ifdef XCE_TESTING /* Test seam (task 2.4): invoke the EXACT durable-gated commit store that the * legacy sync_append branch uses on a majority ACK, so a test can prove the * ACK-without-durability hazard is closed without standing up the replication * wire. Pre-fix this branch stored committed_lsn = written_lsn unconditionally * (over-advancing past durable_lsn). */ void persist_test_sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn) { sync_append_legacy_commit_store(p, written_lsn); } #endif /* XCE_TESTING */ void persist_log_entry_metadata(const wal_entry_hdr_t* hdr) { TRACE("WAL entry: type=%u lsn=%lu term=%lu payload_len=%u crc=%08x", hdr->type, (unsigned long)hdr->lsn, (unsigned long)hdr->term, hdr->payload_len, hdr->crc); } /* ------------------------------------------------------------------ */ /* Hook functions — AFTER_EVENT handlers that write WAL entries */ /* ------------------------------------------------------------------ */ /* Map HOOK_DETAIL_* values to WAL entry types. * Returns 0 for details that have no WAL representation (skip). */ static uint32_t detail_to_wal_type(int detail) { switch (detail) { case HOOK_DETAIL_JOB_NEW: return WAL_JOB_SUBMIT; case HOOK_DETAIL_JOB_START: return WAL_JOB_START; case HOOK_DETAIL_JOB_EXIT: return WAL_JOB_FINISH; case HOOK_DETAIL_JOB_FAIL: return WAL_JOB_FAIL; case HOOK_DETAIL_JOB_STOP: return WAL_JOB_STOP; case HOOK_DETAIL_JOB_RESUME: return WAL_JOB_RESUME; case HOOK_DETAIL_JOB_UPDATE: return WAL_JOB_UPDATE; case HOOK_DETAIL_JOB_CKPT: return WAL_JOB_CKPT; case HOOK_DETAIL_JOB_RUSAGE_UPDATE: return WAL_JOB_RUSAGE; case HOOK_DETAIL_SETPREASON: return WAL_JOB_PREASON; case HOOK_DETAIL_CLRPREASON: return WAL_JOB_PREASON; case HOOK_DETAIL_QUEUE_UPDATE: return WAL_QUEUE_CTRL; case HOOK_DETAIL_WORKER_JOIN: return WAL_WORKER_CTRL; case HOOK_DETAIL_WORKER_UPDATE: return WAL_WORKER_CTRL; case HOOK_DETAIL_WORKER_DEAD: return WAL_WORKER_CTRL; default: return 0; /* unknown — skip */ } } /* Write-path commit classification (ha-commit-barrier-rollout task 1.2). * Pure lookup over the VERIFIED table (design.md §Write-Path * Classification); `detail` disambiguates WAL_WORKER_CTRL only. * Unclassified types fail fast under XCE_TESTING (abort) and fall back to * RETAIN in production — never dropped, never falsely confirmed (Req 2.2). */ wal_commit_mode_t wal_classify(uint32_t wal_type, int detail) { switch (wal_type) { case WAL_JOB_SUBMIT: /* client job submit — request-shaped; keeps the in-hook sync path * (reference implementation, handle_jobsubmit) */ return WAL_MODE_WAIT; case WAL_QUEUE_CTRL: /* client queue control — request-shaped; the handler barrier owes * commit completion before the requester reply (tasks 2-3) */ return WAL_MODE_WAIT; case WAL_WORKER_CTRL: /* the only detail-disambiguated type: JOIN/UPDATE have a requester * awaiting confirmation (WAIT); DEAD is a controller-detected fact. * Unexpected details take the fact-safe RETAIN side. */ if (detail == HOOK_DETAIL_WORKER_JOIN || detail == HOOK_DETAIL_WORKER_UPDATE) return WAL_MODE_WAIT; return WAL_MODE_RETAIN; case WAL_JOB_START: case WAL_JOB_FINISH: case WAL_JOB_FAIL: case WAL_JOB_STOP: case WAL_JOB_RESUME: case WAL_JOB_UPDATE: case WAL_JOB_RUSAGE: case WAL_JOB_CKPT: case WAL_JOB_PREASON: /* worker/scheduler-reported facts — retained until committed. * FAIL/UPDATE are dual-origin; their request-shaped side is * realized by the handler barrier in do_jobcontrol_op * (research.md §2 call-site verification). */ return WAL_MODE_RETAIN; case WAL_NOOP: /* internal promotion no-op — replicated like any committed entry, * retained until committed (ha-commit-barrier-rollout pre-declared * row: promotion | internal | RETAIN; guard forces classification) */ return WAL_MODE_RETAIN; case WAL_CONFIG_MARKER: /* config-change marker (ha-term-safety task 4.2): a committable * (term,lsn) log entry like a job-state fact — replicated and retained * until committed; it commits through the SAME barrier as job state * (R7.1/R7.11), so it is RETAIN, NOT a no-op or a LOCAL marker. The * requester barrier on a config upgrade is owned by the upgrade path * (do_upgrade_config_with_bytes), not the per-entry WAIT hook. */ return WAL_MODE_RETAIN; case WAL_CHECKPOINT: /* local segment marker — never replicated, by design */ return WAL_MODE_LOCAL; default: ERROR("wal_classify: unclassified WAL type %u (detail %d) — " "RETAIN fallback; add it to the classification table", wal_type, detail); #ifdef XCE_TESTING assert(0 && "unclassified WAL type — add it to wal_classify"); #endif return WAL_MODE_RETAIN; } } /* Commit barrier helper (ha-commit-barrier-rollout task 2.2, design.md * §Commit Barrier Helper; Req 1.1, 1.2, 1.3). * * Handler-callable WAIT completion on the appended watermark: snapshot the * last-appended LSN, return 0 when the commit index already covers it, * else block in the commit engine until coverage or deadline. The engine * result is returned UNMODIFIED (0 / -ETIMEDOUT / -XCE_ERR_NOT_READY / * -EINVAL) so callers map it to their own error replies. * * Lock discipline: p->lock is held only for the current_lsn snapshot and * is released BEFORE commit_submit(WAIT) — the engine blocks on ce->lock * + condvar, and p->lock must never be held across it (engine * lock-ordering rule). */ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms) { uint64_t watermark; if (p == NULL) return -EINVAL; /* Snapshot the last-appended watermark under p->lock. current_lsn is * the NEXT LSN to assign (persist.h invariant: committed_lsn <= * current_lsn - 1), so the highest appended entry is current_lsn - 1 * (resolved contract, spec Implementation Notes 2.1). current_lsn == * 0 means nothing was ever appended — vacuously satisfied. */ pthread_mutex_lock(&p->lock); if (p->current_lsn == 0) { pthread_mutex_unlock(&p->lock); return 0; } watermark = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); /* Already covered — committed_lsn is monotone, so coverage observed * here can never regress. This check also absorbs the LSN-0 first * entry (watermark 0 vs committed_lsn 0): that once-per-lifetime entry * has no quorum contract (ha-commit-engine Implementation Notes) and * commit_submit would reject lsn == 0 anyway. */ if (atomic_load_explicit(&p->committed_lsn, memory_order_acquire) >= watermark) return 0; /* Engine not wired: legacy/bootstrap contexts have no engine commit * contract — there is nothing to wait on, so the barrier is a no-op. * Production always wires the engine (cluster_init_phase2). */ if (p->repl.ce == NULL) return 0; /* timeout_ms == 0 borrows the configured sync deadline (Req 1.2). */ return commit_submit(p->repl.ce, watermark, COMMIT_MODE_WAIT, timeout_ms != 0 ? timeout_ms : p->config.sync_timeout_ms); } /* Classified append at the hook seam (barrier-rollout task 1.2). * * WAIT (queue/worker control): the append stays async here — the handler * barrier (tasks 2-3) owes commit completion before the requester reply. * WAL_JOB_SUBMIT never reaches this helper (it keeps the in-hook sync * path in persist_job_hook_fun). * * RETAIN (facts): append + broadcast, then register the appended LSN with * the commit engine. commit_submit(COMMIT_MODE_RETAIN) is registration- * free bookkeeping (commit.c — returns 0 immediately; commit state is * derived from registry watermarks at ACK time), so the call is contract * documentation: the entry stays pending purely by lsn > committed_lsn * and commits later via covering ACKs. Commit-progress results are NEVER * propagated to hook callers — only the local append rc is (Req 3.1). * * Lock context: persist_wal_async takes/releases p->lock internally; the * appended-LSN read below re-takes p->lock briefly (same discipline as * persist_wal_sync_append) and releases it BEFORE commit_submit, so no * p->lock is ever held across ce->lock (engine lock-ordering rule). */ static int wal_hook_append_classified(persist_t* p, uint32_t wal_type, wal_commit_mode_t mode, const void* payload, uint32_t len) { uint64_t appended_lsn; int rc; /* ha-term-safety task 3.1: route the primary's local append through the raft * library (ha_primary_log_append → raft_log_append) so the LIBRARY stamps the * live election term and assigns the LSN (firing on_persist_entry → the WAL * write), keeping p->current_lsn co-tracked with s->log_len. The library is * the SINGLE primary append entry — no append bypasses it on the raft road. * On the non-raft road (library not LEADER) ha_primary_log_append falls back * to persist_wal_async, so engine-less / init-time contexts are unchanged. * When p->cluster is NULL (persist used standalone in lib tests) keep the * direct append (no cluster → no raft state to drive). */ if (p->cluster != NULL) rc = ha_primary_log_append(p->cluster, wal_type, payload, len, NULL); else rc = persist_wal_async(p, wal_type, payload, len); if (rc != 0 || mode != WAL_MODE_RETAIN) return rc; if (p->repl.ce == NULL) return rc; /* engine not wired (bootstrap / engine-less contexts) */ /* current_lsn - 1 is the LSN persist_wal_async just assigned (it * increments under p->lock). A concurrent append may bump it further; * that is benign — contiguous advancement means the higher watermark * still covers this entry. LSN-0 edge: commit_submit rejects lsn == 0 * (first-ever entry of a fresh cluster — accepted interim behavior, * spec Implementation Notes). */ pthread_mutex_lock(&p->lock); appended_lsn = p->current_lsn - 1; pthread_mutex_unlock(&p->lock); if (appended_lsn != 0) (void)commit_submit(p->repl.ce, appended_lsn, COMMIT_MODE_RETAIN, 0); return rc; /* local append rc only — facts have no commit timeout */ } /* Explicit controller-side raft step for a job submission (raft-apply-decoupling * item #1 / bug-1306): WAL-append each job in `jlist` and block on the * majority-commit barrier, returning the commit verdict DIRECTLY to the caller * (handle_jobsubmit) instead of through the xce_run_hooks fan-out (which returns * only the last hook's rc and so clobbered this -ETIMEDOUT with a later notify * hook's 0 — the committed-job-loss root cause). * * Returns 0 when every job's entry committed on a majority; the FIRST negative * rc otherwise (-ETIMEDOUT: no majority within the deadline; -XCE_ERR_NOT_READY: * admission cap; -ENOMEM: encode failure; or a local-write error). The caller * MUST reject the submission on a negative rc (never reply success) to preserve * the Commit Durability invariant. On -ETIMEDOUT / -XCE_ERR_NOT_READY the entry * WAS appended locally and the LSN advanced, so j->submit_lsn is still tagged — * the entry remains a pending, uncommitted tail a new-term leader may discard. */ int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist) { int rc = 0; if (p == NULL || jlist == NULL) return -EINVAL; for (int i = 0; i < jlist->count && rc == 0; i++) { job_t* j = (job_t*)xlist_get(jlist, i); uint8_t* buf = NULL; size_t sz = 0; if (capnp_encode_job(&j->job, &buf, &sz) != 0) { rc = -ENOMEM; break; } /* ha-convergence-recovery (3.3): snapshot the LSN this submit entry * will receive BEFORE the append. persist_wal_async assigns * hdr.lsn = current_lsn then increments, so the pre-append current_lsn * is exactly the written entry's LSN. */ pthread_mutex_lock(&p->lock); uint64_t submit_lsn = p->current_lsn; pthread_mutex_unlock(&p->lock); int sync_rc = 0; rc = persist_wal_sync_append(p, WAL_JOB_SUBMIT, buf, (uint32_t)sz, &sync_rc); free(buf); /* Tag the job with its submit entry's LSN whenever the LOCAL WAL write * succeeded. persist_wal_sync_append returns the persist_wal_async * error only on local-write failure; on a replication-degraded outcome * (0 / -ETIMEDOUT / -ENODEV / -XCE_ERR_NOT_READY) the entry was appended * locally and the LSN advanced, so the egress tag is set. */ if (rc == 0 || rc == -ETIMEDOUT || rc == -XCE_ERR_NOT_READY) j->submit_lsn = submit_lsn; } return rc; } /* P0 payload: just the detail value (4 bytes). * Full Cap'n Proto serialization is wired in Task 15. */ int persist_job_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; /* Write to WAL only after the state change is complete. * BEFORE_EVENT has a different va_list layout and must be ignored. */ if (situation != HOOK_AFTER_EVENT) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); if (mode == WAL_MODE_WAIT) { /* WAL_JOB_SUBMIT durability (WAL append + majority-commit barrier) was * MOVED OUT of this hook into the explicit raft step * persist_submit_wal_and_commit(), called directly by handle_jobsubmit * (raft-apply-decoupling item #1 / bug-1306). Rationale: xce_run_hooks * returns only the LAST-registered hook's rc, so a later JOB AFTER_EVENT * subscriber (sched/apiserver, returning 0) CLOBBERED this branch's * -ETIMEDOUT — handle_jobsubmit then replied `jobsubmit ok` for an * entry that never reached majority, and a new-term leader legally * discarded it (committed-job-loss). The commit verdict must be read * from the protocol step directly, never as a hook return. This branch * is now a no-op; the AFTER_EVENT hook chain is notification-only. */ return 0; } /* All other job events: payload = wal_jobstate_t {jobid, state}. * State is read from the job_t after the AFTER_EVENT hook updates it. */ wal_jobstate_t js = {0}; switch (detail) { case HOOK_DETAIL_JOB_START: { /* args: cluster_t*, job_pidinfo_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_EXIT: { /* args: cluster_t*, xce_jobfinish_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } case HOOK_DETAIL_JOB_FAIL: { /* args: cluster_t*, job_failinfo_t*, job_t* */ (void)va_arg(ap, cluster_t*); (void)va_arg(ap, void*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } default: { /* STOP, RESUME, UPDATE, CKPT, PREASON, RUSAGE — args: cluster_t*, job_t* */ (void)va_arg(ap, cluster_t*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } } /* mode == WAL_MODE_RETAIN — fact-shaped job state transition */ return wal_hook_append_classified(p, wal_type, mode, &js, sizeof(js)); } int persist_queue_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; if (situation != HOOK_AFTER_EVENT) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); /* args: cluster_t*, queue_t* */ (void)va_arg(ap, cluster_t*); queue_t* q = va_arg(ap, queue_t*); /* Payload: [wal_queuehdr_t][queue name NUL-terminated] */ const char* name = q->queue.queue ? q->queue.queue : ""; size_t name_len = strlen(name) + 1; size_t total = sizeof(wal_queuehdr_t) + name_len; uint8_t* pkt = malloc(total); if (!pkt) return -ENOMEM; wal_queuehdr_t hdr = { .status = q->queue.status }; memcpy(pkt, &hdr, sizeof(hdr)); memcpy(pkt + sizeof(hdr), name, name_len); /* mode == WAL_MODE_WAIT — the append stays async; the handler barrier * owes commit completion before the requester reply (task 3) */ int rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total); free(pkt); return rc; } int persist_worker_hook_fun(int type, int situation, int detail, void* cls, va_list ap) { (void)type; if (situation != HOOK_AFTER_EVENT) return 0; persist_t* p = cls; uint32_t wal_type = detail_to_wal_type(detail); wal_commit_mode_t mode; if (wal_type == 0) return 0; mode = wal_classify(wal_type, detail); /* WAL_WORKER_CTRL payload layout: [uint32_t detail][capnp worker OR name] */ uint32_t disc = (uint32_t)detail; uint8_t* buf = NULL; size_t capnp_sz = 0; int rc = 0; /* All worker detail types: args are cluster_t* then worker_t*. * JOIN uses a unified signature with an extra char* name between them; * DEAD and UPDATE use (cluster_t*, worker_t*) directly. */ (void)va_arg(ap, cluster_t*); if (detail == HOOK_DETAIL_WORKER_JOIN) (void)va_arg(ap, char*); /* skip name — present only for JOIN */ worker_t* w = va_arg(ap, worker_t*); if (!w) return 0; /* guard: failed join produced no worker slot */ rc = capnp_encode_worker(&w->host, &buf, &capnp_sz); if (rc != 0) return rc; /* Build [disc | capnp bytes] in one allocation */ size_t total = WORKER_WAL_HDR_SIZE + capnp_sz; uint8_t* pkt = malloc(total); if (!pkt) { free(buf); return -ENOMEM; } memcpy(pkt, &disc, WORKER_WAL_HDR_SIZE); memcpy(pkt + WORKER_WAL_HDR_SIZE, buf, capnp_sz); free(buf); /* JOIN/UPDATE → WAIT (handler barrier owes completion, task 3); * DEAD → RETAIN (controller-detected fact, registered with the engine) */ rc = wal_hook_append_classified(p, wal_type, mode, pkt, (uint32_t)total); free(pkt); return rc; } /* ============================================================ * GC retention predicates (P0-5a / P0-8 / P0-13) * * Placed in persist.c so the unit test binary (which links only * persist.c from the controller sources) can access them without * pulling in RTTI-dependent job.c / worker.c. * ============================================================ */ bool job_is_terminal(const job_t* j) { return (j->job.state & (JOB_STAT_EXIT | JOB_STAT_DONE | JOB_STAT_PERR)) != 0; } bool job_should_retain(const job_t* j, const persist_config_t* cfg) { time_t now = time(NULL); /* TRANSIENT jobs: auto-terminate when TTL elapses after failover. * transient_since == 0 means the timestamp was never set — do not GC. */ if (j->job.state & JOB_STAT_TRANSIENT) { if (j->job.transient_since > 0 && now > (time_t)j->job.transient_since + (time_t)cfg->transient_job_timeout_s) { return false; } return true; } /* Non-terminal jobs are always retained */ if (!job_is_terminal(j)) return true; /* Terminal jobs: retain until finished_retain_s expires */ return (j->job.finish_time + (time_t)cfg->finished_retain_s >= now); } bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg) { time_t now = time(NULL); /* HIDDEN workers: admin explicitly removed — clean up at next checkpoint */ if (w->status & XCE_WORKER_HIDDEN) return false; /* UNAVAILABLE workers: retain until worker_retain_s expires. * dead_since == 0 means not yet set — do not GC. */ if (w->status & XCE_WORKER_UNAVAIL) { if (w->dead_since > 0 && now > w->dead_since + (time_t)cfg->worker_retain_s) { return false; } } return true; } /* ============================================================ * WAL truncation (task 7.5) * * Remove all WAL entries with LSN > cutoff_lsn. * Idempotent: calling twice with same cutoff_lsn returns 0 both times. * ============================================================ */ /* Clamp durable_lsn down to `tip` when the log tip recedes below it (truncate * / divergent-suffix discard). Without this, durable_lsn could exceed * current_lsn - 1 after a truncate, violating its invariant and letting the * commit majority count self at an LSN whose bytes were just removed. MUST be * called with p->lock held (mirrors the committed_lsn clamp in the truncate * paths). durable_lsn only ever decreases here — the sole exception to its * monotonic-non-decreasing rule (ha-commit-durability 2.1 State Management). */ static void clamp_durable_lsn_locked(persist_t* p, uint64_t tip) { uint64_t dur = atomic_load_explicit(&p->durable_lsn, memory_order_acquire); if (dur > tip) atomic_store_explicit(&p->durable_lsn, tip, memory_order_release); } int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn) { pthread_mutex_lock(&p->lock); uint64_t before = p->current_lsn; int rc = truncate_wal_above_locked(p, cutoff_lsn); uint64_t after = p->current_lsn; pthread_mutex_unlock(&p->lock); /* raft-apply-decoupling task 1.4: fire the truncation wake only when a * truncation actually removed entries above cutoff_lsn (current_lsn * strictly decreased -- both truncate_wal_above_locked branches set * current_lsn = cutoff_lsn + 1 on a real truncation) -- mirrors the * durable-advance wake's own before/after guard (see fsync_worker * above) so a no-op truncate (nothing above cutoff_lsn, or an I/O * error before current_lsn is touched) never spuriously fails a * continuation whose entry is still safely in the log. Called AFTER * releasing p->lock: the callback (the commit engine) takes only * ce->lock, preserving the cycle-free p->lock -> ce->lock edge already * established by the durable-advance wake. */ if (rc == 0 && after < before && p->on_truncate != NULL) { p->on_truncate(p->truncate_cb_ctx, cutoff_lsn); } return rc; } /* Lock-held core of persist_truncate_wal_above (see the forward declaration and * the conflict-guard reuse near persist_wal_write_replicated). Caller MUST hold * p->lock; this function never touches the lock so it is reusable from another * already-locked path. */ static int truncate_wal_above_locked(persist_t* p, uint64_t cutoff_lsn) { /* No-op: nothing written yet, or cutoff covers all written entries */ if (p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn) { return 0; } /* --- Collect segment filenames and sort by start LSN --- */ uint64_t seg_lsns[4096]; int nseg = 0; DIR* d = opendir(p->wal_path); if (d) { struct dirent* e; while ((e = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(seg_lsns[0]))) { if (strncmp(e->d_name, "seg-", 4) != 0) continue; uint64_t sl = 0; if (sscanf(e->d_name, "seg-%" SCNx64 ".wal", &sl) != 1) continue; seg_lsns[nseg++] = sl; } closedir(d); } if (nseg == 0) { return 0; } /* Insertion sort ascending by start LSN */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } /* --- Find segment containing cutoff_lsn --- */ int cutoff_seg_idx = -1; for (int i = 0; i < nseg; i++) { if (seg_lsns[i] <= cutoff_lsn) cutoff_seg_idx = i; else break; } if (cutoff_seg_idx < 0) { /* cutoff_lsn is below the oldest retained segment's start LSN, so EVERY * on-disk entry has LSN > cutoff_lsn and must be removed (the function's * contract: "remove all entries with LSN > cutoff_lsn"). This is the * demotion-convergence case (ha-convergence-recovery Req 1.2): a demoted * ex-primary whose committed boundary sits below its entire retained log * must discard all of it and collapse the tip to cutoff_lsn + 1. Delete * every segment, close the active fd, and reset the in-memory tip. */ for (int i = 0; i < nseg; i++) { char del_path[4096]; snprintf(del_path, sizeof(del_path), WAL_SEG_FMT, p->wal_path, seg_lsns[i]); unlink(del_path); } if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } p->seg_offset = 0; p->seg_path[0] = '\0'; p->current_lsn = cutoff_lsn + 1; p->last_crc = WAL_CRC_NONE; atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release); clamp_durable_lsn_locked(p, cutoff_lsn); /* Keep the per-LSN term index consistent: drop terms for removed LSNs. */ truncate_term_index_locked(p, cutoff_lsn); return 0; } /* --- Scan segment to find byte offset of first entry with lsn > cutoff --- */ char cutoff_seg_path[4096]; snprintf(cutoff_seg_path, sizeof(cutoff_seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]); int scan_fd = open(cutoff_seg_path, O_RDONLY); if (scan_fd < 0) { return -errno; } uint32_t scan_off = 0; uint32_t trunc_off = 0; uint32_t new_last_crc = WAL_CRC_NONE; bool found_cut = false; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(scan_fd, &hdr, sizeof(hdr), (off_t)scan_off); if (n < (ssize_t)sizeof(hdr)) break; /* short read or EOF */ /* Pre-allocated zero region: type==0 is not a valid WAL type (min=1) */ if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn > cutoff_lsn) { trunc_off = scan_off; found_cut = true; break; } new_last_crc = hdr.crc; scan_off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(scan_fd); if (!found_cut) trunc_off = scan_off; /* end of valid data in segment */ /* --- Close the current wal_fd --- */ if (p->wal_fd >= 0) { fdatasync(p->wal_fd); close(p->wal_fd); p->wal_fd = -1; } /* --- Truncate the cutoff segment at the computed offset --- */ int new_fd = open(cutoff_seg_path, O_RDWR); if (new_fd < 0) { return -errno; } if (ftruncate(new_fd, (off_t)trunc_off) != 0) { int saved = errno; close(new_fd); return -saved; } fdatasync(new_fd); /* --- Delete all segments after the cutoff segment --- */ for (int i = cutoff_seg_idx + 1; i < nseg; i++) { char del_path[4096]; snprintf(del_path, sizeof(del_path), WAL_SEG_FMT, p->wal_path, seg_lsns[i]); unlink(del_path); } /* --- Update persist_t state under lock --- */ p->wal_fd = new_fd; p->seg_offset = trunc_off; snprintf(p->seg_path, sizeof(p->seg_path), WAL_SEG_FMT, p->wal_path, seg_lsns[cutoff_seg_idx]); p->current_lsn = cutoff_lsn + 1; p->last_crc = new_last_crc; atomic_store_explicit(&p->committed_lsn, cutoff_lsn, memory_order_release); clamp_durable_lsn_locked(p, cutoff_lsn); /* Keep the per-LSN term index consistent: drop terms for removed LSNs. */ truncate_term_index_locked(p, cutoff_lsn); return 0; }