#pragma once #include #include #include #include #include #include #include "config.h" #include "wal_types.h" /* wal_entry_hdr_t, wal_entry_type — P2-4: shared with repl.h */ #include "repl.h" /* replicator_t — P2-4: embedded in persist_t */ #include "xlist.h" /* xlist_t — persist_submit_wal_and_commit job batch */ /* Forward declaration */ typedef struct cluster_s cluster_t; /* ============================================================ * Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1) * * The outcome of WAL replay at startup, distinguishing a structurally * intact log from one whose chain is broken. It is about the STRUCTURE * of the local log, NOT about being behind the cluster: a clean log that * simply ends below the cluster's committed high-water mark is still * RECOVER_CLEAN (the freshness/promotion fence handles "behind"). A * corruption-truncated recovery is NEVER reported as a clean recovery. * ============================================================ */ typedef enum { RECOVER_CLEAN = 0, /* WAL forms a valid CRC chain to a clean entry * boundary: clean EOF, pre-allocated-zero tail, * snapshot-sentinel + clean replay, or an empty * first-boot WAL (no entries != corruption). */ RECOVER_TRUNCATED = 1, /* replay stopped on structural corruption: a CRC * chain break, a CRC mismatch, or a short/partial * entry read — the log cannot be reconstructed * contiguously. Marks the member non-promotable * (consumed by the promotion gate, task 4.3). */ } recover_verdict_t; /* ============================================================ * Boot-policy recovery decision (ha-commit-durability task 3.3, Req 2.4) * * The outcome of evaluating a member's boot/promotion-eligibility state * against the Boot Policy Decision Table (design.md "Boot Policy Decision * Table (task 3.2)"). This task (3.3) owns the FATAL terminal; the fence + * catch-up + release path is task 4.3. * * ELIGIBLE — may promote (no fence). Rows 1, 6. * FENCE — ineligible to promote; catch up from a healthy peer, regain * eligibility on success (Req 2.6). Rows 2, 3, 5. A reachable * healthy peer NEVER yields FENCE_UNRECOVERABLE. * FENCE_UNRECOVERABLE — loud, PERMANENT, NON-terminating fence: the member * cannot recover locally AND cannot catch up (no healthy peer). * It logs LOUDLY, stays PERMANENTLY non-promotable, and keeps * retrying catch-up indefinitely — but the process STAYS ALIVE. * Rows 4, 4b. This is NOT process exit: a controller MUST NEVER * self-terminate as a designed response to an operational * condition (user invariant, 2026-06-18). A self-exited controller * needs MANUAL admin recovery and leaves quorum degraded * meanwhile; a controller that stays alive and keeps retrying * SELF-HEALS when a peer returns. The fence RELEASES (→ ELIGIBLE, * Req 2.6) automatically when catch-up reaches a * committed-consistent state. * * AUTHORITATIVE INVARIANT (design.md): a truncated/behind member resolves to * either FENCE→ELIGIBLE (caught up) or a loud PERMANENT FENCE_UNRECOVERABLE * (stays alive, non-promotable, keeps retrying). It is NEVER silently divergent * (it refuses to lead) and the process NEVER self-exits. * ============================================================ */ typedef enum { RECOVER_DECISION_ELIGIBLE = 0, RECOVER_DECISION_FENCE = 1, RECOVER_DECISION_FENCE_UNRECOVERABLE = 2, } recover_decision_t; /* Policy inputs evaluated at boot / promotion-eligibility (design.md table). * Modelled as a plain value so the decision is a pure, unit-testable function * of its inputs — the live catch-up machinery (replication / task 4.3) supplies * these; this task does not build it. */ typedef struct { recover_verdict_t verdict; /* RECOVER_CLEAN / RECOVER_TRUNCATED (task 3.1) */ bool durable_below_hwm; /* local durable position < cluster committed HWM */ bool healthy_peer_reachable; /* a healthy peer is reachable for catch-up */ bool catch_up_succeeded; /* catch-up reached a committed-consistent state * within the bounded window (only meaningful * when a catch-up was attempted) */ uint64_t durable_floor_lsn; /* highest intact replayed LSN (truncation boundary) */ uint64_t cluster_hwm_lsn; /* cluster committed high-water mark, for logging */ /* raft-apply-decoupling task 5.2 (Req 8.3/8.4): the post-replay * apply_verdict_t computed a semantic mismatch (APPLY_DIVERGENT) — the * materialized state does not match the committed log tip even though * the WAL bytes replayed cleanly. This is a NEW INPUT to the existing * Boot Policy Decision Table, not a new fence mechanism: unlike a * structurally truncated/behind log (which a healthy peer's missing * entries can repair), an apply-time defect is not something streaming * more entries from a peer proves fixed, so persist_recover_decision() * treats this field as an UNCONDITIONAL fence trigger — set it true and * every other field in this struct becomes irrelevant to the outcome * (see persist_recover_decision's implementation). Defaults to false * (an ordinary structural-only evaluation) when left unset by a * designated initializer, matching every pre-existing call site. */ bool apply_divergent; } recover_policy_inputs_t; /* ============================================================ * Post-replay apply-correctness verdict (raft-apply-decoupling task 5.1, * design.md "Restart Replay & Verification", Req 8.1/8.2) * * DISTINCT from recover_verdict_t above, and NOT a replacement for it: * RECOVER_CLEAN/RECOVER_TRUNCATED is a structural verdict over the WAL's * bytes (CRC chain intact or not). apply_verdict_t is a SEMANTIC verdict * over the result of replaying those bytes through apply_committed_entry() — * it proves the materialized state (e.g. the live job table) actually * matches a lightweight expectation derived from the committed log tip, not * merely that the bytes were readable. This catches a class of hidden apply * defect (e.g. a dispatch call that reports success without the entity * actually landing in the live table) that a byte-level CRC check cannot * see. Both verdicts are reported side by side on every restart (Req 8.1); * neither one implies the other. * ============================================================ */ typedef enum { APPLY_CLEAN = 0, /* materialized state matches the log-derived * expectation (Req 8.2). */ APPLY_DIVERGENT = 1, /* materialized state does NOT match the log-derived * expectation (Req 8.3) — a hidden apply defect. * Routed into the existing non-promotable loud-fence * mechanism via recover_policy_inputs_t.apply_divergent * (raft-apply-decoupling task 5.2, Req 8.4) — this * type itself still only classifies; persist_recover() * is what acts on it. */ } apply_verdict_t; /* Lightweight post-replay expectation, derived purely from scanning the * committed log during replay (task 5.1, Req 8.1) — never from trusting * apply_committed_entry()'s own return code, so the comparison is a genuine * second, independent check on the materialized result. * * expected_last_applied_lsn — the highest LSN of any logical (non- * checkpoint, non-unknown-type) entry replay actually dispatched via * apply_committed_entry() in this replay pass: the "committed log tip" * position component of the expectation. * expected_new_job_count — the number of WAL_JOB_SUBMIT entries replay * dispatched in this pass. Every such entry is, by construction, above * the loaded snapshot's checkpoint_lsn (entries at/below it are skipped * before ever reaching dispatch — see replay_wal), so each one names a * job that must now exist in the live job table that did NOT exist * before this replay pass began. */ typedef struct { uint64_t expected_last_applied_lsn; uint64_t expected_new_job_count; } apply_expectation_t; /* ============================================================ * Persistence configuration (Part 2.6) * ============================================================ */ typedef struct { uint32_t wal_sync_ms; /* fsync interval in ms (default 10) */ uint32_t snap_interval_s; /* checkpoint interval in seconds (default 60) */ uint32_t snap_wal_size_mb; /* checkpoint when WAL exceeds this MB (default 64) */ uint32_t snap_keep_n; /* snapshots to retain (default 2) */ uint32_t transient_job_timeout_s; /* seconds before TRANSIENT job auto-terminates (default 300) */ uint32_t finished_retain_s; /* retain finished jobs N seconds (default 3600) */ uint32_t worker_retain_s; /* retain unavailable workers N seconds (default 86400) */ uint32_t sync_timeout_ms; /* sync replication ACK timeout ms (default 50) */ uint32_t max_wal_records; /* checkpoint trigger: records since last snap (default 10000) */ uint32_t wal_segment_bytes; /* physical WAL segment file size (default 64 MB) */ } persist_config_t; static inline persist_config_t persist_config_defaults(void) { persist_config_t c; c.wal_sync_ms = DEFAULT_WAL_SYNC_MS; c.snap_interval_s = DEFAULT_SNAP_INTERVAL_S; c.snap_wal_size_mb = DEFAULT_SNAP_WAL_SIZE_MB; c.snap_keep_n = DEFAULT_SNAP_KEEP_N; c.transient_job_timeout_s = DEFAULT_TRANSIENT_JOB_TIMEOUT_S; c.finished_retain_s = DEFAULT_FINISHED_RETAIN_S; c.worker_retain_s = DEFAULT_WORKER_RETAIN_S; c.sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS; c.max_wal_records = DEFAULT_MAX_WAL_RECORDS; c.wal_segment_bytes = DEFAULT_WAL_SEGMENT_BYTES; return c; } /* * persist_config_set_wal_sync_ms — apply the --wal-sync-ms flush-cadence * override (ha-commit-durability Req 3.1). * * Single override point shared by both persist_config_defaults() callers * (cluster.c phase2 and myapp.c step 6), mirroring the --sync-timeout-ms * convention: override_ms == 0 means "flag not passed" and leaves the * default flush cadence untouched (Req 3.3 — the seam changes timing * controllability only, not the durability contract). A positive value * sets the cadence to exactly that, applied before persistence starts. */ static inline void persist_config_set_wal_sync_ms(persist_config_t *c, uint32_t override_ms) { if (override_ms > 0) c->wal_sync_ms = override_ms; } /* * persist_config_set_max_wal_records — apply the --max-wal-records * checkpoint-trigger override (raft-ha-core-safety task 5.10a / * ha-entity-snapshots task 7.1: forcing a real full-checkpoint catch-up * round in the chaotic soak requires a low, deterministic record-count * threshold rather than waiting on the 10000-record production default). * * Same convention as persist_config_set_wal_sync_ms: override == 0 means * "flag not passed", leaves the default threshold untouched. A positive * value sets it exactly (persist_init's own minimum-100 floor still * applies afterward — this setter does not duplicate that enforcement). */ static inline void persist_config_set_max_wal_records(persist_config_t *c, uint32_t override_n) { if (override_n > 0) c->max_wal_records = override_n; } /* * persist_config_set_wal_segment_bytes — apply the --wal-segment-bytes * override (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: * the physical WAL segment size, not the record-count checkpoint threshold, * gates when trim_wal_segments() has anything to delete — a segment is * never rotated out until it fills, so forcing a real snapshot-triggered * empty-rejoin in the soak requires a small deterministic segment size * rather than waiting on the 64 MB production default). * * Same convention as persist_config_set_max_wal_records: override == 0 * means "flag not passed", leaves the default segment size untouched. */ static inline void persist_config_set_wal_segment_bytes(persist_config_t *c, uint32_t override_bytes) { if (override_bytes > 0) c->wal_segment_bytes = override_bytes; } /* ============================================================ * Key→index mapping for checkpoint serialization (Part 3.8) * ============================================================ */ typedef struct { int index; /* shadow slot index in the frozen zarray */ char key[XCE_MAXNAME_LEN]; /* owned deep copy — not a borrowed kvtab pointer */ size_t key_len; /* length including null terminator */ } slot_key_t; /* ============================================================ * persist_t — concrete definition (exposed for stack/calloc allocation) * The opaque pattern can be restored in a later refactor once the * full API surface is stable. * ============================================================ */ struct persist_s { /* Back-pointer to owning cluster (set by persist_init, used in persist_start) */ cluster_t* cluster; /* WAL state */ int wal_fd; uint64_t current_lsn; uint32_t last_crc; uint32_t last_replicated_crc; uint64_t term; uint32_t seg_offset; uint64_t checkpoint_lsn; _Atomic uint64_t committed_lsn; /* highest LSN known committed; invariant: <= current_lsn - 1 */ _Atomic uint64_t durable_lsn; /* highest LSN whose bytes are fdatasync'd locally (group-commit * prefix watermark). Advanced ONLY by the flusher after a * successful fdatasync. Invariants: monotonic non-decreasing * (except clamp-on-truncate); durable_lsn <= current_lsn - 1. * The commit majority counts self at this LSN, not the * appended tip (ha-commit-durability Req 1.1/1.2; consumed by * task 2.2). */ uint64_t total_wal_bytes; char seg_path[4096]; /* Per-LSN recovered/live term index (ha-term-safety task 1.3, R1.5). * IN-MEMORY ONLY — no on-disk layout change (R1.7). Records each entry's * hdr.term keyed by LSN so the term can be read back after recovery and * fed to the reconcile/promotion logic (on_get_entry_term, task 2.2). * * term_index[lsn - 1] holds the RAW recorded term for a 1-based LSN (LSN 0 * is reserved and never indexed). term_index_count is the highest LSN * recorded (== number of contiguous slots in use), used as the presence * watermark so persist_entry_term distinguishes an ABSENT LSN (returns 0) * from a PRESENT legacy-0 entry (mapped to the lowest-known term). Grown on * demand by record_entry_term during replay_wal and live appends; freed in * persist_fini. Guarded by p->lock (written under the append lock and the * single-threaded recovery path; read by accessors that take p->lock). */ uint64_t* term_index; /* term_index[i] = term of LSN (i+1) */ uint64_t term_index_count; /* highest recorded LSN (slots in use) */ uint64_t term_index_cap; /* allocated slot capacity */ /* Durable committed-config reference carried in the checkpoint sentinel * (ha-term-safety task 4.2 remediation r1 — compaction-safe config derive). * A checkpoint trims WAL segments older than the open one (trim_wal_segments) * and the snapshot sentinel does NOT carry entity state, so a committed * WAL_CONFIG_MARKER in a trimmed segment would vanish and the derived * committed config (config_committed_marker) would regress. To prevent that, * checkpoint stamps the latest-committed marker's 40-byte REFERENCE * ({version_tag, sha256} — a reference, NOT the config bytes / entity * snapshot) into the sentinel; recovery reads it back here; and the derive * falls back to this when no marker survives the live WAL scan. * cfgref_present == 0 means "no committed config marker was known at the last * checkpoint" (a fresh store or a pre-config-marker checkpoint). Written * single-threaded at checkpoint/recovery; read by config_committed_marker. */ int cfgref_present; /* 1 if cfgref_tag/sha are valid */ uint64_t cfgref_tag; /* latest committed marker version_tag */ uint8_t cfgref_sha[32]; /* latest committed marker content hash */ /* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1). * Set by persist_recover/replay_wal at startup: RECOVER_CLEAN when the * local WAL forms a valid CRC chain to a clean entry boundary (incl. an * empty first-boot WAL, a snapshot-sentinel + clean replay, and a * pre-allocated-zero tail), RECOVER_TRUNCATED when replay stopped on a * structural corruption — a CRC chain break, a CRC mismatch, or a * short/partial entry read — so the on-disk log cannot be reconstructed * contiguously. A node that cleanly ends BELOW the cluster high-water is * still CLEAN here (the freshness/promotion fence catches "behind", task * 4.2/4.3 — not this verdict). 3.1 only CLASSIFIES + sets this; it does * NOT FATAL (task 3.3) and does NOT fence promotion (task 4.3, which * CONSUMES this verdict). Defaults to RECOVER_CLEAN before recovery runs * (a fresh store is clean). Written single-threaded during recovery * before the node participates; read by the promotion gate thereafter. */ recover_verdict_t recovery_verdict; /* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4, * NEVER-SELF-EXIT rework 2026-06-18). Set by * persist_recover_loud_fence_if_unrecoverable when a member is * recovery-truncated AND/OR durable below the cluster committed high-water * mark with NO successful catch-up: the member stays ALIVE but is held * PERMANENTLY non-promotable and keeps retrying catch-up. A controller MUST * NEVER self-exit as a designed response to this operational condition; it * self-heals automatically when a healthy peer returns and catch-up reaches * a committed-consistent state — at which point this flag is CLEARED via * persist_clear_recovery_unrecoverable (wired into the fence-release / * catch-up path). Read by the promotion gate as belt-and-suspenders * (a member must never lead while this is set). Atomic: set on the recovery * path, cleared on the catch-up path, read by the promotion/heartbeat * threads. Defaults to false before recovery runs. */ _Atomic bool recovery_unrecoverable; /* Post-replay apply-correctness verdict + cursor (raft-apply-decoupling * task 5.1, design.md "Restart Replay & Verification", Req 8.1/8.2). * last_applied_lsn is the replay-derived LOGICAL apply cursor — the * highest LSN actually dispatched via apply_committed_entry() — seeded * from checkpoint_lsn before replay begins and advanced by replay_wal * per entry; it is DISTINCT from current_lsn (the physical WAL position * owned by the physical layer, bug-778) even though the two coincide on * a clean replay. Never persisted — recomputed fresh every * persist_recover() (design.md State Management), so it cannot itself * drift from disk state. The live apply pump (task 6.1, not yet built) * takes over its advancement once replay completes. apply_verdict is the * sticky APPLY_CLEAN/APPLY_DIVERGENT result of comparing this cursor (and * the live job table's growth) against the log-derived expectation; * defaults to APPLY_CLEAN before recovery runs, mirroring * recovery_verdict's default-clean convention. Written single-threaded * during recovery; read by persist_apply_verdict/persist_last_applied_lsn * thereafter. A DIVERGENT result is additionally routed by persist_recover() * into the existing loud-fence mechanism (task 5.2, Req 8.4) via * recover_policy_inputs_t.apply_divergent. */ _Atomic uint64_t last_applied_lsn; apply_verdict_t apply_verdict; /* bug-1416 (raft-apply-decoupling task 7.2 round 2): closes a boot-time * TOCTOU between persist_recover()'s replay+verification pass above and * two concurrently-reachable live mutation paths that write the SAME * last_applied_lsn field / mutate the SAME live job table this pass * reads: the live apply pump (persist_apply_pump_run, driven by any of * the 6 commit.c committed_lsn-advance sites via persist_apply_pump_kick) * and the standby live-replication apply path (ha_apply_single_entry / * ha_apply_term_entry_via_library, ha.c). Both are reachable the moment * the iomgr event loop starts processing connections in cluster_run() * (myapp.c) -- which happens well BEFORE cluster_init_phase2() reaches * its persist_recover() call (an election round-trip and the whole of * phase2's own setup earlier in the same function separate the two). * Without this flag a fast-electing peer's replicated traffic can mutate * last_applied_lsn/the job table WHILE this node's own persist_recover() * is still computing its replay-derived expectation, producing a FALSE * APPLY_DIVERGENT verdict on a perfectly healthy boot (confirmed live, * reproduced on a fresh 3-node cluster's ordinary two-phase bootstrap -- * committedLsn was identical across all nodes when it fired). * * Defaults to true (persist_init) so every existing unit-test fixture * that calls persist_init() directly and never calls persist_recover() * (the overwhelming majority of this spec's own test suite) sees * IDENTICAL, ungated behavior to before this field existed. persist_ * recover() is the ONLY function that ever sets this false -- for the * exact, single-threaded duration of its own replay+verify pass -- and * always restores it to true (plus issues one deliberate catch-up kick) * before returning, regardless of the resulting verdict. This does NOT * touch the sticky recovery_unrecoverable fence or its clearing logic * (task 5.3) -- a genuinely DIVERGENT verdict still fences exactly as * before; this flag only prevents the comparison ITSELF from racing. */ _Atomic bool recovery_complete; /* Paths (derived from workdir + name via persist_set_workdir) */ char wal_path[4096]; char snap_path[4096]; /* Config */ persist_config_t config; /* Concurrency */ pthread_mutex_t lock; atomic_uint wal_record_count; /* records appended since last checkpoint */ atomic_int checkpoint_in_progress; /* CAS guard: 0=idle, 1=scheduled/running */ atomic_int fsync_stop; atomic_int fsync_thread_running; /* raft-apply-decoupling task 3.2: pending command-apply hand-off list * (producer side; the not-yet-built ordered apply pump, task 6.1/6.2, * is the consumer). Opaque here (pending_command_apply_t is private to * persist.c) — guarded by its OWN dedicated lock, deliberately NOT * `lock` above, since entries are appended from a commit continuation * callback (dispatched off-thread via vtask_enq_classified) that must * never contend with the hot WAL-append path. See * persist_register_pending_apply's doc comment for the full contract. */ void* pending_applies; pthread_mutex_t pending_applies_lock; /* raft-apply-decoupling task 6.1: ordered apply pump. apply_pump_lock * serializes concurrent walk passes -- multiple commit-advance triggers * (from any of the six commit.c paths that can move committed_lsn) can * race to kick the pump; only one walk actually runs at a time (see * persist_apply_pump_run's doc comment). apply_pump_scheduled is a * CAS-coalescing flag: at most one vtask-dispatched walk is ever * in-flight per persist_t regardless of how many trigger events land * concurrently, so a commit burst cannot flood the vtask pool with * mostly-redundant walk requests. Neither field is persisted -- both * reset fresh on process restart (restart applies via replay, not this * pump; see design.md "Restart Replay & Verification"). */ pthread_mutex_t apply_pump_lock; _Atomic int apply_pump_scheduled; /* raft-apply-decoupling task 7.4 perf-fix: apply_pump_read_entry's * segment-lookup cache. apply_pump_read_entry's only caller * (persist_apply_pump_run) calls it once per LSN in strict ascending * order with no gaps in the common case; without this cache every single * lookup redid a full opendir/readdir/segment-sort plus a pread scan * from byte offset 0, making N contiguous applies O(N^2). These fields * cache the currently-open segment fd + byte offset positioned to read * whichever LSN the pump is expected to ask for next, giving amortized * O(1) per entry on the common contiguous-advance path. * * Guarded ENTIRELY by apply_pump_lock (never p->lock) -- the pump's sole * caller already holds apply_pump_lock across the whole walk, including * every apply_pump_read_entry call, so no additional locking is needed * here. apply_pump_cache_fd == -1 means "no cache" -- this is always a * safe, correct fallback trigger (falls through to full segment * rediscovery), never a bug to fall back to. Reset to -1 in persist_init, * persist_fini, persist_recover (a stale pre-recovery cursor must never * be trusted), and persist_truncate_wal_above (a real truncation may * ftruncate()/unlink() the segment the cache still points into). * * Known, accepted, bounded tradeoff vs. checkpoint-driven trim * (trim_wal_segments): trim only unlinks segments below its own trim * floor, and POSIX unlink-keeps-fd-alive semantics mean an fd this cache * already has open into a segment stays readable even after trim unlinks * it on disk. If the pump is lagging behind checkpoint/trim when trim * fires -- reachable in this feature, since local-apply-lagging-behind- * commit is raft-apply-decoupling's entire premise -- the cache can keep * successfully reading that stale, deleted-on-disk data and silently * advance last_applied_lsn past it. Before this cache existed, every * lookup did a fresh opendir/readdir, so the equivalent situation failed * loudly and immediately with -ENOENT on the very next lookup. This is a * real behavior change -- NOT "no worse than before" -- but it is bounded: * a crash before the next checkpoint leaves in-memory apply state ahead * of what disk can prove on restart, and the existing APPLY_DIVERGENT * boot fence (tasks 5.1/5.2, persist_recover) catches exactly this at the * next restart and fences the node loudly rather than serve wrong state. * So the loud failure is deferred from "immediate, at read-time" to * "delayed, at next-restart-boot-time," not eliminated. Redesigning * trim/cache invalidation to close this gap at read-time is a distinct, * pre-existing architectural item (tracked separately) and explicitly * out of scope for this perf fix. */ int apply_pump_cache_fd; uint64_t apply_pump_cache_seg_lsn; uint32_t apply_pump_cache_off; uint64_t apply_pump_cache_next_lsn; #ifdef XCE_TESTING /* Fault-injection seam (ha-commit-durability task 2.1 error-path test): * when non-zero, every fdatasync site treats the sync as FAILED — it * still calls the real fdatasync but forces a non-zero rc, so durable_lsn * must NOT advance and the failure is logged loudly. Compiled out of * production (xce_controller is built without -DXCE_TESTING). */ atomic_int test_force_fsync_fail; /* fdatasync observability counter (ha-commit-durability task 2.3): * incremented at the standby-path inline sync in * persist_wal_write_replicated, BEFORE that function returns (and thus * before the caller can ACK). A persist-before-ACK regression test reads * this to prove a durable sync ran inside the call — page-cache re-reads * cannot distinguish fsync'd from cache-only, so a direct sync signal is * required. Compiled out of production. */ atomic_uint test_replicated_fdatasync_count; /* Liveness observability (ha-commit-durability task 5.1, Req 1.6): * the thread id of the last flusher fdatasync that advanced durable_lsn. * A test reads this to prove fdatasync runs on the dedicated flusher * thread, NOT on the caller/iomgr/heartbeat thread. Compiled out of * production. */ pthread_t test_last_fdatasync_thread; atomic_int test_last_fdatasync_thread_set; /* Fault-injection seam (ha-entity-snapshots task 2.2): force the Nth * (0-based, in retained-job walk order) job's checkpoint encode to fail * inside write_snapshot, proving a genuine mid-loop encode failure * discards the WHOLE checkpoint rather than skipping just one job. * Stored as index+1 so the zero-initialized default (0) means * "disabled" — mirrors test_force_fsync_fail's zero-init-safe * convention (no explicit persist_init reset needed). Compiled out of * production. */ atomic_int test_force_job_encode_fail_at; #endif pthread_t fsync_thread; pthread_mutex_t fsync_mutex; /* guards the fsync timer wait */ pthread_cond_t fsync_cond; /* signaled by persist_stop for prompt join */ /* Durable-advance wake hook (ha-commit-durability task 2.2). * Persistence owns this slot; the commit engine REGISTERS a callback * (ctx = the commit_engine_t) via persist_set_durable_cb. The flusher * invokes it AFTER advancing durable_lsn and AFTER releasing p->lock, * so a COMMIT_MODE_WAIT caller is woken promptly when its entry becomes * durable locally — without it, a commit whose standby ACKs all arrived * before the local fdatasync would stall until the WAIT timeout. The * dependency direction is preserved (commit depends on persist, never * the reverse): persist only stores+calls an opaque function pointer. * NULL until registered (every flush is then a no-op wake). */ void (*on_durable_advance)(void* ctx); void* durable_cb_ctx; /* Truncation wake hook (raft-apply-decoupling task 1.4). Persistence owns * this slot; the commit engine REGISTERS a callback (ctx = the * commit_engine_t) via persist_set_truncate_cb, mirroring * on_durable_advance's registration pattern and dependency direction * (persist only stores+calls an opaque function pointer; it never * depends on commit.c). persist_truncate_wal_above invokes it AFTER it * actually removes entries above the new tail and AFTER releasing * p->lock, so any commit continuation registered for an LSN above the * new tail (which can never commit -- its entry no longer exists) is * resolved FAILED promptly instead of waiting out the commit engine's * timeout sweep. NOT invoked on a no-op truncate (nothing above * cutoff_lsn -- current_lsn unchanged). NULL until registered (a * truncate is then a silent no-op wake). */ void (*on_truncate)(void* ctx, uint64_t cutoff_lsn); void* truncate_cb_ctx; /* Checkpoint (stubs for now — wired in Task 10) */ atomic_int checkpoint_requested; pthread_cond_t checkpoint_done; pthread_mutex_t checkpoint_mutex; pthread_t snap_thread; /* Fan-out replicator — P2-4 */ replicator_t repl; }; typedef struct persist_s persist_t; /* Set workdir and controller name before persist_init(). * Derives wal_path and snap_path as //data/{wal,snap}. */ void persist_set_workdir(persist_t* p, const char* workdir, const char* name); /* ============================================================ * WAL term adapter (ha-term-safety task 1.3, Req 1.1/1.5/1.6/1.7/1.8) * * Today every WAL append stamps hdr.term = p->term, but p->term is never * assigned, so all entries carry term 0 (the G1 bug). The on-disk * wal_entry_hdr_t already has a 40-byte `term` field; this is a VALUE fix, NOT * a layout change (R1.7). * ============================================================ */ /* Legacy-zero-term rule (R1.8) — REVIEW ISSUE 2, made precise: * * A WAL entry that predates this feature carries hdr.term == 0. For (term,lsn) * matching it must map to a SINGLE, well-defined "lowest-known term" so that * (a) recovery never fails on legacy-0 entries, and * (b) a legacy-0 entry is never spuriously treated as divergent from another * legacy-0 entry at the same LSN. * The rule: legacy-0 -> WAL_LEGACY_TERM (1, the lowest real Raft term — Raft * terms start at 1; term 0 is the pre-election "no term" placeholder). It is * applied UNIFORMLY at read-back (persist_entry_term), so every legacy-0 entry * answers the same term 1 and matches another legacy-0 entry at the same LSN. * The raw 0 is still stored in the index (so "present legacy-0" stays * distinguishable from "absent"); only the read-back value is mapped. */ #define WAL_LEGACY_TERM 1u /* Set the controller's live election term so subsequent WAL appends stamp it * into hdr.term (R1.1/R1.6). Called from the append callback (on_persist_entry, * task 2.2) and on term adoption, so no append records a zero placeholder while * a valid term is in effect. No on-disk layout change; value fix only (R1.7). * Safe on a NULL p (no-op). */ void persist_set_term(persist_t* p, uint64_t term); /* Recovered/live per-entry term for a 1-based LSN (lsn == Raft index), feeding * the task-2.2 on_get_entry_term callback and the library's init restore * (R1.5). Returns 0 when the LSN is ABSENT (never recorded). A PRESENT entry * that carries a legacy zero term is mapped to WAL_LEGACY_TERM (R1.8) so it is * never confused with an absent LSN and matches other legacy-0 entries. * Safe on a NULL p (returns 0). */ uint64_t persist_entry_term(const persist_t* p, uint64_t lsn); /* Lock-free counterpart of persist_entry_term for a caller that ALREADY holds * p->lock (calling persist_entry_term there would re-acquire the same * non-recursive mutex and self-deadlock — raft-ha-core-safety task 2.5). Same * absent/legacy-0 semantics, just without the lock/unlock. Caller MUST hold * p->lock; safe on a NULL p (returns 0). */ uint64_t persist_entry_term_locked(const persist_t* p, uint64_t lsn); /* raft-ha-core-safety task 3.2 (Req 3.1/5.1) — Snapshot Boundary Reconcile. * * Authoritatively overwrite the per-LSN term index entry at `checkpoint_lsn` * with `checkpoint_term` (the checkpoint's own true term, as carried on the * wire by the leader that shipped the snapshot). Called from the standby * snapshot-install receive path (ha_handle_snapshot_chunk) immediately after * persist_load_snapshot() succeeds, so a divergent pre-snapshot entry's STALE * local term at the checkpoint LSN is reconciled to the checkpoint's own * term. Establishes a verifiable (term, lsn) boundary: post-install AE intake * (prevLogIndex/prevLogTerm checks) never references an LSN below the * checkpoint against this member again, so only the checkpoint LSN's own * term needs reconciling — not the full compacted history below it. * * `checkpoint_term == 0` means a legacy sender (pre-task-3.2 peer) never * stamped the wire field; treat that as WAL_LEGACY_TERM per the existing * legacy-0 convention (record_entry_term stores the raw value verbatim; the * legacy-0 -> WAL_LEGACY_TERM mapping is applied uniformly at read-back by * persist_entry_term/entry_term_locked, so this call is consistent with that * convention without needing to special-case it here). * * Takes p->lock internally (callers must NOT already hold it). Safe on a * NULL p or checkpoint_lsn == 0 (no-op). */ void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn, uint64_t checkpoint_term); /* Lifecycle */ int persist_init(persist_t* p, persist_config_t* config, cluster_t* c); int persist_start(persist_t* p); void persist_stop(persist_t* p); void persist_fini(persist_t* p); int persist_recover(persist_t* p); /* WAL write paths */ int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len); int persist_wal_sync_append(persist_t* p, uint32_t type, const void* payload, uint32_t len, int* sync_rc_out); /* Write a REPLICATED entry (received from the primary) into the local WAL * at its GIVEN lsn — used by standbys to durably persist streamed entries * BEFORE acking them (persist-before-ACK; an ACK must imply durability). * * Unlike persist_wal_async it does NOT assign a new lsn (the lsn is supplied * by the primary) and does NOT repl_broadcast (standbys never re-replicate). * It builds a SELF-consistent CRC chain (the streamed entry carries no crc), * fsyncs the segment before returning, and is idempotent on re-stream: if the * lsn is already persisted (lsn+1 <= current_lsn) it is a no-op returning 0. * * WAL_CHECKPOINT is rejected with -EINVAL (never streamed to standbys). * Returns 0 on success (incl. dedup no-op), negative errno on failure. */ int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type, const void* payload, uint32_t len); /* ============================================================ * Write-path commit classification (ha-commit-barrier-rollout) * ============================================================ */ typedef enum { WAL_MODE_WAIT, /* request-shaped: barrier before requester reply */ WAL_MODE_RETAIN, /* fact-shaped: retained until committed, no timeout */ WAL_MODE_LOCAL, /* not replicated (WAL_CHECKPOINT) */ } wal_commit_mode_t; /* Pure classification lookup. detail disambiguates WAL_WORKER_CTRL only * (HOOK_DETAIL_WORKER_JOIN/UPDATE -> WAIT, HOOK_DETAIL_WORKER_DEAD -> * RETAIN). Unknown type: XCE_TESTING assert (fail fast); production logs * ERROR and returns WAL_MODE_RETAIN (never dropped, never confirmed early). */ wal_commit_mode_t wal_classify(uint32_t wal_type, int detail); /* Commit barrier — handler-callable WAIT completion on everything appended * so far (ha-commit-barrier-rollout). Snapshots the appended watermark * (current_lsn - 1, under p->lock — matching persist_wal_sync_append's * read); returns 0 without blocking when committed_lsn already covers it * (or nothing was ever appended). Otherwise delegates to * commit_submit(COMMIT_MODE_WAIT) and returns its code unmodified: * 0 — committed on a majority * -ETIMEDOUT — not committed within the deadline (entries remain * in the log; outcome converges per umbrella 2.5) * -XCE_ERR_NOT_READY — engine admission cap exhausted (retriable) * timeout_ms = 0 falls back to p->config.sync_timeout_ms. n_peers <= 1 * returns 0 immediately (engine self-quorum, standalone preserved). * Call only from vtask threads with no entity locks held. */ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms); /* Force WAL fdatasync (for tests and clean shutdown) */ int persist_flush(persist_t* p); /* Highest LSN whose bytes are durably fdatasync'd locally — the * group-commit prefix watermark advanced by the demand-driven flusher * (ha-commit-durability task 2.1). The commit majority self-contribution * caps at this value (task 2.2 consumer). Lock-free atomic read; safe from * any thread. Returns 0 when nothing is yet durable. */ uint64_t persist_durable_lsn(persist_t* p); /* Durable freshness floor (ha-commit-durability task 3.4, Req 2.3). * * The member's durably-backed last-log index (WAL tip): the highest LSN that * is on stable storage locally. persist_recover re-derives it from the * recovered WAL tip (the highest INTACT replayed LSN) so the post-recovery * value is backed by durable on-disk state, NOT a volatile in-memory * applied_lsn / last_known_committed_lsn. The Raft freshness gate (task 4.2, * ha.c / cluster.c) consumes this so a truncated member cannot over-advertise * a tip it does not durably hold. * * Precision: this is the durable LAST-LOG INDEX, NOT the commit position * (committed_lsn). The post-election safe-commit point still comes from voter * replies, never from this floor (CLAUDE.md). After a truncating restart the * floor equals the lower intact tip, never a stale higher in-memory value and * never the commit position. * * Thin wrapper over persist_durable_lsn: at and after recovery the durable * watermark IS the recovered WAL tip (recovered on-disk entries are genuinely * durable), so a separate field would be redundant. Lock-free atomic read; * safe from any thread. Returns 0 when nothing is durable (empty first-boot * WAL). */ uint64_t persist_durable_floor(persist_t* p); /* Recovery-integrity verdict + flag (ha-commit-durability task 3.1, Req 2.1). * persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED * verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh * store before recovery runs). persist_recovery_truncated is the convenience * flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate * (task 4.3) consumes to mark the member non-promotable. Both are pure reads * of recovery state established single-threaded at startup. */ recover_verdict_t persist_recovery_verdict(persist_t* p); bool persist_recovery_truncated(persist_t* p); /* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4; * never-self-exit rework 2026-06-18). * The interval after which a still-unrecoverable, loudly-fenced member RE-EMITS * its operator alert (it does NOT terminate — see RECOVER_DECISION_FENCE_- * UNRECOVERABLE). The member stays alive and keeps retrying catch-up * indefinitely; this window only controls how often the loud operator-visible * alert is repeated. Default 30s. */ #define DEFAULT_RECOVERY_CATCHUP_WINDOW_MS 30000u /* persist_recover_decision (ha-commit-durability task 3.3, Req 2.4). * * Pure decision over the Boot Policy Decision Table: maps the policy inputs to * RECOVER_DECISION_{ELIGIBLE,FENCE,FENCE_UNRECOVERABLE}. No side effects, no * termination — it only classifies, so it is unit-testable in isolation. The * FENCE_UNRECOVERABLE terminal is acted on by * persist_recover_loud_fence_if_unrecoverable (which logs LOUDLY + sets the * sticky non-promotable flag — it NEVER terminates the process). * * Contract (design.md rows): a reachable healthy peer NEVER returns * FENCE_UNRECOVERABLE (rows 3, 5); it is returned ONLY when the member is * truncated or behind the HWM, has NO healthy peer, and catch-up did not reach * a committed-consistent state (rows 4, 4b). */ recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in); /* persist_recover_loud_fence_if_unrecoverable (ha-commit-durability task 3.3, * Req 2.4; never-self-exit rework 2026-06-18). * * Evaluate persist_recover_decision over `in`; if the outcome is * FENCE_UNRECOVERABLE, log LOUDLY at ERROR — INCLUDING the truncation boundary * (durable_floor_lsn) and the cluster high-water mark — stating the member is * staying fenced + non-promotable and will keep retrying catch-up, then SET the * sticky `recovery_unrecoverable` flag on `p` (so the promotion gate refuses * leadership as belt-and-suspenders). Returns the decision. * * CRITICAL — the process STAYS ALIVE. There is NO abort()/exit()/tlog_exit() * here: a controller MUST NEVER self-terminate as a designed response to an * operational condition (user invariant, 2026-06-18). The member self-heals — * the fence releases (→ ELIGIBLE, Req 2.6) and the flag clears via * persist_clear_recovery_unrecoverable — automatically when a healthy peer * returns and catch-up reaches a committed-consistent state. * * `p` may be NULL (pure-decision callers that only want the classification + * loud log without touching a persist_t); the flag set is then skipped. */ recover_decision_t persist_recover_loud_fence_if_unrecoverable(persist_t* p, const recover_policy_inputs_t* in); /* persist_recovery_unrecoverable / persist_clear_recovery_unrecoverable * (ha-commit-durability Req 2.4/2.6, never-self-exit rework 2026-06-18). * * Accessor + clear for the sticky "unrecoverable, loudly fenced" flag. The flag * is SET by persist_recover_loud_fence_if_unrecoverable and CLEARED on the * catch-up / fence-release path once the member reaches a committed-consistent * state (not truncated AND durable_floor >= HWM). While set, the member is held * permanently non-promotable but stays alive. Safe on a NULL p (returns false / * no-op). */ bool persist_recovery_unrecoverable(persist_t* p); void persist_clear_recovery_unrecoverable(persist_t* p); /* persist_recovery_complete (bug-1416, raft-apply-decoupling task 7.2 round * 2) -- true unless a persist_recover() call is CURRENTLY in progress on * this persist_t (see the recovery_complete field doc comment). Consulted * by the standby live-replication apply path (ha_apply_single_entry / * ha_apply_term_entry_via_library, ha.c) to defer in-memory entity mutation * while a boot-time replay+verification pass is running, so it never races * that pass's expectation-vs-actual comparison. Durability (WAL persist) * and the replication ACK are NEVER gated by this -- only the in-memory * apply; a deferred entry is picked up either by the in-progress replay * pass itself (if it observes the entry on disk) or by the single catch-up * kick persist_recover() issues right before returning. Safe on NULL p * (returns true -- nothing to defer against). */ bool persist_recovery_complete(persist_t* p); /* persist_mark_recovery_pending (bug-1416 round 2 -- live-cluster reproduction * confirmed a gap in the original fix). persist_recover() itself already * brackets recovery_complete false/true around its own call, but that window * is too NARROW for the real production boot sequence: a node can be * receiving and durably APPLYING live-replicated entries (ha_apply_single_ * entry / ha_apply_term_entry_via_library -- e.g. while stuck retrying its * own election as a candidate that keeps losing/timing out before it * discovers a live primary and steps down) for an arbitrarily long stretch * BEFORE cluster_init_phase2() ever gets to call persist_recover() for the * first time. During that whole stretch recovery_complete still holds * persist_init()'s default (true), so the live-replication apply path runs * UNGATED and can mutate jobmgr/last_applied_lsn well before persist_ * recover()'s own baseline capture, producing the exact same false * APPLY_DIVERGENT this flag exists to prevent, just from an earlier entry * point than persist_recover() alone can see. * * Call this ONCE, in the real boot path only (myapp_run, immediately after * persist_init()/persist_start() succeed and BEFORE election / cluster_run() * can process any network traffic), to close the gate from the earliest * safe moment instead of only from persist_recover()'s own entry. Safe on * NULL p (no-op). Never called by any test fixture -- every unit test that * constructs a persist_t directly (bypassing myapp_run entirely) is * unaffected and keeps persist_init()'s default (true, ungated). */ void persist_mark_recovery_pending(persist_t* p); /* persist_verify_apply_correctness (raft-apply-decoupling task 5.1, Req * 8.1/8.2). Pure comparison — mirrors persist_recover_decision's separation * of decision from action: no logging, no fence side effects, no mutation of * `p` beyond the read of last_applied_lsn. Compares `exp` (derived by * scanning the committed log during replay — see apply_expectation_t) against * the actual materialized state: p's replay-derived apply cursor, and — when * p->cluster is non-NULL — the live job table's growth since * `jobmgr_count_before_replay` (the job count captured just before replay * began). Returns APPLY_CLEAN on a match. Returns APPLY_DIVERGENT when either * the apply cursor or the live job count disagrees with `exp`. * * Safe to call with p->cluster == NULL (a bare unit-test persist_t, or any * caller with nothing to dispatch into): the job-count half of the check is * skipped — only the position check applies — mirroring * apply_committed_entry()'s own "nothing to dispatch into" convention. Safe * on NULL p or NULL exp (returns APPLY_CLEAN — nothing to verify). */ apply_verdict_t persist_verify_apply_correctness( persist_t* p, const apply_expectation_t* exp, uint64_t jobmgr_count_before_replay); /* Accessor for the sticky verdict computed by the last persist_recover() * (task 5.1). APPLY_CLEAN before recovery has run (fresh store default, * mirrors persist_recovery_verdict's RECOVER_CLEAN default). Safe on NULL p. */ apply_verdict_t persist_apply_verdict(persist_t* p); /* Accessor for the replay-derived logical apply cursor (task 5.1, * design.md "Restart Replay & Verification" state model) — see the * last_applied_lsn field doc comment for its precise semantics and how it * differs from current_lsn/persist_durable_floor. Safe on NULL p (returns * 0). Lock-free atomic read. */ uint64_t persist_last_applied_lsn(persist_t* p); /* Register a durable-advance wake callback (ha-commit-durability task 2.2). * The group-commit flusher calls fn(ctx) after it advances durable_lsn and * after releasing p->lock, so a consumer (the commit engine) can re-evaluate * its commit majority and wake any COMMIT_MODE_WAIT caller whose entry just * became durable locally. fn MUST NOT take p->lock (it runs after release, on * the flusher thread) — the commit engine's callback takes only ce->lock, * keeping the lock order p->lock → ce->lock cycle-free. Pass fn = NULL to * unregister. Safe to call before persist_start (no flusher running yet). */ void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx); /* Register a truncation wake callback (raft-apply-decoupling task 1.4). * persist_truncate_wal_above calls fn(ctx, cutoff_lsn) after it actually * truncates entries above cutoff_lsn (the new log tail) and after releasing * p->lock -- fn MUST NOT take p->lock (same rule as persist_set_durable_cb; * the commit engine's callback takes only ce->lock, keeping the lock order * p->lock → ce->lock cycle-free). Not called for a no-op truncate (nothing * above cutoff_lsn). Pass fn = NULL to unregister. */ void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn), void* ctx); #ifdef XCE_TESTING /* ============================================================ * Durability-verification test seam (ha-commit-durability Req 3.2, 3.3) * * persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL * segments from STABLE STORAGE and return the highest fully-persisted * LSN, independent of the in-memory current_lsn / committed_lsn * watermarks. Lets a test assert that an acknowledged operation is * recoverable from disk after an immediate, ungraceful termination — * without relying on the background flush timer (Req 3.2): * * persist_wal_async(p, ...); // append to page cache * persist_flush(p); // fdatasync → durable on disk * lsn = persist_test_durable_lsn_on_disk(p); // highest durable LSN * * It scans the same on-disk format and CRC chain as recovery, stopping * at the first corruption / short read / pre-allocated zero region, so * it reports only entries that are genuinely on stable storage. It * never mutates persist_t state and fires no hooks (Req 3.3 — the seam * is purely inspective; the acknowledgment contract is unchanged). * * Returns the highest durable LSN, or 0 when nothing is durable (empty * first-boot WAL, no segments). Compiled out of production builds: * the xce_controller target is built without -DXCE_TESTING and links * without this symbol. * ============================================================ */ uint64_t persist_test_durable_lsn_on_disk(persist_t* p); /* Fault-injection toggle for the fdatasync error path (task 2.1). * When `fail` is non-zero, every fdatasync site in persist.c forces a * failed sync result: the real fdatasync still runs, but durable_lsn is * left UNCHANGED and the failure is logged loudly — proving an acknowledged * entry is never counted durable on a sync error (Req 1.1/1.2, fail-loud). * Lock-free atomic write; safe to flip from a test thread at any time. * Compiled out of production (xce_controller links without it). */ void persist_test_set_force_fsync_fail(persist_t* p, int fail); /* Read the standby-path fdatasync counter (task 2.3, persist-before-ACK * regression). Counts the inline fdatasync calls made by * persist_wal_write_replicated. A test snapshots this before the call and * asserts it incremented after the call returns — proving the durable sync * ran inside the persist primitive, BEFORE the caller can send its WalAck. * Lock-free atomic read; compiled out of production. */ unsigned persist_test_replicated_fdatasync_count(persist_t* p); /* Invoke the legacy sync-append commit store (task 2.4) with the durable * precondition applied — the same store persist_wal_sync_append performs on a * majority ACK in its engine-less / LSN-0 branch. Lets a test prove * committed_lsn never advances past durable_lsn on the ACK path without * standing up the replication wire. Compiled out of production. */ void persist_test_sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn); /* Return the thread id of the last flusher fdatasync that advanced * durable_lsn (task 5.1 / Req 1.6 liveness). A test asserts this is NOT the * caller/iomgr thread — proving durability (fdatasync) work runs on the * dedicated flusher thread and never stalls heartbeat/election handling. * Returns pthread_self() if no flusher sync has advanced durable_lsn yet * (so an unprimed read can never falsely look "off-thread"). Compiled out of * production. */ pthread_t persist_test_last_fdatasync_thread(persist_t* p); /* Close the active WAL segment and open a fresh one at the current LSN * (ha-term-safety task 4.2 r1 test seam). Forces a multi-segment layout so a * subsequent persist_checkpoint trims the older segment — exercising the * compaction hazard where a committed marker's segment is unlinked. Returns 0 * on success, negative errno on I/O failure. Compiled out of production. */ int persist_test_force_new_segment(persist_t* p); /* Arm/disarm the job-encode fault-injection seam (see the struct field * comment above). Pass a negative index to disable. Lock-free atomic * write; compiled out of production. */ void persist_test_set_force_job_encode_fail_at(persist_t* p, int index); #endif /* XCE_TESTING */ /* Checkpoint */ int persist_checkpoint(persist_t* p); /* WAL truncation — remove all entries with LSN > cutoff_lsn. * Idempotent; returns 0 on success, negative errno on I/O error. */ int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn); /* Logging helper — never logs payload content (security invariant) */ void persist_log_entry_metadata(const wal_entry_hdr_t* hdr); /* ============================================================ * AFTER_EVENT hook functions — registered by persist_start() * * Signature matches hook_fn_t: int fn(int type, int situation, * int detail, void* data, va_list ap) * cls/data points to the persist_t instance. * * P0: writes a minimal 4-byte payload (detail value) to the WAL. * Full Cap'n Proto serialization is wired in Task 15. * ============================================================ */ /* Explicit controller-side raft step for job submission (raft-apply-decoupling * item #1 / bug-1306): WAL-append every job in `jlist` and block on the * majority-commit barrier, returning the commit verdict DIRECTLY (0 = all * committed on a majority; first negative rc otherwise — -ETIMEDOUT / etc.). * handle_jobsubmit calls this instead of reading the clobber-prone hook return; * a negative rc MUST reject the submission (never reply success). */ int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist); /* Sibling of persist_submit_wal_and_commit for callers that must not * construct any job object before the commit barrier succeeds * (raft-apply-decoupling task 3.1 / Req 2.1, 2.3): `payloads` is an xlist_t * of xce_job_t* — raw submission data that has never been registered in any * job table. WAL-appends + majority-commits each entry in order; on full * success (return 0), out_lsns[i] holds payloads[i]'s committed LSN, for the * caller to construct and tag the corresponding job object AFTER this call * returns — there is nothing to roll back on failure because nothing was * ever constructed. Returns the first negative rc on failure; out_lsns is * only valid up to (not including) the failed index. */ int persist_submit_job_payloads_and_commit(persist_t* p, xlist_t* payloads, uint64_t* out_lsns); /* raft-apply-decoupling task 3.2: outcome delivered to a job submission's * per-entry commit continuation once its commit_submit_async() registration * resolves (or, for the LSN-0 / engine-not-wired edge cases, immediately * and synchronously). rc == 0 means the entry committed on a majority; * rc < 0 carries the failure/timeout error code, the same vocabulary * commit_submit's WAIT mode already uses (-ETIMEDOUT, -XCE_ERR_STANDBY, * -ENOMEM, etc.). `lsn` is 0 for an entry that never reached the point of * being assigned a real LSN (e.g. a local encode/append failure) — 0 is * the reserved "no real entry" sentinel already used throughout persist.h. * * Declared with a single plain rc — NOT commit.h's `commit_outcome_t` + * `int rc` pair — so this header never needs to depend on commit.h, which * already depends on this one (commit_engine_t embeds a `persist_t*`). * persist.c's implementation adapts commit_submit_async's real callback * shape to this one internally (see job_commit_resolve_adapter). */ typedef void (*persist_job_commit_notify_fn)(void* ctx, uint64_t lsn, int rc); /* Async sibling of persist_submit_job_payloads_and_commit (raft-apply- * decoupling task 3.2): WAL-appends each payload the same way (encode + * local append; out_lsns[i] set on a successful append) but never blocks * on the majority-commit barrier. Instead, `fn`/`ctx` is invoked EXACTLY * ONCE per payload in `payloads` — either synchronously, right here (a * local append failure, the once-per-cluster-lifetime LSN-0 edge, an * engine-not-wired context, or a commit_submit_async registration * failure), or later, asynchronously, off this call's stack, once that * payload's entry's commit outcome is known. * * On the FIRST local failure (encode/append), every REMAINING payload in * the batch also resolves immediately with that SAME failure rc — so the * caller always receives exactly `payloads->count` total `fn` invocations, * regardless of where (or whether) a failure occurs; the caller never * needs to separately account for "never going to be called" entries. * * Returns 0 once every payload is durably appended locally and every * payload has an outcome resolution path established (sync or async); the * first negative rc encountered otherwise. Every payload's `fn` invocation * has already happened (or is already guaranteed to happen later) by the * time this function returns, regardless of its return value — the return * value itself carries no information the caller still needs to act on. * * out_lsns[i] is valid for every i whose local append succeeded (same * contract as the synchronous sibling); it is left unset (0, if the caller * zero-initialized it) for a payload whose local append itself failed. */ int persist_submit_job_payloads_async(persist_t* p, xlist_t* payloads, uint64_t* out_lsns, persist_job_commit_notify_fn fn, void* ctx); /* raft-apply-decoupling task 3.2: producer-side hand-off for a command * whose commit_submit_async() continuation resolved COMMITTED. Per task * 3.2's own contract, a COMMITTED outcome must NOT apply state or reply to * the client directly from the commit continuation — both must happen * from the not-yet-built ordered apply pump (task 6.1/6.2), strictly in * commit order relative to every other entry (command or fact). * * persist_register_pending_apply() is the producer: it records that once * `lsn` is applied (via apply_committed_entry(), the same seam restart * replay already uses), `on_applied(ctx, lsn)` must be invoked immediately * afterward — task 6.2's own stated contract: "fire a command's registered * continuation immediately after its entry applies". Until task 6 exists, * registered entries accumulate here inertly: this is the SAME "durably * committed, not yet applied" posture already accepted for retained facts * during quorum loss (Req 6.1), not a leak — task 6.1 is expected to drain * and free each node as it walks committed LSNs. * * bug-1415 self-healing (round 2): the caller (e.g. jobsubmit_commit_ * continuation, dispatched off-thread via commit.c's dispatch_continuation) * and the ordered apply pump's walk (persist_apply_pump_run(), dispatched * off-thread via persist_apply_pump_kick()) are two INDEPENDENT vtasks * enqueued back-to-back onto the same pool with no completion-order * guarantee between them — with n_vcpus > 1 (the production default), the * walk can apply `lsn` and advance last_applied_lsn PAST it before this * function ever runs for that same `lsn`. A registration arriving that late * would otherwise be permanently orphaned: the walk visits each lsn exactly * once, so nothing would ever call persist_take_pending_apply() for it * again, silently hanging the caller forever. This function guards against * that by checking `lsn <= persist_last_applied_lsn(p)` BEFORE linking the * node onto the list; if the walk has already passed `lsn`, `on_applied` is * invoked immediately and synchronously, right here, on the CALLER's thread * (never the apply-pump thread) instead of being linked for a walk pass * that will never revisit it. This is race-safe in both interleavings: * when the walk has NOT reached `lsn` yet, the check is false and * registration proceeds exactly as before (unchanged behavior for the * common, non-racing case); when the walk already passed `lsn`, the walk's * own persist_take_pending_apply() call for that lsn already ran and found * nothing (that is precisely how we know this registration is late), so * invoking `on_applied` here can never double-fire alongside it. * * NOTE for whoever builds task 6.1: apply_committed_entry()'s WAL_JOB_SUBMIT * case (persist.c) already constructs the job (via restore_job()) AND fires * the HOOK_DETAIL_JOB_NEW AFTER_EVENT notification, per entry, independently * of any sibling array element — calling apply_committed_entry() for this * lsn is very likely ALL the "apply" work `on_applied`'s caller needs to do * before invoking `on_applied`; `on_applied` itself should very likely only * need to send the client's success reply, not reconstruct or re-notify * anything (see job.c's jobsubmit_apply_complete for the concrete case). */ typedef void (*pending_apply_notify_fn)(void* ctx, uint64_t lsn); void persist_register_pending_apply(persist_t* p, uint64_t lsn, pending_apply_notify_fn on_applied, void* ctx); /* Non-destructive lookup: 1 if `lsn` has a registered, not-yet-consumed * pending-apply hand-off, 0 otherwise. Test/introspection helper only — * task 6.1's real consumer will need its own draining accessor (e.g. "take * and remove"), not built by this task. */ int persist_has_pending_apply(persist_t* p, uint64_t lsn); /* raft-apply-decoupling task 6.1: "take and remove" counterpart to * persist_register_pending_apply()/persist_has_pending_apply() (task 3.2), * built for task 6.2 to consume — task 3.2 explicitly left this accessor * unbuilt ("task 6.1 needs its own 'take and remove' accessor, not built * here"). Atomically finds the pending-apply hand-off node registered for * `lsn` (if any), unlinks and frees the list node, and returns its * `on_applied`/`ctx` pair via the output parameters. * * Returns 1 and populates *out_fn and *out_ctx when a registration was found * and removed; returns 0 and leaves *out_fn and *out_ctx untouched when no * registration exists for `lsn` (the common case for a fact entry, which * never registers one). * * Contract for task 6.2: call this AFTER apply_committed_entry() has already * applied `lsn` — this accessor does not itself apply anything, it is pure * bookkeeping over the hand-off list. When it returns 1, invoke * `(*out_fn)(*out_ctx, lsn)` exactly once; a second call for the same lsn is * guaranteed to return 0 (the node is already removed), so the hand-off * fires exactly once per registration. Safe to call for every applied lsn * unconditionally (facts and commands alike): it is a cheap linked-list scan * under pending_applies_lock and a correct no-op (returns 0) when nothing is * registered. Not yet called by anything — task 6.2's job, not this one's. */ int persist_take_pending_apply(persist_t* p, uint64_t lsn, pending_apply_notify_fn* out_fn, void** out_ctx); /* raft-apply-decoupling task 6.1: strict-order walk that applies every newly * committed entry, from persist_last_applied_lsn()+1 up to p->committed_lsn * inclusive, invoking apply_committed_entry() for each in ascending LSN * order — regardless of the entry's commit mode (WAIT or RETAIN): a command * entry that is *also* still tracked in the pending_applies hand-off list * (task 3.2) is applied exactly like any other entry here; task 6.2 (not * built by this function) is responsible for separately calling * persist_take_pending_apply() and firing that entry's reply continuation * after this function (or whatever drives it) applies it. * * Synchronous, single pass: walks until no further committed-but-unapplied * entry exists (as of the moment each check is made), then returns. Safe to * call directly — e.g. from a test, without ever going through * persist_apply_pump_kick's async dispatch — serializing against any * concurrent walk via a dedicated lock (never `p->lock`: this performs * blocking disk reads, and `p->lock` is the hot single-writer append lock — * see the lock-ordering invariant, persist.c ~3347-3350). * * Returns the number of entries applied (>= 0) on success, or a negative * errno if reading a committed entry's bytes back off the WAL failed — a * genuine, unexpected condition for an already-committed entry (logged * loud; per design.md's Error Handling, this is treated as an apply-time * failure, not a retry-immediately condition). The walk stops at the failed * LSN; last_applied_lsn is left unadvanced past it, so the SAME entry is * retried, from the same position, by the next kick — never silently * skipped. */ int persist_apply_pump_run(persist_t* p); /* raft-apply-decoupling task 6.1: async trigger for persist_apply_pump_run() * above. Intended to be called from every commit.c-internal path that can * advance p->committed_lsn (commit_resolve_pending, after ce->lock is * released) so the pump reacts to ALL SIX advance paths task 1.2 identified * (durable-advance, self-quorum commit_submit, ack-driven advance, the * writable-lsn gate, the raft-library commit-index feed, and membership * change) — not just the one raft-library-fed path * (ha_cb_commit_advanced_stub/on_commit_advanced) design.md names * explicitly, which alone would starve the pump under a single-node/ * self-quorum cluster that never exercises that specific callback. * * No-op when p is NULL or p->cluster is NULL (mirrors apply_committed_entry's * own bare-persist-fixture convention) — every existing commit-engine-only / * persist-only unit fixture that never wires a cluster_t is completely * unaffected by this call; the pump only ever activates for a real cluster * context. * * `pool` is a `vtaskmgr_t*` (declared void* here to avoid persist.h * depending on vtaskmgr.h, mirroring pending_apply_notify_fn's existing * avoidance of a commit.h dependency) used to dispatch the walk off the * calling thread — never inline on what may be the replication-ACK/iomgr * thread — via vtask_enq_classified(VTASK_LIVE_BLOCK): same class and the * same inline-on-dispatch-failure fallback discipline as commit.c's own * dispatch_continuation, so a walk is never silently dropped even under OOM * or a NULL/not-yet-wired pool (a NULL pool falls back to an inline, * synchronous run — exactly the "commit-only unit fixture" case * dispatch_continuation's own doc comment already documents). * * Never blocks the caller beyond an O(1) CAS + enqueue (the actual walk * always runs asynchronously on a pool worker, except the documented * NULL-pool/allocation-failure fallback). Coalesces concurrent kicks: at * most one walk is ever scheduled or running at a time per persist_t. */ void persist_apply_pump_kick(persist_t* p, void* pool); int persist_job_hook_fun(int type, int situation, int detail, void* cls, va_list ap); int persist_queue_hook_fun(int type, int situation, int detail, void* cls, va_list ap); /* Durable-append-only entry point for a worker-reported job-state fact * (raft-apply-decoupling task 4.1, Req 5.1): encodes {jobid, state} into the * shared wal_jobstate_t wire format and appends it via the same * wal_hook_append_classified() path persist_job_hook_fun already uses for * these WAL types (always WAL_MODE_RETAIN for WAL_JOB_{START,FINISH,FAIL, * UPDATE} — no commit timeout, never blocks) — but bypasses the hook fan-out * entirely, so calling this fires NO AFTER_EVENT notification. The * mutation this fact eventually drives, and the AFTER_EVENT notification * itself, both happen later from apply_committed_entry() once this entry * commits to a majority (task 2.2's existing dispatch). `state` is the * caller-computed resultant state (e.g. JOB_STAT_RUN for a START) — this * function performs no guard checks or merge logic of its own, it only * encodes and appends. Thin wrapper over persist_submit_job_fact_ex() with * pad = 0 (the "legacy absolute state" discriminator — see wal_types.h's * doc comment on wal_jobstate_t._pad) — unchanged contract, every existing * caller (START/FINISH/postdone's old shape) keeps working identically. * Returns the local-append rc (0 on success); facts have no commit timeout * to report, mirroring wal_hook_append_classified's own contract. */ int persist_submit_job_fact(persist_t* p, uint32_t wal_type, uint64_t jobid, int state); /* Full-control sibling of persist_submit_job_fact() (raft-apply-decoupling * task 6.1 round 2, Req 3.3/7.1-7.3): lets the caller set wal_jobstate_t's * `_pad` discriminator explicitly, for the two fact producers whose real * mutation must be re-derived at apply time against the job's CURRENT * state rather than trusting a receipt-time snapshot (record_job_fail_fact, * record_job_postdone_fact — see wal_types.h's doc comment on * wal_jobstate_t._pad for the full rationale). `state` is ignored by * apply_committed_entry() whenever `pad` selects one of these apply-time- * recomputed shapes (its value is not part of the wire contract in that * case, but is still written for forward-compatibility/diagnostics). */ int persist_submit_job_fact_ex(persist_t* p, uint32_t wal_type, uint64_t jobid, int state, uint32_t pad); int persist_worker_hook_fun(int type, int situation, int detail, void* cls, va_list ap); /* Apply a single WAL entry to in-memory state (LOGICAL entity restore only). * No hooks fire — hook system is empty on standbys until promotion. * * This restores entity state from (type, lsn, payload) ONLY. It does NOT touch * the on-disk WAL header, the CRC chain (p->last_crc), or current_lsn — those * are owned exclusively by the physical layer: persist_wal_write_replicated on * the live replicated path, and replay_wal/persist_recover on recovery. (bug-778) * * Returns 0 on success (including skip for checkpoint/unknown types), * negative errno on error (oversized payload, NULL persist). */ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn, const void* payload, uint32_t payload_len); /* apply_committed_entry — raft-apply-decoupling task 2.2, design.md * "Apply Seam". The single dispatch function that mutates in-memory state * for ONE committed WAL entry, shared (once tasks 2.3/6.1 wire their call * sites) by restart replay and the live commit-advance apply pump. Routes: * WAL_JOB_SUBMIT -> restore_job() (existing * idempotent, jobid-addressable construction primitive; -EEXIST is a * no-op re-apply, not an error) * WAL_JOB_{START,FINISH,STOP,RESUME, * RUSAGE,CKPT,PREASON} -> apply_job_update_transition() * (the guard-only core extracted in task 2.1) — these types' payloads * are deterministic/state-independent, so a `state` value baked at * receipt time never goes stale * WAL_JOB_FAIL -> dispatches on wal_jobstate_t. * _pad (task 6.1 round 2 — see wal_types.h's doc comment on * wal_jobstate_t._pad): WAL_JOB_FAIL_STAGE_FLAG set -> reconstructs a * minimal job_failinfo_t{jobid, stage} and calls the REAL * apply_job_fail_transition() against the job's ACTUAL CURRENT state * at apply time — fixes a merge-staleness corruption where a FAIL's * PERR merge depended on prior state that had not actually applied * yet when the fact was received; _pad == 0 (legacy live path, * job_ctrl's mark_job_fail) -> apply_job_update_transition() as an * absolute overwrite, unchanged * WAL_JOB_UPDATE -> also dispatches on _pad: * WAL_JOB_UPDATE_POSTDONE_MARK -> apply_job_postdone_transition(), * which re-derives the postdone merge (conditional EXIT backstop + OR * JOB_STAT_PDONE) fresh against current state; _pad == 0 (legacy live * path, job_ctrl's mark_job_update) -> apply_job_update_transition() * as an absolute overwrite, unchanged * WAL_QUEUE_CTRL -> restore_queue_state() * WAL_WORKER_CTRL -> worker_from_capnp() * and fires the entry type's AFTER_EVENT notification hook exactly once, * strictly after the mutation completes, and ONLY when a genuine (non- * duplicate, non-guard-rejected) mutation actually occurred. * * Deviation from a literal per-type hook fan-out: the on-disk fact payload * (wal_jobstate_t {jobid, state}) carries only the job's resulting state, not * the original job_pidinfo_t / xce_jobfinish_t / job_failinfo_t a type- * specific mark_job_* hook (HOOK_DETAIL_JOB_START/EXIT/FAIL) expects its * subscribers to dereference. The real risk is sched.c's sched_job_start_fun/ * sched_job_exit_fun/sched_job_fail_fun, which dereference info->jobid * directly in a struct initializer — before even their `situation` switch, * with no NULL guard (apiserver.c's equivalent subscribers are NOT at risk: * they store the `info` pointer itself into the notify union (e.g. * `.data = { .pidinfo = info }`) inside `case HOOK_AFTER_EVENT:`, without * dereferencing any of its fields at this call site — unlike sched.c's * unconditional `info->jobid` access in a top-level struct initializer). * Every job-state fact type therefore fires the same * HOOK_DETAIL_JOB_UPDATE, whose subscribers take only (cluster_t*, job_t*) — * matching what this wire format can actually supply without fabricating * data — regardless of which of the guard-only cores above (apply_job_ * update_transition / apply_job_fail_transition / apply_job_postdone_ * transition) actually performed the mutation. No task in the current plan * (2.3, 4.1, 4.2, 6.1, 6.2, 6.3) closes this hook-granularity gap — task 4.1 * is exclusively about removing state mutation/hook- * firing from worker-fact receipt-time handlers, not about enriching the * wire format. Closing it would require either a design.md correction * (formally accepting this HOOK_DETAIL_JOB_UPDATE substitution as final — * design.md's Apply Seam section and Fact Ingestion Event Contract * currently describe per-type hook dispatch this wire format cannot * support) or a new task to enrich the WAL wire format for job-state facts. * * NOT this function's concern (owned by whichever caller drives it — the * replay segment reader today, the live apply pump added by task 6.1): * advancing p->current_lsn, oversized-payload rejection, WAL_CHECKPOINT * skip. See apply_wal_entry's doc comment above for why those stay * physical-layer-owned. * * Returns 0 on success (including a no-op idempotent re-apply or a guard- * rejected/duplicate transition), negative on a genuine apply failure. */ int apply_committed_entry(persist_t* p, uint32_t wal_type, uint64_t lsn, const void* payload, uint32_t len); /* Load a snapshot file from path, set p->checkpoint_lsn and p->current_lsn. * lsn hint is informational — the actual LSN is read from the file header. * * ha-entity-snapshots (task 5.2 owns the restore-before-advance * implementation; this is the public signature it will fill in): * out_entities_restored reports whether this call actually restored * job/queue entity state (true) or only observed a legacy position marker * (false) — set ONLY when this function returns 0. On a non-zero return, * p->checkpoint_lsn/p->current_lsn are left completely unchanged and * *out_entities_restored is left untouched (undefined) — callers must not * read it in that case. * * Returns 0 on success, -errno or -EINVAL/-EIO on error. */ int persist_load_snapshot(persist_t* p, const char* path, uint64_t lsn, bool* out_entities_restored); /* GC retention predicates (P0-5a / P0-8 / P0-13) */ #include #include "xce/types.h" bool job_is_terminal(const job_t* j); bool job_should_retain(const job_t* j, const persist_config_t* cfg); bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg);