#pragma once #include #include #include #include #include #include #include "config.h" #include "wal_types.h" /* wal_entry_hdr_t, wal_entry_type — P2-4: shared with repl.h */ #include "repl.h" /* replicator_t — P2-4: embedded in persist_t */ #include "xlist.h" /* xlist_t — persist_submit_wal_and_commit job batch */ /* Forward declaration */ typedef struct cluster_s cluster_t; /* ============================================================ * Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1) * * The outcome of WAL replay at startup, distinguishing a structurally * intact log from one whose chain is broken. It is about the STRUCTURE * of the local log, NOT about being behind the cluster: a clean log that * simply ends below the cluster's committed high-water mark is still * RECOVER_CLEAN (the freshness/promotion fence handles "behind"). A * corruption-truncated recovery is NEVER reported as a clean recovery. * ============================================================ */ typedef enum { RECOVER_CLEAN = 0, /* WAL forms a valid CRC chain to a clean entry * boundary: clean EOF, pre-allocated-zero tail, * snapshot-sentinel + clean replay, or an empty * first-boot WAL (no entries != corruption). */ RECOVER_TRUNCATED = 1, /* replay stopped on structural corruption: a CRC * chain break, a CRC mismatch, or a short/partial * entry read — the log cannot be reconstructed * contiguously. Marks the member non-promotable * (consumed by the promotion gate, task 4.3). */ } recover_verdict_t; /* ============================================================ * Boot-policy recovery decision (ha-commit-durability task 3.3, Req 2.4) * * The outcome of evaluating a member's boot/promotion-eligibility state * against the Boot Policy Decision Table (design.md "Boot Policy Decision * Table (task 3.2)"). This task (3.3) owns the FATAL terminal; the fence + * catch-up + release path is task 4.3. * * ELIGIBLE — may promote (no fence). Rows 1, 6. * FENCE — ineligible to promote; catch up from a healthy peer, regain * eligibility on success (Req 2.6). Rows 2, 3, 5. A reachable * healthy peer NEVER yields FENCE_UNRECOVERABLE. * FENCE_UNRECOVERABLE — loud, PERMANENT, NON-terminating fence: the member * cannot recover locally AND cannot catch up (no healthy peer). * It logs LOUDLY, stays PERMANENTLY non-promotable, and keeps * retrying catch-up indefinitely — but the process STAYS ALIVE. * Rows 4, 4b. This is NOT process exit: a controller MUST NEVER * self-terminate as a designed response to an operational * condition (user invariant, 2026-06-18). A self-exited controller * needs MANUAL admin recovery and leaves quorum degraded * meanwhile; a controller that stays alive and keeps retrying * SELF-HEALS when a peer returns. The fence RELEASES (→ ELIGIBLE, * Req 2.6) automatically when catch-up reaches a * committed-consistent state. * * AUTHORITATIVE INVARIANT (design.md): a truncated/behind member resolves to * either FENCE→ELIGIBLE (caught up) or a loud PERMANENT FENCE_UNRECOVERABLE * (stays alive, non-promotable, keeps retrying). It is NEVER silently divergent * (it refuses to lead) and the process NEVER self-exits. * ============================================================ */ typedef enum { RECOVER_DECISION_ELIGIBLE = 0, RECOVER_DECISION_FENCE = 1, RECOVER_DECISION_FENCE_UNRECOVERABLE = 2, } recover_decision_t; /* Policy inputs evaluated at boot / promotion-eligibility (design.md table). * Modelled as a plain value so the decision is a pure, unit-testable function * of its inputs — the live catch-up machinery (replication / task 4.3) supplies * these; this task does not build it. */ typedef struct { recover_verdict_t verdict; /* RECOVER_CLEAN / RECOVER_TRUNCATED (task 3.1) */ bool durable_below_hwm; /* local durable position < cluster committed HWM */ bool healthy_peer_reachable; /* a healthy peer is reachable for catch-up */ bool catch_up_succeeded; /* catch-up reached a committed-consistent state * within the bounded window (only meaningful * when a catch-up was attempted) */ uint64_t durable_floor_lsn; /* highest intact replayed LSN (truncation boundary) */ uint64_t cluster_hwm_lsn; /* cluster committed high-water mark, for logging */ } recover_policy_inputs_t; /* ============================================================ * Persistence configuration (Part 2.6) * ============================================================ */ typedef struct { uint32_t wal_sync_ms; /* fsync interval in ms (default 10) */ uint32_t snap_interval_s; /* checkpoint interval in seconds (default 60) */ uint32_t snap_wal_size_mb; /* checkpoint when WAL exceeds this MB (default 64) */ uint32_t snap_keep_n; /* snapshots to retain (default 2) */ uint32_t transient_job_timeout_s; /* seconds before TRANSIENT job auto-terminates (default 300) */ uint32_t finished_retain_s; /* retain finished jobs N seconds (default 3600) */ uint32_t worker_retain_s; /* retain unavailable workers N seconds (default 86400) */ uint32_t sync_timeout_ms; /* sync replication ACK timeout ms (default 50) */ uint32_t max_wal_records; /* checkpoint trigger: records since last snap (default 10000) */ uint32_t wal_segment_bytes; /* physical WAL segment file size (default 64 MB) */ } persist_config_t; static inline persist_config_t persist_config_defaults(void) { persist_config_t c; c.wal_sync_ms = DEFAULT_WAL_SYNC_MS; c.snap_interval_s = DEFAULT_SNAP_INTERVAL_S; c.snap_wal_size_mb = DEFAULT_SNAP_WAL_SIZE_MB; c.snap_keep_n = DEFAULT_SNAP_KEEP_N; c.transient_job_timeout_s = DEFAULT_TRANSIENT_JOB_TIMEOUT_S; c.finished_retain_s = DEFAULT_FINISHED_RETAIN_S; c.worker_retain_s = DEFAULT_WORKER_RETAIN_S; c.sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS; c.max_wal_records = DEFAULT_MAX_WAL_RECORDS; c.wal_segment_bytes = DEFAULT_WAL_SEGMENT_BYTES; return c; } /* * persist_config_set_wal_sync_ms — apply the --wal-sync-ms flush-cadence * override (ha-commit-durability Req 3.1). * * Single override point shared by both persist_config_defaults() callers * (cluster.c phase2 and myapp.c step 6), mirroring the --sync-timeout-ms * convention: override_ms == 0 means "flag not passed" and leaves the * default flush cadence untouched (Req 3.3 — the seam changes timing * controllability only, not the durability contract). A positive value * sets the cadence to exactly that, applied before persistence starts. */ static inline void persist_config_set_wal_sync_ms(persist_config_t *c, uint32_t override_ms) { if (override_ms > 0) c->wal_sync_ms = override_ms; } /* * persist_config_set_max_wal_records — apply the --max-wal-records * checkpoint-trigger override (raft-ha-core-safety task 5.10a / * ha-entity-snapshots task 7.1: forcing a real full-checkpoint catch-up * round in the chaotic soak requires a low, deterministic record-count * threshold rather than waiting on the 10000-record production default). * * Same convention as persist_config_set_wal_sync_ms: override == 0 means * "flag not passed", leaves the default threshold untouched. A positive * value sets it exactly (persist_init's own minimum-100 floor still * applies afterward — this setter does not duplicate that enforcement). */ static inline void persist_config_set_max_wal_records(persist_config_t *c, uint32_t override_n) { if (override_n > 0) c->max_wal_records = override_n; } /* * persist_config_set_wal_segment_bytes — apply the --wal-segment-bytes * override (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1: * the physical WAL segment size, not the record-count checkpoint threshold, * gates when trim_wal_segments() has anything to delete — a segment is * never rotated out until it fills, so forcing a real snapshot-triggered * empty-rejoin in the soak requires a small deterministic segment size * rather than waiting on the 64 MB production default). * * Same convention as persist_config_set_max_wal_records: override == 0 * means "flag not passed", leaves the default segment size untouched. */ static inline void persist_config_set_wal_segment_bytes(persist_config_t *c, uint32_t override_bytes) { if (override_bytes > 0) c->wal_segment_bytes = override_bytes; } /* ============================================================ * Key→index mapping for checkpoint serialization (Part 3.8) * ============================================================ */ typedef struct { int index; /* shadow slot index in the frozen zarray */ char key[XCE_MAXNAME_LEN]; /* owned deep copy — not a borrowed kvtab pointer */ size_t key_len; /* length including null terminator */ } slot_key_t; /* ============================================================ * persist_t — concrete definition (exposed for stack/calloc allocation) * The opaque pattern can be restored in a later refactor once the * full API surface is stable. * ============================================================ */ struct persist_s { /* Back-pointer to owning cluster (set by persist_init, used in persist_start) */ cluster_t* cluster; /* WAL state */ int wal_fd; uint64_t current_lsn; uint32_t last_crc; uint32_t last_replicated_crc; uint64_t term; uint32_t seg_offset; uint64_t checkpoint_lsn; _Atomic uint64_t committed_lsn; /* highest LSN known committed; invariant: <= current_lsn - 1 */ _Atomic uint64_t durable_lsn; /* highest LSN whose bytes are fdatasync'd locally (group-commit * prefix watermark). Advanced ONLY by the flusher after a * successful fdatasync. Invariants: monotonic non-decreasing * (except clamp-on-truncate); durable_lsn <= current_lsn - 1. * The commit majority counts self at this LSN, not the * appended tip (ha-commit-durability Req 1.1/1.2; consumed by * task 2.2). */ uint64_t total_wal_bytes; char seg_path[4096]; /* Per-LSN recovered/live term index (ha-term-safety task 1.3, R1.5). * IN-MEMORY ONLY — no on-disk layout change (R1.7). Records each entry's * hdr.term keyed by LSN so the term can be read back after recovery and * fed to the reconcile/promotion logic (on_get_entry_term, task 2.2). * * term_index[lsn - 1] holds the RAW recorded term for a 1-based LSN (LSN 0 * is reserved and never indexed). term_index_count is the highest LSN * recorded (== number of contiguous slots in use), used as the presence * watermark so persist_entry_term distinguishes an ABSENT LSN (returns 0) * from a PRESENT legacy-0 entry (mapped to the lowest-known term). Grown on * demand by record_entry_term during replay_wal and live appends; freed in * persist_fini. Guarded by p->lock (written under the append lock and the * single-threaded recovery path; read by accessors that take p->lock). */ uint64_t* term_index; /* term_index[i] = term of LSN (i+1) */ uint64_t term_index_count; /* highest recorded LSN (slots in use) */ uint64_t term_index_cap; /* allocated slot capacity */ /* Durable committed-config reference carried in the checkpoint sentinel * (ha-term-safety task 4.2 remediation r1 — compaction-safe config derive). * A checkpoint trims WAL segments older than the open one (trim_wal_segments) * and the snapshot sentinel does NOT carry entity state, so a committed * WAL_CONFIG_MARKER in a trimmed segment would vanish and the derived * committed config (config_committed_marker) would regress. To prevent that, * checkpoint stamps the latest-committed marker's 40-byte REFERENCE * ({version_tag, sha256} — a reference, NOT the config bytes / entity * snapshot) into the sentinel; recovery reads it back here; and the derive * falls back to this when no marker survives the live WAL scan. * cfgref_present == 0 means "no committed config marker was known at the last * checkpoint" (a fresh store or a pre-config-marker checkpoint). Written * single-threaded at checkpoint/recovery; read by config_committed_marker. */ int cfgref_present; /* 1 if cfgref_tag/sha are valid */ uint64_t cfgref_tag; /* latest committed marker version_tag */ uint8_t cfgref_sha[32]; /* latest committed marker content hash */ /* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1). * Set by persist_recover/replay_wal at startup: RECOVER_CLEAN when the * local WAL forms a valid CRC chain to a clean entry boundary (incl. an * empty first-boot WAL, a snapshot-sentinel + clean replay, and a * pre-allocated-zero tail), RECOVER_TRUNCATED when replay stopped on a * structural corruption — a CRC chain break, a CRC mismatch, or a * short/partial entry read — so the on-disk log cannot be reconstructed * contiguously. A node that cleanly ends BELOW the cluster high-water is * still CLEAN here (the freshness/promotion fence catches "behind", task * 4.2/4.3 — not this verdict). 3.1 only CLASSIFIES + sets this; it does * NOT FATAL (task 3.3) and does NOT fence promotion (task 4.3, which * CONSUMES this verdict). Defaults to RECOVER_CLEAN before recovery runs * (a fresh store is clean). Written single-threaded during recovery * before the node participates; read by the promotion gate thereafter. */ recover_verdict_t recovery_verdict; /* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4, * NEVER-SELF-EXIT rework 2026-06-18). Set by * persist_recover_loud_fence_if_unrecoverable when a member is * recovery-truncated AND/OR durable below the cluster committed high-water * mark with NO successful catch-up: the member stays ALIVE but is held * PERMANENTLY non-promotable and keeps retrying catch-up. A controller MUST * NEVER self-exit as a designed response to this operational condition; it * self-heals automatically when a healthy peer returns and catch-up reaches * a committed-consistent state — at which point this flag is CLEARED via * persist_clear_recovery_unrecoverable (wired into the fence-release / * catch-up path). Read by the promotion gate as belt-and-suspenders * (a member must never lead while this is set). Atomic: set on the recovery * path, cleared on the catch-up path, read by the promotion/heartbeat * threads. Defaults to false before recovery runs. */ _Atomic bool recovery_unrecoverable; /* Paths (derived from workdir + name via persist_set_workdir) */ char wal_path[4096]; char snap_path[4096]; /* Config */ persist_config_t config; /* Concurrency */ pthread_mutex_t lock; atomic_uint wal_record_count; /* records appended since last checkpoint */ atomic_int checkpoint_in_progress; /* CAS guard: 0=idle, 1=scheduled/running */ atomic_int fsync_stop; atomic_int fsync_thread_running; #ifdef XCE_TESTING /* Fault-injection seam (ha-commit-durability task 2.1 error-path test): * when non-zero, every fdatasync site treats the sync as FAILED — it * still calls the real fdatasync but forces a non-zero rc, so durable_lsn * must NOT advance and the failure is logged loudly. Compiled out of * production (xce_controller is built without -DXCE_TESTING). */ atomic_int test_force_fsync_fail; /* fdatasync observability counter (ha-commit-durability task 2.3): * incremented at the standby-path inline sync in * persist_wal_write_replicated, BEFORE that function returns (and thus * before the caller can ACK). A persist-before-ACK regression test reads * this to prove a durable sync ran inside the call — page-cache re-reads * cannot distinguish fsync'd from cache-only, so a direct sync signal is * required. Compiled out of production. */ atomic_uint test_replicated_fdatasync_count; /* Liveness observability (ha-commit-durability task 5.1, Req 1.6): * the thread id of the last flusher fdatasync that advanced durable_lsn. * A test reads this to prove fdatasync runs on the dedicated flusher * thread, NOT on the caller/iomgr/heartbeat thread. Compiled out of * production. */ pthread_t test_last_fdatasync_thread; atomic_int test_last_fdatasync_thread_set; /* Fault-injection seam (ha-entity-snapshots task 2.2): force the Nth * (0-based, in retained-job walk order) job's checkpoint encode to fail * inside write_snapshot, proving a genuine mid-loop encode failure * discards the WHOLE checkpoint rather than skipping just one job. * Stored as index+1 so the zero-initialized default (0) means * "disabled" — mirrors test_force_fsync_fail's zero-init-safe * convention (no explicit persist_init reset needed). Compiled out of * production. */ atomic_int test_force_job_encode_fail_at; #endif pthread_t fsync_thread; pthread_mutex_t fsync_mutex; /* guards the fsync timer wait */ pthread_cond_t fsync_cond; /* signaled by persist_stop for prompt join */ /* Durable-advance wake hook (ha-commit-durability task 2.2). * Persistence owns this slot; the commit engine REGISTERS a callback * (ctx = the commit_engine_t) via persist_set_durable_cb. The flusher * invokes it AFTER advancing durable_lsn and AFTER releasing p->lock, * so a COMMIT_MODE_WAIT caller is woken promptly when its entry becomes * durable locally — without it, a commit whose standby ACKs all arrived * before the local fdatasync would stall until the WAIT timeout. The * dependency direction is preserved (commit depends on persist, never * the reverse): persist only stores+calls an opaque function pointer. * NULL until registered (every flush is then a no-op wake). */ void (*on_durable_advance)(void* ctx); void* durable_cb_ctx; /* Truncation wake hook (raft-apply-decoupling task 1.4). Persistence owns * this slot; the commit engine REGISTERS a callback (ctx = the * commit_engine_t) via persist_set_truncate_cb, mirroring * on_durable_advance's registration pattern and dependency direction * (persist only stores+calls an opaque function pointer; it never * depends on commit.c). persist_truncate_wal_above invokes it AFTER it * actually removes entries above the new tail and AFTER releasing * p->lock, so any commit continuation registered for an LSN above the * new tail (which can never commit -- its entry no longer exists) is * resolved FAILED promptly instead of waiting out the commit engine's * timeout sweep. NOT invoked on a no-op truncate (nothing above * cutoff_lsn -- current_lsn unchanged). NULL until registered (a * truncate is then a silent no-op wake). */ void (*on_truncate)(void* ctx, uint64_t cutoff_lsn); void* truncate_cb_ctx; /* Checkpoint (stubs for now — wired in Task 10) */ atomic_int checkpoint_requested; pthread_cond_t checkpoint_done; pthread_mutex_t checkpoint_mutex; pthread_t snap_thread; /* Fan-out replicator — P2-4 */ replicator_t repl; }; typedef struct persist_s persist_t; /* Set workdir and controller name before persist_init(). * Derives wal_path and snap_path as //data/{wal,snap}. */ void persist_set_workdir(persist_t* p, const char* workdir, const char* name); /* ============================================================ * WAL term adapter (ha-term-safety task 1.3, Req 1.1/1.5/1.6/1.7/1.8) * * Today every WAL append stamps hdr.term = p->term, but p->term is never * assigned, so all entries carry term 0 (the G1 bug). The on-disk * wal_entry_hdr_t already has a 40-byte `term` field; this is a VALUE fix, NOT * a layout change (R1.7). * ============================================================ */ /* Legacy-zero-term rule (R1.8) — REVIEW ISSUE 2, made precise: * * A WAL entry that predates this feature carries hdr.term == 0. For (term,lsn) * matching it must map to a SINGLE, well-defined "lowest-known term" so that * (a) recovery never fails on legacy-0 entries, and * (b) a legacy-0 entry is never spuriously treated as divergent from another * legacy-0 entry at the same LSN. * The rule: legacy-0 -> WAL_LEGACY_TERM (1, the lowest real Raft term — Raft * terms start at 1; term 0 is the pre-election "no term" placeholder). It is * applied UNIFORMLY at read-back (persist_entry_term), so every legacy-0 entry * answers the same term 1 and matches another legacy-0 entry at the same LSN. * The raw 0 is still stored in the index (so "present legacy-0" stays * distinguishable from "absent"); only the read-back value is mapped. */ #define WAL_LEGACY_TERM 1u /* Set the controller's live election term so subsequent WAL appends stamp it * into hdr.term (R1.1/R1.6). Called from the append callback (on_persist_entry, * task 2.2) and on term adoption, so no append records a zero placeholder while * a valid term is in effect. No on-disk layout change; value fix only (R1.7). * Safe on a NULL p (no-op). */ void persist_set_term(persist_t* p, uint64_t term); /* Recovered/live per-entry term for a 1-based LSN (lsn == Raft index), feeding * the task-2.2 on_get_entry_term callback and the library's init restore * (R1.5). Returns 0 when the LSN is ABSENT (never recorded). A PRESENT entry * that carries a legacy zero term is mapped to WAL_LEGACY_TERM (R1.8) so it is * never confused with an absent LSN and matches other legacy-0 entries. * Safe on a NULL p (returns 0). */ uint64_t persist_entry_term(const persist_t* p, uint64_t lsn); /* raft-ha-core-safety task 3.2 (Req 3.1/5.1) — Snapshot Boundary Reconcile. * * Authoritatively overwrite the per-LSN term index entry at `checkpoint_lsn` * with `checkpoint_term` (the checkpoint's own true term, as carried on the * wire by the leader that shipped the snapshot). Called from the standby * snapshot-install receive path (ha_handle_snapshot_chunk) immediately after * persist_load_snapshot() succeeds, so a divergent pre-snapshot entry's STALE * local term at the checkpoint LSN is reconciled to the checkpoint's own * term. Establishes a verifiable (term, lsn) boundary: post-install AE intake * (prevLogIndex/prevLogTerm checks) never references an LSN below the * checkpoint against this member again, so only the checkpoint LSN's own * term needs reconciling — not the full compacted history below it. * * `checkpoint_term == 0` means a legacy sender (pre-task-3.2 peer) never * stamped the wire field; treat that as WAL_LEGACY_TERM per the existing * legacy-0 convention (record_entry_term stores the raw value verbatim; the * legacy-0 -> WAL_LEGACY_TERM mapping is applied uniformly at read-back by * persist_entry_term/entry_term_locked, so this call is consistent with that * convention without needing to special-case it here). * * Takes p->lock internally (callers must NOT already hold it). Safe on a * NULL p or checkpoint_lsn == 0 (no-op). */ void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn, uint64_t checkpoint_term); /* Lifecycle */ int persist_init(persist_t* p, persist_config_t* config, cluster_t* c); int persist_start(persist_t* p); void persist_stop(persist_t* p); void persist_fini(persist_t* p); int persist_recover(persist_t* p); /* WAL write paths */ int persist_wal_async(persist_t* p, uint32_t type, const void* payload, uint32_t len); int persist_wal_sync_append(persist_t* p, uint32_t type, const void* payload, uint32_t len, int* sync_rc_out); /* Write a REPLICATED entry (received from the primary) into the local WAL * at its GIVEN lsn — used by standbys to durably persist streamed entries * BEFORE acking them (persist-before-ACK; an ACK must imply durability). * * Unlike persist_wal_async it does NOT assign a new lsn (the lsn is supplied * by the primary) and does NOT repl_broadcast (standbys never re-replicate). * It builds a SELF-consistent CRC chain (the streamed entry carries no crc), * fsyncs the segment before returning, and is idempotent on re-stream: if the * lsn is already persisted (lsn+1 <= current_lsn) it is a no-op returning 0. * * WAL_CHECKPOINT is rejected with -EINVAL (never streamed to standbys). * Returns 0 on success (incl. dedup no-op), negative errno on failure. */ int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type, const void* payload, uint32_t len); /* ============================================================ * Write-path commit classification (ha-commit-barrier-rollout) * ============================================================ */ typedef enum { WAL_MODE_WAIT, /* request-shaped: barrier before requester reply */ WAL_MODE_RETAIN, /* fact-shaped: retained until committed, no timeout */ WAL_MODE_LOCAL, /* not replicated (WAL_CHECKPOINT) */ } wal_commit_mode_t; /* Pure classification lookup. detail disambiguates WAL_WORKER_CTRL only * (HOOK_DETAIL_WORKER_JOIN/UPDATE -> WAIT, HOOK_DETAIL_WORKER_DEAD -> * RETAIN). Unknown type: XCE_TESTING assert (fail fast); production logs * ERROR and returns WAL_MODE_RETAIN (never dropped, never confirmed early). */ wal_commit_mode_t wal_classify(uint32_t wal_type, int detail); /* Commit barrier — handler-callable WAIT completion on everything appended * so far (ha-commit-barrier-rollout). Snapshots the appended watermark * (current_lsn - 1, under p->lock — matching persist_wal_sync_append's * read); returns 0 without blocking when committed_lsn already covers it * (or nothing was ever appended). Otherwise delegates to * commit_submit(COMMIT_MODE_WAIT) and returns its code unmodified: * 0 — committed on a majority * -ETIMEDOUT — not committed within the deadline (entries remain * in the log; outcome converges per umbrella 2.5) * -XCE_ERR_NOT_READY — engine admission cap exhausted (retriable) * timeout_ms = 0 falls back to p->config.sync_timeout_ms. n_peers <= 1 * returns 0 immediately (engine self-quorum, standalone preserved). * Call only from vtask threads with no entity locks held. */ int persist_commit_barrier(persist_t* p, uint32_t timeout_ms); /* Force WAL fdatasync (for tests and clean shutdown) */ int persist_flush(persist_t* p); /* Highest LSN whose bytes are durably fdatasync'd locally — the * group-commit prefix watermark advanced by the demand-driven flusher * (ha-commit-durability task 2.1). The commit majority self-contribution * caps at this value (task 2.2 consumer). Lock-free atomic read; safe from * any thread. Returns 0 when nothing is yet durable. */ uint64_t persist_durable_lsn(persist_t* p); /* Durable freshness floor (ha-commit-durability task 3.4, Req 2.3). * * The member's durably-backed last-log index (WAL tip): the highest LSN that * is on stable storage locally. persist_recover re-derives it from the * recovered WAL tip (the highest INTACT replayed LSN) so the post-recovery * value is backed by durable on-disk state, NOT a volatile in-memory * applied_lsn / last_known_committed_lsn. The Raft freshness gate (task 4.2, * ha.c / cluster.c) consumes this so a truncated member cannot over-advertise * a tip it does not durably hold. * * Precision: this is the durable LAST-LOG INDEX, NOT the commit position * (committed_lsn). The post-election safe-commit point still comes from voter * replies, never from this floor (CLAUDE.md). After a truncating restart the * floor equals the lower intact tip, never a stale higher in-memory value and * never the commit position. * * Thin wrapper over persist_durable_lsn: at and after recovery the durable * watermark IS the recovered WAL tip (recovered on-disk entries are genuinely * durable), so a separate field would be redundant. Lock-free atomic read; * safe from any thread. Returns 0 when nothing is durable (empty first-boot * WAL). */ uint64_t persist_durable_floor(persist_t* p); /* Recovery-integrity verdict + flag (ha-commit-durability task 3.1, Req 2.1). * persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED * verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh * store before recovery runs). persist_recovery_truncated is the convenience * flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate * (task 4.3) consumes to mark the member non-promotable. Both are pure reads * of recovery state established single-threaded at startup. */ recover_verdict_t persist_recovery_verdict(persist_t* p); bool persist_recovery_truncated(persist_t* p); /* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4; * never-self-exit rework 2026-06-18). * The interval after which a still-unrecoverable, loudly-fenced member RE-EMITS * its operator alert (it does NOT terminate — see RECOVER_DECISION_FENCE_- * UNRECOVERABLE). The member stays alive and keeps retrying catch-up * indefinitely; this window only controls how often the loud operator-visible * alert is repeated. Default 30s. */ #define DEFAULT_RECOVERY_CATCHUP_WINDOW_MS 30000u /* persist_recover_decision (ha-commit-durability task 3.3, Req 2.4). * * Pure decision over the Boot Policy Decision Table: maps the policy inputs to * RECOVER_DECISION_{ELIGIBLE,FENCE,FENCE_UNRECOVERABLE}. No side effects, no * termination — it only classifies, so it is unit-testable in isolation. The * FENCE_UNRECOVERABLE terminal is acted on by * persist_recover_loud_fence_if_unrecoverable (which logs LOUDLY + sets the * sticky non-promotable flag — it NEVER terminates the process). * * Contract (design.md rows): a reachable healthy peer NEVER returns * FENCE_UNRECOVERABLE (rows 3, 5); it is returned ONLY when the member is * truncated or behind the HWM, has NO healthy peer, and catch-up did not reach * a committed-consistent state (rows 4, 4b). */ recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in); /* persist_recover_loud_fence_if_unrecoverable (ha-commit-durability task 3.3, * Req 2.4; never-self-exit rework 2026-06-18). * * Evaluate persist_recover_decision over `in`; if the outcome is * FENCE_UNRECOVERABLE, log LOUDLY at ERROR — INCLUDING the truncation boundary * (durable_floor_lsn) and the cluster high-water mark — stating the member is * staying fenced + non-promotable and will keep retrying catch-up, then SET the * sticky `recovery_unrecoverable` flag on `p` (so the promotion gate refuses * leadership as belt-and-suspenders). Returns the decision. * * CRITICAL — the process STAYS ALIVE. There is NO abort()/exit()/tlog_exit() * here: a controller MUST NEVER self-terminate as a designed response to an * operational condition (user invariant, 2026-06-18). The member self-heals — * the fence releases (→ ELIGIBLE, Req 2.6) and the flag clears via * persist_clear_recovery_unrecoverable — automatically when a healthy peer * returns and catch-up reaches a committed-consistent state. * * `p` may be NULL (pure-decision callers that only want the classification + * loud log without touching a persist_t); the flag set is then skipped. */ recover_decision_t persist_recover_loud_fence_if_unrecoverable(persist_t* p, const recover_policy_inputs_t* in); /* persist_recovery_unrecoverable / persist_clear_recovery_unrecoverable * (ha-commit-durability Req 2.4/2.6, never-self-exit rework 2026-06-18). * * Accessor + clear for the sticky "unrecoverable, loudly fenced" flag. The flag * is SET by persist_recover_loud_fence_if_unrecoverable and CLEARED on the * catch-up / fence-release path once the member reaches a committed-consistent * state (not truncated AND durable_floor >= HWM). While set, the member is held * permanently non-promotable but stays alive. Safe on a NULL p (returns false / * no-op). */ bool persist_recovery_unrecoverable(persist_t* p); void persist_clear_recovery_unrecoverable(persist_t* p); /* Register a durable-advance wake callback (ha-commit-durability task 2.2). * The group-commit flusher calls fn(ctx) after it advances durable_lsn and * after releasing p->lock, so a consumer (the commit engine) can re-evaluate * its commit majority and wake any COMMIT_MODE_WAIT caller whose entry just * became durable locally. fn MUST NOT take p->lock (it runs after release, on * the flusher thread) — the commit engine's callback takes only ce->lock, * keeping the lock order p->lock → ce->lock cycle-free. Pass fn = NULL to * unregister. Safe to call before persist_start (no flusher running yet). */ void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx); /* Register a truncation wake callback (raft-apply-decoupling task 1.4). * persist_truncate_wal_above calls fn(ctx, cutoff_lsn) after it actually * truncates entries above cutoff_lsn (the new log tail) and after releasing * p->lock -- fn MUST NOT take p->lock (same rule as persist_set_durable_cb; * the commit engine's callback takes only ce->lock, keeping the lock order * p->lock → ce->lock cycle-free). Not called for a no-op truncate (nothing * above cutoff_lsn). Pass fn = NULL to unregister. */ void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn), void* ctx); #ifdef XCE_TESTING /* ============================================================ * Durability-verification test seam (ha-commit-durability Req 3.2, 3.3) * * persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL * segments from STABLE STORAGE and return the highest fully-persisted * LSN, independent of the in-memory current_lsn / committed_lsn * watermarks. Lets a test assert that an acknowledged operation is * recoverable from disk after an immediate, ungraceful termination — * without relying on the background flush timer (Req 3.2): * * persist_wal_async(p, ...); // append to page cache * persist_flush(p); // fdatasync → durable on disk * lsn = persist_test_durable_lsn_on_disk(p); // highest durable LSN * * It scans the same on-disk format and CRC chain as recovery, stopping * at the first corruption / short read / pre-allocated zero region, so * it reports only entries that are genuinely on stable storage. It * never mutates persist_t state and fires no hooks (Req 3.3 — the seam * is purely inspective; the acknowledgment contract is unchanged). * * Returns the highest durable LSN, or 0 when nothing is durable (empty * first-boot WAL, no segments). Compiled out of production builds: * the xce_controller target is built without -DXCE_TESTING and links * without this symbol. * ============================================================ */ uint64_t persist_test_durable_lsn_on_disk(persist_t* p); /* Fault-injection toggle for the fdatasync error path (task 2.1). * When `fail` is non-zero, every fdatasync site in persist.c forces a * failed sync result: the real fdatasync still runs, but durable_lsn is * left UNCHANGED and the failure is logged loudly — proving an acknowledged * entry is never counted durable on a sync error (Req 1.1/1.2, fail-loud). * Lock-free atomic write; safe to flip from a test thread at any time. * Compiled out of production (xce_controller links without it). */ void persist_test_set_force_fsync_fail(persist_t* p, int fail); /* Read the standby-path fdatasync counter (task 2.3, persist-before-ACK * regression). Counts the inline fdatasync calls made by * persist_wal_write_replicated. A test snapshots this before the call and * asserts it incremented after the call returns — proving the durable sync * ran inside the persist primitive, BEFORE the caller can send its WalAck. * Lock-free atomic read; compiled out of production. */ unsigned persist_test_replicated_fdatasync_count(persist_t* p); /* Invoke the legacy sync-append commit store (task 2.4) with the durable * precondition applied — the same store persist_wal_sync_append performs on a * majority ACK in its engine-less / LSN-0 branch. Lets a test prove * committed_lsn never advances past durable_lsn on the ACK path without * standing up the replication wire. Compiled out of production. */ void persist_test_sync_append_legacy_commit_store(persist_t* p, uint64_t written_lsn); /* Return the thread id of the last flusher fdatasync that advanced * durable_lsn (task 5.1 / Req 1.6 liveness). A test asserts this is NOT the * caller/iomgr thread — proving durability (fdatasync) work runs on the * dedicated flusher thread and never stalls heartbeat/election handling. * Returns pthread_self() if no flusher sync has advanced durable_lsn yet * (so an unprimed read can never falsely look "off-thread"). Compiled out of * production. */ pthread_t persist_test_last_fdatasync_thread(persist_t* p); /* Close the active WAL segment and open a fresh one at the current LSN * (ha-term-safety task 4.2 r1 test seam). Forces a multi-segment layout so a * subsequent persist_checkpoint trims the older segment — exercising the * compaction hazard where a committed marker's segment is unlinked. Returns 0 * on success, negative errno on I/O failure. Compiled out of production. */ int persist_test_force_new_segment(persist_t* p); /* Arm/disarm the job-encode fault-injection seam (see the struct field * comment above). Pass a negative index to disable. Lock-free atomic * write; compiled out of production. */ void persist_test_set_force_job_encode_fail_at(persist_t* p, int index); #endif /* XCE_TESTING */ /* Checkpoint */ int persist_checkpoint(persist_t* p); /* WAL truncation — remove all entries with LSN > cutoff_lsn. * Idempotent; returns 0 on success, negative errno on I/O error. */ int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn); /* Logging helper — never logs payload content (security invariant) */ void persist_log_entry_metadata(const wal_entry_hdr_t* hdr); /* ============================================================ * AFTER_EVENT hook functions — registered by persist_start() * * Signature matches hook_fn_t: int fn(int type, int situation, * int detail, void* data, va_list ap) * cls/data points to the persist_t instance. * * P0: writes a minimal 4-byte payload (detail value) to the WAL. * Full Cap'n Proto serialization is wired in Task 15. * ============================================================ */ /* Explicit controller-side raft step for job submission (raft-apply-decoupling * item #1 / bug-1306): WAL-append every job in `jlist` and block on the * majority-commit barrier, returning the commit verdict DIRECTLY (0 = all * committed on a majority; first negative rc otherwise — -ETIMEDOUT / etc.). * handle_jobsubmit calls this instead of reading the clobber-prone hook return; * a negative rc MUST reject the submission (never reply success). */ int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist); int persist_job_hook_fun(int type, int situation, int detail, void* cls, va_list ap); int persist_queue_hook_fun(int type, int situation, int detail, void* cls, va_list ap); int persist_worker_hook_fun(int type, int situation, int detail, void* cls, va_list ap); /* Apply a single WAL entry to in-memory state (LOGICAL entity restore only). * No hooks fire — hook system is empty on standbys until promotion. * * This restores entity state from (type, lsn, payload) ONLY. It does NOT touch * the on-disk WAL header, the CRC chain (p->last_crc), or current_lsn — those * are owned exclusively by the physical layer: persist_wal_write_replicated on * the live replicated path, and replay_wal/persist_recover on recovery. (bug-778) * * Returns 0 on success (including skip for checkpoint/unknown types), * negative errno on error (oversized payload, NULL persist). */ int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn, const void* payload, uint32_t payload_len); /* Load a snapshot file from path, set p->checkpoint_lsn and p->current_lsn. * lsn hint is informational — the actual LSN is read from the file header. * * ha-entity-snapshots (task 5.2 owns the restore-before-advance * implementation; this is the public signature it will fill in): * out_entities_restored reports whether this call actually restored * job/queue entity state (true) or only observed a legacy position marker * (false) — set ONLY when this function returns 0. On a non-zero return, * p->checkpoint_lsn/p->current_lsn are left completely unchanged and * *out_entities_restored is left untouched (undefined) — callers must not * read it in that case. * * Returns 0 on success, -errno or -EINVAL/-EIO on error. */ int persist_load_snapshot(persist_t* p, const char* path, uint64_t lsn, bool* out_entities_restored); /* GC retention predicates (P0-5a / P0-8 / P0-13) */ #include #include "xce/types.h" bool job_is_terminal(const job_t* j); bool job_should_retain(const job_t* j, const persist_config_t* cfg); bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg);