forked from lei.chen/myxce
Rongsong Shen
9b4b805a27
commit_resolve_pending_above(ce, lsn_floor, rc) fails every registered continuation for lsn > lsn_floor, reusing commit_resolve()'s existing exactly-once/idempotent guarantee (no parallel bookkeeping path). Truncation: a new on_truncate/truncate_cb_ctx callback on persist_t (same inversion-of-control shape as the existing on_durable_advance hook) fires from persist_truncate_wal_above after releasing p->lock, only when entries were actually removed. This one wrapper covers all three real truncation call sites: raft_cb_become_leader's post-election truncate, the library's on_truncate_from conflict-reconcile, and ha_demotion_convergence_fn's post-demotion truncate. Demotion: raft_cb_become_follower calls commit_resolve_pending_above(&c->commit, 0, -XCE_ERR_STANDBY) -- lsn_floor=0 fails everything, since a demoted node can never authoritatively confirm anything still pending (CP posture). Without this, a demoted or truncated node would leave any pending async command permanently unresolved -- the requester would simply never get a reply, worse than today's bounded-timeout behavior. Also closes bug-1407 (pre-existing test_config_repl flake, unrelated to this feature) -- confirmed fixed as a side effect of prior tasks in this series, 70/70 passing across 3 independent runs.
763 lines
42 KiB
C
763 lines
42 KiB
C
#pragma once
|
|
#include <pthread.h>
|
|
#include <stdarg.h>
|
|
#include <stdatomic.h>
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "config.h"
|
|
#include "wal_types.h" /* wal_entry_hdr_t, wal_entry_type — P2-4: shared with repl.h */
|
|
#include "repl.h" /* replicator_t — P2-4: embedded in persist_t */
|
|
#include "xlist.h" /* xlist_t — persist_submit_wal_and_commit job batch */
|
|
|
|
/* Forward declaration */
|
|
typedef struct cluster_s cluster_t;
|
|
|
|
/* ============================================================
|
|
* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1)
|
|
*
|
|
* The outcome of WAL replay at startup, distinguishing a structurally
|
|
* intact log from one whose chain is broken. It is about the STRUCTURE
|
|
* of the local log, NOT about being behind the cluster: a clean log that
|
|
* simply ends below the cluster's committed high-water mark is still
|
|
* RECOVER_CLEAN (the freshness/promotion fence handles "behind"). A
|
|
* corruption-truncated recovery is NEVER reported as a clean recovery.
|
|
* ============================================================ */
|
|
typedef enum {
|
|
RECOVER_CLEAN = 0, /* WAL forms a valid CRC chain to a clean entry
|
|
* boundary: clean EOF, pre-allocated-zero tail,
|
|
* snapshot-sentinel + clean replay, or an empty
|
|
* first-boot WAL (no entries != corruption). */
|
|
RECOVER_TRUNCATED = 1, /* replay stopped on structural corruption: a CRC
|
|
* chain break, a CRC mismatch, or a short/partial
|
|
* entry read — the log cannot be reconstructed
|
|
* contiguously. Marks the member non-promotable
|
|
* (consumed by the promotion gate, task 4.3). */
|
|
} recover_verdict_t;
|
|
|
|
/* ============================================================
|
|
* Boot-policy recovery decision (ha-commit-durability task 3.3, Req 2.4)
|
|
*
|
|
* The outcome of evaluating a member's boot/promotion-eligibility state
|
|
* against the Boot Policy Decision Table (design.md "Boot Policy Decision
|
|
* Table (task 3.2)"). This task (3.3) owns the FATAL terminal; the fence +
|
|
* catch-up + release path is task 4.3.
|
|
*
|
|
* ELIGIBLE — may promote (no fence). Rows 1, 6.
|
|
* FENCE — ineligible to promote; catch up from a healthy peer, regain
|
|
* eligibility on success (Req 2.6). Rows 2, 3, 5. A reachable
|
|
* healthy peer NEVER yields FENCE_UNRECOVERABLE.
|
|
* FENCE_UNRECOVERABLE — loud, PERMANENT, NON-terminating fence: the member
|
|
* cannot recover locally AND cannot catch up (no healthy peer).
|
|
* It logs LOUDLY, stays PERMANENTLY non-promotable, and keeps
|
|
* retrying catch-up indefinitely — but the process STAYS ALIVE.
|
|
* Rows 4, 4b. This is NOT process exit: a controller MUST NEVER
|
|
* self-terminate as a designed response to an operational
|
|
* condition (user invariant, 2026-06-18). A self-exited controller
|
|
* needs MANUAL admin recovery and leaves quorum degraded
|
|
* meanwhile; a controller that stays alive and keeps retrying
|
|
* SELF-HEALS when a peer returns. The fence RELEASES (→ ELIGIBLE,
|
|
* Req 2.6) automatically when catch-up reaches a
|
|
* committed-consistent state.
|
|
*
|
|
* AUTHORITATIVE INVARIANT (design.md): a truncated/behind member resolves to
|
|
* either FENCE→ELIGIBLE (caught up) or a loud PERMANENT FENCE_UNRECOVERABLE
|
|
* (stays alive, non-promotable, keeps retrying). It is NEVER silently divergent
|
|
* (it refuses to lead) and the process NEVER self-exits.
|
|
* ============================================================ */
|
|
typedef enum {
|
|
RECOVER_DECISION_ELIGIBLE = 0,
|
|
RECOVER_DECISION_FENCE = 1,
|
|
RECOVER_DECISION_FENCE_UNRECOVERABLE = 2,
|
|
} recover_decision_t;
|
|
|
|
/* Policy inputs evaluated at boot / promotion-eligibility (design.md table).
|
|
* Modelled as a plain value so the decision is a pure, unit-testable function
|
|
* of its inputs — the live catch-up machinery (replication / task 4.3) supplies
|
|
* these; this task does not build it. */
|
|
typedef struct {
|
|
recover_verdict_t verdict; /* RECOVER_CLEAN / RECOVER_TRUNCATED (task 3.1) */
|
|
bool durable_below_hwm; /* local durable position < cluster committed HWM */
|
|
bool healthy_peer_reachable; /* a healthy peer is reachable for catch-up */
|
|
bool catch_up_succeeded; /* catch-up reached a committed-consistent state
|
|
* within the bounded window (only meaningful
|
|
* when a catch-up was attempted) */
|
|
uint64_t durable_floor_lsn; /* highest intact replayed LSN (truncation boundary) */
|
|
uint64_t cluster_hwm_lsn; /* cluster committed high-water mark, for logging */
|
|
} recover_policy_inputs_t;
|
|
|
|
/* ============================================================
|
|
* Persistence configuration (Part 2.6)
|
|
* ============================================================ */
|
|
typedef struct {
|
|
uint32_t wal_sync_ms; /* fsync interval in ms (default 10) */
|
|
uint32_t snap_interval_s; /* checkpoint interval in seconds (default 60) */
|
|
uint32_t snap_wal_size_mb; /* checkpoint when WAL exceeds this MB (default 64) */
|
|
uint32_t snap_keep_n; /* snapshots to retain (default 2) */
|
|
uint32_t transient_job_timeout_s; /* seconds before TRANSIENT job auto-terminates (default 300) */
|
|
uint32_t finished_retain_s; /* retain finished jobs N seconds (default 3600) */
|
|
uint32_t worker_retain_s; /* retain unavailable workers N seconds (default 86400) */
|
|
uint32_t sync_timeout_ms; /* sync replication ACK timeout ms (default 50) */
|
|
uint32_t max_wal_records; /* checkpoint trigger: records since last snap (default 10000) */
|
|
uint32_t wal_segment_bytes; /* physical WAL segment file size (default 64 MB) */
|
|
} persist_config_t;
|
|
|
|
static inline persist_config_t persist_config_defaults(void)
|
|
{
|
|
persist_config_t c;
|
|
c.wal_sync_ms = DEFAULT_WAL_SYNC_MS;
|
|
c.snap_interval_s = DEFAULT_SNAP_INTERVAL_S;
|
|
c.snap_wal_size_mb = DEFAULT_SNAP_WAL_SIZE_MB;
|
|
c.snap_keep_n = DEFAULT_SNAP_KEEP_N;
|
|
c.transient_job_timeout_s = DEFAULT_TRANSIENT_JOB_TIMEOUT_S;
|
|
c.finished_retain_s = DEFAULT_FINISHED_RETAIN_S;
|
|
c.worker_retain_s = DEFAULT_WORKER_RETAIN_S;
|
|
c.sync_timeout_ms = DEFAULT_SYNC_TIMEOUT_MS;
|
|
c.max_wal_records = DEFAULT_MAX_WAL_RECORDS;
|
|
c.wal_segment_bytes = DEFAULT_WAL_SEGMENT_BYTES;
|
|
return c;
|
|
}
|
|
|
|
/*
|
|
* persist_config_set_wal_sync_ms — apply the --wal-sync-ms flush-cadence
|
|
* override (ha-commit-durability Req 3.1).
|
|
*
|
|
* Single override point shared by both persist_config_defaults() callers
|
|
* (cluster.c phase2 and myapp.c step 6), mirroring the --sync-timeout-ms
|
|
* convention: override_ms == 0 means "flag not passed" and leaves the
|
|
* default flush cadence untouched (Req 3.3 — the seam changes timing
|
|
* controllability only, not the durability contract). A positive value
|
|
* sets the cadence to exactly that, applied before persistence starts.
|
|
*/
|
|
static inline void persist_config_set_wal_sync_ms(persist_config_t *c,
|
|
uint32_t override_ms)
|
|
{
|
|
if (override_ms > 0)
|
|
c->wal_sync_ms = override_ms;
|
|
}
|
|
|
|
/*
|
|
* persist_config_set_max_wal_records — apply the --max-wal-records
|
|
* checkpoint-trigger override (raft-ha-core-safety task 5.10a /
|
|
* ha-entity-snapshots task 7.1: forcing a real full-checkpoint catch-up
|
|
* round in the chaotic soak requires a low, deterministic record-count
|
|
* threshold rather than waiting on the 10000-record production default).
|
|
*
|
|
* Same convention as persist_config_set_wal_sync_ms: override == 0 means
|
|
* "flag not passed", leaves the default threshold untouched. A positive
|
|
* value sets it exactly (persist_init's own minimum-100 floor still
|
|
* applies afterward — this setter does not duplicate that enforcement).
|
|
*/
|
|
static inline void persist_config_set_max_wal_records(persist_config_t *c,
|
|
uint32_t override_n)
|
|
{
|
|
if (override_n > 0)
|
|
c->max_wal_records = override_n;
|
|
}
|
|
|
|
/*
|
|
* persist_config_set_wal_segment_bytes — apply the --wal-segment-bytes
|
|
* override (raft-ha-core-safety task 5.10a / ha-entity-snapshots task 7.1:
|
|
* the physical WAL segment size, not the record-count checkpoint threshold,
|
|
* gates when trim_wal_segments() has anything to delete — a segment is
|
|
* never rotated out until it fills, so forcing a real snapshot-triggered
|
|
* empty-rejoin in the soak requires a small deterministic segment size
|
|
* rather than waiting on the 64 MB production default).
|
|
*
|
|
* Same convention as persist_config_set_max_wal_records: override == 0
|
|
* means "flag not passed", leaves the default segment size untouched.
|
|
*/
|
|
static inline void persist_config_set_wal_segment_bytes(persist_config_t *c,
|
|
uint32_t override_bytes)
|
|
{
|
|
if (override_bytes > 0)
|
|
c->wal_segment_bytes = override_bytes;
|
|
}
|
|
|
|
/* ============================================================
|
|
* Key→index mapping for checkpoint serialization (Part 3.8)
|
|
* ============================================================ */
|
|
typedef struct {
|
|
int index; /* shadow slot index in the frozen zarray */
|
|
char key[XCE_MAXNAME_LEN]; /* owned deep copy — not a borrowed kvtab pointer */
|
|
size_t key_len; /* length including null terminator */
|
|
} slot_key_t;
|
|
|
|
/* ============================================================
|
|
* persist_t — concrete definition (exposed for stack/calloc allocation)
|
|
* The opaque pattern can be restored in a later refactor once the
|
|
* full API surface is stable.
|
|
* ============================================================ */
|
|
struct persist_s {
|
|
/* Back-pointer to owning cluster (set by persist_init, used in persist_start) */
|
|
cluster_t* cluster;
|
|
|
|
/* WAL state */
|
|
int wal_fd;
|
|
uint64_t current_lsn;
|
|
uint32_t last_crc;
|
|
uint32_t last_replicated_crc;
|
|
uint64_t term;
|
|
uint32_t seg_offset;
|
|
uint64_t checkpoint_lsn;
|
|
_Atomic uint64_t committed_lsn; /* highest LSN known committed; invariant: <= current_lsn - 1 */
|
|
_Atomic uint64_t durable_lsn; /* highest LSN whose bytes are fdatasync'd locally (group-commit
|
|
* prefix watermark). Advanced ONLY by the flusher after a
|
|
* successful fdatasync. Invariants: monotonic non-decreasing
|
|
* (except clamp-on-truncate); durable_lsn <= current_lsn - 1.
|
|
* The commit majority counts self at this LSN, not the
|
|
* appended tip (ha-commit-durability Req 1.1/1.2; consumed by
|
|
* task 2.2). */
|
|
uint64_t total_wal_bytes;
|
|
char seg_path[4096];
|
|
|
|
/* Per-LSN recovered/live term index (ha-term-safety task 1.3, R1.5).
|
|
* IN-MEMORY ONLY — no on-disk layout change (R1.7). Records each entry's
|
|
* hdr.term keyed by LSN so the term can be read back after recovery and
|
|
* fed to the reconcile/promotion logic (on_get_entry_term, task 2.2).
|
|
*
|
|
* term_index[lsn - 1] holds the RAW recorded term for a 1-based LSN (LSN 0
|
|
* is reserved and never indexed). term_index_count is the highest LSN
|
|
* recorded (== number of contiguous slots in use), used as the presence
|
|
* watermark so persist_entry_term distinguishes an ABSENT LSN (returns 0)
|
|
* from a PRESENT legacy-0 entry (mapped to the lowest-known term). Grown on
|
|
* demand by record_entry_term during replay_wal and live appends; freed in
|
|
* persist_fini. Guarded by p->lock (written under the append lock and the
|
|
* single-threaded recovery path; read by accessors that take p->lock). */
|
|
uint64_t* term_index; /* term_index[i] = term of LSN (i+1) */
|
|
uint64_t term_index_count; /* highest recorded LSN (slots in use) */
|
|
uint64_t term_index_cap; /* allocated slot capacity */
|
|
|
|
/* Durable committed-config reference carried in the checkpoint sentinel
|
|
* (ha-term-safety task 4.2 remediation r1 — compaction-safe config derive).
|
|
* A checkpoint trims WAL segments older than the open one (trim_wal_segments)
|
|
* and the snapshot sentinel does NOT carry entity state, so a committed
|
|
* WAL_CONFIG_MARKER in a trimmed segment would vanish and the derived
|
|
* committed config (config_committed_marker) would regress. To prevent that,
|
|
* checkpoint stamps the latest-committed marker's 40-byte REFERENCE
|
|
* ({version_tag, sha256} — a reference, NOT the config bytes / entity
|
|
* snapshot) into the sentinel; recovery reads it back here; and the derive
|
|
* falls back to this when no marker survives the live WAL scan.
|
|
* cfgref_present == 0 means "no committed config marker was known at the last
|
|
* checkpoint" (a fresh store or a pre-config-marker checkpoint). Written
|
|
* single-threaded at checkpoint/recovery; read by config_committed_marker. */
|
|
int cfgref_present; /* 1 if cfgref_tag/sha are valid */
|
|
uint64_t cfgref_tag; /* latest committed marker version_tag */
|
|
uint8_t cfgref_sha[32]; /* latest committed marker content hash */
|
|
|
|
/* Recovery-integrity verdict (ha-commit-durability task 3.1, Req 2.1).
|
|
* Set by persist_recover/replay_wal at startup: RECOVER_CLEAN when the
|
|
* local WAL forms a valid CRC chain to a clean entry boundary (incl. an
|
|
* empty first-boot WAL, a snapshot-sentinel + clean replay, and a
|
|
* pre-allocated-zero tail), RECOVER_TRUNCATED when replay stopped on a
|
|
* structural corruption — a CRC chain break, a CRC mismatch, or a
|
|
* short/partial entry read — so the on-disk log cannot be reconstructed
|
|
* contiguously. A node that cleanly ends BELOW the cluster high-water is
|
|
* still CLEAN here (the freshness/promotion fence catches "behind", task
|
|
* 4.2/4.3 — not this verdict). 3.1 only CLASSIFIES + sets this; it does
|
|
* NOT FATAL (task 3.3) and does NOT fence promotion (task 4.3, which
|
|
* CONSUMES this verdict). Defaults to RECOVER_CLEAN before recovery runs
|
|
* (a fresh store is clean). Written single-threaded during recovery
|
|
* before the node participates; read by the promotion gate thereafter. */
|
|
recover_verdict_t recovery_verdict;
|
|
|
|
/* Sticky "unrecoverable, loudly fenced" flag (ha-commit-durability Req 2.4,
|
|
* NEVER-SELF-EXIT rework 2026-06-18). Set by
|
|
* persist_recover_loud_fence_if_unrecoverable when a member is
|
|
* recovery-truncated AND/OR durable below the cluster committed high-water
|
|
* mark with NO successful catch-up: the member stays ALIVE but is held
|
|
* PERMANENTLY non-promotable and keeps retrying catch-up. A controller MUST
|
|
* NEVER self-exit as a designed response to this operational condition; it
|
|
* self-heals automatically when a healthy peer returns and catch-up reaches
|
|
* a committed-consistent state — at which point this flag is CLEARED via
|
|
* persist_clear_recovery_unrecoverable (wired into the fence-release /
|
|
* catch-up path). Read by the promotion gate as belt-and-suspenders
|
|
* (a member must never lead while this is set). Atomic: set on the recovery
|
|
* path, cleared on the catch-up path, read by the promotion/heartbeat
|
|
* threads. Defaults to false before recovery runs. */
|
|
_Atomic bool recovery_unrecoverable;
|
|
|
|
/* Paths (derived from workdir + name via persist_set_workdir) */
|
|
char wal_path[4096];
|
|
char snap_path[4096];
|
|
|
|
/* Config */
|
|
persist_config_t config;
|
|
|
|
/* Concurrency */
|
|
pthread_mutex_t lock;
|
|
atomic_uint wal_record_count; /* records appended since last checkpoint */
|
|
atomic_int checkpoint_in_progress; /* CAS guard: 0=idle, 1=scheduled/running */
|
|
atomic_int fsync_stop;
|
|
atomic_int fsync_thread_running;
|
|
#ifdef XCE_TESTING
|
|
/* Fault-injection seam (ha-commit-durability task 2.1 error-path test):
|
|
* when non-zero, every fdatasync site treats the sync as FAILED — it
|
|
* still calls the real fdatasync but forces a non-zero rc, so durable_lsn
|
|
* must NOT advance and the failure is logged loudly. Compiled out of
|
|
* production (xce_controller is built without -DXCE_TESTING). */
|
|
atomic_int test_force_fsync_fail;
|
|
/* fdatasync observability counter (ha-commit-durability task 2.3):
|
|
* incremented at the standby-path inline sync in
|
|
* persist_wal_write_replicated, BEFORE that function returns (and thus
|
|
* before the caller can ACK). A persist-before-ACK regression test reads
|
|
* this to prove a durable sync ran inside the call — page-cache re-reads
|
|
* cannot distinguish fsync'd from cache-only, so a direct sync signal is
|
|
* required. Compiled out of production. */
|
|
atomic_uint test_replicated_fdatasync_count;
|
|
/* Liveness observability (ha-commit-durability task 5.1, Req 1.6):
|
|
* the thread id of the last flusher fdatasync that advanced durable_lsn.
|
|
* A test reads this to prove fdatasync runs on the dedicated flusher
|
|
* thread, NOT on the caller/iomgr/heartbeat thread. Compiled out of
|
|
* production. */
|
|
pthread_t test_last_fdatasync_thread;
|
|
atomic_int test_last_fdatasync_thread_set;
|
|
/* Fault-injection seam (ha-entity-snapshots task 2.2): force the Nth
|
|
* (0-based, in retained-job walk order) job's checkpoint encode to fail
|
|
* inside write_snapshot, proving a genuine mid-loop encode failure
|
|
* discards the WHOLE checkpoint rather than skipping just one job.
|
|
* Stored as index+1 so the zero-initialized default (0) means
|
|
* "disabled" — mirrors test_force_fsync_fail's zero-init-safe
|
|
* convention (no explicit persist_init reset needed). Compiled out of
|
|
* production. */
|
|
atomic_int test_force_job_encode_fail_at;
|
|
#endif
|
|
pthread_t fsync_thread;
|
|
pthread_mutex_t fsync_mutex; /* guards the fsync timer wait */
|
|
pthread_cond_t fsync_cond; /* signaled by persist_stop for prompt join */
|
|
|
|
/* Durable-advance wake hook (ha-commit-durability task 2.2).
|
|
* Persistence owns this slot; the commit engine REGISTERS a callback
|
|
* (ctx = the commit_engine_t) via persist_set_durable_cb. The flusher
|
|
* invokes it AFTER advancing durable_lsn and AFTER releasing p->lock,
|
|
* so a COMMIT_MODE_WAIT caller is woken promptly when its entry becomes
|
|
* durable locally — without it, a commit whose standby ACKs all arrived
|
|
* before the local fdatasync would stall until the WAIT timeout. The
|
|
* dependency direction is preserved (commit depends on persist, never
|
|
* the reverse): persist only stores+calls an opaque function pointer.
|
|
* NULL until registered (every flush is then a no-op wake). */
|
|
void (*on_durable_advance)(void* ctx);
|
|
void* durable_cb_ctx;
|
|
|
|
/* Truncation wake hook (raft-apply-decoupling task 1.4). Persistence owns
|
|
* this slot; the commit engine REGISTERS a callback (ctx = the
|
|
* commit_engine_t) via persist_set_truncate_cb, mirroring
|
|
* on_durable_advance's registration pattern and dependency direction
|
|
* (persist only stores+calls an opaque function pointer; it never
|
|
* depends on commit.c). persist_truncate_wal_above invokes it AFTER it
|
|
* actually removes entries above the new tail and AFTER releasing
|
|
* p->lock, so any commit continuation registered for an LSN above the
|
|
* new tail (which can never commit -- its entry no longer exists) is
|
|
* resolved FAILED promptly instead of waiting out the commit engine's
|
|
* timeout sweep. NOT invoked on a no-op truncate (nothing above
|
|
* cutoff_lsn -- current_lsn unchanged). NULL until registered (a
|
|
* truncate is then a silent no-op wake). */
|
|
void (*on_truncate)(void* ctx, uint64_t cutoff_lsn);
|
|
void* truncate_cb_ctx;
|
|
|
|
/* Checkpoint (stubs for now — wired in Task 10) */
|
|
atomic_int checkpoint_requested;
|
|
pthread_cond_t checkpoint_done;
|
|
pthread_mutex_t checkpoint_mutex;
|
|
pthread_t snap_thread;
|
|
|
|
/* Fan-out replicator — P2-4 */
|
|
replicator_t repl;
|
|
};
|
|
typedef struct persist_s persist_t;
|
|
|
|
/* Set workdir and controller name before persist_init().
|
|
* Derives wal_path and snap_path as <workdir>/<name>/data/{wal,snap}. */
|
|
void persist_set_workdir(persist_t* p, const char* workdir, const char* name);
|
|
|
|
/* ============================================================
|
|
* WAL term adapter (ha-term-safety task 1.3, Req 1.1/1.5/1.6/1.7/1.8)
|
|
*
|
|
* Today every WAL append stamps hdr.term = p->term, but p->term is never
|
|
* assigned, so all entries carry term 0 (the G1 bug). The on-disk
|
|
* wal_entry_hdr_t already has a 40-byte `term` field; this is a VALUE fix, NOT
|
|
* a layout change (R1.7).
|
|
* ============================================================ */
|
|
|
|
/* Legacy-zero-term rule (R1.8) — REVIEW ISSUE 2, made precise:
|
|
*
|
|
* A WAL entry that predates this feature carries hdr.term == 0. For (term,lsn)
|
|
* matching it must map to a SINGLE, well-defined "lowest-known term" so that
|
|
* (a) recovery never fails on legacy-0 entries, and
|
|
* (b) a legacy-0 entry is never spuriously treated as divergent from another
|
|
* legacy-0 entry at the same LSN.
|
|
* The rule: legacy-0 -> WAL_LEGACY_TERM (1, the lowest real Raft term — Raft
|
|
* terms start at 1; term 0 is the pre-election "no term" placeholder). It is
|
|
* applied UNIFORMLY at read-back (persist_entry_term), so every legacy-0 entry
|
|
* answers the same term 1 and matches another legacy-0 entry at the same LSN.
|
|
* The raw 0 is still stored in the index (so "present legacy-0" stays
|
|
* distinguishable from "absent"); only the read-back value is mapped. */
|
|
#define WAL_LEGACY_TERM 1u
|
|
|
|
/* Set the controller's live election term so subsequent WAL appends stamp it
|
|
* into hdr.term (R1.1/R1.6). Called from the append callback (on_persist_entry,
|
|
* task 2.2) and on term adoption, so no append records a zero placeholder while
|
|
* a valid term is in effect. No on-disk layout change; value fix only (R1.7).
|
|
* Safe on a NULL p (no-op). */
|
|
void persist_set_term(persist_t* p, uint64_t term);
|
|
|
|
/* Recovered/live per-entry term for a 1-based LSN (lsn == Raft index), feeding
|
|
* the task-2.2 on_get_entry_term callback and the library's init restore
|
|
* (R1.5). Returns 0 when the LSN is ABSENT (never recorded). A PRESENT entry
|
|
* that carries a legacy zero term is mapped to WAL_LEGACY_TERM (R1.8) so it is
|
|
* never confused with an absent LSN and matches other legacy-0 entries.
|
|
* Safe on a NULL p (returns 0). */
|
|
uint64_t persist_entry_term(const persist_t* p, uint64_t lsn);
|
|
|
|
/* raft-ha-core-safety task 3.2 (Req 3.1/5.1) — Snapshot Boundary Reconcile.
|
|
*
|
|
* Authoritatively overwrite the per-LSN term index entry at `checkpoint_lsn`
|
|
* with `checkpoint_term` (the checkpoint's own true term, as carried on the
|
|
* wire by the leader that shipped the snapshot). Called from the standby
|
|
* snapshot-install receive path (ha_handle_snapshot_chunk) immediately after
|
|
* persist_load_snapshot() succeeds, so a divergent pre-snapshot entry's STALE
|
|
* local term at the checkpoint LSN is reconciled to the checkpoint's own
|
|
* term. Establishes a verifiable (term, lsn) boundary: post-install AE intake
|
|
* (prevLogIndex/prevLogTerm checks) never references an LSN below the
|
|
* checkpoint against this member again, so only the checkpoint LSN's own
|
|
* term needs reconciling — not the full compacted history below it.
|
|
*
|
|
* `checkpoint_term == 0` means a legacy sender (pre-task-3.2 peer) never
|
|
* stamped the wire field; treat that as WAL_LEGACY_TERM per the existing
|
|
* legacy-0 convention (record_entry_term stores the raw value verbatim; the
|
|
* legacy-0 -> WAL_LEGACY_TERM mapping is applied uniformly at read-back by
|
|
* persist_entry_term/entry_term_locked, so this call is consistent with that
|
|
* convention without needing to special-case it here).
|
|
*
|
|
* Takes p->lock internally (callers must NOT already hold it). Safe on a
|
|
* NULL p or checkpoint_lsn == 0 (no-op). */
|
|
void persist_reconcile_checkpoint_term(persist_t* p, uint64_t checkpoint_lsn,
|
|
uint64_t checkpoint_term);
|
|
|
|
/* Lifecycle */
|
|
int persist_init(persist_t* p, persist_config_t* config, cluster_t* c);
|
|
int persist_start(persist_t* p);
|
|
void persist_stop(persist_t* p);
|
|
void persist_fini(persist_t* p);
|
|
int persist_recover(persist_t* p);
|
|
|
|
/* WAL write paths */
|
|
int persist_wal_async(persist_t* p, uint32_t type,
|
|
const void* payload, uint32_t len);
|
|
int persist_wal_sync_append(persist_t* p, uint32_t type,
|
|
const void* payload, uint32_t len,
|
|
int* sync_rc_out);
|
|
|
|
/* Write a REPLICATED entry (received from the primary) into the local WAL
|
|
* at its GIVEN lsn — used by standbys to durably persist streamed entries
|
|
* BEFORE acking them (persist-before-ACK; an ACK must imply durability).
|
|
*
|
|
* Unlike persist_wal_async it does NOT assign a new lsn (the lsn is supplied
|
|
* by the primary) and does NOT repl_broadcast (standbys never re-replicate).
|
|
* It builds a SELF-consistent CRC chain (the streamed entry carries no crc),
|
|
* fsyncs the segment before returning, and is idempotent on re-stream: if the
|
|
* lsn is already persisted (lsn+1 <= current_lsn) it is a no-op returning 0.
|
|
*
|
|
* WAL_CHECKPOINT is rejected with -EINVAL (never streamed to standbys).
|
|
* Returns 0 on success (incl. dedup no-op), negative errno on failure. */
|
|
int persist_wal_write_replicated(persist_t* p, uint64_t lsn, uint32_t type,
|
|
const void* payload, uint32_t len);
|
|
|
|
/* ============================================================
|
|
* Write-path commit classification (ha-commit-barrier-rollout)
|
|
* ============================================================ */
|
|
typedef enum {
|
|
WAL_MODE_WAIT, /* request-shaped: barrier before requester reply */
|
|
WAL_MODE_RETAIN, /* fact-shaped: retained until committed, no timeout */
|
|
WAL_MODE_LOCAL, /* not replicated (WAL_CHECKPOINT) */
|
|
} wal_commit_mode_t;
|
|
|
|
/* Pure classification lookup. detail disambiguates WAL_WORKER_CTRL only
|
|
* (HOOK_DETAIL_WORKER_JOIN/UPDATE -> WAIT, HOOK_DETAIL_WORKER_DEAD ->
|
|
* RETAIN). Unknown type: XCE_TESTING assert (fail fast); production logs
|
|
* ERROR and returns WAL_MODE_RETAIN (never dropped, never confirmed early). */
|
|
wal_commit_mode_t wal_classify(uint32_t wal_type, int detail);
|
|
|
|
/* Commit barrier — handler-callable WAIT completion on everything appended
|
|
* so far (ha-commit-barrier-rollout). Snapshots the appended watermark
|
|
* (current_lsn - 1, under p->lock — matching persist_wal_sync_append's
|
|
* read); returns 0 without blocking when committed_lsn already covers it
|
|
* (or nothing was ever appended). Otherwise delegates to
|
|
* commit_submit(COMMIT_MODE_WAIT) and returns its code unmodified:
|
|
* 0 — committed on a majority
|
|
* -ETIMEDOUT — not committed within the deadline (entries remain
|
|
* in the log; outcome converges per umbrella 2.5)
|
|
* -XCE_ERR_NOT_READY — engine admission cap exhausted (retriable)
|
|
* timeout_ms = 0 falls back to p->config.sync_timeout_ms. n_peers <= 1
|
|
* returns 0 immediately (engine self-quorum, standalone preserved).
|
|
* Call only from vtask threads with no entity locks held. */
|
|
int persist_commit_barrier(persist_t* p, uint32_t timeout_ms);
|
|
|
|
/* Force WAL fdatasync (for tests and clean shutdown) */
|
|
int persist_flush(persist_t* p);
|
|
|
|
/* Highest LSN whose bytes are durably fdatasync'd locally — the
|
|
* group-commit prefix watermark advanced by the demand-driven flusher
|
|
* (ha-commit-durability task 2.1). The commit majority self-contribution
|
|
* caps at this value (task 2.2 consumer). Lock-free atomic read; safe from
|
|
* any thread. Returns 0 when nothing is yet durable. */
|
|
uint64_t persist_durable_lsn(persist_t* p);
|
|
|
|
/* Durable freshness floor (ha-commit-durability task 3.4, Req 2.3).
|
|
*
|
|
* The member's durably-backed last-log index (WAL tip): the highest LSN that
|
|
* is on stable storage locally. persist_recover re-derives it from the
|
|
* recovered WAL tip (the highest INTACT replayed LSN) so the post-recovery
|
|
* value is backed by durable on-disk state, NOT a volatile in-memory
|
|
* applied_lsn / last_known_committed_lsn. The Raft freshness gate (task 4.2,
|
|
* ha.c / cluster.c) consumes this so a truncated member cannot over-advertise
|
|
* a tip it does not durably hold.
|
|
*
|
|
* Precision: this is the durable LAST-LOG INDEX, NOT the commit position
|
|
* (committed_lsn). The post-election safe-commit point still comes from voter
|
|
* replies, never from this floor (CLAUDE.md). After a truncating restart the
|
|
* floor equals the lower intact tip, never a stale higher in-memory value and
|
|
* never the commit position.
|
|
*
|
|
* Thin wrapper over persist_durable_lsn: at and after recovery the durable
|
|
* watermark IS the recovered WAL tip (recovered on-disk entries are genuinely
|
|
* durable), so a separate field would be redundant. Lock-free atomic read;
|
|
* safe from any thread. Returns 0 when nothing is durable (empty first-boot
|
|
* WAL). */
|
|
uint64_t persist_durable_floor(persist_t* p);
|
|
|
|
/* Recovery-integrity verdict + flag (ha-commit-durability task 3.1, Req 2.1).
|
|
* persist_recovery_verdict returns the RECOVER_CLEAN / RECOVER_TRUNCATED
|
|
* verdict computed by the last persist_recover (RECOVER_CLEAN on a fresh
|
|
* store before recovery runs). persist_recovery_truncated is the convenience
|
|
* flag form (true iff the verdict is RECOVER_TRUNCATED) the promotion gate
|
|
* (task 4.3) consumes to mark the member non-promotable. Both are pure reads
|
|
* of recovery state established single-threaded at startup. */
|
|
recover_verdict_t persist_recovery_verdict(persist_t* p);
|
|
bool persist_recovery_truncated(persist_t* p);
|
|
|
|
/* Bounded catch-up RE-ALERT window (ha-commit-durability task 3.3, Req 2.4;
|
|
* never-self-exit rework 2026-06-18).
|
|
* The interval after which a still-unrecoverable, loudly-fenced member RE-EMITS
|
|
* its operator alert (it does NOT terminate — see RECOVER_DECISION_FENCE_-
|
|
* UNRECOVERABLE). The member stays alive and keeps retrying catch-up
|
|
* indefinitely; this window only controls how often the loud operator-visible
|
|
* alert is repeated. Default 30s. */
|
|
#define DEFAULT_RECOVERY_CATCHUP_WINDOW_MS 30000u
|
|
|
|
/* persist_recover_decision (ha-commit-durability task 3.3, Req 2.4).
|
|
*
|
|
* Pure decision over the Boot Policy Decision Table: maps the policy inputs to
|
|
* RECOVER_DECISION_{ELIGIBLE,FENCE,FENCE_UNRECOVERABLE}. No side effects, no
|
|
* termination — it only classifies, so it is unit-testable in isolation. The
|
|
* FENCE_UNRECOVERABLE terminal is acted on by
|
|
* persist_recover_loud_fence_if_unrecoverable (which logs LOUDLY + sets the
|
|
* sticky non-promotable flag — it NEVER terminates the process).
|
|
*
|
|
* Contract (design.md rows): a reachable healthy peer NEVER returns
|
|
* FENCE_UNRECOVERABLE (rows 3, 5); it is returned ONLY when the member is
|
|
* truncated or behind the HWM, has NO healthy peer, and catch-up did not reach
|
|
* a committed-consistent state (rows 4, 4b). */
|
|
recover_decision_t persist_recover_decision(const recover_policy_inputs_t* in);
|
|
|
|
/* persist_recover_loud_fence_if_unrecoverable (ha-commit-durability task 3.3,
|
|
* Req 2.4; never-self-exit rework 2026-06-18).
|
|
*
|
|
* Evaluate persist_recover_decision over `in`; if the outcome is
|
|
* FENCE_UNRECOVERABLE, log LOUDLY at ERROR — INCLUDING the truncation boundary
|
|
* (durable_floor_lsn) and the cluster high-water mark — stating the member is
|
|
* staying fenced + non-promotable and will keep retrying catch-up, then SET the
|
|
* sticky `recovery_unrecoverable` flag on `p` (so the promotion gate refuses
|
|
* leadership as belt-and-suspenders). Returns the decision.
|
|
*
|
|
* CRITICAL — the process STAYS ALIVE. There is NO abort()/exit()/tlog_exit()
|
|
* here: a controller MUST NEVER self-terminate as a designed response to an
|
|
* operational condition (user invariant, 2026-06-18). The member self-heals —
|
|
* the fence releases (→ ELIGIBLE, Req 2.6) and the flag clears via
|
|
* persist_clear_recovery_unrecoverable — automatically when a healthy peer
|
|
* returns and catch-up reaches a committed-consistent state.
|
|
*
|
|
* `p` may be NULL (pure-decision callers that only want the classification +
|
|
* loud log without touching a persist_t); the flag set is then skipped. */
|
|
recover_decision_t
|
|
persist_recover_loud_fence_if_unrecoverable(persist_t* p,
|
|
const recover_policy_inputs_t* in);
|
|
|
|
/* persist_recovery_unrecoverable / persist_clear_recovery_unrecoverable
|
|
* (ha-commit-durability Req 2.4/2.6, never-self-exit rework 2026-06-18).
|
|
*
|
|
* Accessor + clear for the sticky "unrecoverable, loudly fenced" flag. The flag
|
|
* is SET by persist_recover_loud_fence_if_unrecoverable and CLEARED on the
|
|
* catch-up / fence-release path once the member reaches a committed-consistent
|
|
* state (not truncated AND durable_floor >= HWM). While set, the member is held
|
|
* permanently non-promotable but stays alive. Safe on a NULL p (returns false /
|
|
* no-op). */
|
|
bool persist_recovery_unrecoverable(persist_t* p);
|
|
void persist_clear_recovery_unrecoverable(persist_t* p);
|
|
|
|
/* Register a durable-advance wake callback (ha-commit-durability task 2.2).
|
|
* The group-commit flusher calls fn(ctx) after it advances durable_lsn and
|
|
* after releasing p->lock, so a consumer (the commit engine) can re-evaluate
|
|
* its commit majority and wake any COMMIT_MODE_WAIT caller whose entry just
|
|
* became durable locally. fn MUST NOT take p->lock (it runs after release, on
|
|
* the flusher thread) — the commit engine's callback takes only ce->lock,
|
|
* keeping the lock order p->lock → ce->lock cycle-free. Pass fn = NULL to
|
|
* unregister. Safe to call before persist_start (no flusher running yet). */
|
|
void persist_set_durable_cb(persist_t* p, void (*fn)(void* ctx), void* ctx);
|
|
|
|
/* Register a truncation wake callback (raft-apply-decoupling task 1.4).
|
|
* persist_truncate_wal_above calls fn(ctx, cutoff_lsn) after it actually
|
|
* truncates entries above cutoff_lsn (the new log tail) and after releasing
|
|
* p->lock -- fn MUST NOT take p->lock (same rule as persist_set_durable_cb;
|
|
* the commit engine's callback takes only ce->lock, keeping the lock order
|
|
* p->lock → ce->lock cycle-free). Not called for a no-op truncate (nothing
|
|
* above cutoff_lsn). Pass fn = NULL to unregister. */
|
|
void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff_lsn),
|
|
void* ctx);
|
|
|
|
#ifdef XCE_TESTING
|
|
/* ============================================================
|
|
* Durability-verification test seam (ha-commit-durability Req 3.2, 3.3)
|
|
*
|
|
* persist_test_durable_lsn_on_disk — re-read and CRC-validate the WAL
|
|
* segments from STABLE STORAGE and return the highest fully-persisted
|
|
* LSN, independent of the in-memory current_lsn / committed_lsn
|
|
* watermarks. Lets a test assert that an acknowledged operation is
|
|
* recoverable from disk after an immediate, ungraceful termination —
|
|
* without relying on the background flush timer (Req 3.2):
|
|
*
|
|
* persist_wal_async(p, ...); // append to page cache
|
|
* persist_flush(p); // fdatasync → durable on disk
|
|
* lsn = persist_test_durable_lsn_on_disk(p); // highest durable LSN
|
|
*
|
|
* It scans the same on-disk format and CRC chain as recovery, stopping
|
|
* at the first corruption / short read / pre-allocated zero region, so
|
|
* it reports only entries that are genuinely on stable storage. It
|
|
* never mutates persist_t state and fires no hooks (Req 3.3 — the seam
|
|
* is purely inspective; the acknowledgment contract is unchanged).
|
|
*
|
|
* Returns the highest durable LSN, or 0 when nothing is durable (empty
|
|
* first-boot WAL, no segments). Compiled out of production builds:
|
|
* the xce_controller target is built without -DXCE_TESTING and links
|
|
* without this symbol.
|
|
* ============================================================ */
|
|
uint64_t persist_test_durable_lsn_on_disk(persist_t* p);
|
|
|
|
/* Fault-injection toggle for the fdatasync error path (task 2.1).
|
|
* When `fail` is non-zero, every fdatasync site in persist.c forces a
|
|
* failed sync result: the real fdatasync still runs, but durable_lsn is
|
|
* left UNCHANGED and the failure is logged loudly — proving an acknowledged
|
|
* entry is never counted durable on a sync error (Req 1.1/1.2, fail-loud).
|
|
* Lock-free atomic write; safe to flip from a test thread at any time.
|
|
* Compiled out of production (xce_controller links without it). */
|
|
void persist_test_set_force_fsync_fail(persist_t* p, int fail);
|
|
|
|
/* Read the standby-path fdatasync counter (task 2.3, persist-before-ACK
|
|
* regression). Counts the inline fdatasync calls made by
|
|
* persist_wal_write_replicated. A test snapshots this before the call and
|
|
* asserts it incremented after the call returns — proving the durable sync
|
|
* ran inside the persist primitive, BEFORE the caller can send its WalAck.
|
|
* Lock-free atomic read; compiled out of production. */
|
|
unsigned persist_test_replicated_fdatasync_count(persist_t* p);
|
|
|
|
/* Invoke the legacy sync-append commit store (task 2.4) with the durable
|
|
* precondition applied — the same store persist_wal_sync_append performs on a
|
|
* majority ACK in its engine-less / LSN-0 branch. Lets a test prove
|
|
* committed_lsn never advances past durable_lsn on the ACK path without
|
|
* standing up the replication wire. Compiled out of production. */
|
|
void persist_test_sync_append_legacy_commit_store(persist_t* p,
|
|
uint64_t written_lsn);
|
|
|
|
/* Return the thread id of the last flusher fdatasync that advanced
|
|
* durable_lsn (task 5.1 / Req 1.6 liveness). A test asserts this is NOT the
|
|
* caller/iomgr thread — proving durability (fdatasync) work runs on the
|
|
* dedicated flusher thread and never stalls heartbeat/election handling.
|
|
* Returns pthread_self() if no flusher sync has advanced durable_lsn yet
|
|
* (so an unprimed read can never falsely look "off-thread"). Compiled out of
|
|
* production. */
|
|
pthread_t persist_test_last_fdatasync_thread(persist_t* p);
|
|
|
|
/* Close the active WAL segment and open a fresh one at the current LSN
|
|
* (ha-term-safety task 4.2 r1 test seam). Forces a multi-segment layout so a
|
|
* subsequent persist_checkpoint trims the older segment — exercising the
|
|
* compaction hazard where a committed marker's segment is unlinked. Returns 0
|
|
* on success, negative errno on I/O failure. Compiled out of production. */
|
|
int persist_test_force_new_segment(persist_t* p);
|
|
|
|
/* Arm/disarm the job-encode fault-injection seam (see the struct field
|
|
* comment above). Pass a negative index to disable. Lock-free atomic
|
|
* write; compiled out of production. */
|
|
void persist_test_set_force_job_encode_fail_at(persist_t* p, int index);
|
|
#endif /* XCE_TESTING */
|
|
|
|
/* Checkpoint */
|
|
int persist_checkpoint(persist_t* p);
|
|
|
|
/* WAL truncation — remove all entries with LSN > cutoff_lsn.
|
|
* Idempotent; returns 0 on success, negative errno on I/O error. */
|
|
int persist_truncate_wal_above(persist_t* p, uint64_t cutoff_lsn);
|
|
|
|
/* Logging helper — never logs payload content (security invariant) */
|
|
void persist_log_entry_metadata(const wal_entry_hdr_t* hdr);
|
|
|
|
/* ============================================================
|
|
* AFTER_EVENT hook functions — registered by persist_start()
|
|
*
|
|
* Signature matches hook_fn_t: int fn(int type, int situation,
|
|
* int detail, void* data, va_list ap)
|
|
* cls/data points to the persist_t instance.
|
|
*
|
|
* P0: writes a minimal 4-byte payload (detail value) to the WAL.
|
|
* Full Cap'n Proto serialization is wired in Task 15.
|
|
* ============================================================ */
|
|
/* Explicit controller-side raft step for job submission (raft-apply-decoupling
|
|
* item #1 / bug-1306): WAL-append every job in `jlist` and block on the
|
|
* majority-commit barrier, returning the commit verdict DIRECTLY (0 = all
|
|
* committed on a majority; first negative rc otherwise — -ETIMEDOUT / etc.).
|
|
* handle_jobsubmit calls this instead of reading the clobber-prone hook return;
|
|
* a negative rc MUST reject the submission (never reply success). */
|
|
int persist_submit_wal_and_commit(persist_t* p, xlist_t* jlist);
|
|
|
|
int persist_job_hook_fun(int type, int situation, int detail,
|
|
void* cls, va_list ap);
|
|
int persist_queue_hook_fun(int type, int situation, int detail,
|
|
void* cls, va_list ap);
|
|
int persist_worker_hook_fun(int type, int situation, int detail,
|
|
void* cls, va_list ap);
|
|
|
|
/* Apply a single WAL entry to in-memory state (LOGICAL entity restore only).
|
|
* No hooks fire — hook system is empty on standbys until promotion.
|
|
*
|
|
* This restores entity state from (type, lsn, payload) ONLY. It does NOT touch
|
|
* the on-disk WAL header, the CRC chain (p->last_crc), or current_lsn — those
|
|
* are owned exclusively by the physical layer: persist_wal_write_replicated on
|
|
* the live replicated path, and replay_wal/persist_recover on recovery. (bug-778)
|
|
*
|
|
* Returns 0 on success (including skip for checkpoint/unknown types),
|
|
* negative errno on error (oversized payload, NULL persist). */
|
|
int apply_wal_entry(persist_t* p, uint32_t type, uint64_t lsn,
|
|
const void* payload, uint32_t payload_len);
|
|
|
|
/* Load a snapshot file from path, set p->checkpoint_lsn and p->current_lsn.
|
|
* lsn hint is informational — the actual LSN is read from the file header.
|
|
*
|
|
* ha-entity-snapshots (task 5.2 owns the restore-before-advance
|
|
* implementation; this is the public signature it will fill in):
|
|
* out_entities_restored reports whether this call actually restored
|
|
* job/queue entity state (true) or only observed a legacy position marker
|
|
* (false) — set ONLY when this function returns 0. On a non-zero return,
|
|
* p->checkpoint_lsn/p->current_lsn are left completely unchanged and
|
|
* *out_entities_restored is left untouched (undefined) — callers must not
|
|
* read it in that case.
|
|
*
|
|
* Returns 0 on success, -errno or -EINVAL/-EIO on error. */
|
|
int persist_load_snapshot(persist_t* p, const char* path, uint64_t lsn,
|
|
bool* out_entities_restored);
|
|
|
|
/* GC retention predicates (P0-5a / P0-8 / P0-13) */
|
|
#include <stdbool.h>
|
|
#include "xce/types.h"
|
|
bool job_is_terminal(const job_t* j);
|
|
bool job_should_retain(const job_t* j, const persist_config_t* cfg);
|
|
bool worker_should_retain(const xce_worker_t* w, const persist_config_t* cfg);
|