#ifndef HA_H #define HA_H /* * XCE Controller HA — role management and election (P1). * * P1-1 audit: the existing create_ssl_context() + iomgr_set_ssl_ctx() path * in cluster.c (line 559, set at line 583) covers all controller-to-controller SSL. * HA listeners (heartbeat, election probe, replication accept) reuse the same * ssl_ctx already installed on c->iomgr. * No new TLS infrastructure is needed — XCE runs on a trusted management LAN. * Peer identity is validated by host:port match against the --peers candidate list. * See design doc Part 5.9 and memory: Persistence & HA Trusted LAN Decision. * * P1-2: full struct definitions and API. * endpoint_t* (primary_ep) is intentionally omitted here — including netio.h * would pull in heavy OpenSSL headers and create circular dependency risks. * The watchdog (P1-3) will include netio.h directly where it needs endpoint_t. */ #include #include #include #include #include #include #include "config.h" /* XCE_MAXNAME_LEN, MAX_STANDBYS, etc. */ #include "myconf.h" /* myconf_t, config_version_tag */ #include "persist.h" /* persist_t, apply_wal_entry */ #include "typemap.h" /* wal_catchup_entry_t, state_report_t, xce_reply_t */ #include "xce/cluster.h" /* xce_candidate_t */ #include "zarray.h" #include "raft/raft.h" /* raft_state_t, raft_init */ /* Forward declaration — avoids circular include with cluster_def.h */ typedef struct cluster_s cluster_t; /* ── Inline PRNG for election jitter ───────────────────────── */ static inline uint64_t xorshift64_next(uint64_t *state) { uint64_t x = *state; x ^= x << 13; x ^= x >> 7; x ^= x << 17; *state = x; return x; } /* * ha_seed_jitter: seed the xorshift64 PRNG from /dev/urandom (8 bytes). * Falls back to CLOCK_MONOTONIC ^ priority ^ pid if urandom is unavailable. * Not inline — contains syscalls (open/read/close/clock_gettime/getpid). */ void ha_seed_jitter(uint64_t *seed, int my_priority); /* ── Role enum ─────────────────────────────────────────────── */ typedef enum { CTRL_PRIMARY = 0, CTRL_STANDBY = 1, CTRL_INIT = 2, /* Startup: listener open, awaiting heartbeat or election timeout */ CTRL_ELECTING = 3 /* Running quorum election via OP_requestVote */ } ctrl_role_t; /* ── Election types ────────────────────────────────────────── */ /* * ha_vote_entry_t: per-peer state accumulated by the election candidate. * One entry per peer in ha_elect_ctx_t.votes[] (flexible array). */ /* Forward declaration to avoid pulling netio.h into ha.h */ struct endpoint_t; typedef struct { xce_candidate_t *peer; /* Peer being solicited */ uint64_t my_term; /* Term sent in OP_requestVote */ uint64_t my_tag; /* config_tag sent in request */ uint64_t my_lsn; /* WAL LSN sent in request */ int vote_granted; /* 1 if peer granted vote */ int replied; /* 1 if peer replied (granted or denied) */ uint64_t peer_tag; /* peer's config_tag from reply */ uint64_t peer_lsn; /* peer's WAL LSN from reply */ struct endpoint_t *ep; /* Outgoing vote endpoint; NULL if connect failed or not yet connected */ } ha_vote_entry_t; /* * ha_elect_ctx_t: shared context for one quorum election round. * Allocated as: calloc(1, sizeof(*ctx) + (n_peers-1) * sizeof(ctx->votes[0])) * Stored in ha_state_t.elect_ctx (void * _Atomic) during election; NULL otherwise. */ typedef struct { int n_peers; /* Total peer count (excluding self) */ int votes_granted; /* Votes received; starts at 1 (self-vote) */ int n_replied; /* Number of peers that replied */ uint64_t max_resp_tag; /* Highest config_tag seen in any reply */ uint64_t max_resp_lsn; /* Highest WAL LSN seen among max_resp_tag peers */ int result; /* 1=primary, 0=standby, <0=error */ int done_posted; /* Guard: non-zero once sem_post(&done) has been called */ pthread_mutex_t lock; /* Protects votes_granted/n_replied/max_resp_tag/max_resp_lsn/result */ sem_t done; /* Startup thread blocks; dispatch posts when done */ /* Fix 4 (HIGH): pointer back to the election_timeout_ctx_t so that task 4.1 * (quorum-reached path) can set timeout_ctx->cancelled = 1 to cancel the * watchdog. Set by ha_start_election_async after scheduling the timeout. */ void *timeout_ctx; /* election_timeout_ctx_t*; NULL until watchdog scheduled */ /* Task 4.1: cluster_t* so ha_elect_evaluate_and_signal can call * cluster_init_phase2(c). * NULL in unit tests (c=NULL bypasses phase2). */ cluster_t *c; /* Task 6.3: candidate's own LSN/tag, stored unconditionally so that * ha_compute_safe_committed_lsn can use them even for single-node elections * where votes[] is empty (n_peers=0). */ uint64_t self_lsn; /* Candidate's own WAL LSN at election start */ uint64_t self_tag; /* Candidate's own config_tag at election start */ /* ha-elect-ctx-lifecycle task 2.1: reference count governing this * carrier's lifetime. Starts at 1 (bias reference for "currently * published in ha->elect_ctx"), incremented once per vote_ep_ctx_t that * captures this carrier (raft_cb_send_vote_request / * raft_cb_send_pre_vote_request), decremented once by that endpoint's * on_fini completion hook and once when a later round's atomic_exchange * supersedes this carrier in ha->elect_ctx. Freed via * ha_elect_ctx_release() when it reaches zero, from whichever of those * two release kinds fires last — see design.md's Architecture §Mechanism * for the full rationale (a pure per-endpoint count would allow a * spurious free before the round is superseded). */ _Atomic int refcount; ha_vote_entry_t votes[]; /* C99 flexible array: n_peers-1 entries */ } ha_elect_ctx_t; /* ── HA configuration ──────────────────────────────────────── */ /* Default values used by cluster_init(). Tuned for a LAN with ≤5ms RTT. */ #define HA_DEFAULT_HEARTBEAT_INTERVAL_MS 3000 #define HA_DEFAULT_HEARTBEAT_TIMEOUT_MS 10000 #define HA_DEFAULT_ELECTION_TIMEOUT_MS 5000 #define HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS 500 /* Log the first connect failure per peer, then every Nth attempt. */ #define HA_CONNECT_LOG_THROTTLE 10 #define HA_DEFAULT_MAX_PROMOTION_RETRIES 5 #define HA_DEFAULT_ELECTION_JITTER_MAX_MS 200 #define HA_DEFAULT_ELECTION_PRIORITY_STEP_MS 500 typedef struct { uint32_t heartbeat_interval_ms; /* Primary heartbeat send interval (min 2000 ms) */ uint32_t heartbeat_timeout_ms; /* Standby timeout before promoting (>= interval) */ uint32_t election_timeout_ms; /* Election probe shared deadline (min 500 ms) */ uint32_t probe_connect_timeout_ms; /* Per-probe TCP connect timeout */ uint32_t max_promotion_retries; /* Max consecutive failed promotion attempts */ uint32_t election_jitter_max_ms; /* random jitter added to election timeout (default 200) */ uint32_t election_priority_step_ms; /* extra ms per priority level (default 500) */ } ha_config_t; /* probe_ctx_t deleted — only used by am_i_primary() (election-iomgr-refactor) */ /* ── HA runtime state (embedded in cluster_t as `ha_state_t ha` in P1-5) ── */ typedef struct { _Atomic uint64_t term; /* Election term — fencing token */ _Atomic int32_t role; /* Cast to ctrl_role_t at access sites */ int my_priority; /* Index in --peers list (lower = higher priority) */ _Atomic uint64_t applied_lsn; /* Standby: last applied WAL LSN */ char primary_host[XCE_MAXNAME_LEN]; /* Hostname of current primary */ uint32_t primary_port; /* Port of current primary */ /* last_heartbeat deleted — heartbeat freshness now derives from last_activity */ _Atomic int promotion_failed; _Atomic int promotion_retry_count; _Atomic int in_catch_up; /* Non-zero while processing historical WAL entries */ /* failover_requested deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */ int hooks_registered; /* Idempotency guard for xce_add_hook calls */ uint32_t last_received_crc; /* Chained CRC of last received replicated entry */ ha_config_t config; pthread_mutex_t lock; /* Protects primary_host/port + voted_for_term/name */ /* watchdog_stop / watchdog_thread deleted — replaced by ha_arm_heartbeat_timeout_vtask */ void *ssl_ctx; /* SSL_CTX* cast; avoids pulling openssl headers here */ uint64_t jitter_seed; /* xorshift64 PRNG state for election jitter */ _Atomic uint64_t last_failover_ms; /* CLOCK_MONOTONIC ms of last failover trigger; RDHUP debounce (5.10b) */ zarray_t *standby_endpoints; /* connected standby controller endpoints for config push */ /* ── New fields added by ha-config-bootstrap spec ──────── */ uint64_t voted_for_term; /* Req 9: term of last granted vote */ char voted_for_name[XCE_MAXNAME_LEN]; /* Req 9: candidate of last granted vote */ int primary_ctrl_fd; /* Req 6: persistent channel fd (-1 = none) */ /* bug-1865: the netio endpoint primary_ctrl_fd belongs to, held with a * reference (handle_ctrlhello) so its socket -- which netio owns and * closes -- cannot be closed and its fd number reused while HA still * treats it as the primary channel. Dropped via ha_drop_primary_ctrl_conn, * never by close(primary_ctrl_fd). NULL = none. */ struct endpoint_t* primary_ctrl_ep; bool defer_config_fetch; /* Req 3: tag=0 node defers fetch to post-election */ void * _Atomic elect_ctx; /* Req 2: ha_elect_ctx_t* during election; NULL otherwise */ _Atomic uint64_t last_known_committed_lsn; /* Req 5: from primary heartbeats */ sem_t heartbeat_seen; /* Req 1: posted by heartbeat handler during CTRL_INIT */ _Atomic uint64_t last_known_committed_config_tag; /* Req 8: from primary heartbeats */ _Atomic uint64_t committed_config_tag; /* Req 8.1: set on quorum ACK */ /* ── New field added by controller-status spec delta (Req 12, task 7.1) ── */ _Atomic uint64_t last_known_elected_at_ms; /* Currently-known primary's * election time (epoch ms): self-stamped * by both promotion roads at the instant * each commits to CTRL_PRIMARY (task 7.2, * not yet wired), unconditionally learned * off-primary via heartbeat (task 7.3, not * yet wired) -- mirrors last_known_committed_lsn/ * last_known_committed_config_tag above. * 0/unset at init (ha_init memset) until * the first promotion or heartbeat. */ /* ── New fields added by election-iomgr-refactor spec (Task 2.1) ── */ void *client_ssl_ctx; /* SSL_CTX* (TLS_client_method); created in myapp_run */ /* ── New field added by raft-drive-async spec (Task 3.1) ──────── */ void *pool; /* vtaskmgr_t* (cast; avoids pulling vtaskmgr.h * into every ha.h consumer). Set once at init * (cluster_init_phase2 → &c->vtaskmgr). Used ONLY * by peer_ctrl_msg_handler when peer-control * dispatch is enabled; NULL otherwise (gate OFF). */ int phase2_done; /* 1 after cluster_init_phase2 runs */ int n_peers; /* total cluster size (including self), matches raft convention */ /* ── New fields added by raft-library spec (Task 6.1) ──────── */ raft_state_t raft; /* Raft consensus state (wired in task 6.2) */ /* bug-1435 (raft-drive-async concurrency gap): raft_state_t's own fields * (term/role/votes_granted/replies_received/election_decided/vote_replies[] * etc.) are plain, non-atomic and have NO protection of their own — the * Raft library assumes a single-threaded/serialized caller (the classic * "single event-loop" consensus model). But raft-drive-async's default-ON * pool dispatch (peer_ctrl_dispatch_to_pool, VTASK_LIVENESS) lets messages * from DIFFERENT peer endpoints (e.g. two RequestVoteReply's from two * different voters, or a vote reply racing a vote request) run * CONCURRENTLY on different vtask pool workers — per-endpoint * serialization (bind_vtask) only orders messages on the SAME endpoint, * not across endpoints. Two concurrent raft_on_vote_reply() calls can * race on votes_granted/replies_received/election_decided: e.g. one * worker's grant reaches quorum and fires on_become_leader (role→PRIMARY) * while a second worker, concurrently processing another peer's reply, * reads pre-increment votes_granted and independently satisfies the * "all-replied, no quorum" branch, firing on_become_follower (role→ * STANDBY) an instant later for the SAME term — silently undoing the * promotion with no further election. raft_lock serializes every mutating * entry point into `raft` (election/vote-reply/vote-request path only — * see call sites) so at most one thread evaluates the state machine at a * time, matching the raft library's single-threaded design assumption. * NOT the same lock as `lock` above (which protects unrelated fields) — * kept separate so the two never nest in conflicting order. Initialized * PTHREAD_MUTEX_RECURSIVE (ha_init): a WON/quorum outcome can legitimately * re-enter the state machine synchronously on the SAME thread (PreVote WON * -> real election, within one call chain) — that nesting must not * self-deadlock, while a genuinely different thread still fully blocks. */ pthread_mutex_t raft_lock; /* ── New fields added by raft-library spec (Task 6.2) ──────── */ cluster_t *raft_c; /* cluster_t* set by ha_start_election_async; used by raft callbacks */ void *raft_election_timeout_ctx; /* election_timeout_ctx_t* for cancel (task 6.2) */ /* ── New fields added by ha-heartbeat-liveness spec (Task 1.1) ── */ time_t last_activity; /* CLOCK_MONOTONIC seconds of last primary message; 0 = none yet */ /* ── New fields added by ha-repl-hardening spec (Task 1.4) ──────── */ uint64_t standby_reported_tag[MAX_HA_CANDIDATES]; /* Last config_tag reported by each standby */ uint64_t standby_reported_lsn[MAX_HA_CANDIDATES]; /* Last applied_lsn reported by each standby */ bool standby_converged[MAX_HA_CANDIDATES]; /* Whether standby[i] has converged with primary */ int peer_connect_fail_count[MAX_HA_CANDIDATES]; /* throttle log spam on repeated connect failure */ /* ── New field added by ha-commit-engine spec (Task 4.2) ──────── */ replicator_t *repl; /* Back-pointer to the standby registry (persist_t.repl). * When set, ha_has_write_quorum counts registry truth via * repl_active_standby_count (ep != NULL && !in_catch_up); * NULL → legacy standby_endpoints->count fallback. * (Req 3.1-3.3, 6.1, 6.2; wired at init by task 5.3.) */ /* ── New field added by ha-convergence-recovery spec (Task 3.x) ──────── */ _Atomic int rejoin_reconciled; /* Defensive once-guard for rejoin * reconciliation (Req 2.1): set after the * first truncate+self-restart so a second * CtrlHelloReply in the same process * lifetime cannot re-fire the restart. * 0 at init (cleared by ha_init memset). */ /* ── New field added by ha-convergence-recovery spec (Task 4.x) ──────── */ _Atomic int catchup_inflight[MAX_STANDBYS]; /* Per-standby single-flight * guard for ACK-driven catch-up * continuation (Req 5.2): set when a * continuation is scheduled for a * standby slot, cleared when it * completes; a duplicate walAck while * the flag is set schedules nothing. * Slot identity (the repl registry slot * index) is stable across schedule/run. * 0 at init (cleared by ha_init memset). */ /* ── New field added by ha-convergence-recovery spec (Task 5.x) ──────── */ _Atomic int quorum_lost; /* Quorum-transition edge detector (Req * 8.1-8.3): 0 = quorum held (or unknown * at init), 1 = quorum lost. Flipped by * ha_log_quorum_transition via CAS so the * loss/restore WARN is emitted exactly * once per edge (no log storms across * repeated dispatch-gate rejections). * 0 at init (cleared by ha_init memset). */ /* ── New field (2026-07-10, ha-convergence-recovery Req 1.4 supersession) ── */ _Atomic int self_restart_pending; /* Set when ha_self_restart's execv() * fails (a demoted ex-primary could not * re-exec to discard provisional state). * Checked unconditionally, alongside * apply_divergent, by raft_cb_become_ * leader and promotion_fence_timeout_fn * -- NEVER live-cleared (no clear site * exists anywhere): a periodic re-check * like promotion_fence_timeout_fn's * catch-up clear must NOT lift this, since * this node's WAL/durability state can * look perfectly healthy while its * in-memory state is still the stale * pre-demotion primary-era state the * restart exists to discard. Clears only * via a fresh process (a successful * retry replaces the image entirely). * 0 at init (cleared by ha_init memset). */ /* ── New field added by ha-commit-durability spec (Task 4.4) ──────── */ void * _Atomic promotion_fence_ctx; /* promotion_fence_ctx_t* for the * bounded-window catch-up timer (Req * 2.2/2.4): a recovery-truncated/behind * member that wins an election with no * reachable peer is NOT FATAL'd at the * promotion instant; it is fenced * (in_catch_up) and this one-shot timer * is armed for DEFAULT_RECOVERY_CATCHUP_- * WINDOW_MS. The 3.3 FATAL is reached * ONLY from that timer iff still * hollow + no peer + no catch-up * progress when it fires; cleared/ * cancelled on recovery or shutdown so a * caught-up or stopped member never * FATALs. NULL at init / when no fence * timer is pending. */ /* ── New field added by ha-term-safety spec (Task 4.3) ──────── */ _Atomic int config_serve_fence; /* Config byte-delivery serve-fence * (R7.9): set when the latest committed * WAL_CONFIG_MARKER references config * bytes whose sha256 does NOT match the * locally-held bytes (or the bytes are * absent locally). While set, the member * MUST NOT serve under the new * configuration; the bytes travel * out-of-band via the existing config-push * transport, so the member STAYS ALIVE and * config_activate_committed KEEPS RETRYING * until the matching bytes arrive and * hash-verify, at which point the fence is * cleared and the config is activated. * NEVER self-terminates (loud non-serving * fence only). 0 at init (ha_init memset). */ /* ── New field added by config-upgrade-snapshot spec (Task 6.1) ────── */ _Atomic int upgrade_materialization_fence; /* Upgrade-snapshot * materialization-and-hash-verify * fence (Req 1.8, 3.2, 3.4) — a NEW, * INDEPENDENT fence, deliberately NOT * merged into config_serve_fence * above: that fence tracks whether the * out-of-band config-PUSH bytes match * the committed marker; THIS fence * tracks whether THIS MEMBER'S OWN * upgrade-snapshot materialization * (the marker-triggered checkpoint, * config-upgrade-snapshot task 5) has * actually completed for the CURRENT * committed marker, with the live * on-disk config bytes still * hash-verifying against it right * now. These are logically distinct * and can disagree in either * direction (e.g. bytes can already * hash-match — config_serve_fence * clear — while this member's own * checkpoint for that exact marker * has not run yet, immediately after * a marker commits but before its * forced checkpoint fires). Set by * upgrade_materialization_verify() * on materialization-incomplete or a * hash mismatch against a FRESH * re-read of the live config bytes * (never a cached/embedded copy); * cleared on success. While set, the * member MUST NOT serve under the new * configuration or become primary; it * STAYS ALIVE and keeps retrying. * NEVER self-terminates (loud * non-serving fence only). 0 at init * (ha_init memset). */ /* ── New field added by raft-drive-async spec (Task 2.1) ──────── */ _Atomic int reconnect_inflight[MAX_HA_CANDIDATES]; /* Per-peer * single-flight guard for the offloaded * delaymgr reconnect (Req 2.5): set (CAS * 0->1) when a reconnect to peer[i] is * submitted to the slow lane, cleared by * the lane worker on completion (success * or failure) so the next heartbeat tick * can retry. While set, the heartbeat * reconnect scan enqueues no duplicate for * that peer. Indexed by ABSOLUTE candidate * index (peers_cfg.candidates[]). 0 at * init (ha_init memset). */ /* ── New field added by raft-drive-async spec (Task 3.2) ──────── */ _Atomic int snapshot_inflight[MAX_HA_CANDIDATES]; /* Per-peer * single-flight guard for the offloaded * snapshot send (Req 1.5): set (CAS * 0->1) when a snapshot to peer[i] is * submitted to the slow lane, cleared by * the lane worker on completion (success * or failure). While set, the heartbeat * decision loop enqueues no duplicate * snapshot for that peer. Indexed by * ABSOLUTE candidate index. 0 at * init (ha_init memset). */ /* ── New field added by raft-drive-async spec (Task 3.3) ──────── */ _Atomic int configpush_inflight; /* CLUSTER-WIDE single-flight guard * for the offloaded requester-less * config push. Config push is a * cluster-wide fan-out (one push thread * per standby), so at most ONE is in * flight at a time — hence a scalar, not * a per-peer array. Set (CAS 0->1) when * a config push is submitted to the slow * lane, cleared by the lane worker on * completion (success or failure). While * set, the heartbeat decision loop / * promotion enqueues no duplicate config * push. 0 at init (ha_init memset). */ /* ── New field added by xce-audit-invariants controller pass (thread-role * violation 1 fix, 2026-07-18) ──────────── */ _Atomic int demotion_convergence_inflight; /* CLUSTER-WIDE single- * flight guard for the offloaded * demotion-convergence task (truncate * above committed_lsn + self-restart). * Like configpush_inflight, this is a * single scalar, not a per-peer array — * demotion convergence is a one-shot, * whole-node event, never per-peer. Set * (CAS 0->1) when the task is submitted * to the slow lane, cleared by the lane * worker on completion. Defensive only: * raft_cb_become_follower's was-PRIMARY * check already makes scheduling this * task structurally single-shot (role * flips to STANDBY synchronously before * scheduling), so this guard should never * actually observe a collision in * practice. 0 at init (ha_init memset). */ /* ── New field added by ha-entity-snapshots spec (Task 4.1) ──────── */ _Atomic uint64_t snapshot_inflight_lsn[MAX_HA_CANDIDATES]; /* Per-peer * checkpoint LSN referenced by an * in-flight snapshot transfer (Req * 3.1-3.3): set in * ha_snapshot_send_dispatch at the * SAME point snapshot_inflight[peer_idx] * is set to 1 (before dispatch), * cleared at the same points the flag * is cleared to 0 (on completion or * CAS-release failure) — identical * lifecycle, one more field. Consulted * by rotate_snapshots (task 4.3) to * skip deleting a checkpoint file a * peer transfer is still reading from. * Explicitly zeroed on every clear so * a stale value can never be mistaken * for a live in-flight LSN. 0 at * init (ha_init memset). */ /* ── New fields added by bug-1389 fix ──────────────────────────────── */ uint32_t snapshot_recv_count; /* Standby receive side: count of * OPCode_snapshotTransfer chunks * structurally accounted for (written * to snap.part) in the CURRENT * transfer attempt. The vtask pool can * dispatch chunks of one transfer to * different workers, so chunks are NOT * guaranteed to arrive/process in * chunk_index order (bug-1389: chunks * observed out of order in a live * soak) — ha_handle_snapshot_chunk * writes each chunk at its own * chunk_offset (pwrite, order- * independent) and uses this counter, * not "this call happened to carry * is_last", to decide when every chunk * has actually arrived. Reset to 0 * when chunk_index==0 starts a fresh * transfer (Req 4.4). Guarded by * ha->lock. 0 at init (ha_init * memset). */ uint32_t snapshot_recv_total; /* Standby receive side: total chunk * count for the current transfer, * known only once the is_last chunk * has been seen (= its chunk_index+1, * chunks are 0-based and contiguous). * 0 = not yet known (is_last not seen * yet) OR no transfer in progress — * ha_handle_snapshot_chunk finalizes * (verify/rename/load) only once * snapshot_recv_count reaches this * value, then resets it to 0 so a * late straggler chunk cannot trigger * a second finalize. Guarded by * ha->lock. 0 at init (ha_init * memset). */ } ha_state_t; /* ── Public API ────────────────────────────────────────────── */ /* * ha_init: zero-initialise *ha, validate cfg, store config and my_priority. * Returns 0 on success, -EINVAL on bad config. * Initial role is always CTRL_STANDBY. */ int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority); /* * ha_fini: destroy mutex. Watchdog thread join added in P1-3. */ void ha_fini(ha_state_t *ha); /* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */ /* * ha_promote_to_primary: stub — atomically sets role to PRIMARY. * Full promotion sequence (hooks, WAL flush, peer notify) added in P1-3a. */ int ha_promote_to_primary(cluster_t *c); /* * ha_demote_to_standby: stub — atomically sets role to STANDBY. * Full demotion sequence added in P1-3a. */ int ha_demote_to_standby(cluster_t *c); /* ── Internal promotion/demotion (P1-3a) ───────────────────── */ /* * ha_do_promote: increment term, set role=PRIMARY, reset last_received_crc. * No cluster_t dependency — fully testable standalone. * Callers: ha_promote_to_primary() (P1-5), watchdog failover path. */ void ha_do_promote(ha_state_t *ha); /* * ha_do_demote: set role=STANDBY, reset hooks_registered idempotency guard. * Hook disable requires cluster_t and is done in ha_demote_to_standby() (P1-5). */ void ha_do_demote(ha_state_t *ha); /* ── Demotion convergence + self-restart primitive ───────────── * (ha-convergence-recovery Requirement 1: a demoted ex-primary discards its * uncommitted, potentially divergent state and rejoins from committed state * only via truncation + process self-restart through boot recovery.) */ /* * controller_saved_argv: accessor for the NULL-terminated argv captured by * main() before argp_parse, used by ha_self_restart to re-exec the controller * (ha-convergence-recovery Req 1.3, design "Self-Restart Primitive"). * Returns NULL if argv was never saved (e.g. unit-test binaries). */ char **controller_saved_argv(void); /* * ha_self_restart: the truncate-and-restart primitive's restart half. * * Production contract (Req 1.3, 1.4): flush logs, execv(controller_saved_argv()) * to rebuild in-memory state through boot recovery from the persisted committed * state only; if exec cannot be performed, terminate FATALLY (abort) rather than * continue running with potentially divergent state — NEVER returns in * production. * * Test contract (Req 1.6): under XCE_TESTING, invoke ha_restart_hook instead of * execv and return its result (0 on the success path); a failing ha_restart_hook * routes to ha_fatal_hook in place of abort so tests can assert the FATAL path * without dying. * * Preconditions: persistence hooks already disabled; truncation completed * before this call. Invariant: no new WAL entry is appended by this node * between hook-disable and exec. */ int ha_self_restart(cluster_t *c); /* * ha_primary_log_append (ha-term-safety task 3.1, R1.1/R1.2/R2.1): the single * primary append entry point. Routes a local primary append (job-state entry or * the own-term promotion no-op) through the raft library (raft_log_append) so * the library stamps the live election term and assigns the LSN, firing * on_persist_entry for the actual WAL write — making the library the sole * LSN/term source and keeping p->current_lsn co-tracked with s->log_len. When * the raft library is not RAFT_LEADER (the non-raft promotion road / engine-less * tests), falls back to a legacy local persist_wal_async append (the library is * not tracking the log, so no co-tracking hazard). `type` is a wal_entry_type; * `payload`/`len` may be NULL/0 (e.g. WAL_NOOP). out_lsn (optional) receives the * assigned 1-based LSN. Returns 0 on success; propagates a non-zero persist rc. */ int ha_primary_log_append(cluster_t *c, uint32_t type, const void *payload, uint32_t len, uint64_t *out_lsn); /* * ha_raft_on_repl_ack (ha-term-safety task 3.3, R4.1/R4.5): drive the library on * a replication ACK. Routes a successful WalAck through raft_on_append_entries_reply * (updating the library's match_index[]/next_index[] at the absolute peer index) * then raft_advance_commit, so the library recomputes the current-term-gated * commit (the Figure-8 rule) and MAY fire on_commit_advanced — the task-2.4 * callback that feeds the existing synchronous commit barrier. A current-term * quorum advances committed only when the agreed entry is current-term; a * prior-term agreed entry is held; a stale-term reply is dropped by the library. * * `abs_peer_idx` is the ABSOLUTE library peer index (j != self_idx), the same * index space as match_index[]/next_index[]. The CALLER resolves it per endpoint * type via ha_repl_ack_abs_peer_idx — promoted-vote eps carry a relative votes * index, heartbeat standby eps carry an already-absolute index. This seam must * NOT remap again. An out-of-range / self index is rejected loudly (returns * -EINVAL, no library accounting) so the library's unchecked match_index[] write * is never corrupted (no false quorum, no OOB). `acked_lsn` is the standby's * matched index (WalAckReply.lsn); `reply_term` is the reply's term. * * Leader-only effect: raft_advance_commit returns -EPERM off the leader (no * commit change, no callback) — harmless on a node that is no longer primary. * Non-blocking (ha-async-io): the library performs no I/O; on_commit_advanced * only records the barrier ceiling. Returns 0 on success; -EINVAL on a * no-cluster context or a rejected peer index. */ int ha_raft_on_repl_ack(cluster_t *c, int abs_peer_idx, uint64_t acked_lsn, uint64_t reply_term); /* * ha_raft_on_repl_nack (bug-1479): drive the library on a REJECTED * AppendEntries (WalAckReply.success == 0, i.e. the standby's log conflicts * at the sent prevLogIndex/prevLogTerm). Routes the rejection through * raft_on_append_entries_reply(success=0), which backs off * next_index[abs_peer_idx] by one (raft.tla's AppendEntries failure branch, * floored at 1) -- mirrors ha_raft_on_repl_ack's bounds-checked peer-index * resolution exactly, but does NOT call raft_advance_commit (a rejection * can never advance commit). * * Before this fix, a NACK was only ever used for stale-term discovery * (raft_on_higher_term_observed) and otherwise silently dropped -- the * library's own, already-correct nextIndex backoff mechanism was never * invoked, so a standby stuck in a genuine log conflict (an ordinary * post-election occurrence) never had its resend position corrected: the * primary kept broadcasting new entries assuming agreement at an index the * standby had already rejected, forever. * * Returns the corrected resume LSN (next_index[abs_peer_idx] - 1, i.e. the * position the standby is now believed caught up through) via * *corrected_lsn_out on success; returns -EINVAL (leaving *corrected_lsn_out * untouched) on a no-cluster context or a rejected peer index, matching * ha_raft_on_repl_ack's defense-in-depth bounds check. */ int ha_raft_on_repl_nack(cluster_t *c, int abs_peer_idx, uint64_t reply_term, uint64_t *corrected_lsn_out); /* * ha_repl_ack_abs_peer_idx (ha-term-safety task 3.3 remediation): resolve the * ABSOLUTE library peer index from an ACK endpoint's stored peer_idx, branching * on endpoint type (mirrors the standby_endpoints scan at ha.c:6187-6189): * - ep_is_vote != 0 (promoted vote ep, vote_ep_ctx_t): peer_idx is a * self-EXCLUDED votes index → remap abs = peer_idx < self_idx ? peer_idx * : peer_idx+1. * - ep_is_vote == 0 (heartbeat standby ep, standby_ep_ctx_t, ctx==NULL): * peer_idx is ALREADY absolute → pass through. * self_idx == ha->my_priority. Pure (no side effects); the single source of * truth shared by the production ACK call site and the unit test so the * remap-vs-passthrough distinction cannot drift. */ int ha_repl_ack_abs_peer_idx(ha_state_t *ha, int ep_is_vote, int peer_idx); /* * ha_schedule_demotion_convergence: schedule the one-shot delaymgr task that * truncates above committed_lsn (Req 1.2), WARN-logs the discarded range * (Req 1.5), and self-restarts (Req 1.3). Called from raft_cb_become_follower's * was-PRIMARY branch AFTER the persistence hooks are disabled inline (Req 1.1), * so all file I/O and the exec run off the become-follower callback thread (D1). * Returns 0 on success, negative on alloc/schedule failure (suffix is then * reconciled by rejoin reconciliation at the next join). * * The scheduled task body (ha_demotion_convergence_fn) CAN run off the * delaymgr thread entirely — see ha_set_demotion_offload_for_test: with the * offload flag ON, it hands the truncate + self-restart off to the slow * lane and returns immediately (flag currently defaults OFF pending a * cluster-wide test-suite compatibility audit — see the flag's doc comment). */ int ha_schedule_demotion_convergence(cluster_t *c); #ifdef XCE_TESTING /* * ha_demotion_convergence_fn: the delaymgr task body — exposed under XCE_TESTING * only so the demotion-convergence suite can reference its contract; the test * fixture drives the queued task through its stored entry pointer. * * With the offload flag ON, this function itself only does fast, * non-blocking admin work (ctx unpack, single-flight CAS, slowlane_submit) * and returns immediately; the truncate + ha_self_restart body runs on a * slow-lane worker (see ha_set_demotion_offload_for_test). */ int ha_demotion_convergence_fn(void *arg); #endif /* * ha_rejoin_reconcile: converge a rejoining node whose local WAL log runs ahead * of the cluster's committed boundary (ha-convergence-recovery Req 2.1–2.3, * design "Rejoin Reconciliation", research D5). * * Called from the standby's CtrlHelloReply intake (cluster.c) after the existing * last_known_committed_lsn store, with primary_committed_lsn = the boundary the * primary reported. * * Contract (task 3.2 GREEN implements; task 3.1 ships a RED no-op stub): * - local_tip = persist.current_lsn - 1. * - If local_tip > primary_committed_lsn AND the rejoin_reconciled once-guard * is clear: cutoff = max(primary_committed_lsn, local committed_lsn) — the * CLAMP that never discards the locally known committed prefix (Req 2.2) — * persist_truncate_wal_above(cutoff), WARN the discarded range, set the * once-guard, then ha_self_restart (deferred to delaymgr if the intake runs * on iomgr, same rule as demotion). Returns the restart result. * - Else (tip <= boundary): return 0 with NO side effects (Req 2.3). * - A second call in the same process lifetime takes no restart (Req 2.1 — * defensive once-guard; "once" is also structural since post-restart the tip * equals the boundary). * * Generalizes the RDHUP truncation pattern (ha_on_primary_ctrl_fd_rdhup), which * remains in place unchanged for the failover path. */ int ha_rejoin_reconcile(cluster_t *c, uint64_t primary_committed_lsn); /* * cluster_apply_ctrlhello_reply: standby-side CtrlHelloReply intake body * (ha-convergence-recovery task 3.1). Stores the primary-reported committed * boundary + config tag; called from cluster.c's myreq_handler on every * CtrlHelloReply. Defined in ha_convergence.c (a unit-linkable TU) so the * rejoin intake-integration test can drive the real path. Task 3.2 adds the * ha_rejoin_reconcile trigger after the store. */ void cluster_apply_ctrlhello_reply(cluster_t *c, uint64_t committed_lsn, uint64_t config_tag); /* * ha_walack_on_ack: the post-ACK catch-up decision (ha-convergence-recovery * task 4.2, Req 5.1, 5.2, 7.1, design D6). Called from the walAck branch of * repl_ack_decode_and_dispatch (ha.c) AFTER repl_on_ack, and from the * ha_walack_continuation_for_test shim so production and tests exercise the * same logic. Defined in config_repl.c (alongside ha_stream_wal_to_standby and * the continuation task body). * * target = max(committed_lsn, last_written) (same derivation as ha.c:2771-2776) * if acked_lsn < target → schedule ONE catch-up continuation that streams * [acked_lsn+1, target] on delaymgr (never inline on * iomgr), UNLESS c->ha.catchup_inflight[peer_idx] is * already set (per-standby single-flight, Req 5.2). * The scheduled task body (ha_catchup_continuation_fn) * CAN run off the delaymgr thread entirely — see * ha_set_catchup_offload_for_test: with the * offload flag ON, the actual WAL streaming runs * on the slow lane (flag currently defaults OFF * pending a cluster-wide test-suite compatibility * audit — see the flag's doc comment). * if acked_lsn >= target → repl_complete_catchup(&c->persist.repl, ep) to * re-include the standby in live replication (Req 7.1). * * c — cluster_t* (real persist + repl registry; never NULL). * ep — repl registry endpoint key for the standby (endpoint_t* / * repl_ep_t); identifies the standby slot for streaming + parity. * peer_idx — 0-based standby slot index (into c->ha.catchup_inflight[]). * acked_lsn— the LSN the standby just acknowledged. * * Returns 0 on the parity / single-flight-suppressed paths and on a successfully * scheduled continuation; negative on a scheduling/allocation failure. Never * performs file I/O itself (streaming runs in the deferred continuation task). */ int ha_walack_on_ack(cluster_t *c, void *ep, int peer_idx, uint64_t acked_lsn); /* * ha_log_quorum_transition: emit an operator-visible WARN on each write-quorum * loss/restore EDGE (ha-convergence-recovery task 5.x, Req 8.1, 8.2, 8.3). * * Evaluates the current write-quorum state and compares it against the * ha_state_t.quorum_lost edge-detector flag. On a TRANSITION (and only on a * transition), it atomically flips the flag via CAS and emits exactly one WARN: * - loss: "HA quorum lost: connected=%d required=%d" (Req 8.1) * - restore: "HA quorum restored: connected=%d required=%d" (Req 8.2) * The flag makes the logging an edge detector, NOT a latch: repeated calls while * the state is unchanged emit nothing (Req 8.3 — no log storms across repeated * dispatch-gate rejections), but a later loss/restore cycle logs again. The CAS * guarantees once-per-transition under concurrency. * * Call sites (task 5.2): the standby_endpoints mutation points (promotion * registration, RDHUP removal, on_standby_connected) plus the ha_dispatch_gate * quorum-rejection branch as a backstop. Observability only — never gates an * operation outcome. * * Defined in ha_convergence.c (a unit-linkable TU) so the quorum-transition * tests can drive it directly against a real persist/repl fixture. * * RED scaffold (task 5.1): the current definition is a NO-OP — it performs no * edge detection, flips no flag, and emits no WARN — so the quorum-transition * tests FAIL until task 5.2 lands the real edge-detection logic and wires the * call sites. */ void ha_log_quorum_transition(ha_state_t *ha); /* ── Redirect helpers used by join.c ──────────────────────── */ /* * ha_should_redirect: returns 1 if this controller should redirect join * requests to the primary. * Returns 0 if: (a) role is PRIMARY, or (b) primary_host is empty / * primary_port is 0 (no primary known yet — client should retry). */ int ha_should_redirect(const ha_state_t *ha); /* * ha_no_primary_known: returns 1 if role is STANDBY and primary_host is * empty or primary_port is 0. Caller should reply E_XCE_NOMASTER so the * client knows to retry later rather than treating the error as permanent. */ int ha_no_primary_known(const ha_state_t *ha); /* * ha_is_primary: returns 1 if ha->role is CTRL_PRIMARY, 0 otherwise. * Lock-free. Safe to call from any thread. */ int ha_is_primary(const ha_state_t *ha); /* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */ /* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */ /* * ha_dispatch_gate: role-based operation filter for the vtaskmgr dispatch loop. * Returns: * 0 — operation is permitted for the current role * -E_XCE_NOT_READY — role is CTRL_INIT or CTRL_ELECTING and opcode is not * OP_requestVote or OP_probeConfigTag * -E_XCE_STANDBY — role is CTRL_STANDBY and opcode is a primary-only op * CTRL_PRIMARY always returns 0. */ int ha_dispatch_gate(ha_state_t *ha, int opcode); /* * ha_config_promotion_fenced: 1 when this member is config serve-fenced and * therefore NON-PROMOTABLE — its latest committed WAL_CONFIG_MARKER references * config bytes it does not hold/match, so it must not become authoritative * CTRL_PRIMARY (ha-term-safety task 4.3, R7.9). 0 otherwise. The pre-promotion * gate in raft_cb_become_leader consults this; a fenced member stays ALIVE and * non-promotable until the correct bytes arrive out-of-band and re-verify clears * the fence (controller-never-self-terminate). Pure atomic read (no I/O). */ int ha_config_promotion_fenced(const ha_state_t *ha); /* * upgrade_materialization_verify (config-upgrade-snapshot task 6.1, Req 1.8, * 3.2, 3.4): the Activation Fence's own verify function. Mirrors * config_activate_committed's exact shape (config_repl.c:304-377) for a * DIFFERENT, independently-tracked condition — see ha_state_t's * upgrade_materialization_fence field doc comment above for the full * rationale on why this must be a separate fence from config_serve_fence. * * On every call: reads the current committed WAL_CONFIG_MARKER (the * authority for the target upgrade), checks whether THIS MEMBER'S OWN * checkpoint machinery has already captured a materialization reference for * that EXACT marker (c->persist.cfgref_present/cfgref_tag/cfgref_sha, * stamped by every checkpoint via persist.c's scan_latest_committed_config_ * ref), and separately re-reads the LIVE, currently-active config bytes off * disk FRESH (never a cached/embedded copy) to hash-verify them against the * marker too. Both must hold for the fence to clear. * * Returns 0 (fence cleared or absent-marker benign case) on success; * -EAGAIN when materialization is incomplete or the hash mismatches (fence * SET, logged loud only on the 0->1 transition); -EINVAL/-EIO on a genuine * error reading the committed marker. NEVER exits/aborts on this condition * (controller/CLAUDE.md never-self-terminate invariant) — always fail loud + * fence + stay alive + retriable. */ int upgrade_materialization_verify(cluster_t *c); /* * upgrade_materialization_fenced: 1 when this member's own upgrade-snapshot * materialization for the current committed config marker is incomplete or * hash-mismatched (upgrade_materialization_fence set); 0 otherwise. Trivial * atomic-read accessor — mirrors config_serve_fenced (config_repl.c:262-268) * exactly, for the new independent fence. Pure atomic read (no I/O). */ int upgrade_materialization_fenced(const cluster_t *c); /* * upgrade_materialization_retry_fn (config-upgrade-snapshot task 6.2, Req * 3.4, 5.5): re-runs upgrade_materialization_verify so a member whose * materialization was incomplete or hash-mismatched re-checks and clears the * fence once conditions allow. Does the actual blocking disk I/O; production * callers invoke it off the delaymgr thread via * schedule_upgrade_materialization_retry's trampoline (below) — mirrors * config_activate_retry_fn (config_repl.h). Public so tests can drive it * directly without a vtask pool. `arg` is a `cluster_t *`. Returns 0 on * success/already-unfenced, -EAGAIN while still fenced, negative errno on a * genuine error. */ int upgrade_materialization_retry_fn(void *arg); /* * schedule_upgrade_materialization_retry (config-upgrade-snapshot task 6.2, * Req 3.4, 5.5): non-blocking, self-triggering retry for the Activation * Fence — mirrors config_schedule_activate_retry's exact delaymgr- * trampoline-then-vtask shape (config_repl.c:409-507): a one-shot delaymgr * task (NON-BLOCKING — only enqueues) offloads the actual blocking re-verify * work (upgrade_materialization_verify's disk reads) onto a VTASK_LIVE_BLOCK * vtask so the delaymgr thread itself never blocks (ha-async-io.md * thread-role contract). No-op when the member is NOT upgrade-materialization * fenced (steady state schedules nothing — no busy-spin) or when the * delaymgr is not running (unit-test fixture path; caller drives the retry * directly instead). Falls back to running the verify inline (stay alive, * never silently drop a scheduled retry) if the vtask pool is unavailable. * Returns 0 on enqueue/no-op, negative on enqueue failure. */ int schedule_upgrade_materialization_retry(cluster_t *c); /* * ha_phase2_retry_fn (bug-1477): re-attempts cluster_init_phase2 for a node * whose earlier attempt(s) failed and never set ha->phase2_done. Idempotent * -- a no-op once phase2_done is already set by ANY call site (the shared * once-per-process contract every cluster_init_phase2 caller relies on). * Does the actual blocking work (cluster_init_phase2 runs persist_recover, * wires the commit engine, etc.); production callers invoke it off the * delaymgr thread via ha_schedule_phase2_retry's trampoline (below) -- * mirrors upgrade_materialization_retry_fn/config_activate_retry_fn exactly. * Public so tests can drive it directly without a vtask pool. `arg` is a * `cluster_t *`. Returns 0 on success/already-done, the underlying negative * errno on a genuine cluster_init_phase2 failure (and self-reschedules * before returning, so the caller need not). * * Root cause this closes: a node that takes myapp_run's skip_election * passive path (bug-849) -- because a leader was already known after the * startup stagger -- makes exactly ONE cluster_init_phase2 attempt. If that * attempt races config availability and fails, NOTHING else ever retries * it: raft_cb_become_follower / raft_cb_become_leader / * ha_elect_evaluate_and_signal (the other three call sites) only fire via * THIS node's own raft-library state transitions, which never happen for a * node that stays passively CTRL_STANDBY while its primary keeps * heartbeating normally. The node is then permanently stuck without a * wired commit engine -- silently unable to apply ANY committed entry for * the rest of its process lifetime (never self-terminates, per * controller/CLAUDE.md's Process Lifecycle invariant, but never recovers * either without this retry). */ int ha_phase2_retry_fn(void *arg); /* * ha_schedule_phase2_retry (bug-1477): non-blocking, self-rescheduling retry * for a node whose cluster_init_phase2 has not yet completed -- mirrors * config_schedule_activate_retry / schedule_upgrade_materialization_retry's * delaymgr-trampoline-then-vtask shape, except it re-arms ITSELF on failure * (via ha_phase2_retry_fn) since -- unlike those two, which are re-triggered * externally by every commit-advance/config-push event -- there is no * other recurring event that would naturally re-trigger this for a node * stuck on the passive skip_election path. Retries at ha->config. * heartbeat_interval_ms cadence (the natural rhythm config/connectivity * changes arrive on), indefinitely (never gives up -- never-self-terminate * invariant). No-op when phase2_done is already set (steady state * schedules nothing -- no busy-spin) or when the delaymgr is not running * (unit-test fixture path; caller drives the retry directly instead). * Falls back to running the retry inline (stay alive, never silently drop * a scheduled retry) if the vtask pool is unavailable. Returns 0 on * enqueue/no-op, negative on enqueue failure. */ int ha_schedule_phase2_retry(cluster_t *c); /* * ha_apply_wal_catchup_entry: apply a single WAL catch-up entry to local * persistent storage and advance ha->applied_lsn on success. * * Called from cluster.c OPCode_walCatchupEntry dispatch handler (Task 2.1). * On success (rc == 0): applied_lsn advances to entry->lsn+1 and a * WalAckReply {term, lsn, success=1} is sent back to the primary via * xce_reply(ep, c->myhost, &reply) (Task 3.1, Req 3.1). * On a genuine apply/persist ERROR (rc < 0): logs ERROR, returns error; no ACK * sent at all (Task 3.2) -- there is nothing coherent to report back yet. * On a term-carrying entry's AppendEntries REJECT (logOk fails -- forward gap * or a genuine (term,lsn) conflict; rc == 0, entry not accepted): bug-1479 -- * a WalAckReply {term, lsn, success=0} (a NACK) IS now sent, so the primary's * ha_raft_on_repl_nack can back off nextIndex and immediately mark this * standby for catch-up. Before this fix the reject was dropped with no reply * at all, so the primary's NACK-handling could never engage no matter how * correct it was -- the standby would loop rejecting the identical * re-broadcast entry forever. * applied_lsn is in-memory only — not persisted to disk. * ep and c may be NULL in unit-test builds (XCE_TESTING); ACK/NACK is skipped * (or captured via ha_wal_ack_send_hook when set). * Requirements: 3.1, 3.2, 3.3, 3.4, 3.5 */ int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p, wal_catchup_entry_t *entry, struct endpoint_t *ep, cluster_t *c); #ifdef XCE_TESTING /* * ha_wal_ack_send_hook: test-only seam for the outbound WalAckReply built by * ha_send_wal_ack_reply_ex (bug-1479). When non-NULL, called with * (term, lsn, success) INSTEAD OF building/sending the real reply over ep -- * lets a unit test assert that an AppendEntries reject now sends a NACK * (success=0), without needing a live endpoint. NULL in production. */ extern void (*ha_wal_ack_send_hook)(uint64_t term, uint64_t lsn, int success); #endif /* ha_nop_reply_state_report — populate a heartbeat reply with stateReport. * Fills *sr from ha->applied_lsn (atomic read) and myconf->config_version_tag, * then sets reply_out->data_which = stateReportMsg and * reply_out->data.state_report = sr. * Caller must keep *sr live until xce_reply() serialises the reply. */ void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf, state_report_t *sr, xce_reply_t *reply_out); /* ── Task 4.1 (ha-repl-hardening): safe commit point computation ───────────── * * ha_compute_safe_committed_lsn: determine safe_committed_lsn and safe_config_tag * from vote replies collected in ctx after winning an election. * * Collects LSNs from all granted voters (self + peers with vote_granted=1), * sorts descending, and returns the majority-threshold element. * * single-node (n_peers=0): returns ctx->self_lsn and ctx->self_tag directly — * self is the only majority; returning 0 would truncate all WAL on re-election. * * Requirements: 2.3 (ha-repl-hardening spec) * Task 6.3: single-node fix (self_lsn/self_tag fields added to ha_elect_ctx_t) */ void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx, uint64_t *safe_lsn_out, uint64_t *safe_tag_out); /* ── bug-1060: Leader Completeness guard for promotion-time WAL truncation ──── * * ha_promotion_truncate_floor: clamp the post-election WAL truncation cutoff so * a newly-elected leader NEVER truncates entries it holds durably. * * Returns max(safe_lsn, durable_tip). safe_lsn is the majority-threshold commit * point from the vote replies (ha_compute_safe_committed_lsn); durable_tip is the * leader's own durable WAL tip (persist_durable_floor). Because the cutoff is * never below the leader's durable tip, persist_truncate_wal_above is a no-op on * the leader's own durable log — preserving every entry committed by a prior * leader (Raft §5.4 Leader Completeness). safe_lsn only governs removal of a * divergent, non-durable tail STRICTLY ABOVE the leader's durable tip. * * Root cause it fixes: truncating to the bare majority element destroyed * committed entries the new leader had acked as a standby (granted voters * {self=47, peer=44} → safe_lsn=44 cut the leader's own committed 45..47), * producing divergent WALs and a permanent prevote deadlock. */ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip); /* ── bug-1462: config_tag floor for post-election committed_config_tag ─────── * * ha_promotion_config_tag_floor: clamp the post-election committed_config_tag * baseline so it never drops below the config the new leader already has * loaded. * * Returns max(safe_tag, self_tag). safe_tag is the majority-threshold config * tag from the vote replies (ha_compute_safe_committed_lsn); self_tag is the * candidate's own actively-loaded config tag at election time. Unlike WAL LSN * (where an unacknowledged entry is genuinely speculative), a node's own * loaded config is never speculative — so the majority floor, correct for * LSN, is wrong for config_tag. * * Root cause it fixes: on a cluster where no live admin config push had ever * happened, every peer voter reported config_tag=0 while the new leader's own * self_tag was a real, non-zero, loaded tag. The majority floor picked 0, * permanently stalling the heartbeat-driven HA_REPL_PUSH_CONFIG decision for * any standby not connected at the exact instant of promotion (a late * joiner) — WAL/job replication was unaffected since it correctly floors at * genuinely-speculative LSN 0. */ uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag); /* ── raft-drive-async task 1.1: peer-scaled liveness reservation ───────────── * * drive_async_liveness_reserved: PURE derivation of VTASK_LIVENESS_RESERVED — * the number of vtask-pool workers kept free for liveness-critical handling * (heartbeat/vote/ack) so they are never starved by blocking offloaded tasks. * * reserved = clamp(n_peers - 1, 1, n_vcpus - 1) * = max(1, min(n_peers - 1, n_vcpus - 1)) * * - n_peers is the TOTAL cluster size including self (matches ha->n_peers). * Per-endpoint serialization (endpoint_block_tasks) caps in-flight work at * one per peer connection, so worst-case concurrent liveness work is the * n_peers-1 OTHER peers (one ack/vote each) — reserving n_peers-1 means no * peer's liveness message queues behind a busy reserved worker (a LATENCY * optimization). * - Floor 1 is the hard CORRECTNESS invariant (deadlock-freedom): one * guaranteed-free worker + liveness-never-gated + sub-ms cycling is * deadlock-free regardless of peer count. Applied LAST so it always wins. * - Ceiling n_vcpus-1 keeps >=1 REQ_BLOCK (commit) slot, so the commit cap * (n_vcpus - reserved) is never 0. * * On an under-provisioned box (n_vcpus < n_peers) the ceiling clamps the * result below n_peers-1; the result is still correct via fast cycling. * The CALLER (controller init) is responsible for emitting a loud WARN in * that case so the under-provisioning is operator-visible — this helper is * pure and never logs. Controllers NEVER self-terminate on this condition. */ int drive_async_liveness_reserved(int n_peers, int n_vcpus); /* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ────────── * * ha_promotion_gate_decision: PURE pre-promotion eligibility decision (Req 2.2, * 2.5, 2.6). Maps the promoting member's recovery/durability state to the Boot * Policy Decision Table outcome by building recover_policy_inputs_t and * delegating to persist_recover_decision (DRY — the 3.2/3.3 decision table is * the single source of truth; this gate does not re-derive it). Note the * FENCE_UNRECOVERABLE terminal is a LOUD PERMANENT NON-promotable fence that * STAYS ALIVE and keeps retrying catch-up — it is NEVER process exit * (never-self-exit invariant, 2026-06-18): * - recovery_truncated → the task-3.1 structural-corruption flag. * - durable_floor → the local CLAMPED durable position (read AFTER the * promotion-path persist_truncate_wal_above so the * 2.2 truncate-clamp is reflected). * - safe_hwm → the cluster committed high-water mark from the voter * replies (ha_compute_safe_committed_lsn), NOT a stale * ctrlhello. * - peer_reachable → a healthy peer is reachable to catch up from. * - catch_up_succeeded → catch-up reached a committed-consistent state. * A member is "behind the HWM" iff durable_floor < safe_hwm. Returns * RECOVER_DECISION_{ELIGIBLE,FENCE,FATAL}; a hollow node (truncated OR behind) * is never ELIGIBLE until it catches up, regardless of quorum size (Req 2.5). */ recover_decision_t ha_promotion_gate_decision(bool recovery_truncated, uint64_t durable_floor, uint64_t safe_hwm, bool peer_reachable, bool catch_up_succeeded); /* ha_evaluate_promotion_fence: live pre-promotion gate over ha->in_catch_up * (Req 2.2, 2.6). Evaluates ha_promotion_gate_decision and applies the fence: * - ELIGIBLE → clears the in_catch_up fence and returns ELIGIBLE (promotion * may proceed; a member that has caught up past the committed * boundary regains eligibility — Req 2.6). * - FENCE → sets the in_catch_up fence (refuse promotion; catch up from a * healthy peer) and returns FENCE. * - FENCE_UNRECOVERABLE → routes through * persist_recover_loud_fence_if_unrecoverable (logs LOUDLY at * ERROR + sets the sticky persist recovery_unrecoverable flag), * also sets the in_catch_up fence, and returns * FENCE_UNRECOVERABLE. The process STAYS ALIVE, non-promotable, * retrying catch-up — it NEVER self-exits (Req 2.4). * Reuses the existing in_catch_up fence — does NOT invent a parallel fence. */ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha, bool recovery_truncated, uint64_t durable_floor, uint64_t safe_hwm, bool peer_reachable, bool catch_up_succeeded); /* ── ha-commit-durability task 4.4: catch-up re-alert window ───────────────── * (never-self-exit rework 2026-06-18.) * * When a recovery-truncated / behind member wins an election with no reachable * peer AT THAT INSTANT (a transient no-peer snapshot during aggressive * failover), it FENCES (sets in_catch_up, does NOT become primary) and arms a * delaymgr re-alert timer for DEFAULT_RECOVERY_CATCHUP_WINDOW_MS. When the * timer fires, if the member is STILL recovery-truncated/behind AND no healthy * peer is reachable AND no catch-up progress was made, it RE-EMITS the loud * operator alert (persist_recover_loud_fence_if_unrecoverable) and RE-ARMS — * the process STAYS ALIVE and keeps retrying catch-up INDEFINITELY. It NEVER * terminates: a controller must never self-exit as a designed response to an * operational condition. Catch-up that brings durable_floor >= HWM releases * the fence + clears the sticky flag and stops the re-alert (Req 2.6); shutdown * cancels it too. * * ha_promotion_fence_window_ms: the re-alert interval in milliseconds. * Defaults to DEFAULT_RECOVERY_CATCHUP_WINDOW_MS; tests override it (via * ha_set_promotion_fence_window_ms) to a small value so the timer fires fast. */ uint32_t ha_promotion_fence_window_ms(void); /* ha_cancel_promotion_fence_timer: cancel a pending re-alert timer (idempotent; * no-op when none is armed). Called when the member catches up (fence released) * or on shutdown, so a stopped member stops re-alerting. It never gated a * process exit — the timer never terminates the process. */ void ha_cancel_promotion_fence_timer(ha_state_t *ha); #ifdef XCE_TESTING /* Test seam: override the re-alert window so the timer fires deterministically * fast. 0 restores the DEFAULT_RECOVERY_CATCHUP_WINDOW_MS default. XCE_TESTING * only — compiled out of production. */ void ha_set_promotion_fence_window_ms(uint32_t ms); #endif /* ── Task 3.2: Unified replication decision enum and evaluator ────────────── */ /* * ha_repl_action_t: result of ha_evaluate_repl_action(). * * Encodes the single replication action the primary should take for a standby * on the current heartbeat cycle, based solely on the standby's reported * (config_tag, applied_lsn) vs the primary's committed state. * * Decision tree (design §Unified Heartbeat-Driven Protocol): * standby_tag < committed_tag → PUSH_CONFIG * standby_lsn < oldest_wal_lsn → SEND_SNAPSHOT * standby_lsn < committed_lsn → STREAM_WAL * tag >= committed AND lsn >= committed → CONVERGED * * Requirements: 2.2, 2.3, 2.4 (ha-repl-hardening spec) */ typedef enum { HA_REPL_PUSH_CONFIG, /* standby config tag is behind — push config first */ HA_REPL_SEND_SNAPSHOT, /* standby LSN predates oldest WAL — transfer snapshot */ HA_REPL_STREAM_WAL, /* standby LSN is within WAL window — stream entries */ HA_REPL_CONVERGED, /* standby tag and LSN are both current */ } ha_repl_action_t; /* * ha_evaluate_repl_action: pure decision function — no side effects. * * Evaluates the standby's reported (standby_tag, standby_lsn) against the * primary's committed state (committed_tag, committed_lsn, oldest_wal_lsn) * and returns the single action the primary should take this heartbeat cycle. * * Arguments: * standby_tag — config_tag last reported by this standby * standby_lsn — applied_lsn last reported by this standby * committed_tag — primary's committed config_tag * committed_lsn — primary's committed WAL LSN * oldest_wal_lsn — oldest retained WAL LSN (entries before this are gone) * * Requirements: 2.2, 2.3, 2.4 (ha-repl-hardening spec) * Boundary: Primary Heartbeat Handler (ha.c) */ ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag, uint64_t standby_lsn, uint64_t committed_tag, uint64_t committed_lsn, uint64_t oldest_wal_lsn); /* * ha_process_standby_state: update per-standby convergence tracking. * * Called on each heartbeat reply from a standby while the primary is in * CTRL_PRIMARY. Stores the standby's reported * (config_tag, applied_lsn) and evaluates whether the standby has converged * with the primary's committed state. * * Convergence criteria (Req 1.3): * reported_tag >= committed_tag AND reported_lsn >= committed_lsn * * Logs INFO on the first transition from not-converged → converged (Req 7.2). * Regression (converged → not-converged) clears standby_converged[peer_idx] * and re-evaluates (Req 2.5). * * peer_idx: 0-based index into ha->standby_reported_* arrays; must be in * [0, MAX_HA_CANDIDATES). * * Requirements: 1.2, 2.2, 2.5 (ha-repl-hardening spec) * Boundary: Primary Heartbeat Handler (ha.c) */ void ha_process_standby_state(ha_state_t *ha, int peer_idx, uint64_t reported_tag, uint64_t reported_lsn, uint64_t committed_tag, uint64_t committed_lsn); /* * ha_compute_repl_target_lsn: the replication/convergence target used by * ha_dispatch_repl_action_for_standby's heartbeat-driven decision — normally * committed_lsn itself, falling back to last_written (the raw locally- * appended WAL tip) ONLY when committed_lsn is genuinely stuck at 0 * (cluster bootstrap, before any standby has ever acked anything). * * bug-1478: the fallback must NOT apply once committed_lsn is already > 0 — * see ha.c's doc comment on this function for the full mechanism (a * local-only WAL_CHECKPOINT trailing an already-advancing committed_lsn * otherwise leaks an unstreamable/unreachable target into every subsequent * heartbeat decision, permanently wedging catch-up). * * Pure, side-effect-free. */ uint64_t ha_compute_repl_target_lsn(uint64_t committed_lsn, uint64_t last_written); /* * ha_compute_repl_stream_ceiling_lsn (bug-1479, gap 3): the upper bound of * what a leader may actually STREAM to a standby via ha_stream_wal_to_standby * — distinct from ha_compute_repl_target_lsn's CONVERGENCE target above. * * Bounding the stream range at committed_lsn is a permanent deadlock whenever * the very standby being streamed to must ACK before committed_lsn can * advance at all (e.g. a 3-node cluster with one member down and this * standby log-conflicted after a chaotic election): it can never receive the * entries it needs to ack to unblock the majority. raft.tla's real * AppendEntries replicates the leader's FULL log — leaderCommit is advisory * to the follower, never a cap on what gets sent. Streaming past commit is * always safe: a later leadership change's own (term,lsn) conflict-truncate * (ha_consume_append_entries) correctly unwinds anything that turns out not * to have been committed. * * bug-1478 guard: never widen the ceiling to exactly the trailing local-only * checkpoint's own consumed LSN slot — no standby's applied_lsn can ever * legitimately reach that value (WAL_CHECKPOINT is never streamed or applied * by any standby, by design), so doing so would reintroduce bug-1478's * permanent-non-convergence trap. Falling back to committed_lsn for just * that transient instant is safe — the very next real entry appended after * the checkpoint widens the ceiling normally again. * * Pure, side-effect-free — safe to unit test directly. */ uint64_t ha_compute_repl_stream_ceiling_lsn(uint64_t committed_lsn, uint64_t last_written, uint64_t checkpoint_lsn); /* ── Task 3.5c: Standby snapshot receive handler ──────────────────────────── */ /* * ha_handle_snapshot_chunk: receive one snapshot transfer chunk on the standby. * * Each chunk's payload bytes are written (or appended) to snap.part in snap_dir. * When chunk->is_last is non-zero: * 1. Verify total bytes written == chunk->total_size. * 2. Atomically rename snap.part → snap-{snapshot_lsn:016x}.bin. * 3. Load the snapshot via persist_load_snapshot(). * 4. Set ha->applied_lsn = p->checkpoint_lsn (in-memory only). * * On mid-transfer reconnect (caller resets state), snap.part is discarded. * Returns 0 on success, negative errno on error. * * Requirements: 4.2, 4.3, 4.4 (ha-repl-hardening spec) * Boundary: Standby Dispatch (cluster.c, ha.c) */ int ha_handle_snapshot_chunk(ha_state_t *ha, persist_t *p, const char *snap_dir, snapshot_transfer_t *chunk, const uint8_t *payload_bytes, size_t payload_len); /* ── HA Config Bootstrap & Term Durability (ha-config-bootstrap spec) ── */ /* * ha_persist_term: write ha-term.bin atomically to {workdir}/ha-term.bin. * Uses write + rename for durability (existing pattern in the codebase). * Must be called before any vote is granted or term is incremented. * If workdir is NULL, the file write is skipped (safe for unit tests) and * this returns 0 -- that is a deliberate no-op, not a failure. * Returns 0 on success (including the NULL-workdir skip); a negative errno * on a genuine I/O failure (open/write/fsync/rename). Callers on the * vote-granting path (raft_cb_persist) MUST propagate a non-zero return so * the raft library denies the vote instead of granting one with no durable * record (bug-1684). */ int ha_persist_term(ha_state_t *ha, const char *workdir); /* * ha_init_raft: initialize the embedded raft state machine with cluster topology. * * Must be called ONCE at controller startup after config is loaded and * ha_restore_term() has run. Subsequent elections (initial and re-elections * after primary loss) reuse the same raft state — only raft_start_election() * and raft_on_leader_lost() are called for each election cycle. * * Calling raft_init() more than once would reset raft's internal state * (term, voted_for, role) and could lose durable election progress. * * n_peers: total cluster size including self (candidates_count from config). */ void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers); /* * ha_restore_term: read {workdir}/ha-term.bin at startup and restore * ha->term and ha->voted_for_* into ha_state_t. * * On missing file (ENOENT): no-op — term stays 0, voted_for_name stays empty. * Returns 0 (safe first-boot path). Req 9.3. * On CRC/magic mismatch: log WARN, zero ha->term and ha->voted_for_* for * safety and return 0. Req 9.2. * On valid file: atomically restore ha->term; copy voted_for_term * and voted_for_name. Returns 0. Req 9.2. * On unexpected I/O error: return -errno. * * Must be called after ha_init() and before any election or vote processing. * If workdir is NULL, the function is a no-op (safe for unit tests). * Requirements: 9.1, 9.2, 9.3. */ int ha_restore_term(ha_state_t *ha, const char *workdir); /* * handle_request_vote: process an incoming OP_requestVote RPC. * Applies term/voted_for/freshness gate; persists state via ha_persist_term(). * Returns 1 = vote granted, 0 = vote denied, <0 = error. * Requirements: 2.2, 2.3, 2.4, 2.5, 2.6, 9.1 * * TEST-ONLY (XCE_TESTING): production vote dispatch is cluster.c → * raft_on_vote_request. This is the legacy entry point for test-only * role-gate unit tests (test_ha_vote_gate.c, test_ha_election.c). */ #ifdef XCE_TESTING int handle_request_vote(ha_state_t *ha, uint64_t term, const char *candidate_id, uint64_t config_tag, uint64_t last_lsn, uint64_t my_config_tag, uint64_t my_last_lsn, int in_catch_up_flag); #endif #ifdef XCE_TESTING /* * ha_cb_persist_entry_for_test (ha-term-safety task 2.1): invoke the WIRED * on_persist_entry callback (s_real_cbs.on_persist_entry) directly so a unit * test exercises the real adapter body — term stamping + WAL write at the given * 1-based index == LSN — without driving raft_log_append (task 3.1). `ha` must * have raft_c set to a cluster whose persist module is started. The `entry` * blob is the adapter envelope (raft_entry_blob_t prefix + payload). Returns * the callback rc (0 on durable persist; non-zero on persist failure, in which * case nothing is ACKed). TEST-ONLY. */ int ha_cb_persist_entry_for_test(ha_state_t *ha, uint64_t index, uint64_t term, const void *entry, size_t len); /* * ha_cb_get_entry_term_for_test (ha-term-safety task 2.2): invoke the WIRED * on_get_entry_term callback directly so a unit test exercises the real * entry-term accessor (persist_entry_term at the given 1-based index == LSN). * Returns the recorded term (legacy-0 mapped to WAL_LEGACY_TERM), or 0 when the * LSN is absent. TEST-ONLY. */ uint64_t ha_cb_get_entry_term_for_test(ha_state_t *ha, uint64_t index); /* * ha_cb_truncate_from_for_test (ha-term-safety task 2.2): invoke the WIRED * on_truncate_from callback directly so a unit test exercises the real conflict * truncation (delete local entries at LSN >= index). Returns the truncate rc * (0 on success; non-zero propagates so the library/leader retries while the * member stays non-promotable). TEST-ONLY. */ int ha_cb_truncate_from_for_test(ha_state_t *ha, uint64_t index); /* * ha_cb_send_append_entries_for_test (ha-term-safety task 2.3): invoke the WIRED * on_send_append_entries callback directly so a unit test exercises the real * AppendEntries send body — peer→endpoint mapping, prev-tuple + leader_commit * carry, and (for n_entries==1) sourcing the single entry at prev_log_index+1 * from the leader's own WAL. Combine with repl_send_ae_capture_hook to assert * the built envelope content without network I/O. Returns the callback rc. * TEST-ONLY. */ int ha_cb_send_append_entries_for_test(ha_state_t *ha, int peer_idx, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, int n_entries); /* * ha_cb_commit_advanced_for_test (ha-term-safety task 2.4): invoke the WIRED * on_commit_advanced callback directly so a unit test exercises the real * commit-source trigger — feeding the raft library's current-term-gated * commit_index into the cluster's synchronous commit barrier as the commit * source. TEST-ONLY. */ void ha_cb_commit_advanced_for_test(ha_state_t *ha, uint64_t commit_index); /* * ha_arm_own_term_gate_for_test (ha-term-safety task 3.1): drive the static * own-term gate arm (the promotion WAL_NOOP, now routed through * ha_primary_log_append) so a unit test can assert the no-op records the live * term via the library. TEST-ONLY. */ void ha_arm_own_term_gate_for_test(cluster_t *c); /* * ha_send_append_entries_stub_call_count (raft-adapter-conformance task 4.1, * Requirement 3.3): test-only seam counting invocations of * ha_cb_send_append_entries_stub, the on_send_append_entries callback. That * callback is reachable only via the raft library's raft_send_append_entries * (zero production callers) or ha_cb_send_append_entries_for_test above — * see the DORMANT IN PRODUCTION comment on ha_cb_send_append_entries_stub in * ha.c for the full finding. A test that drives only the production * replication path (repl_broadcast / ha_stream_wal_to_standby) must observe * this counter stay at 0; a future change that accidentally gives * raft_send_append_entries a production caller moves it off zero. NULL in * production (the whole symbol does not exist outside XCE_TESTING). * TEST-ONLY. */ extern uint64_t ha_send_append_entries_stub_call_count; #endif /* * ha_vote_self_freshness: compute the voter's effective freshness floor. * * Election safety: a voter must deny candidates that are outdated against the * strongest knowledge it has of the cluster's COMMITTED state — not merely its * own local config file / applied LSN. The floor is the per-field max of the * voter's local state and its learned-committed state: * * *out_tag = max(local_config_tag, atomic committed_config_tag) * *out_lsn = max(local_applied_lsn, atomic last_known_committed_lsn) * * Uses acquire loads of the atomic learned-committed fields; never reads * last_known_committed_config_tag (a heartbeat-learned config tag, not the * vote freshness floor). Pure aside from the atomic reads — no state mutation. */ void ha_vote_self_freshness(ha_state_t *ha, uint64_t local_config_tag, uint64_t local_applied_lsn, uint64_t *out_tag, uint64_t *out_lsn); /* * ha_candidate_advertise_lsn: the LSN a candidate advertises in a RequestVote / * PreVote payload (ha-commit-durability task 4.2). * * Returns the BARE durable WAL tip persist_durable_floor(p) — NOT the folded * max(durable_floor, last_known_committed_lsn) ha_vote_self_freshness computes * for the voter floor. A candidate may only claim a log position it can * durably back, so a node restarted with a truncated log cannot over-advertise * a stale volatile learned-committed value and win (design §1a/§5a). The * candidate's TAG axis still uses ha_vote_self_freshness (config epoch is * orthogonal to log durability). NULL-safe (→ 0). */ uint64_t ha_candidate_advertise_lsn(persist_t *p); /* ha_sync_applied_lsn_from_recovery (bug-848) — after local WAL recovery on * (re)start, set ha->applied_lsn to the position the member actually recovered * (persist_durable_floor(p)), instead of the ha_init default of 0. * * Why: a restarted standby advertises ha->applied_lsn to the primary in its * state report; the primary then streams the catch-up range [applied_lsn+1 .. * committed]. If a node that already holds [1..N] on disk reports 0, the * primary re-streams the WHOLE log and the receiver O_TRUNC-overwrites already- * consistent records (instead of Raft incremental catch-up of only the missing * suffix). This mirrors the value the election/PreVote path already trusts * (ha_candidate_advertise_lsn == persist_durable_floor), so the catch-up and * vote-freshness advertisements finally agree. NULL-safe; never lowers an * already-higher applied_lsn (monotonic). */ void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p); /* ha_sync_raft_log_from_recovery (bug-1042) — after local WAL recovery on a * (re)start, raise the raft LIBRARY's log_len / commit_index / term to the * recovered position. ha_init_raft (before the election) sources the library's * initial_log_len from persist_durable_floor, but on a standby-rejoin boot * persist_recover runs LATER (cluster_init_phase2), so the library would * otherwise stay at log_len=0 and reject every incoming AppendEntries (logOk * fails for prevLogIndex>=1), wedging the rejoined member as a permanently non- * converging standby. Call this right after persist_recover. Monotonic + NULL- * safe; clamps commit_index<=log_len. */ void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p); /* * ha_elect_evaluate_and_signal: evaluate accumulated ha_elect_ctx_t votes. * Sets ctx->result (1=primary, 0=standby) and posts ctx->done semaphore. * Applies quorum check, fresher-peer yield gate, and config_tag=0 fatal gate. * Requirements: 2.1, 2.7, 2.8 */ ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx); /* * handle_request_vote_reply: process an incoming reply to an OP_requestVote RPC. * * Called by each per-peer vote-request thread once the peer replies (or times out). * Updates the entry in ctx->votes[peer_idx] with the reply fields. * Increments ctx->votes_granted if vote_granted is non-zero. * Increments ctx->n_replied unconditionally. * Updates ctx->max_resp_tag / ctx->max_resp_lsn to track the freshest peer seen. * When quorum is reached (votes_granted >= quorum) OR all peers have replied * (n_replied >= n_peers), calls ha_elect_evaluate_and_signal() to post ctx->done. * * peer_idx: 0-based index into ctx->votes[] identifying the replying peer. * term: term the peer reported in its reply (used for fencing if > ha->term). * vote_granted: non-zero if the peer granted the vote, 0 if denied. * peer_config_tag: peer's current config_tag at reply time. * peer_lsn: peer's current WAL LSN at reply time. * * Requirements: 2.2, 2.3, 2.4, 2.5, 2.6, 9.1 */ void handle_request_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_config_tag, uint64_t peer_lsn); /* * ha_record_vote_reply: record-only vote-reply seam (bug-828). * * Records ctx->votes[peer_idx] (granted flag + peer_config_tag + peer_lsn) under * ctx->lock with the same idempotency guard as handle_request_vote_reply, and * updates the vote tally / freshest-responder tracking — but does NOT call * ha_elect_evaluate_and_signal (never posts ctx->done, never runs the legacy * promotion side-effects). The PRODUCTION vote_reply_dispatch path calls this so * the granting voter's advertised LSN reaches ha_compute_safe_committed_lsn via * ctx->votes[], while the raft library remains the sole production promotion * driver (raft_on_vote_reply → on_become_leader) — no double-promotion. * * peer_idx: 0-based index into ctx->votes[] (same convention as * handle_request_vote_reply; the absolute→local mapping is done by the caller). */ void ha_record_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_config_tag, uint64_t peer_lsn); /* * xce_probe_config_tag: single-RTT OP_probeConfigTag RPC to one peer. * Fills *tag_out with the peer's current config_tag on success. * Returns 0 on success, -errno on connection/protocol error. */ int xce_probe_config_tag(const char *host, uint16_t port, uint64_t *tag_out); /* * ha_wait_for_heartbeat_or_timeout: wait in CTRL_INIT for an incoming heartbeat. * * Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where * deadline = now(CLOCK_REALTIME) + election_timeout_ms + xorshift64_jitter * * Returns CTRL_STANDBY — heartbeat arrived; ha_do_demote() was called * (role is now CTRL_STANDBY). * Returns CTRL_ELECTING — deadline expired; caller must drive the election. * * The heartbeat handler posts heartbeat_seen only when role == CTRL_INIT. * OPCode_heartbeat is allowed through ha_dispatch_gate() in CTRL_INIT state * so that the primary's periodic heartbeat can reach the handler. * * Requirements: 1.1 (heartbeat → CTRL_STANDBY), 1.2 (timeout → CTRL_ELECTING). * Boundary: Controller FSM (ha.c). */ ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha); /* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout (Req 6.2–6.4, 6.8) ─ * ha_on_primary_ctrl_fd_rdhup: EPOLLRDHUP event handler. * Drops the primary channel (ha_drop_primary_ctrl_conn), transitions to * CTRL_ELECTING, truncates WAL to last_known_committed_lsn, fires * ha_start_election_async (via cluster when non-NULL). * Pass c == NULL to skip persist/election (unit test mode). */ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c); /* bug-1865: shut down + release the standby's primary channel. Never * close()s primary_ctrl_fd -- netio owns that socket. */ void ha_drop_primary_ctrl_conn(ha_state_t *ha); /* * ha_arm_heartbeat_timeout_vtask: start/re-arm the heartbeat watchdog thread. * Fires ha_on_primary_ctrl_fd_rdhup after election_timeout_ms + rand_jitter of * silence. Calling ha_on_heartbeat_received() resets the deadline. */ void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c); /* ha_discover_primary: step down from CTRL_ELECTING to CTRL_STANDBY when a * live primary message (NOP, config push, ctrlhello, WAL, snapshot) arrives. * Safe to call from any state — no-ops unless role == CTRL_ELECTING. */ void ha_discover_primary(ha_state_t *ha, cluster_t *c); /* ha_on_heartbeat_received: reset the heartbeat deadline (Req 6.3). */ void ha_on_heartbeat_received(ha_state_t *ha); /* ha_msg_term_is_stale: true iff msg_term is a real (non-zero) term stamp * strictly less than this node's own current ha->term -- i.e. the sender is * provably behind and any liveness-reset / step-down reaction to this * specific message would be acting on stale information. msg_term == 0 * means the message type carries no comparable term (e.g. ctrlhello, * snapshotTransfer) -- always non-stale by this check since there is * nothing to compare (callers keep their existing behavior for those). * * Exists to break a real liveness deadlock (bug-1418 family): a standby * that races its own real election against the peer that actually wins can * end up at a HIGHER term than the winning primary (a legitimate, if rare, * outcome of a near-simultaneous election race). The raft library correctly * rejects the winning primary's now-stale-term AppendEntries/heartbeats — * but every one of those messages was ALSO resetting the standby's own * liveness deadline before this check existed, so its liveness timeout * never fired and it could never start a fresh, higher-term election to * let the cluster self-heal. Callers use this to skip the liveness reset * (and ha_discover_primary step-down) specifically for messages already * known stale by term — never to change how the message itself is applied * or rejected. */ bool ha_msg_term_is_stale(ha_state_t *ha, uint64_t msg_term); /* * ha_observe_heartbeat_committed: reconcile-free monotonic update of a standby's * learned committed position from a primary heartbeat * (ha-heartbeat-committed-propagation, Req 2.1, 2.2, 2.3, 3.1, 3.3). * * Advances ha->last_known_committed_lsn to committed_lsn ONLY when it is * strictly greater than the current value (monotonic); when it advances, the * carried committed_config_tag is stored together with it (they move as a pair — * the tag never regresses independently). No-op when committed_lsn is 0 or * <= the current learned LSN. * * This is the reconcile-free sibling of cluster_apply_ctrlhello_reply: it MUST * NOT call ha_rejoin_reconcile, truncate the WAL, or modify ANY other state * (produced/persisted committed, role, term, applied, etc.). The value is * advisory — for observability and vote-floor freshness only. The caller (the * OP_nop receive path) applies the term-guard before invoking this; the helper * itself is term-agnostic and single-receiver-thread safe. */ void ha_observe_heartbeat_committed(cluster_t *c, uint64_t committed_lsn, uint64_t committed_config_tag); /* * ha_consume_heartbeat_committed: the standby-side OP_nop consume branch * (ha-heartbeat-committed-propagation, Req 1.2, 3.2, 3.4, 4.2). * * Safe to call unconditionally from the OP_nop handler with the decoded * heartbeatmsg body. It applies the two safety gates before delegating to the * monotonic ha_observe_heartbeat_committed: * - non-primary gate (3.4): a primary uses its own authoritative committed and * never consumes an inbound value; * - term-guard (3.2): a heartbeat whose advertised term is behind this node's * term is rejected, fencing a stale/partitioned primary from inflating the * vote-freshness floor. * NULL body, a failed gate, or a zero committed value updates nothing (4.2); * the helper never touches liveness or any other OP_nop behaviour (4.1 — the * caller invokes this AFTER the unchanged liveness/reply path). */ void ha_consume_heartbeat_committed(cluster_t *c, const heartbeat_msg_t *hb); /* * ha_consume_append_entries (ha-term-safety task 3.2): route ONE incoming * heartbeat (entry->n_entries==0) or replication (entry->n_entries>0, one entry) * AppendEntries through the raft library (raft_on_append_entries_request) so * UpdateTerm (G5), logOk (G2), and the (term,lsn) conflict-truncate (G3) happen * via the wired callbacks — replacing the term-blind term handling in * ha_consume_heartbeat_committed and the committed-boundary truncate in * ha_rejoin_reconcile. `entry` carries the AppendEntries metadata (prev tuple + * leader_commit) on its top-level fields. `leader_idx` is the sending leader's * raft peer index. Returns 0 on every path (never crashes the receive thread); * the AE result (term adoption / append / truncate / reject) is reflected in the * library + WAL state. The ACK reply that carries success/match_index is task 3.3. */ int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry, int leader_idx); /* ha_heartbeat_is_fresh: 1 if a primary message arrived within * heartbeat_timeout_ms (leader-stickiness predicate). Exposed for PreVote * denial-reason logging in cluster.c; pure read, no state change. */ int ha_heartbeat_is_fresh(ha_state_t *ha); /* ha_disarm_heartbeat_timeout_vtask: stop the heartbeat watchdog thread. */ void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha); /* * ha_start_primary_heartbeat: arm a repeating delaymgr task that sends * OPCode_nop to every standby in ha->standby_endpoints every * heartbeat_interval_ms. Keeps the standby-side watchdog from timing out * when the cluster is idle (no WAL entries, no config pushes). * Must be called after ha->role is set to CTRL_PRIMARY. * No-op when c == NULL (test mode). */ void ha_start_primary_heartbeat(cluster_t *c); /* * ha_set_reconnect_offload_for_test / ha_reconnect_offload_enabled: * Feature-flag accessors (raft-drive-async Task 2.1). The offload flag gates * whether primary_heartbeat_fn submits the peer reconnect to the slow lane (ON, * the production default — the delaymgr thread never blocks on connect) or runs * the legacy inline blocking connect on the delaymgr thread (OFF, retained only * so the TDD RED test can demonstrate the pre-change stall). These accessors * are compiled in ALL builds; the setter is used by tests to flip the flag. */ void ha_set_reconnect_offload_for_test(int on); int ha_reconnect_offload_enabled(void); /* * ha_set_snapshot_offload_for_test / ha_snapshot_offload_enabled: * TDD kill-switch for raft-drive-async task 3.2 snapshot slow-lane offload. * OFF (0, PRODUCTION DEFAULT): snapshot send runs INLINE on the caller * (io_worker/drive) thread — byte-for-byte the legacy path. The offload * lands DORMANT because it borrows a standby endpoint that is not yet * pool-lifetime-safe (bug-1197); so bug-1198 stays open by design until * Task 3.4 flips this ON together with the peer-control dispatch flag. * ON (1): snapshot send runs on a slow-lane worker, off the drive thread. */ void ha_set_snapshot_offload_for_test(int on); int ha_snapshot_offload_enabled(void); /* * ha_set_configpush_offload_for_test / ha_configpush_offload_enabled: * TDD kill-switch for raft-drive-async task 3.3 requester-less config-push * slow-lane offload. * OFF (0, PRODUCTION DEFAULT): the config push runs INLINE on the caller * (io_worker/drive) thread — byte-for-byte the legacy path (the * requester-less push_config_transport_only fan-out + its blocking * wait_all_with_timeout up to CONF_PUSH_TIMEOUT_MS). The offload lands * DORMANT because the lane task borrows standby endpoints it does not own * (bug-1197 family) and reads c->ha.standby_endpoints; moving the push to a * lane thread while the io_worker may RDHUP-free those endpoints is a * cross-thread UAF. Flag-ON is therefore only safe once Task 3.4 * establishes endpoint-lifetime discipline, and flipping it ON is owned by * Task 3.4 (the single gate that flips the snapshot + peerctrl flags too). * ON (1): the config push runs on a slow-lane worker, off the drive thread. * NOTE: the admin-upgrade path (push_config_to_standbys, which has a waiting * requester / barrier) is NOT affected by this flag. */ void ha_set_configpush_offload_for_test(int on); int ha_configpush_offload_enabled(void); /* * ha_set_demotion_offload_for_test / ha_demotion_offload_enabled: * TDD kill-switch (xce-audit-invariants controller pass, thread-role * violation 1 fix, 2026-07-18) for the demotion-convergence slow-lane * offload. ha_demotion_convergence_fn is scheduled directly as a delaymgr * task body; it must not block that thread (it also drives heartbeats and * election timeouts cluster-wide — ha-async-io.md thread-role contract). * ON: the truncate-above-committed_lsn + ha_self_restart (tlog_exit + * execv) body runs on a slow-lane worker; ha_demotion_convergence_fn * itself only does the fast admin work (CAS + slowlane_submit) and * returns immediately — this closes the thread-role violation. * OFF (0, PRODUCTION DEFAULT FOR NOW): legacy inline path — the blocking * body runs directly on the caller (delaymgr) thread. * Mirrors ha_set_reconnect_offload_for_test / ha_set_configpush_offload_for_test * (raft-drive-async); demotion convergence is a 4th instance of the SAME * slow-lane pattern, not a new mechanism — a prior VTASK_LIVE_BLOCK vtask- * pool attempt here (2026-07-10) deadlocked controller_unit_test and was * reverted (bug-1429). The slow lane's own submit contract cannot * deadlock a caller that way, but flipping this flag ON by default * reproduced a DIFFERENT real hang: raft_cb_become_follower (the caller) * is exercised by a very broad set of pre-existing controller_unit_test * fixtures that never call slowlane_init, so the offloaded work is * silently queued and never runs, and downstream code waiting on that * now-never-happening completion hangs. OFF-by-default matches the * ORIGINAL rollout discipline of all three reference offloads (every one * shipped OFF, flipped ON only in a later dedicated compatibility-audit * pass — raft-drive-async task 3.4). Flipping this default to ON is the * same kind of follow-up, out of scope here (bug-1450, buglog.json). */ void ha_set_demotion_offload_for_test(int on); int ha_demotion_offload_enabled(void); /* * ha_set_catchup_offload_for_test / ha_catchup_offload_enabled: * TDD kill-switch (xce-audit-invariants controller pass, thread-role * violation 2 fix, 2026-07-18) for the catch-up continuation slow-lane * offload. ha_catchup_continuation_fn is scheduled directly as a delaymgr * task body; catchup_stream_range -> ha_stream_wal_to_standby does blocking * WAL file reads + socket writes to a standby, which can stream a variable, * potentially large amount of WAL data — an unbounded block on the SAME * single thread that drives heartbeats and election timeouts cluster-wide * is a real spurious-election risk (ha-async-io.md thread-role contract). * ON: the streaming body runs on a slow-lane worker; ha_catchup_ * continuation_fn itself only does the fast admin work (slowlane_submit) * and returns immediately — this closes the thread-role violation. * c->ha.catchup_inflight[peer_idx] (set by ha_walack_on_ack before * scheduling) remains the authoritative per-standby single-flight guard; * the lane's own per-key dedup (key = peer_idx+1) is a redundant backstop, * same relationship as the reconnect offload. * OFF (0, PRODUCTION DEFAULT FOR NOW): legacy inline path — the blocking * body runs directly on the caller (delaymgr) thread. * 5th instance of the SAME established slow-lane pattern (see * ha_set_demotion_offload_for_test above for why NOT the vtask pool, and * for why OFF-by-default for now — the identical hazard applies here: * test_convergence_verification.c's pre-existing catch-up fixtures never * call slowlane_init, and flipping this flag ON by default reproduced a * real hang against them). Flipping the default to ON is a follow-up * compatibility audit, out of scope here (bug-1451, buglog.json). */ void ha_set_catchup_offload_for_test(int on); int ha_catchup_offload_enabled(void); /* * ha_set_peerctrl_dispatch_for_test / ha_peerctrl_dispatch_enabled: * Feature-flag accessors (raft-drive-async Task 3.1). The dispatch flag gates * whether the registered peer-control handler (peer_ctrl_msg_handler) runs its * routing body INLINE on the iomgr io_worker thread (OFF — the production * default for this commit; behavior is byte-for-byte identical to before the * task) or hands the decoded packet to the vtask pool via vtask_enq_classified * and returns promptly (ON — the target behavior, validated by the task 3.4 * deadlock gate before it becomes the default). These accessors are compiled * in ALL builds; the setter is used by tests to flip RED<->GREEN. * * DEFAULT: OFF (0). Production behavior is UNCHANGED by this commit. */ void ha_set_peerctrl_dispatch_for_test(int on); int ha_peerctrl_dispatch_enabled(void); /* * ha_peerctrl_body_probe: test-only observation seam (raft-drive-async Task * 3.1). When non-NULL it is invoked at the TOP of the peer-control routing * body (peer_ctrl_msg_handler_inline) with the endpoint, BEFORE any role * routing. Tests use it to (a) count body invocations, (b) capture the thread * the body ran on (proving inline-vs-pool execution), and (c) optionally park * the body to prove per-endpoint arrival-order serialization. NULL in * production (zero cost) — a plain global like ha_nop_req_hook. */ extern void (*ha_peerctrl_body_probe)(endpoint_t *ep); /* * ha_vep_teardown_probe / ha_vote_ep_teardown_for_test (bug-1197): test seams * for the deferred vote-ep teardown. ha_vep_teardown_probe (NULL in production) * fires with the real ep at the top of the teardown vtask body, before * endpoint_fini, so a test can assert teardown ran strictly after the last live * same-ep use. ha_vote_ep_teardown_for_test drives the flag-ON per-ep async * teardown branch (mark + iomgr_unmonitor + enqueue) for a single ep without * staging a full election. Both compiled only under XCE_TESTING. */ extern void (*ha_vep_teardown_probe)(endpoint_t *ep); void ha_vote_ep_teardown_for_test(ha_state_t *ha, endpoint_t *ep); /* * ha_peerctrl_predispatch_probe (bug-1197): test seam fired inside the dispatch * path AFTER the tearing_down check passes but BEFORE the pool enqueue (within * the ep->dispatch_inflight bracket). A test installs it to park a dispatch in * exactly the drained window and prove endpoint_mark_tearing_down blocks until * the enqueue completes. NULL in production; compiled only under XCE_TESTING. */ extern void (*ha_peerctrl_predispatch_probe)(endpoint_t *ep); /* ── Task 2.4: Non-blocking async election initiator ────────────────────── */ /* * ha_start_election_async: initiate a quorum election without blocking. * * Steps: * 1. Store n_peers in ha->n_peers. * 2. For single-node (n_peers == 1): call ha_elect_evaluate_and_signal directly * with a self-only ha_elect_ctx_t — shortcircuit, no iomgr/delaymgr needed. * 3. For multi-node: for each peer in peers[], allocate vote_ep_ctx_t, init * endpoint with client_ssl_ctx, set on_vote_connected + vote_ep_err_handler, * call endpoint_connect + iomgr_monitor; skip peer on connect failure. * Allocate election_timeout_ctx_t (cancelled=0) and schedule * delay_task_ms(delaymgr, election_timeout_ms, 1, election_timeout_fn, ctx). * 4. Returns immediately; election result delivered via ha_elect_evaluate_and_signal. * * Observable: returns in < 5ms; endpoints registered in iomgr; watchdog in delaymgr. * Requirements: 1.3 (non-blocking), 2.1 (single-node shortcircuit), 2.4 (async vote) * Boundary: ha.c (election orchestration) */ void ha_start_election_async(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name); /* * ha_start_prevote_async: start a side-effect-free PreVote round before any * real election (ha-prevote task 4.1). * * Snapshots self tag/lsn, calls raft_begin_pre_vote (self-grant + per-peer * preVoteRequest send), and arms a round timeout bounded by election_timeout_ms * (Req 5.4). The node stays CTRL_STANDBY and mutates NO persistent state * (no term / voted_for / role change) for the duration of the round. Replies * are fed to the reply handler (pre_vote_reply_dispatch); on a majority WIN the * unchanged real-election sequence runs (truncate + CTRL_ELECTING + * ha_start_election_async), on a LOSS / round-timeout the node stays * CTRL_STANDBY and re-arms the liveness watchdog. * * The failover triggers are rewired to call this in task 4.2; until then the * round is inert in production. c may be NULL in unit-test mode. * Requirements: 1.1, 1.4, 1.5, 5.4, 7.3. */ void ha_start_prevote_async(ha_state_t *ha, cluster_t *c); /* * ha_begin_failover: single failover entry point (ha-prevote task 4.2). * * Called by both failover triggers (ha_on_primary_ctrl_fd_rdhup and, through * it, the liveness-timeout poll) AFTER connection-teardown bookkeeping and the * config-less voter-only guard. Starts a side-effect-free PreVote round * (ha_start_prevote_async); does NOT truncate the WAL or transition to * CTRL_ELECTING — those destructive steps are deferred to PreVote success. * c may be NULL in unit-test mode. Requirements: 1.1, 4.3, 4.4. */ void ha_begin_failover(ha_state_t *ha, cluster_t *c); /* * ha_election_stagger_ms: per-node desync delay (ms) for a FAILOVER re-election. * * Mirrors the startup stagger (myapp.c): my_priority * election_priority_step_ms * plus a fresh random jitter in [0, election_jitter_max_ms]. Applied before the * PreVote start on the failover path so two survivors that lose the primary at * the same instant do not campaign in lock-step and split the vote forever * (Raft §5.2 randomized election timeout; bug-606). Re-randomized each call — * the jitter_seed advances — so repeated collisions still converge. */ uint32_t ha_election_stagger_ms(ha_state_t *ha); /* * ha_jittered_election_delay: the per-CAMPAIGN election-timeout watchdog delay, * randomized to the canonical Raft interval [base, 2*base] (Ongaro §3.4) and * re-drawn on every campaign. * * task 5.7: the campaign watchdog (raft_start_election → on_schedule_timeout → * raft_cb_schedule_timeout) was previously armed with a FIXED election_timeout_ms. * Under cold-connection churn a survivor's single needed grant can land a hair * AFTER a constant deadline on EVERY campaign — a resonance that times out, * re-escalates the term, and then drops the now-stale grant (DropStaleResponse) * forever (unbounded term climb; the final round-4 chaotic-soak stall). * Randomizing the watchdog per campaign breaks the resonance: some campaign's * deadline falls AFTER its grant RTT, the grant lands in-window, quorum is * reached. Independent of election_jitter_max_ms (which tunes only the * sub-second startup/stagger jitter) so it still spreads when jmax is small/0. * Never returns below `base`, so genuine-loss failover is never sped up past the * configured election_timeout_ms floor. base==0 returns 0 (no modulo-by-zero). */ uint32_t ha_jittered_election_delay(ha_state_t *ha, uint32_t base); /* * ha_dispatch_repl_action_for_standby: evaluate the replication decision for * one standby and execute the resulting action. * * Reads ha->standby_reported_tag[peer_idx] and ha->standby_reported_lsn[peer_idx] * (stored by ha_process_standby_state on the previous call), calls * ha_evaluate_repl_action, and acts on the result: * - HA_REPL_PUSH_CONFIG: In production builds, calls push_config_to_standbys(c, …). * In XCE_TESTING builds, calls ha_repl_push_config_for_standby_hook(c, peer_idx). * - HA_REPL_STREAM_WAL: calls ha_stream_wal_to_standby(ep, wal_path, standby_lsn, * stream_ceiling_lsn) to batch-stream WAL entries via xce_request() * (non-blocking). stream_ceiling_lsn (bug-1479 gap 3) is * ha_compute_repl_stream_ceiling_lsn(committed_lsn, last_written, * checkpoint_lsn) when c is non-NULL — the leader's own log tail, NOT * just committed_lsn, so a standby whose own ack is required to advance * committed_lsn can still receive the entries it needs to ack. Falls * back to committed_lsn when c is NULL (test fixtures with no real * persist_t). In XCE_TESTING builds, calls ha_repl_stream_wal_hook if * non-NULL. * - HA_REPL_SEND_SNAPSHOT: calls ha_send_snapshot_to_standby(ep, snap_path, lsn). * In XCE_TESTING builds, calls ha_repl_send_snapshot_hook(peer_idx) if non-NULL. * - HA_REPL_CONVERGED: no-op. * * One action per heartbeat cycle per standby (design §Unified Heartbeat-Driven * Protocol). CONVERGED is a no-op. * * ep — outbound endpoint to the standby (used for STREAM_WAL; may be * NULL in tests that only exercise PUSH_CONFIG or CONVERGED paths) * peer_idx — 0-based index into ha->standby_reported_* arrays * committed_tag — primary's committed config_tag (from c->myconf or test fixture) * committed_lsn — primary's committed WAL LSN (from c->persist or test fixture) * oldest_wal_lsn— oldest retained WAL LSN (0 = all WAL retained, skip snapshot) * * Returns 0 on success; negative on internal error. * Requirements: 1.4, 5.1–5.5 (ha-repl-hardening spec) */ int ha_dispatch_repl_action_for_standby(ha_state_t *ha, cluster_t *c, endpoint_t *ep, int peer_idx, uint64_t committed_tag, uint64_t committed_lsn, uint64_t oldest_wal_lsn); /* * ha_send_ctrlhello_to_all_standbys: iterate ha->standby_endpoints and send * an OP_hello CtrlHelloRequest to each connected standby. * This triggers the existing catch-up replication path in handle_ctrlhello * (Req 7.2). Defined in cluster.c; called from ha_elect_evaluate_and_signal * in the post-election path. */ void ha_send_ctrlhello_to_all_standbys(cluster_t *c); /* ── P3-6: Test-only failover shim ────────────────────────────── */ #ifdef XCE_TESTING /* * ha_phase2_hook: test-only seam for cluster_init_phase2. * When non-NULL, ha_elect_evaluate_and_signal() calls this INSTEAD OF the * real cluster_init_phase2() so unit tests can count phase2 calls without * requiring a real cluster_t. Pattern mirrors xce_vote_request_hook. */ extern int (*ha_phase2_hook)(void *c); /* * ha_send_ctrlhello_hook: test-only seam for ha_send_ctrlhello_to_all_standbys. * When non-NULL, ha_elect_evaluate_and_signal() calls this in the post-election * path instead of the real function (cluster.c not linked in test binary). * Requirements: 7.2. */ extern int (*ha_send_ctrlhello_hook)(void *c); /* * ha_push_config_hook: test-only seam for push_config_to_standbys. * When non-NULL, ha_elect_evaluate_and_signal() calls this in the post-election * path instead of the real function. Requirements: 7.1. */ extern int (*ha_push_config_hook)(void *c); /* * ha_heartbeat_req_hook: test-only seam for the per-standby HA heartbeat send * in primary_heartbeat_fn (Task 3.1). When non-NULL, primary_heartbeat_fn * calls this INSTEAD OF xce_ha_heartbeat_req, passing the SAME four values it * stamps onto the wire — the primary's CURRENT own term / committed_lsn / * committed_config_tag / elected_at_ms. A test installs it to capture those * args and assert the heartbeat carries the primary's own committed state and * election time (Req 1.1, 3.4, 12.2). * * Precedence (XCE_TESTING only): ha_heartbeat_req_hook (if set) takes priority, * else the legacy ha_nop_req_hook (if set), else the real xce_ha_heartbeat_req. * Keeping ha_nop_req_hook as a fallback preserves the existing liveness tests. */ extern int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term, uint64_t committed_lsn, uint64_t committed_config_tag, uint64_t elected_at_ms); /* * ha_reconnect_connect_hook: test-only seam for the BLOCKING half of the * offloaded peer reconnect (raft-drive-async Task 2.1). The reconnect splits * into a non-blocking prep (alloc endpoint, set callbacks) on the delaymgr * tick and a blocking connect+monitor that runs on a slow-lane worker. When * non-NULL, the lane worker calls this INSTEAD OF the real bounded * endpoint_connect + iomgr_monitor, so a test can drive the reconnect path * deterministically (e.g. park the worker to simulate an unreachable peer) * without a real socket. * * Signature: int hook(cluster_t *c, int peer_idx) * returns 0 to simulate a successful connect, <0 to simulate failure. * The lane worker clears reconnect_inflight[peer_idx] after this returns * regardless of outcome, so dedup never wedges (Req 2.5). */ extern int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx); /* * ha_repl_push_config_for_standby_hook: test-only seam for the per-standby * config push triggered by the heartbeat-reply decision loop (Task 3.3). * * When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF * push_config_to_standbys() when the decision is HA_REPL_PUSH_CONFIG, so unit * tests can verify the wiring without requiring a real cluster_t or network. * * Signature: int hook(void *c, int peer_idx) * c — cluster_t* (may be NULL in test mode) * peer_idx — 0-based index of the standby that triggered the push * Requirements: 1.4 (ha-repl-hardening) */ extern int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx); /* * ha_repl_stream_wal_hook: test-only seam for ha_stream_wal_to_standby. * * When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF * ha_stream_wal_to_standby() when the decision is HA_REPL_STREAM_WAL, so unit * tests can count batch-send invocations without requiring a real endpoint or * WAL directory. * * Signature: int hook(void *ep, const char *wal_path, * uint64_t standby_lsn, uint64_t committed_lsn) * ep — endpoint_t* (may be NULL in test mode) * wal_path — WAL directory path * standby_lsn — standby's last applied LSN * committed_lsn— primary's committed LSN * Requirements: 5.1–5.5 (ha-repl-hardening) */ extern int (*ha_repl_stream_wal_hook)(void *ep, const char *wal_path, uint64_t standby_lsn, uint64_t committed_lsn); /* * ha_repl_send_snapshot_hook: test-only seam for ha_send_snapshot_to_standby. * * When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF * ha_send_snapshot_to_standby() when the decision is HA_REPL_SEND_SNAPSHOT, * so unit tests can verify the wiring without requiring a real endpoint or * snapshot directory. * * Signature: int hook(int peer_idx) * peer_idx — 0-based index of the standby that triggered the snapshot * Requirements: 4.1–4.5 (ha-repl-hardening) */ extern int (*ha_repl_send_snapshot_hook)(int peer_idx); /* * ha_tcp_connect_hook: test-only seam for the raw TCP connect inside * ha_find_and_connect_to_primary. When non-NULL, the function calls this * INSTEAD OF the real getaddrinfo/connect path, allowing unit tests to inject * socketpair fds without real network I/O. * Signature: int hook(const char *host, int port) * Returns fd >= 0 on success, or -errno on failure. * Requirements: task 9.2. */ extern int (*ha_tcp_connect_hook)(const char *host, int port); /* Test-only shim: demote then re-promote; returns new term. * Exercises the manual failover state machine without a live cluster_t. * Calls ha_do_demote() then ha_do_promote(); validates role transitions * and term monotonicity. See design doc P3-6 and src/docs/persistence-and-ha-design.md. */ uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha); /* * ha_rdhup_election_hook: test-only seam for ha_start_election_async in the * RDHUP failover path. * * When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls this INSTEAD OF the real * ha_start_election_async() so unit tests can verify the RDHUP path triggers * async re-election without requiring a live iomgr/delaymgr. * * The hook receives the same arguments that would be passed to ha_start_election_async. * Requirements: 6.3, 6.5, 9.3. */ extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name); /* * ha_elect_ctx_release_for_test (ha-elect-ctx-lifecycle task 2.1): drive the * static ha_elect_ctx_release() release/free helper directly, so unit tests * can verify the reference-count-to-zero mechanism without staging a full * election round. */ void ha_elect_ctx_release_for_test(ha_elect_ctx_t *ctx); /* * vote_ep_ctx_set_cluster_for_test: set the cluster_t pointer inside the * vote_ep_ctx_t stored at ep->data. Used by unit tests that need the * promoted-endpoint dispatch path to reach ha_promote_to_primary(c) with a * live cluster pointer. * The vote_ep_ctx_t struct is internal to ha.c; this helper avoids exposing it. * Requirements: task 7.3 (ha-peer-connection-reuse). */ void vote_ep_ctx_set_cluster_for_test(struct endpoint_t *ep, cluster_t *c); /* * ha_restart_hook: test-only seam for ha_self_restart's process re-execution * (ha-convergence-recovery Req 1.6). * * When non-NULL, ha_self_restart() invokes this INSTEAD OF execv() so unit * tests can observe the eventual restart deterministically (the demotion * convergence task and rejoin reconciliation both end in ha_self_restart). * The hook's return value is ha_self_restart's return value on the * test-success path; a NON-ZERO return models an exec failure and routes * ha_self_restart to the FATAL contract via ha_fatal_hook (Req 1.4). * * Signature: int hook(cluster_t *c) * Returns 0 to model a successful restart; non-zero to model exec failure. */ extern int (*ha_restart_hook)(cluster_t *c); /* * ha_fatal_hook: test-only seam for the FATAL (abort) contract of * ha_self_restart (ha-convergence-recovery Req 1.4). * * When non-NULL, ha_self_restart() invokes this INSTEAD OF abort() when the * restart cannot be performed, so a test binary can assert the FATAL path was * taken without the process dying. ha_self_restart returns after invoking it * (test builds only — production aborts). * * Signature: void hook(const char *msg) */ extern void (*ha_fatal_hook)(const char *msg); /* * raft_cb_become_follower_with_cluster_for_test: drive the follower path with * a LIVE cluster pointer (ha-convergence-recovery task 2.1 demotion-sequence * tests). Unlike raft_cb_become_follower_for_test (which forces raft_c = NULL * to suppress production calls), this shim sets ha->raft_c = c first so the * cluster-dependent demotion-convergence branch (task 2.2: hook disable + * scheduled truncate/restart) is exercised against a real persist/hooksys/ * delaymgr fixture. Mirrors the real-cluster seam ha_promote_to_primary(c) * used by the promotion-side lifecycle tests. */ void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term); /* * raft_cb_become_leader_with_cluster_for_test: drive the leader promotion path * with a LIVE cluster pointer (fix-promotion-standby-registration). Unlike * raft_cb_become_leader_for_test (which forces raft_c = NULL and so skips the * c-dependent standby-registration branch entirely), this shim sets * ha->raft_c = c first so the promotion-time repl_add_standby loop — which * registers each surviving vote endpoint (votes[i].ep != NULL) carrying a * peer pointer (votes[i].peer != NULL) into the WAL replicator — is exercised * against a real persist/repl/commit fixture. Symmetric with * raft_cb_become_follower_with_cluster_for_test. */ void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, uint64_t term); /* * ha_walack_continuation_for_test: test-only entry point for the ACK-driven * catch-up continuation logic (ha-convergence-recovery task 4.x, Req 5.1, 5.2, * 7.1). * * The production trigger lives in the walAck branch of * repl_ack_decode_and_dispatch (ha.c), which decodes a real Cap'n Proto * WalAckReply off the wire — too heavy to drive from a unit test. This shim * exposes the SAME post-ACK decision the walAck branch performs after * repl_on_ack, so the continuation / single-flight / parity behaviors can be * exercised directly against a unit fixture: * * target = max(committed_lsn, last_written) (same derivation as ha.c:2771-2776) * if acked_lsn < target → schedule ONE continuation that streams * [acked_lsn+1, target] via the catch-up streaming * path, UNLESS ha.catchup_inflight[peer_idx] is set * (single-flight, Req 5.2); the continuation must not * dispatch any heartbeat/stateReport action (Req 5.1). * if acked_lsn >= target → repl_complete_catchup(&c->persist.repl, ep), which * clears the standby's in_catch_up (Req 7.1). * * c — cluster_t* (real persist + repl registry; never NULL in tests) * ep — repl registry endpoint key for the standby (the same pointer * passed to repl_add_standby); identifies the standby slot. * peer_idx — 0-based standby slot index (into ha.catchup_inflight[]). * acked_lsn— the LSN the standby just acknowledged. * * Returns 0 on the no-op / parity paths and on a successfully scheduled * continuation; negative on a scheduling failure. * * RED scaffold (task 4.1): ha_convergence.c currently records the ACK only and * neither schedules a continuation nor calls repl_complete_catchup — task 4.2 * implements the real decision. The continuation / single-flight / parity tests * therefore FAIL until 4.2 lands. */ int ha_walack_continuation_for_test(cluster_t *c, void *ep, int peer_idx, uint64_t acked_lsn); /* * ha_quorum_transition_log_hook: test-only emission-count seam for * ha_log_quorum_transition (ha-convergence-recovery task 5.x, Req 8.1-8.3). * * There is NO log-capture harness in this testsuite, so the quorum-transition * tests use a counter/edge-observable instead (the task permits "log-capture OR * counter-hook pattern"). When non-NULL, ha_log_quorum_transition invokes this * hook EXACTLY ONCE on each transition it WARNs about, passing the new state and * the connected/required counts that go into the WARN line. Tests count * invocations to assert "logged once per transition" (Req 8.3) and inspect the * arguments to assert the loss line carries the right counts (Req 8.1). * * Signature: void hook(int lost, int connected, int required) * lost — 1 on a quorum-LOSS edge, 0 on a quorum-RESTORE edge. * connected — connected standby count reported in the WARN line. * required — required standby count (quorum threshold) in the WARN line. * * RED scaffold (task 5.1): ha_log_quorum_transition is a no-op stub that never * invokes this hook, so the transition tests' emission-count assertions FAIL * until task 5.2 implements the real edge detection. */ extern void (*ha_quorum_transition_log_hook)(int lost, int connected, int required); #endif #endif /* HA_H */