lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/ha.c
Rongsong Shen 8d1ee8b761 feat(ha-entity-snapshots): track in-flight snapshot transfer LSN per peer (task 4.1) 
Adds snapshot_inflight_lsn[MAX_HA_CANDIDATES] to ha_state_t, alongside the
existing snapshot_inflight[] flag it mirrors exactly: set to the
checkpoint's LSN at ha_snapshot_send_dispatch before a transfer's first
byte goes out, cleared to 0 at every completion/failure path
(snapshot_lane_fn's normal completion, and all 3 early-return failure
sites: malloc failure, slowlane dedup-refuse, slowlane submit failure).

This is the foundation task 4.3 will consult from rotate_snapshots to
avoid deleting a checkpoint file a peer is still mid-transfer reading
from -- the one genuinely new retention guard this feature needs (the
existing write_snapshot-before-trim_wal_segments sequencing already
satisfies the WAL-trim side of Requirement 3).

Proven via a new gated unit test (test_ha_liveness.c) that observes the
tracked LSN mid-flight while the send body is parked at a test gate, not
just before/after -- confirms the lifecycle ordering, not just the
endpoints.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JDYFbrgyDHbvNYcxApKhrX
2026-07-01 07:26:44 +00:00

7979 lines
359 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#define _GNU_SOURCE
#include "ha.h"
#include <arpa/inet.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <netdb.h>
#include <poll.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>
#include <zlib.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
/* Circular-include safety: cluster_def.h includes ha.h (include-guarded),
* and ha.h only forward-declares cluster_t — no cycle. */
#include "xce/cluster_def.h"
#include "xce/errnox.h"
#include "controller.capnp.h"
#include "persist.h"
#include "request.h"
#include "utils.h"
/* ── HA dispatch gate error codes (defined here; declared extern in errnox.h) ── */
int XCE_ERR_NOT_READY = E_XCE_NOT_READY;
int XCE_ERR_STANDBY = E_XCE_STANDBY;
/* No-op stubs removed: s_real_cbs (below) is used exclusively since task 6.2. */
/* ── Forward declarations needed by real raft callbacks (Task 6.2) ────────
* These static functions and types are defined later in this file; declared
* here so the callback implementations can reference them. */
/* vote_ep_ctx_t: per-peer context for outgoing vote endpoints (full def below) */
typedef struct {
ha_state_t *ha;
cluster_t *c;
ha_elect_ctx_t *ctx;
int peer_idx;
uint64_t my_term;
uint64_t my_tag;
uint64_t my_lsn;
/* ha-term-safety task 1.2: candidate's own Raft (lastLogTerm, lastLogIndex)
* advertised in its RequestVote/PreVote so a voter's library logOk gate can
* compare. my_last_log_index == candidate WAL log_len; my_last_log_term ==
* term of that last entry (recovered/live WAL term). */
uint64_t my_last_log_term;
uint64_t my_last_log_index;
const char *my_name;
} vote_ep_ctx_t;
/* election_timeout_ctx_t: delaymgr callback context for raft election timeout */
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
ha_elect_ctx_t *ctx;
} election_timeout_ctx_t;
/* cluster_init_phase2 / push_config_transport_only /
* ha_send_ctrlhello_to_all_standbys: defined in cluster.c / config_repl.c;
* only referenced in production builds.
* (XCE_TESTING uses hook shims and does not link cluster.c.)
* ha.c uses push_config_transport_only ONLY (task 5.2 remediation r1):
* every ha.c push site is reachable from peer_ctrl_msg_handler / io_worker
* (vote replies, heartbeat replies), where the push_config_to_standbys
* barrier wait would block the single iomgr epoll thread — and the config
* ACK it waits for would arrive on that same blocked thread (structural
* deadlock). Convergence is owned by the OP_replConfig anti-entropy pull
* and the heartbeat decision loop, not by these best-effort pushes. */
#ifndef XCE_TESTING
extern int cluster_init_phase2(cluster_t *c);
extern void ha_send_ctrlhello_to_all_standbys(cluster_t *c);
extern int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag, int *n_out);
#endif
/* raft-drive-async Task 3.3: requester-less config-push offload router (defined
* below, near the snapshot helpers). Forward-declared because
* raft_cb_become_leader (earlier) is one of its two call sites. */
static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag);
/* config_repl.c is linked in BOTH production and XCE_TESTING builds.
* ha-term-safety task 4.3 (remediation r1): the commit-advance hook activates a
* newly-committed config marker (Finding B); ha_dispatch_gate / the pre-promotion
* gate read the serve-fence (Finding C). */
extern int config_activate_committed(cluster_t *c);
extern int config_serve_fenced(const cluster_t *c);
/* task 4.3 (remediation r2, Finding D): the commit-advance path schedules an
* off-iomgr retry that re-activates a newly-committed config marker and clears a
* stale serve-fence (non-blocking ordq_put only). */
extern int config_schedule_activate_retry(cluster_t *c);
static void on_vote_connected(endpoint_t *ep);
static void on_pre_vote_connected(endpoint_t *ep);
static int vote_ep_err_handler(void *ep_raw);
/* bug-1197: defer a vote-ep teardown onto the per-ep FIFO of the vtask pool so
* it runs strictly after the last live use of the ep (flag-ON loser path). */
static void vote_ep_teardown_async(ha_state_t *ha, endpoint_t *ep);
/* ha-single-leader-role task 2.2: shared promotion seam — every road to
* PRIMARY registers the persist hooks and arms the own-term gate. */
static void ha_register_persist_hooks(cluster_t *c);
static void ha_arm_own_term_gate(cluster_t *c);
int ha_heartbeat_is_fresh(ha_state_t *ha); /* exposed for PreVote denial-reason logging (cluster.c) */
static int primary_heartbeat_fn(void *arg);
static void on_standby_connected(endpoint_t *ep);
static int standby_ep_err_handler_primary(void *ep_raw);
/* ── Real raft callback implementations (Task 6.2) ─────────────
*
* These are registered as s_real_cbs in ha_init, replacing the no-op stubs.
* They are wired into the active election path in task 6.3.
*
* All callbacks receive ctx = ha_state_t*.
* ha->raft_c is set by ha_start_election_async before any raft_* calls.
* When ha->raft_c is NULL (unit test mode), callbacks that require cluster_t
* are no-ops.
*/
/*
* raft_cb_send_vote_request: encode and send a capnp VoteRequest to peer.
*
* peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[].
* Uses the existing iomgr vote-send path (on_vote_connected / vote_ep_ctx_t).
* Reads the current election context from ha->elect_ctx (set before raft_start_election).
* Returns 0 on success; non-zero if no cluster context or alloc failure.
*/
static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c || !c->myconf) return 1; /* test mode */
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
if (peer_idx < 0 || peer_idx >= pcfg->candidates_count)
return 1;
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (!ectx) return 1; /* no election in progress */
xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx];
/* Compute the local index of peer_idx within ectx->votes[]:
* votes[] has n_peers - 1 = n_other entries for peers excluding self.
* Map: votes_idx = peer_idx if peer_idx < self_idx, else peer_idx - 1. */
int self_idx = ha->my_priority; /* self_idx == my_priority by convention */
int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1;
if (votes_idx < 0 || votes_idx >= ectx->n_peers)
return 1;
/* Allocate per-peer context */
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
if (!vctx) {
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
return 1;
}
vctx->ha = ha;
vctx->c = c;
vctx->ctx = ectx;
vctx->peer_idx = votes_idx;
vctx->my_term = term;
/* tag and lsn come from votes[0] which holds our own values */
vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0;
vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0;
/* ha-term-safety task 1.2: advertise our Raft (lastLogTerm, lastLogIndex).
* lastLogIndex == our WAL log length (the library's log_len == WAL tip LSN);
* lastLogTerm == the term STAMPED ON THAT LAST ENTRY.
*
* bug-851 / task 5.5: lastLogTerm MUST be the term of the entry at log_len —
* sourced via persist_entry_term, the SAME accessor voters use to compute
* their own last-log term (on_get_entry_term → persist_entry_term, ha.c:1556).
* The old stopgap advertised c->persist.term (the persist layer's RUNNING
* term), which is stale/0 after a restart and does NOT equal the last entry's
* stored term. That made a freshly-restarted but freshest survivor advertise
* a too-low lastLogTerm, so every voter's logOk rejected it (asymmetric with
* how the voter computes its own term) and the election stalled. */
vctx->my_last_log_index = ha->raft.log_len;
vctx->my_last_log_term = (c && ha->raft.log_len > 0)
? persist_entry_term(&c->persist, ha->raft.log_len)
: 0;
vctx->my_name = c->myconf ? c->myconf->name : "";
/* Get a fresh endpoint */
endpoint_t *ep = endpoint_get();
if (!ep) {
free(vctx);
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
return 1;
}
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
if (c) endpoint_set_peer(ep, c->myhost);
ep->data = vctx;
ep->iomgr = &c->iomgr;
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
if (ep->ssl)
SSL_set_connect_state(ep->ssl);
}
endpoint_set_on_connected(ep, on_vote_connected);
endpoint_set_err_handler(ep, vote_ep_err_handler);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) {
endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (not monitored) */
if (!c)
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
return 1;
}
rc = iomgr_monitor(ep);
if (rc < 0) {
endpoint_unref(ep); /* bug-1298: fini frees ep->data (vctx), then frees ep */
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
return 1;
}
/* Track endpoint for promotion/close after election result.
* Also record the candidate pointer: raft_cb_become_leader uses
* votes[i].peer->hostname / ->port to register the surviving vote
* endpoint with the WAL replicator (repl_add_standby). Without this,
* the promotion registration block (guarded by votes[i].peer != NULL)
* is skipped — the standbys never enter c->persist.repl.standbys[],
* commit/quorum see zero standbys, and every write fails NOT_READY.
* `peer` points into c->myconf->peers_cfg.candidates[] (persistent;
* outlives the election — repl_add_standby copies host/port).
* (fix-promotion-standby-registration, Req 1.1) */
ectx->votes[votes_idx].ep = ep;
ectx->votes[votes_idx].peer = peer;
return 0;
}
/*
* raft_cb_send_pre_vote_request: encode and send a capnp PreVote request to a
* peer (ha-prevote task 4.1).
*
* Mirrors raft_cb_send_vote_request exactly — same iomgr connect path, same
* vote_ep_ctx_t carrier, same per-peer endpoint tracking — but the connected
* handler (on_pre_vote_connected) emits OPCode_preVoteRequest instead of
* OPCode_requestVote. prospective_term is the candidate's CURRENT+1 term; it
* is NEVER persisted (PreVote is a side-effect-free straw poll).
*
* peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[].
* Returns 0 on success; non-zero in test mode (no cluster) or on alloc failure.
*
* Transport-side reply routing for preVoteReply is wired separately in task 5.1;
* this callback only emits the request.
*/
static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx,
uint64_t prospective_term)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c || !c->myconf) return 1; /* test mode — caller owns transport */
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
if (peer_idx < 0 || peer_idx >= pcfg->candidates_count)
return 1;
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (!ectx) return 1; /* no round metadata carrier */
xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx];
/* Map absolute peer_idx → votes[] local index (same convention as the
* real-vote send callback). */
int self_idx = ha->my_priority;
int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1;
if (votes_idx < 0 || votes_idx >= ectx->n_peers)
return 1;
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
if (!vctx) return 1;
vctx->ha = ha;
vctx->c = c;
vctx->ctx = ectx;
/* Carry the ABSOLUTE peer index: the PreVote tally in raft_state_t is keyed
* by absolute cluster index (raft_begin_pre_vote loops over i in
* [0, n_peers) and raft_on_pre_vote_reply dedups by that same index). */
vctx->peer_idx = peer_idx;
vctx->my_term = prospective_term;
vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0;
vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0;
/* ha-term-safety task 1.2 + bug-851/task 5.5: advertise lastLogTerm as the
* term of the entry at log_len (persist_entry_term — the same source voters
* use), NOT the stale c->persist.term. Identical sourcing to the real-vote
* send path above; see the full rationale there. */
vctx->my_last_log_index = ha->raft.log_len;
vctx->my_last_log_term = (c && ha->raft.log_len > 0)
? persist_entry_term(&c->persist, ha->raft.log_len)
: 0;
vctx->my_name = c->myconf ? c->myconf->name : "";
endpoint_t *ep = endpoint_get();
if (!ep) { free(vctx); return 1; }
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
if (c) endpoint_set_peer(ep, c->myhost);
ep->data = vctx;
ep->iomgr = &c->iomgr;
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
if (ep->ssl)
SSL_set_connect_state(ep->ssl);
}
endpoint_set_on_connected(ep, on_pre_vote_connected);
endpoint_set_err_handler(ep, vote_ep_err_handler);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) { endpoint_unref(ep); return 1; } /* bug-1298: fini+free */
rc = iomgr_monitor(ep);
if (rc < 0) { endpoint_unref(ep); return 1; } /* bug-1298: fini+free */
return 0;
}
/*
* raft_cb_vote_eligible: freshness gate delegated from raft library.
*
* from_idx is the absolute candidate index of the requester.
* Applies the same checks as handle_request_vote:
* - role gate: PRIMARY → deny
* - fresh heartbeat on STANDBY → deny
* - in_catch_up → deny
* - freshness: candidate config_tag/lsn must be ≥ ours
* Returns 1 to grant, 0 to deny.
*/
static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term)
{
(void)from_idx; (void)term;
ha_state_t *ha = (ha_state_t *)ctx;
/* Role gate */
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_PRIMARY)
return 0;
/* Fresh heartbeat on STANDBY WITH a live primary connection → primary is
* alive, deny. BOTH conditions are required (mirrors the handle_request_vote
* gate at ~ha.c:2205). Without the primary_ctrl_fd>=0 guard a node whose
* primary has DIED keeps a falsely-"fresh" status: ha_heartbeat_is_fresh
* derives from last_activity, which ha_arm_heartbeat_timeout_vtask re-arms to
* now on every failover retry — so the node would deny every legitimate
* higher-term PreVote for minutes, preventing convergence (bug-660).
* primary_ctrl_fd is -1 after the primary RDHUPs / the liveness watchdog
* fires / on a fresh boot, so a dead-or-absent primary correctly does NOT
* deny. */
if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(ha))
return 0;
/* In catch-up WHILE a live primary is feeding us → deny.
*
* raft-ha-core-safety task 5.3 (bug-1272): this deny used to be
* UNCONDITIONAL, which deadlocks the election under overlapping churn — after
* several rapid kills the only fully-up-to-date node is killed and the two
* survivors are both restarted-and-lagging (in_catch_up=1) with NO live
* primary left to catch up FROM (so the flag never clears). Each holds the
* full committed prefix and is a SAFE candidate, yet they mutually deny every
* PreVote → permanent stall (round-N election stall, 7.2 soak).
*
* Gate the deny on a LIVE primary, mirroring the heartbeat-fresh deny above:
* the purpose of the in_catch_up deny is to avoid disrupting a primary that is
* ACTIVELY replicating to us — not to fence a survivor whose primary is gone.
* Once the primary is unreachable, raft_log_ok (already applied in Step 2 of
* raft_on_pre_vote_request / raft_on_vote_request, BEFORE this callback)
* guarantees the candidate's (lastLogTerm,lastLogIndex) >= ours, so granting a
* fresher candidate is safe (Leader Completeness preserved). */
if (atomic_load_explicit(&ha->in_catch_up, memory_order_acquire) &&
ha->primary_ctrl_fd >= 0 && ha_heartbeat_is_fresh(ha))
return 0;
/* Freshness gate requires knowing the candidate's config_tag/lsn.
* The raft library does not pass these; they are carried in the capnp
* RequestVoteRequest fields. For the adapter, freshness is checked in
* handle_request_vote (called from the iomgr decode path in task 6.3).
* Here we return 1 (eligible) so the raft library proceeds; the full
* freshness gate is enforced by the caller before invoking raft_on_vote_request.
* This is safe because the caller (task 6.3 dispatcher) already applies
* the freshness check from handle_request_vote before delegating here. */
return 1;
}
/* raft_election_timeout_fn: delaymgr callback for raft election timeout.
* Fires raft_on_election_timeout after delay_ms if not cancelled.
* Owns the election_timeout_ctx_t — frees it on exit. */
static int raft_election_timeout_fn(void *arg)
{
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
ha_state_t *ha = t->ha;
/* Clear the stored pointer before checking cancelled so on_cancel_timeout
* is a safe no-op if it races with this callback. */
ha->raft_election_timeout_ctx = NULL;
if (atomic_load(&t->cancelled)) {
free(t);
return 0;
}
raft_on_election_timeout(&ha->raft);
free(t);
return 0;
}
/*
* raft_cb_schedule_timeout: schedule election timeout via delaymgr.
*
* Creates an election_timeout_ctx_t and calls delay_task_ms.
* The pointer is stored in ha->raft_election_timeout_ctx for cancellation.
*/
static void raft_cb_schedule_timeout(void *ctx, uint32_t delay_ms)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c) return; /* test mode */
election_timeout_ctx_t *t = calloc(1, sizeof(*t));
if (!t) return;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->ctx = NULL; /* raft path does not use ha_elect_ctx_t */
/* task 5.7: randomize the per-campaign watchdog to [delay_ms, 2*delay_ms]
* (Ongaro §3.4). The library passes a FIXED election_timeout_ms; arming the
* watchdog at exactly that value lets a constant cold-connection grant RTT
* land just past every deadline forever (resonance → unbounded term climb).
* Re-randomizing here breaks the resonance without touching the pure
* (sim-tested) library. */
delay_ms = ha_jittered_election_delay(ha, delay_ms);
int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1,
raft_election_timeout_fn, t);
if (rc < 0) {
WARN("raft_cb_schedule_timeout: delay_task_ms failed: %d", rc);
free(t);
return;
}
ha->raft_election_timeout_ctx = t;
}
/*
* raft_cb_cancel_timeout: cancel the pending election timeout.
* Sets cancelled=1 on the stored election_timeout_ctx_t.
*/
static void raft_cb_cancel_timeout(void *ctx)
{
ha_state_t *ha = (ha_state_t *)ctx;
election_timeout_ctx_t *t =
(election_timeout_ctx_t *)ha->raft_election_timeout_ctx;
if (t)
atomic_store(&t->cancelled, 1);
ha->raft_election_timeout_ctx = NULL;
}
/* raft_rdhup_delay_fn: delaymgr callback for RDHUP-triggered delayed election.
* Owns its ctx (election_timeout_ctx_t) — frees it on exit.
* Calls raft_check_and_start_election to start election only if not already voted. */
static int raft_rdhup_delay_fn(void *arg)
{
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
ha_state_t *ha = t->ha;
if (!atomic_load(&t->cancelled))
raft_check_and_start_election(&ha->raft);
free(t);
return 0;
}
/*
* raft_cb_start_election_after_delay: schedule RDHUP retry timer.
* After delay_ms, calls raft_check_and_start_election to start election
* only if this node hasn't already voted for another candidate.
*/
static void raft_cb_start_election_after_delay(void *ctx, uint32_t delay_ms)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c) return; /* test mode */
/* Priority 0 → delay_ms == 0: highest-priority node starts immediately. */
if (delay_ms == 0) {
raft_check_and_start_election(&ha->raft);
return;
}
election_timeout_ctx_t *t = calloc(1, sizeof(*t));
if (!t) return;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->ctx = NULL;
int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1,
raft_rdhup_delay_fn, t);
if (rc < 0) {
WARN("raft_cb_start_election_after_delay: delay_task_ms failed: %d", rc);
free(t);
}
}
/* ── Task 4.1 (ha-repl-hardening): safe commit point computation ─────────────
*
* ha_compute_safe_committed_lsn: determine the safe committed LSN and config
* tag after winning an election, using the vote replies collected in ctx.
*
* Algorithm (design §Flow 0: Post-Election — Determine Safe Commit Point):
* 1. Collect LSNs from all granted voters:
* - Self always granted: self_lsn = ctx->self_lsn (stored unconditionally)
* - Each peer with vote_granted=1: lsn = votes[i].peer_lsn
* 2. Sort granted LSNs descending.
* 3. safe_committed_lsn = sorted[floor(n_granted / 2)]
* (the majority-threshold element — the highest LSN held by a majority)
* 4. Same logic for config tags (ctx->self_tag / votes[i].peer_tag).
*
* Single-node (n_peers == 0): votes[] is empty; safe_committed_lsn = ctx->self_lsn.
* This prevents WAL truncation to 0 on re-election in a single-node cluster.
*
* Requirements: 2.3 (ha-repl-hardening spec)
* Boundary: Controller FSM (ha.c)
*/
static int _cmp_u64_desc(const void *a, const void *b)
{
uint64_t ua = *(const uint64_t *)a;
uint64_t ub = *(const uint64_t *)b;
if (ua > ub) return -1;
if (ua < ub) return 1;
return 0;
}
void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
uint64_t *safe_lsn_out,
uint64_t *safe_tag_out)
{
if (!ctx || !safe_lsn_out || !safe_tag_out) return;
/* Single-node: no votes[] entries — self is the only majority.
* Return self_lsn so WAL is not truncated to 0 on re-election. */
if (ctx->n_peers == 0) {
*safe_lsn_out = ctx->self_lsn;
*safe_tag_out = ctx->self_tag;
return;
}
/* Collect granted voter LSNs and tags.
* Maximum granted voters = n_peers + 1 (self + all other peers). */
int max_voters = ctx->n_peers + 1;
uint64_t *lsns = calloc((size_t)max_voters, sizeof(uint64_t));
uint64_t *tags = calloc((size_t)max_voters, sizeof(uint64_t));
if (!lsns || !tags) {
free(lsns);
free(tags);
*safe_lsn_out = 0;
*safe_tag_out = 0;
return;
}
int n_granted = 0;
/* Self always granted: self's lsn/tag from the dedicated self_lsn/self_tag fields */
lsns[n_granted] = ctx->self_lsn;
tags[n_granted] = ctx->self_tag;
n_granted++;
/* Each peer with vote_granted=1 */
for (int i = 0; i < ctx->n_peers; i++) {
if (ctx->votes[i].vote_granted) {
lsns[n_granted] = ctx->votes[i].peer_lsn;
tags[n_granted] = ctx->votes[i].peer_tag;
n_granted++;
}
}
/* Sort descending */
qsort(lsns, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc);
qsort(tags, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc);
/* Majority-threshold element: index = floor(n_granted / 2)
* For n_granted=3: index=1 (2nd element, 0-based) — majority=2 out of 3.
* For n_granted=1: index=0 (self only). */
int idx = n_granted / 2;
*safe_lsn_out = lsns[idx];
*safe_tag_out = tags[idx];
free(lsns);
free(tags);
INFO("HA: safe_committed_lsn=%" PRIu64 " safe_config_tag=%" PRIu64
" (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
}
/* ── bug-1060: never truncate the new leader's own durable log ───────────────
*
* The post-election WAL truncation cutoff must never fall below the leader's own
* durable tip, or it destroys entries committed by a prior leader (which this
* node had acked as a standby, and so are committed) — a Raft §5.4 Leader
* Completeness violation that diverges WALs and deadlocks the next election.
* See ha.h for the full rationale. */
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
{
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
}
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
*
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
* applied LAST so deadlock-freedom (>=1 guaranteed-free liveness worker) holds
* even when the n_vcpus ceiling would otherwise drive the value to 0 (e.g.
* n_vcpus == 1). See ha.h for the full rationale; the caller WARNs on the
* under-provisioned (n_vcpus < n_peers) clamp — this helper never logs. */
int drive_async_liveness_reserved(int n_peers, int n_vcpus)
{
int desired = n_peers - 1; /* worst-case concurrent liveness work */
int ceiling = n_vcpus - 1; /* keep >=1 commit (REQ_BLOCK) slot */
int reserved = desired < ceiling ? desired : ceiling; /* min(desired, ceiling) */
if (reserved < 1) /* floor applied LAST — always wins */
reserved = 1;
return reserved;
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
*
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
* Policy Decision Table inputs from the promoting member's recovery/durability
* state and delegate to persist_recover_decision so the 3.2/3.3 table is the
* single source of truth (DRY). "Behind the HWM" is durable_floor < safe_hwm;
* a truncated log is treated as behind regardless of its volatile position
* (design.md note). A hollow node (truncated OR behind) can never be ELIGIBLE
* until it catches up — independent of quorum size (Req 2.5).
*/
recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
uint64_t durable_floor,
uint64_t safe_hwm,
bool peer_reachable,
bool catch_up_succeeded)
{
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < safe_hwm),
.healthy_peer_reachable = peer_reachable,
.catch_up_succeeded = catch_up_succeeded,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
return persist_recover_decision(&in);
}
/* ha_evaluate_promotion_fence: live gate over the existing in_catch_up fence
* (Req 2.2, 2.6). Reuses ha->in_catch_up — does NOT invent a parallel fence.
* - ELIGIBLE → clear the fence, return ELIGIBLE (a member that caught up past
* the committed boundary regains eligibility, Req 2.6).
* - FENCE → set the fence, return FENCE (refuse promotion; catch up from a
* healthy peer).
* - FENCE_UNRECOVERABLE → run the 3.3 loud-fence action (logs LOUDLY + sets
* the sticky non-promotable flag) and ALSO set the in_catch_up
* fence. The member STAYS ALIVE, non-promotable, retrying
* catch-up — it NEVER self-exits (Req 2.4).
*/
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
bool recovery_truncated,
uint64_t durable_floor,
uint64_t safe_hwm,
bool peer_reachable,
bool catch_up_succeeded)
{
recover_decision_t d = ha_promotion_gate_decision(
recovery_truncated, durable_floor, safe_hwm, peer_reachable,
catch_up_succeeded);
switch (d) {
case RECOVER_DECISION_ELIGIBLE:
/* Release the fence: catch-up brought the member to a
* committed-consistent state at/above the cluster boundary (Req 2.6). */
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
break;
case RECOVER_DECISION_FENCE:
/* Refuse promotion: extend the catch-up fence so the member catches up
* from a healthy peer before it can lead (Req 2.2). */
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
INFO("HA: promotion FENCED — recovery_truncated=%d durable_floor=%"
PRIu64 " < cluster HWM=%" PRIu64 " (catch up before leading; "
"ha-commit-durability Req 2.2)",
recovery_truncated, durable_floor, safe_hwm);
break;
case RECOVER_DECISION_FENCE_UNRECOVERABLE: {
/* Unrecoverable: route through the 3.3 loud-fence action — logs LOUDLY
* and sets the sticky non-promotable flag. Also hold the in_catch_up
* fence so the member refuses promotion. The process STAYS ALIVE and
* keeps retrying catch-up; it NEVER self-exits (Req 2.4). */
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < safe_hwm),
.healthy_peer_reachable = peer_reachable,
.catch_up_succeeded = catch_up_succeeded,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
persist_t *p = (ha != NULL && ha->raft_c != NULL) ? &ha->raft_c->persist
: NULL;
persist_recover_loud_fence_if_unrecoverable(p, &in);
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
break;
}
}
return d;
}
/* ── ha-commit-durability task 4.4: bounded-window catch-up re-alert ─────────
*
* (never-self-exit rework 2026-06-18.) A member that wins an election while
* recovery-truncated / behind the HWM with no peer reachable AT THE PROMOTION
* INSTANT is FENCED (in_catch_up set, non-promotable) and STAYS ALIVE, retrying
* catch-up indefinitely. This one-shot timer exists ONLY for operator
* VISIBILITY: when it fires, if the member is STILL unrecoverable (still
* truncated/behind AND no peer AND no progress) it RE-EMITS the loud,
* operator-visible alert via persist_recover_loud_fence_if_unrecoverable (which
* also keeps the sticky non-promotable flag set) — it NEVER terminates the
* process. If the member has caught up (not truncated AND durable_floor >=
* HWM) the timer RELEASES the fence (Req 2.6). If progress is possible (peer
* back, or durable floor advanced) it leaves the member fenced under the
* existing catch-up machinery.
*
* Earlier this timer DEFERRED-then-ABORTED. The abort is GONE: a controller
* MUST NEVER self-terminate as a designed response to an operational condition
* (user invariant, 2026-06-18). The task-5.2 soak quorum collapse was caused
* by that self-abort; staying alive + retrying self-heals when a peer returns.
*
* The window is a settable seam so the timer test runs fast. Default =
* DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */
static _Atomic uint32_t s_promotion_fence_window_ms =
DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
uint32_t ha_promotion_fence_window_ms(void)
{
uint32_t w = atomic_load_explicit(&s_promotion_fence_window_ms,
memory_order_acquire);
return w ? w : DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
}
#ifdef XCE_TESTING
void ha_set_promotion_fence_window_ms(uint32_t ms)
{
atomic_store_explicit(&s_promotion_fence_window_ms, ms,
memory_order_release);
}
#endif
/* promotion_fence_ctx_t: delaymgr context for the bounded catch-up window.
* Captures the cluster committed high-water mark (target_hwm) and the durable
* floor AT SCHEDULE TIME (baseline_durable_floor) so the timer can detect
* catch-up progress (durable_floor advanced toward the HWM). `cancelled` is set
* by ha_cancel_promotion_fence_timer (recovery / shutdown) so a fired-but-
* cancelled callback is a safe no-op. */
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
bool was_truncated; /* recovery-truncated at schedule time */
uint64_t target_hwm; /* cluster committed high-water mark */
uint64_t baseline_durable_floor; /* durable floor at schedule time */
} promotion_fence_ctx_t;
/* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer.
* Idempotent; safe to call when no timer is armed. Used when the member catches
* up (fence released) or when the controller is shutting down — so neither a
* caught-up nor a stopped member ever reaches the deferred FATAL. */
void ha_cancel_promotion_fence_timer(ha_state_t *ha)
{
if (!ha) return;
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)atomic_exchange_explicit(
&ha->promotion_fence_ctx, NULL, memory_order_acq_rel);
if (t)
atomic_store_explicit(&t->cancelled, 1, memory_order_release);
}
/* Forward decl: the timeout callback re-arms itself for periodic re-alert. */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm);
/* promotion_fence_timeout_fn: the bounded-window re-alert timer callback. Owns
* its ctx — frees it on exit. Re-evaluates the LIVE recovery/durability state.
* It NEVER terminates the process (never-self-exit invariant, 2026-06-18):
* - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
* in_catch_up fence + clear the sticky unrecoverable flag; eligible again
* (Req 2.6).
* - a healthy peer is reachable now, OR catch-up is making progress → leave
* the member fenced under the existing catch-up machinery (it will release
* via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not
* re-arm the re-alert while progress is happening.
* - STILL unrecoverable (truncated/behind AND no peer AND no progress) →
* RE-EMIT the loud operator alert via persist_recover_loud_fence_if_-
* unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the
* timer so the alert keeps repeating until a peer returns. The process
* STAYS ALIVE and keeps retrying — it NEVER aborts. */
static int promotion_fence_timeout_fn(void *arg)
{
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg;
ha_state_t *ha = t->ha;
cluster_t *c = t->c;
/* Clear the stored handle before checking cancelled so a concurrent
* ha_cancel_promotion_fence_timer is a safe no-op (matches the
* raft_election_timeout_fn convention). */
promotion_fence_ctx_t *cur = (promotion_fence_ctx_t *)atomic_load_explicit(
&ha->promotion_fence_ctx, memory_order_acquire);
if (cur == t)
atomic_store_explicit(&ha->promotion_fence_ctx, NULL,
memory_order_release);
if (atomic_load_explicit(&t->cancelled, memory_order_acquire)) {
free(t);
return 0;
}
if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */
/* Re-read the LIVE state after the window. */
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool below_hwm = durable_floor < t->target_hwm;
bool made_progress = durable_floor > t->baseline_durable_floor;
bool peer_reachable = (ha->repl != NULL)
? (repl_active_standby_count(ha->repl) > 0)
: false;
/* Caught up: not truncated AND durable position reached the boundary →
* release the fence + clear the sticky flag; the member is eligible on its
* next election (Req 2.6). */
if (!truncated && !below_hwm) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)",
durable_floor, t->target_hwm);
free(t);
return 0;
}
/* Still hollow, but a peer is reachable or catch-up advanced the durable
* floor: progress is possible. Leave the member fenced (in_catch_up stays
* set) under the existing catch-up machinery; do not re-arm the re-alert
* while progress is happening. */
if (peer_reachable || made_progress) {
INFO("HA: promotion fence catch-up window elapsed but progress is "
"possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, catching up "
"(ha-commit-durability Req 2.2)",
peer_reachable, durable_floor, t->baseline_durable_floor,
t->target_hwm);
free(t);
return 0;
}
/* STILL unrecoverable: truncated/behind AND no healthy peer AND no catch-up
* progress. RE-EMIT the loud operator alert and keep the member fenced +
* non-promotable + ALIVE (Req 2.4) — NEVER terminate. Re-arm the re-alert
* timer so the operator keeps hearing about it until a peer returns. */
uint64_t target_hwm = t->target_hwm; /* capture before free (UAF guard) */
{
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = below_hwm,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
/* Loud, operator-visible alert + sticky non-promotable flag. STAYS
* ALIVE: no abort/exit anywhere on this path. */
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
}
free(t);
/* Re-arm so the loud alert repeats periodically while the member remains
* unrecoverable. The member keeps retrying catch-up; the next firing
* re-evaluates and releases the fence once a peer returns and it catches up. */
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, target_hwm);
return 0;
}
/* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule
* the one-shot bounded-window RE-ALERT timer. Cancels any previously-armed
* timer first so a re-evaluation never leaks or double-fires. On a failure to
* allocate/schedule the timer it emits the loud unrecoverable fence ONCE (stays
* alive, non-promotable) — it NEVER aborts (never-self-exit invariant). */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm)
{
/* Fence first (Req 2.2): refuse promotion while hollow. */
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
/* Cancel a stale timer (e.g. a prior promotion attempt). */
ha_cancel_promotion_fence_timer(ha);
promotion_fence_ctx_t *t = calloc(1, sizeof(*t));
if (!t)
goto loud_fence_no_timer;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->was_truncated = truncated;
t->target_hwm = target_hwm;
t->baseline_durable_floor = durable_floor;
uint32_t window_ms = ha_promotion_fence_window_ms();
int rc = delay_task_ms(&c->delaymgr, (int)window_ms, 1,
promotion_fence_timeout_fn, t);
if (rc < 0) {
WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc);
free(t);
goto loud_fence_no_timer;
}
atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release);
INFO("HA: promotion FENCED + catch-up re-alert window armed (%u ms) — "
"recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64
"; staying ALIVE + non-promotable, retrying catch-up "
"(ha-commit-durability Req 2.2/2.4)",
window_ms, truncated, durable_floor, target_hwm);
return;
loud_fence_no_timer:
{
/* Could not arm the re-alert timer — still emit the loud unrecoverable
* fence once so the operator sees it; the member stays fenced
* (in_catch_up set above) + non-promotable + ALIVE. NO abort. */
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < target_hwm),
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
}
}
/*
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
*
* Equivalent of ha_elect_evaluate_and_signal CTRL_PRIMARY path:
* - enter CTRL_PRIMARY (no transitional state)
* - call cluster_init_phase2 on first boot
* - push config, send ctrlhello
*
* ha->raft_c must be non-NULL (production path only; test mode uses noop stubs).
*/
static void raft_cb_become_leader(void *ctx, uint64_t term)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
/* Cancel any pending election timeout */
raft_cb_cancel_timeout(ctx);
/* Sync ha->term from raft term (raft owns term increments) */
atomic_store_explicit(&ha->term, term, memory_order_release);
/* The cluster committed high-water mark for this promotion — derived from
* the voter replies (NOT a stale ctrlhello). The task-4.3 pre-promotion
* eligibility gate (below) compares the local CLAMPED durable position
* against it. Lifted to function scope so the gate can read it after the
* truncate-clamp. */
uint64_t promotion_safe_lsn = 0;
/* Task 4.1 (ha-repl-hardening): compute safe_committed_lsn from vote replies.
* Uses voter LSNs collected during election (ha->elect_ctx->votes[]) to find
* the highest LSN held by a majority without an extra round-trip.
* Design §Flow 0: Post-Election — Determine Safe Commit Point. */
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
uint64_t safe_lsn = 0;
uint64_t safe_tag = 0;
ha_compute_safe_committed_lsn(ectx, &safe_lsn, &safe_tag);
promotion_safe_lsn = safe_lsn;
/* Store results for use in WAL truncation and post-election commit.
* safe_lsn is the highest LSN on a majority; entries above it
* may not have been committed on all nodes and must be truncated.
*
* bug-1060 (Leader Completeness, Raft §5.4): the truncation cutoff is
* clamped to the leader's OWN durable tip so it never destroys entries
* this node holds durably — those were acked by this node as a standby
* and so were committed by the prior leader. Truncating to the bare
* majority element (which can sit BELOW the leader's durable tip when
* a slower peer is in the granting majority) diverged WALs and dead-
* locked the next election. With the clamp, persist_truncate_wal_above
* is a no-op on the leader's durable log; safe_lsn only governs removal
* of a non-durable divergent tail strictly above the durable tip. */
if (c) {
uint64_t durable_tip = persist_durable_floor(&c->persist);
uint64_t trunc_floor =
ha_promotion_truncate_floor(safe_lsn, durable_tip);
persist_truncate_wal_above(&c->persist, trunc_floor);
}
/* Task 4.2 (ha-repl-hardening): set committed_lsn = safe_lsn.
* persist_truncate_wal_above already sets c->persist.committed_lsn
* in the production path (c != NULL). Also update
* ha->last_known_committed_lsn so the test-mode decision loop
* (XCE_TESTING path in standby heartbeat reply handler) and any
* code reading ha->last_known_committed_lsn see the safe commit
* point immediately after promotion.
* Design §Flow 0 step 5: committed_lsn = safe_committed_lsn.
* Req 2.3 (ha-repl-hardening). */
atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn,
memory_order_release);
/* Update committed config tag baseline for the post-election decision loop */
atomic_store_explicit(&ha->committed_config_tag, safe_tag,
memory_order_release);
}
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────
* Refuse promotion BEFORE the node becomes authoritative CTRL_PRIMARY if it
* carries the recovery-truncated flag (task 3.1 — a structurally corrupt log
* must never lead) OR its CLAMPED durable position is below the cluster
* committed high-water mark (promotion_safe_lsn, from the voter replies).
* The durable position is read AFTER persist_truncate_wal_above above, so it
* reflects the 2.2 truncate-clamp (a node whose durable tip was clamped down
* is correctly seen as behind). A hollow node (truncated OR behind) is
* refused regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote (Req 2.5). On refusal we set
* the in_catch_up fence and return WITHOUT becoming primary; the member
* catches up from a healthy peer and a later promotion attempt passes the
* gate (Req 2.2/2.6). Only the production path (real persist) is gated; the
* c==NULL unit shim has no persist and skips it.
*
* Belt-and-suspenders (never-self-exit rework 2026-06-18): a member that has
* been marked sticky-unrecoverable (persist_recovery_unrecoverable) must
* NEVER become primary while that flag is set — it stays alive, fenced, and
* keeps retrying catch-up. The flag clears automatically on catch-up. */
if (c != NULL) {
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool unrecoverable = persist_recovery_unrecoverable(&c->persist);
/* A healthy peer is reachable for catch-up iff some voter granted with a
* live endpoint survives this election (these become the standby
* channels promoted below). */
bool peer_reachable = false;
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
for (int i = 0; i < ectx->n_peers; i++) {
if (ectx->votes[i].vote_granted && ectx->votes[i].ep != NULL) {
peer_reachable = true;
break;
}
}
}
}
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
* a hollow member never silently proceeds to leadership. We evaluate
* the PURE 3.2/3.3 decision here. A FENCE or an UNRECOVERABLE
* (truncated/behind + no peer right now) both fence the member
* (in_catch_up) and arm the catch-up re-alert timer; the member STAYS
* ALIVE and keeps retrying — it NEVER self-exits (never-self-exit
* invariant, 2026-06-18; this is what fixes the task-5.2 soak collapse
* where a self-aborting recovering winner repeatedly dropped quorum). */
recover_decision_t gate = ha_promotion_gate_decision(
truncated, durable_floor, promotion_safe_lsn,
peer_reachable, /*catch_up_succeeded=*/false);
if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable) {
/* ELIGIBLE and not sticky-unrecoverable: release the fence (a
* caught-up member regains eligibility, Req 2.6), clear the sticky
* flag if set, and cancel any pending re-alert timer. Proceed to
* authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE, UNRECOVERABLE, or sticky-unrecoverable: refuse promotion —
* do NOT become CTRL_PRIMARY. Fence the member and arm the catch-up
* re-alert timer. The member STAYS ALIVE, non-promotable, retrying
* catch-up (Req 2.2/2.4). NEVER self-exit, never fence silently. */
WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d) — "
"recovery_truncated=%d durable_floor=%" PRIu64
" cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + arming "
"catch-up re-alert window, staying ALIVE "
"(ha-commit-durability Req 2.2/2.4)",
(int)gate, unrecoverable, truncated, durable_floor,
promotion_safe_lsn, peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;
}
}
/* ── ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): config
* serve-fence promotion gate ──────────────────────────────────────────────
* A member whose latest committed config marker references bytes it does not
* hold (config_serve_fenced) MUST NOT become authoritative CTRL_PRIMARY — it
* cannot serve under the committed configuration. Refuse promotion and STAY
* ALIVE (controller-never-self-terminate): the member remains non-promotable
* and keeps retrying; once the correct bytes arrive out-of-band and re-verify
* clears the fence, a later promotion attempt passes this gate. The check is a
* pure atomic read (no I/O) — safe on the election path. */
if (config_serve_fenced(c)) {
WARN("HA: promotion REFUSED — config serve-FENCED (committed config "
"marker references bytes not present/matching locally); member "
"stays ALIVE, non-promotable, awaiting out-of-band config bytes "
"(ha-term-safety R7.9)");
return; /* do NOT become CTRL_PRIMARY; never self-exit */
}
int is_first_boot = !ha->phase2_done;
/* First boot: call cluster_init_phase2 */
if (is_first_boot) {
#ifdef XCE_TESTING
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
}
/* ha-single-leader-role (Req 1.1): enter CTRL_PRIMARY directly — there is
* no transitional COMMITTING state. New-leader safety comes from the
* own-term commit rule (the WAL_NOOP + writable_lsn gate armed below),
* not from a fourth role value. The raft-owned term was already synced
* at entry (~line 463), so we do NOT call ha_do_promote here (it would
* double-increment the term, research.md §4 finding 2). This store sits
* OUTSIDE the c != NULL guard so the c==NULL test shim
* (raft_cb_become_leader_for_test) reaches it. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
memory_order_release);
INFO("HA FSM (raft): → CTRL_PRIMARY (term=%" PRIu64
", first_boot=%d)", term, is_first_boot);
/* Req 2.1 (ha-repl-hardening): re-initialize per-standby convergence
* tracking for THIS primary term. Stale converged flags from a
* previous term (demote → re-elect cycle) must not satisfy the
* majority-convergence gate before standbys re-report their state. */
memset(ha->standby_reported_tag, 0, sizeof(ha->standby_reported_tag));
memset(ha->standby_reported_lsn, 0, sizeof(ha->standby_reported_lsn));
memset(ha->standby_converged, 0, sizeof(ha->standby_converged));
/* Promote surviving vote endpoints to standby replication channels.
* Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to
* ha->standby_endpoints so push_config and ctrlhello reach them. */
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx && ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ectx->n_peers; i++) {
if (ectx->votes[i].ep != NULL) {
zarray_store(ha->standby_endpoints, ectx->votes[i].ep);
INFO("HA: promoted vote ep[%d] → standby channel", i);
/* Task 2.1: register promoted ep with WAL replicator.
* Req 1.1: primary must register standby with repl_add_standby
* when establishing the standby control channel.
* peer->hostname / peer->port from the vote entry's peer pointer. */
if (c && ectx->votes[i].peer != NULL) {
int rc = repl_add_standby(&c->persist.repl,
ectx->votes[i].ep,
ectx->votes[i].peer->hostname,
(uint32_t)ectx->votes[i].peer->port);
if (rc >= 0)
INFO("HA: repl registered standby ep[%d] (slot %d, %s:%u)",
i, rc,
ectx->votes[i].peer->hostname,
ectx->votes[i].peer->port);
else
WARN("HA: repl_add_standby ep[%d] failed: %d", i, rc);
}
}
}
pthread_mutex_unlock(&ha->lock);
}
}
/* Quorum-transition observability (Req 8.1, 8.2): promotion registered the
* surviving vote endpoints as standbys — the write-quorum count just changed.
* Record any loss/restore edge once (no-op when quorum is held at promotion). */
ha_log_quorum_transition(ha);
if (c) {
/* Push config to standbys — read raw bytes from disk so standbys
* can save the file. NULL bytes are passed only in test mode. */
#ifdef XCE_TESTING
if (ha_push_config_hook != NULL)
ha_push_config_hook(c);
#else
{
uint8_t *cfg_bytes = NULL;
size_t cfg_len = 0;
extern int conf_read_raw(const char *, uint8_t **, size_t *);
if (c->myconf && c->myconf->conffile &&
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
/* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON).
* The offload deep-copies cfg_bytes, so this free is still
* correct regardless of routing. */
ha_config_push_offloadable(c, cfg_bytes, cfg_len,
c->myconf->config_sha256,
c->myconf->config_version_tag);
free(cfg_bytes);
} else {
ha_config_push_offloadable(c, NULL, 0, NULL,
c->myconf ? c->myconf->config_version_tag : 0);
}
}
#endif
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up */
#ifdef XCE_TESTING
if (ha_send_ctrlhello_hook != NULL)
ha_send_ctrlhello_hook(c);
#else
ha_send_ctrlhello_to_all_standbys(c);
#endif
/* ha-single-leader-role task 2.2: register the persistence hooks FIRST,
* before the own-term no-op append, so the persistence seam is live
* before any racing client write can reach the cluster. Idempotent
* via the hooks_registered guard. */
ha_register_persist_hooks(c);
/* Start primary heartbeat immediately so standbys receive NOP and their
* liveness poll does not fire. ha_start_primary_heartbeat is also
* called in ha_promote_to_primary; the idempotency guard (role check
* at fn top) prevents double-arming. */
ha_start_primary_heartbeat(c);
/* ha-single-leader-role task 2.2 (Req 1.4, 2.1, 2.2): append the
* own-term WAL_NOOP and arm the commit engine's writable_lsn gate.
* This is the shared promotion tail both roads to PRIMARY run.
* Non-blocking: promotion never waits for the no-op to commit. */
ha_arm_own_term_gate(c);
}
}
/*
* raft_cb_become_follower: node lost election or saw higher term — enter STANDBY.
*
* Syncs ha->term from the raft-reported term, sets role to CTRL_STANDBY,
* calls cluster_init_phase2 on first boot (standbys also need it to
* receive config replication push from the new primary).
*/
static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
{
(void)leader_idx;
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
/* Cancel any pending election timeout */
raft_cb_cancel_timeout(ctx);
/* Sync ha->term */
atomic_store_explicit(&ha->term, term, memory_order_release);
int is_first_boot = !ha->phase2_done;
/* First boot: call cluster_init_phase2 so structures are ready */
if (is_first_boot) {
#ifdef XCE_TESTING
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
}
/* Close all outgoing vote endpoints (loser path).
* The primary will push config and ctrlhello via promoted connections;
* the standby waits passively on its listener for the inbound connection.
*
* FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197): when
* g_drive_async_peerctrl_dispatch_enabled is ON this loop runs on a POOL
* worker and frees ep->data while the io_worker may still deliver a queued
* peer-control event on another ep → UAF. Before task 3.4 flips the flag
* ON, each endpoint_fini below MUST be preceded by iomgr_unmonitor(ep). See
* the full hazard note at peer_ctrl_dispatch_to_pool. Safe today (flag OFF). */
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
for (int i = 0; i < ectx->n_peers; i++) {
endpoint_t *ep = ectx->votes[i].ep;
if (ep == NULL)
continue;
INFO("HA: closing vote ep[%d] (loser path)", i);
if (ha_peerctrl_dispatch_enabled() && ha->pool != NULL) {
/* bug-1197: this callback runs on a pool worker; another
* worker may be mid-vtask (or have a queued vtask) on this
* same ep reading ep->data. Defer teardown onto the ep's
* per-ep FIFO so endpoint_fini+free runs strictly after the
* last live use, and gate late dispatch (tearing_down). */
vote_ep_teardown_async(ha, ep);
} else {
/* bug-1298: drop the owner ref (deferred fini+free if a pool
* vtask still references this vote-ep). Also fixes the old
* flag-OFF struct leak (was endpoint_fini-only). */
endpoint_unref(ep);
}
ectx->votes[i].ep = NULL;
}
}
}
/* ha-convergence-recovery Req 1.1-1.3, 1.5: demotion convergence.
*
* Read the PRIOR role BEFORE the CTRL_STANDBY store below. If this node was
* the PRIMARY, it may hold an uncommitted, potentially divergent WAL suffix
* and provisional in-memory state. Synchronously disable the three
* persistence hooks HERE (before any further event can be persisted — Req
* 1.1, reusing ha_demote_to_standby's disable sequence), then schedule a
* one-shot delaymgr task that truncates above committed_lsn and self-restarts
* (Req 1.2/1.3/1.5) off this callback thread (D1: no file I/O or exec on the
* iomgr/raft callback thread).
*
* Non-PRIMARY demotions (STANDBY/INIT/ELECTING) take the unchanged path —
* they have no registered hooks and no provisional suffix to discard, so
* their behavior is byte-equivalent to before this spec. */
ctrl_role_t prior_role = (ctrl_role_t)atomic_load_explicit(
&ha->role, memory_order_acquire);
if (prior_role == CTRL_PRIMARY && c != NULL) {
/* Req 1.1: disable persistence hooks SYNCHRONOUSLY, before the role
* store and before scheduling, so no provisional entry can be produced
* after the higher term was observed. Mirrors ha_demote_to_standby
* (ha.c) — disable each hook, then clear the idempotency guard. */
if (ha->hooks_registered) {
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun);
ha->hooks_registered = 0;
}
}
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
INFO("HA FSM (raft): → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
term, is_first_boot);
/* Req 1.2/1.3/1.5: schedule the deferred truncate-above-committed-LSN +
* self-restart now that hooks are disabled and the role is STANDBY. Only
* the was-PRIMARY edge converges; all other edges skip this entirely. */
if (prior_role == CTRL_PRIMARY && c != NULL) {
WARN("HA: PRIMARY demoted (term=%" PRIu64 ") — scheduling demotion "
"convergence (truncate above committed_lsn + self-restart)", term);
(void)ha_schedule_demotion_convergence(c);
}
/* Arm heartbeat watchdog — fires ha_on_primary_ctrl_fd_rdhup if primary
* sends no ctrlhello within election_timeout_ms. (This also resets
* last_activity = now, granting a full grace window before the first poll.) */
ha_arm_heartbeat_timeout_vtask(ha, c);
}
/*
* raft_cb_persist: persist term + voted_for to disk before granting vote.
*
* Maps raft integer voted_for index to a candidate name using cluster config.
* Updates ha->term, ha->voted_for_term, ha->voted_for_name then calls
* ha_persist_term(ha, workdir).
*
* voted_for == -1 means "no vote"; voted_for >= 0 is the candidate index.
* Returns 0 on success, non-zero on error (causes raft library to deny vote).
*/
static int raft_cb_persist(void *ctx, uint64_t term, int voted_for)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
/* Store ha-term.bin under {workdir}/{myname}/data/ alongside WAL/snapshot. */
char data_dir[1024] = {0};
if (c && c->myconf && c->myconf->workdir && c->myconf->name)
snprintf(data_dir, sizeof(data_dir), "%s/%s/data",
c->myconf->workdir, c->myconf->name);
const char *workdir = data_dir[0] ? data_dir : NULL;
/* Update ha->term */
atomic_store_explicit(&ha->term, term, memory_order_release);
/* Map voted_for index to name */
pthread_mutex_lock(&ha->lock);
if (voted_for == -1) {
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
} else {
ha->voted_for_term = term;
/* Look up candidate name by index */
if (c && c->myconf &&
voted_for < c->myconf->peers_cfg.candidates_count) {
const char *name = c->myconf->peers_cfg.candidates[voted_for].name;
strncpy(ha->voted_for_name, name, sizeof(ha->voted_for_name) - 1);
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
} else {
/* Test mode or out-of-range: use "self" placeholder */
if (voted_for == ha->my_priority) {
strncpy(ha->voted_for_name, "self",
sizeof(ha->voted_for_name) - 1);
} else {
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
}
}
}
pthread_mutex_unlock(&ha->lock);
ha_persist_term(ha, workdir);
return 0;
}
/* raft_cb_trace: emit raft state-machine events at DEBUG3 level.
* Active when controller is started with -vvv; suppressed otherwise. */
static void raft_cb_trace(void *ctx, const char *event, const char *detail)
{
ha_state_t *ha = (ha_state_t *)ctx;
tlog(TLOG_DEBUG3, "RAFT[%s] term=%" PRIu64 " event=%s%s%s",
ha->my_priority == 0 ? "p0" :
ha->my_priority == 1 ? "p1" : "p2",
atomic_load_explicit(&ha->term, memory_order_relaxed),
event,
detail ? " " : "", detail ? detail : "");
}
/* ── ha-term-safety: MINIMAL log/commit callback placeholders ────────────────
*
* TODO(ha-term-safety task 2.x): REPLACE these stubs with the real bodies.
* The raft library's raft_init() now REQUIRES all five non-NULL, and
* raft_on_vote_request() internally calls on_get_entry_term for its logOk gate.
* These placeholders exist ONLY to restore the controller build (tasks 1.1/1.2);
* they perform NO persistence, NO replication, and NO truncation. See CONCERNS.
*
* on_persist_entry → task 2.1 (durable WAL append/overwrite) — IMPLEMENTED
* on_get_entry_term → task 2.2 (WAL entry-term lookup; persist_entry_term, 1.3) — IMPLEMENTED
* on_send_append_entries→ task 2.3 (AppendEntries transmit) — IMPLEMENTED
* on_truncate_from → task 2.2 (conflict truncation) — IMPLEMENTED
* on_commit_advanced → task 2.4 (commit-barrier trigger / commit source) — IMPLEMENTED
*
* on_get_entry_term returns 0 (== "absent") for now: with the log-replication
* path unwired, the library's local last-log term reads as 0, so the logOk gate
* grants on lastLogTerm >= 0 — i.e. it does NOT yet tighten the existing
* config_tag/last_log_lsn freshness gate already enforced in cluster.c. The real
* accessor (task 2.2/1.3) makes the gate effective. */
/* on_persist_entry (ha-term-safety task 2.1, R1.1–R1.4): durably append/overwrite
* the log entry at 1-based `index` (== WAL LSN) stamping the LEADER-supplied
* `term` into hdr.term, then return the persist rc.
*
* Persist-before-act: this is called BEFORE the library reports the append/accept
* (raft_log_append / raft_on_append_entries_request). On a non-zero (failure) rc
* the library does NOT advance its log and a standby does NOT ACK (the existing
* persist-before-ACK + bug-850 contiguity guard sit beneath the WAL write and are
* preserved — we do not bypass them). We never false-ACK on failure.
*
* `entry` is the adapter envelope (raft_entry_blob_t prefix + payload) packed by
* the append/replication driver (tasks 3.1/3.2); the library holds no entry bytes
* and treats it as opaque. We decode (type, payload, payload_len) from it for the
* existing persist write path.
*
* Path selection by role:
* - PRIMARY own-append: persist_wal_async — assigns the contiguous next LSN
* (== index here, since the library's index == log_len+1 == current_lsn) and
* defers fsync to the group-commit flusher (the steady-state append path).
* - STANDBY replicated apply: persist_wal_write_replicated — writes at the GIVEN
* lsn=index with the contiguity guard + inline persist-before-ACK fsync.
*
* NOTE (deferred to 2.3/3.2): the leader's per-entry term is carried on the repl
* envelope by task 2.3/3.2; until then the standby path stamps the term we set
* here via persist_set_term — which IS the supplied `term` argument — so a test
* that drives this callback already records the leader term. The envelope wiring
* makes the live replication path source the same value. */
static int ha_cb_persist_entry_stub(void *ctx, uint64_t index, uint64_t term,
const void *entry, size_t len)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return -EINVAL; /* test mode without a cluster: cannot persist */
persist_t *p = &ha->raft_c->persist;
/* Decode the adapter entry envelope: a fixed raft_entry_blob_t prefix
* carrying the WAL type + payload_len, followed by the payload bytes. */
uint32_t type = WAL_NOOP;
const void *payload = NULL;
uint32_t plen = 0;
if (entry && len >= sizeof(raft_entry_blob_t)) {
const raft_entry_blob_t *blob = (const raft_entry_blob_t *)entry;
type = blob->type;
plen = blob->payload_len;
/* Reject a malformed envelope whose declared payload exceeds the buffer
* — fail loud rather than read past the blob. */
if ((size_t)plen + sizeof(*blob) > len)
return -EINVAL;
if (plen > 0)
payload = (const uint8_t *)entry + sizeof(*blob);
} else if (entry && len > 0) {
/* Defensive: an un-enveloped blob is treated as malformed. */
return -EINVAL;
}
/* Stamp the live election term so the WAL header records the LEADER's term
* for this entry (R1.1/R1.2/R1.4). */
persist_set_term(p, term);
/* Persist at LSN=index through the existing write path (persist-before-ACK
* and the bug-850 contiguity guard are preserved beneath these). */
if (ha_is_primary(ha)) {
/* Own-append: persist_wal_async assigns the contiguous next LSN, which
* equals `index` for the leader (library index == current_lsn). */
return persist_wal_async(p, type, payload, plen);
}
return persist_wal_write_replicated(p, index, type, payload, plen);
}
/* on_get_entry_term (ha-term-safety task 2.2, R1.5/R3.1): return the recorded
* Raft term for the entry at 1-based LSN `index`, sourced from the WAL term
* index via persist_entry_term. Returns 0 when the LSN is absent (no entry).
* The library calls this for its logOk / (term,lsn) reconcile gate.
*
* Legacy-0 caveat (from the task 1.3 review MEDIUM): persist_entry_term maps a
* present legacy-0 entry (hdr.term == 0, written before this feature) to
* WAL_LEGACY_TERM (== 1). In a mixed pre/post-feature cluster where a
* post-feature leader is genuinely in term 1, a (term,lsn) EQUALITY check
* cannot tell a legacy-0 entry (mapped to 1) apart from a real term-1 entry by
* term alone. This is safe because the Raft log-matching invariant (same term
* + same index ⇒ same entry, enforced across the prefix) plus the bounded
* one-time upgrade window (every entry written after upgrade carries a real
* term) mean the only collision is at the legacy/term-1 boundary, where the
* prefixes already agree. We note it explicitly rather than special-casing it. */
static uint64_t ha_cb_get_entry_term_stub(void *ctx, uint64_t index)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return 0; /* test mode without a cluster: no log → absent */
return persist_entry_term(&ha->raft_c->persist, index);
}
/* ha_read_wal_entry_at: source ONE local WAL entry at LSN `lsn` for the
* AppendEntries send path (ha-term-safety task 2.3). The library holds no entry
* bytes, so the leader reads its own WAL segment files (seg-%016llx.wal under
* wal_path) and returns the entry's type + payload. The per-LSN term is NOT read
* from the on-disk header here — the caller takes it from persist_entry_term
* (the authoritative in-memory term index), per the library contract that the
* wire entry's term == the term recorded for that LSN. `payload` is malloc'd and
* owned by the caller (freed after the send). Returns 0 on success, negative on
* absence/read error. */
static int ha_read_wal_entry_at(const char *wal_path, uint64_t lsn,
uint8_t *type_out, uint8_t **payload_out,
uint32_t *plen_out)
{
*type_out = 0;
*payload_out = NULL;
*plen_out = 0;
DIR *d = opendir(wal_path);
if (!d)
return -errno ? -errno : -EIO;
/* Enumerate segment start-LSNs, descending, so we open the latest segment
* whose start LSN is <= the target first. */
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent *de;
while ((de = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(*seg_lsns))) {
uint64_t sl = 0;
if (sscanf(de->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
seg_lsns[nseg++] = sl;
}
closedir(d);
if (nseg == 0)
return -ENOENT;
/* insertion sort ascending */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
seg_lsns[j + 1] = key;
}
int rc = -ENOENT;
for (int si = 0; si < nseg; si++) {
/* Skip a segment whose successor already starts past the target. */
if (si + 1 < nseg && seg_lsns[si + 1] <= lsn)
continue;
if (seg_lsns[si] > lsn)
break;
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path),
"%s/seg-%016" PRIx64 ".wal", wal_path, seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0)
continue;
uint32_t off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
if (n < (ssize_t)sizeof(hdr)) break;
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
if (hdr.lsn == lsn) {
*type_out = (uint8_t)hdr.type;
if (hdr.payload_len > 0) {
uint8_t *pbuf = malloc(hdr.payload_len);
if (!pbuf) { rc = -ENOMEM; }
else {
ssize_t pr = pread(fd, pbuf, hdr.payload_len,
(off_t)(off + sizeof(hdr)));
if (pr == (ssize_t)hdr.payload_len) {
*payload_out = pbuf;
*plen_out = hdr.payload_len;
rc = 0;
} else {
free(pbuf);
rc = -EIO;
}
}
} else {
rc = 0; /* zero-payload entry (e.g. WAL_NOOP) */
}
close(fd);
return rc;
}
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
return rc;
}
/* on_send_append_entries (ha-term-safety task 2.3, R1.3/R3.1): transmit ONE
* AppendEntries to the standby at slot `peer_idx`, carrying the prev tuple
* (prev_log_index, prev_log_term) + leader_commit, and — for n_entries==1 — the
* single entry at LSN prev_log_index+1 sourced from this leader's OWN WAL (the
* library holds no entry bytes). The entry's term is taken from
* persist_entry_term (the authoritative per-LSN term index), so the standby
* persists the LEADER's recorded term, not a substitute. n_entries==0 emits a
* heartbeat (zero entries) carrying only the prev tuple + leader_commit.
*
* Non-blocking (ha-async-io): delegates to repl_send_append_entries which
* enqueues via the async repl transport — never blocks in the callback.
* peer_idx is the library's raft peer index, mapped 1:1 to the repl standby
* registry slot (the same index match_index/next_index use). */
static int ha_cb_send_append_entries_stub(void *ctx, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return -EINVAL; /* test mode without a cluster: cannot send */
persist_t *p = &ha->raft_c->persist;
replicator_t *r = ha->repl ? ha->repl : &p->repl;
if (n_entries == 0) {
/* Heartbeat: prev tuple + leader_commit, no entry. */
return repl_send_append_entries(r, peer_idx, prev_log_index,
prev_log_term, leader_commit,
/*n_entries*/0,
/*entry_lsn*/0, /*type*/0, /*term*/0,
NULL, 0);
}
/* n_entries == 1: source the entry at prev_log_index+1 from the local WAL. */
uint64_t entry_lsn = prev_log_index + 1;
uint64_t entry_term = persist_entry_term(p, entry_lsn);
uint8_t type = 0;
uint8_t *payload = NULL;
uint32_t plen = 0;
int rc = ha_read_wal_entry_at(p->wal_path, entry_lsn, &type, &payload, &plen);
if (rc != 0) {
WARN("on_send_append_entries: cannot source local WAL entry at lsn=%lu "
"(rc=%d) — not sending to peer %d",
(unsigned long)entry_lsn, rc, peer_idx);
return rc;
}
rc = repl_send_append_entries(r, peer_idx, prev_log_index, prev_log_term,
leader_commit, /*n_entries*/1,
entry_lsn, type, entry_term, payload, plen);
free(payload);
return rc;
}
/* on_truncate_from (ha-term-safety task 2.2, R1.8/R3.2): delete the local log
* entries at LSN `index..end` because they conflict with the leader's log at a
* (term,lsn) the library detected. persist_truncate_wal_above(p, cutoff) removes
* every entry with LSN > cutoff, so to drop LSN >= index we pass cutoff =
* index - 1. This REPLACES the committed-boundary truncate heuristic on the
* library-driven path: the library, not a watermark, decides the cut point.
*
* Returns the truncate rc. A non-zero rc propagates so the library/leader
* retries; the member stays non-promotable per R3.5 (that fence is existing
* behavior and is not touched here). persist_truncate_wal_above also shrinks
* the per-LSN term index for the removed LSNs, so a later on_get_entry_term on
* a truncated LSN reads back 0 (absent). */
static int ha_cb_truncate_from_stub(void *ctx, uint64_t index)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return -EINVAL; /* test mode without a cluster: cannot truncate */
if (index == 0)
return -EINVAL; /* LSN 0 is reserved — never an entry to truncate */
return persist_truncate_wal_above(&ha->raft_c->persist, index - 1);
}
/* on_commit_advanced (ha-term-safety task 2.4, R4.1/R4.5/R5.4): feed the raft
* library's current-term-gated `commit_index` into the existing synchronous
* commit barrier as the commit SOURCE. The library fired this ONLY on a strict
* advance of its Figure-8 current-term commit index (raft.c), so the barrier
* now advances committed_lsn only when BOTH a durable majority agrees AND the
* library has admitted the entry — the term-blind safe-commit
* (ha_compute_safe_committed_lsn) is no longer the steady-state commit source.
*
* This callback only TRIGGERS the barrier (records the ceiling + re-evaluates);
* it performs NO durability itself. The barrier mechanism — majority self-cap at
* the durable floor, persist-before-ACK, monotonic committed_lsn — is unchanged
* (ha-commit-durability). The own-term writable_lsn gate is RETAINED as a
* complementary WAIT-completion predicate (it never caps committed_lsn, so it
* cannot deadlock against this source; see ha_arm_own_term_gate).
*
* Non-blocking (ha-async-io): commit_set_raft_commit_index takes only ce->lock
* and never performs I/O — safe on the iomgr ACK path that drove
* raft_advance_commit. */
static void ha_cb_commit_advanced_stub(void *ctx, uint64_t commit_index)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return; /* test mode without a cluster: no barrier to trigger */
commit_set_raft_commit_index(&ha->raft_c->commit, commit_index);
/* ha-term-safety task 4.3 (remediation r1, Finding B): a config marker may
* have just entered the committed prefix. Activation is NOT performed HERE:
* this callback runs on the iomgr ACK path, where the async-I/O invariant
* forbids the disk reads config_activate_committed performs
* (scan_latest_committed_marker pread + load_primary_config). The committed
* marker is instead consumed off the iomgr thread by the call sites that make
* §7.10 true in production:
* - boot/recovery (myapp.c step 7): activates the latest committed marker,
* hash-verified, after persist recovery.
* - handle_repl_config_push: after the out-of-band bytes are saved, gates
* activation on the committed marker (verify-before-activate).
*
* ha-term-safety task 4.3 (remediation r2, Finding D / R7.9): the two sites
* above are NOT sufficient when the BYTES arrive (and are saved) BEFORE the
* NEW marker commits — the config MARKER (WAL) and the config BYTES (out-of-
* band push) travel on INDEPENDENT transports. In that ordering the push path
* hash-checks the freshly-saved NEW bytes against the still-OLDER committed
* marker, MISMATCHES, and sets the serve-fence; when the NEW marker later
* commits HERE, nothing re-runs activation (this callback can't do disk I/O,
* and the primary's config-behind re-push net does not fire because the push
* already advanced this member's advertised config_version_tag) — so the
* standby would stay PERMANENTLY fenced. Close the gap: while serve-fenced,
* schedule an OFF-IOMGR delaymgr task (config_schedule_activate_retry, NON-
* BLOCKING ordq_put only) that re-runs config_activate_committed on a thread
* allowed to block on disk; once the NEW marker is in the committed prefix and
* the saved bytes verify, the retry clears the fence and activates. No-op once
* unfenced (no busy-spin); a still-genuine mismatch reschedules on the next
* commit advance. */
if (config_serve_fenced(ha->raft_c))
config_schedule_activate_retry(ha->raft_c);
}
static const raft_callbacks_t s_real_cbs = {
.on_send_vote_request = raft_cb_send_vote_request,
.on_send_pre_vote_request = raft_cb_send_pre_vote_request,
.on_vote_eligible = raft_cb_vote_eligible,
.on_become_leader = raft_cb_become_leader,
.on_become_follower = raft_cb_become_follower,
.on_persist = raft_cb_persist,
.on_schedule_timeout = raft_cb_schedule_timeout,
.on_cancel_timeout = raft_cb_cancel_timeout,
.on_start_election_after_delay = raft_cb_start_election_after_delay,
/* ha-term-safety: MINIMAL placeholders to satisfy raft_init's non-NULL
* requirement — REPLACE in task 2.x (see stubs + CONCERNS). */
.on_persist_entry = ha_cb_persist_entry_stub,
.on_get_entry_term = ha_cb_get_entry_term_stub,
.on_send_append_entries = ha_cb_send_append_entries_stub,
.on_truncate_from = ha_cb_truncate_from_stub,
.on_commit_advanced = ha_cb_commit_advanced_stub,
.on_trace = raft_cb_trace,
};
/* ── PRNG seeding ───────────────────────────────────────────── */
void ha_seed_jitter(uint64_t *seed, int my_priority)
{
uint8_t buf[8] = {0};
int fd = open("/dev/urandom", O_RDONLY);
if (fd >= 0) {
ssize_t n = 0;
while (n < 8) {
ssize_t r = read(fd, buf + n, (size_t)(8 - n));
if (r <= 0) break;
n += r;
}
close(fd);
if (n == 8) {
memcpy(seed, buf, 8);
if (*seed != 0) return;
}
}
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
*seed = (uint64_t)ts.tv_nsec
^ ((uint64_t)my_priority << 56)
^ ((uint64_t)getpid() << 16);
}
int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority)
{
if (!ha || !cfg)
return -EINVAL;
/* Zero first so ha_fini() is safe even if validation fails below */
memset(ha, 0, sizeof(*ha));
if (cfg->heartbeat_interval_ms < 2000)
return -EINVAL;
if (cfg->heartbeat_timeout_ms < cfg->heartbeat_interval_ms)
return -EINVAL;
if (cfg->election_timeout_ms < 500)
return -EINVAL;
ha->config = *cfg;
ha->my_priority = my_priority;
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
atomic_store_explicit(&ha->applied_lsn, 0ULL, memory_order_release);
atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed);
atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed);
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed);
/* failover_requested / watchdog_stop deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */
pthread_mutex_init(&ha->lock, NULL);
ha->ssl_ctx = NULL;
ha->jitter_seed = 0;
ha->standby_endpoints = NULL;
ha->primary_ctrl_fd = -1;
/* election-iomgr-refactor Task 2.1: explicit zero-init for documentation clarity
* (memset above already zeroes these, but being explicit is better documentation) */
ha->client_ssl_ctx = NULL;
ha->phase2_done = 0;
ha->n_peers = 0;
/* ha-heartbeat-liveness Task 1.1: zero-init liveness timestamp.
* Written by ha_on_heartbeat_received (task 2.1); read by liveness_poll_fn. */
ha->last_activity = 0;
/* ha-repl-hardening Task 1.4: per-standby convergence tracking arrays.
* memset(ha, 0, sizeof(*ha)) above already zeroes these arrays; the explicit
* note documents the invariant: on promotion to CTRL_PRIMARY every
* standby starts as not-converged (tag=0, lsn=0, converged=false). */
/* heartbeat_seen starts at 0: no heartbeat seen yet.
* ha_wait_for_heartbeat_or_timeout() calls sem_timedwait on this. */
sem_init(&ha->heartbeat_seen, 0, 0);
/* ── Task 6.1/6.2: Initialize embedded raft_state_t ─────────────────────
*
* Wire real callbacks (s_real_cbs) so the state machine is ready when
* ha_start_election_async re-initialises raft with the correct n_peers /
* self_idx / initial_term after ha_restore_term().
*
* n_peers = 1 and self_idx = 0 are safe placeholders here; they are
* overwritten by ha_raft_reinit() (called from ha_start_election_async)
* which runs after ha_restore_term() and knows the real cluster size.
*
* initial_term = 0 / initial_voted_for = -1 are first-boot defaults;
* ha_raft_reinit() restores the persisted values before any election fires.
*
* ha->raft_c is NULL until ha_start_election_async sets it.
*/
ha->raft_c = NULL;
ha->raft_election_timeout_ctx = NULL;
raft_config_t raft_cfg;
memset(&raft_cfg, 0, sizeof(raft_cfg));
raft_cfg.n_peers = 1; /* placeholder; overwritten by ha_raft_reinit */
raft_cfg.self_idx = my_priority; /* self_idx == priority by convention */
raft_cfg.election_timeout_ms = cfg->election_timeout_ms;
raft_cfg.my_priority = my_priority;
raft_cfg.initial_term = 0; /* ha_restore_term() updates ha->term after ha_init */
raft_cfg.initial_voted_for = -1; /* no vote on fresh init */
raft_cfg.cbs = &s_real_cbs;
raft_cfg.ctx = ha;
raft_init(&ha->raft, &raft_cfg);
return 0;
}
/* SIGUSR1 handler / ha_install_sigusr1() deleted — failover_requested replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
void ha_do_promote(ha_state_t *ha)
{
/* Monotonically increasing fencing token — standbys reject entries from lower terms */
atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel);
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
/* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */
ha->last_received_crc = WAL_CRC_NONE;
/* Req 10.5: log FSM transition to PRIMARY */
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")",
atomic_load_explicit(&ha->term, memory_order_acquire));
}
void ha_do_demote(ha_state_t *ha)
{
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
/* Reset so the next ha_promote_to_primary() re-registers hooks idempotently */
ha->hooks_registered = 0;
/* Hook disable requires hooksys_t — done by ha_demote_to_standby(cluster_t*) */
/* Req 10.5: log FSM transition to STANDBY */
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ")",
atomic_load_explicit(&ha->term, memory_order_acquire));
}
/* probe_thread() / wait_any_probe() deleted — only called by am_i_primary() (election-iomgr-refactor) */
/* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */
/*
* ha_is_primary: lock-free role check.
*/
int ha_is_primary(const ha_state_t *ha)
{
return (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire)
== CTRL_PRIMARY;
}
/* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */
/* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */
/* ── P3-6: Test-only failover shim ────────────────────────────── */
#ifdef XCE_TESTING
uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha)
{
ha_do_demote(ha);
ha_do_promote(ha);
return atomic_load_explicit(&ha->term, memory_order_acquire);
}
/* ha-term-safety task 2.1: drive the WIRED on_persist_entry callback so a unit
* test exercises the real adapter body (term stamp + WAL write) without
* raft_log_append (task 3.1). Exposed only under XCE_TESTING. */
int ha_cb_persist_entry_for_test(ha_state_t *ha, uint64_t index, uint64_t term,
const void *entry, size_t len)
{
return s_real_cbs.on_persist_entry(ha, index, term, entry, len);
}
/* ha-term-safety task 2.2: drive the WIRED on_get_entry_term callback so a unit
* test exercises the real entry-term accessor. Exposed only under XCE_TESTING. */
uint64_t ha_cb_get_entry_term_for_test(ha_state_t *ha, uint64_t index)
{
return s_real_cbs.on_get_entry_term(ha, index);
}
/* ha-term-safety task 2.2: drive the WIRED on_truncate_from callback so a unit
* test exercises the real conflict truncation. Exposed only under XCE_TESTING. */
int ha_cb_truncate_from_for_test(ha_state_t *ha, uint64_t index)
{
return s_real_cbs.on_truncate_from(ha, index);
}
/* ha-term-safety task 2.3: drive the WIRED on_send_append_entries callback so a
* unit test exercises the real AppendEntries send body (peer→ep map, prev tuple
* + leader_commit carry, local-WAL entry sourcing). Pair with
* repl_send_ae_capture_hook to assert envelope content. Exposed only under
* XCE_TESTING. */
int ha_cb_send_append_entries_for_test(ha_state_t *ha, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries)
{
return s_real_cbs.on_send_append_entries(ha, peer_idx, prev_log_index,
prev_log_term, leader_commit,
n_entries);
}
/* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback so a unit
* test exercises the real commit-source trigger (feed the library commit_index
* into the cluster's commit barrier). Exposed only under XCE_TESTING. */
void ha_cb_commit_advanced_for_test(ha_state_t *ha, uint64_t commit_index)
{
s_real_cbs.on_commit_advanced(ha, commit_index);
}
/* ha-term-safety task 3.1: drive the (static) own-term gate arm — the WAL_NOOP
* append now routed through ha_primary_log_append — so a unit test can assert
* the promotion no-op records the live term via the library. Exposed only under
* XCE_TESTING. */
void ha_arm_own_term_gate_for_test(cluster_t *c)
{
ha_arm_own_term_gate(c);
}
#endif
/* ha_check_heartbeat_nocluster() / ha_watchdog_thread() / ha_start_watchdog() /
* ha_stop_watchdog() deleted — watchdog pthread replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
void ha_fini(ha_state_t *ha)
{
if (!ha)
return;
/* ha-commit-durability task 4.4: cancel any pending bounded-window FATAL
* timer so a stopped member never reaches the deferred FATAL. The delaymgr
* task ctx is freed by promotion_fence_timeout_fn when it fires; cancelling
* makes that firing a no-op (it observes cancelled=1 and frees). */
ha_cancel_promotion_fence_timer(ha);
pthread_mutex_destroy(&ha->lock);
sem_destroy(&ha->heartbeat_seen);
/* Fix ASAN leak: free the SSL context if one was allocated */
if (ha->ssl_ctx) {
SSL_CTX_free((SSL_CTX *)ha->ssl_ctx);
ha->ssl_ctx = NULL;
}
}
/* ── Task 2.1/2.2: WAL catch-up entry application (standby side) ─────────────
*
* ha_apply_wal_catchup_entry: apply WAL catch-up entries received from the
* primary to local persistent storage and advance applied_lsn on success.
*
* applied_lsn semantic (bug-991): the LAST CONTIGUOUSLY-HELD LSN — the same
* meaning recovery (= durable_floor), checkpoint (= checkpoint_lsn), and init
* (= 0) assign it, and the watermark the standby advertises to the leader
* (sr->applied_lsn) so the leader's catch-up streamer streams the half-open
* range (applied_lsn, committed]. NEVER lsn+1 (next-to-apply): a +1 makes the
* standby advertise one LSN too high, so the leader skips the real next-needed
* entry and the standby's library logOk wedges forever (no WalAck → commit stall).
*
* Two modes (Req 5.3, 5.5):
*
* Batch mode (entry->n_entries > 0):
* Iterate entry->entries[0..n_entries-1] in order. Apply each sub-entry
* individually via apply_wal_entry(). NULL sub-entries are treated as
* corrupt and skipped (Req 3.3). After ALL sub-entries are processed,
* advance applied_lsn to the last-successfully-applied lsn so the next
* heartbeat reply reflects post-batch state (Req 5.3).
*
* Single-entry mode (entry->n_entries == 0):
* Backward-compatible path from task 2.1. Apply entry->lsn/entry_type
* directly and advance applied_lsn to lsn on success.
*
* In both modes:
* - applied_lsn is advanced atomically and monotonically.
* - On error for any individual entry: log ERROR, skip, continue (Req 3.3).
* - applied_lsn is in-memory only; not persisted to disk (design §Durability).
* - Standby stays in CTRL_STANDBY throughout (Req 3.4).
*
* Returns 0 on success (even if some batch entries were skipped due to errors).
* Returns -EINVAL if ha, p, or entry is NULL.
*/
/* ha_advance_applied_lsn: atomically advance ha->applied_lsn to new_lsn
* if new_lsn > current value (monotonic). */
static void ha_advance_applied_lsn(ha_state_t *ha, uint64_t new_lsn)
{
uint64_t old_lsn = (uint64_t)atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
while (new_lsn > old_lsn) {
if (atomic_compare_exchange_weak_explicit(
&ha->applied_lsn, &old_lsn, new_lsn,
memory_order_acq_rel, memory_order_acquire))
break;
}
}
/* ha_apply_single_entry: apply one decoded wal_catchup_entry_t sub-entry.
* Payload bytes (raw_payload/raw_payload_len, extracted by xce_decode)
* are passed through to apply_wal_entry so entity state is restored —
* not just the LSN chain (task 7.3).
* Returns 0 on success, non-zero on error (caller logs and skips). */
static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e)
{
/* Persist-before-apply-before-ACK: durably write the replicated entry to
* this standby's own WAL BEFORE applying it in memory (and before the
* caller ACKs). The ACK feeds the primary's commit majority, so it must
* imply durability — a standby must not ACK an entry it could lose on
* restart (Raft log[i] includes standbys; ha-async-io.md).
*
* persist_wal_write_replicated does NOT assign a new lsn (uses e->lsn) and
* does NOT re-broadcast; it fsyncs before returning and is idempotent on
* re-stream. WAL_CHECKPOINT is never streamed, but it would be rejected
* by both this primitive and apply_wal_entry. */
int prc = persist_wal_write_replicated(p, e->lsn, (uint32_t)e->entry_type,
e->raw_payload,
(uint32_t)e->raw_payload_len);
if (prc == -EAGAIN) {
/* Contiguity guard (bug-850): this entry is ahead of our log tail
* (a forward gap) — e.g. a live broadcast that arrived before the
* in-order catch-up of the prefix. NOT an error: drop it quietly and do
* NOT ack, so the catch-up continuation re-streams from current_lsn and
* the log fills in order. */
TRACE("ha_apply_single_entry: lsn=%" PRIu64 " ahead of log tail — "
"dropped (will arrive via in-order catch-up)", e->lsn);
return prc;
}
if (prc != 0) {
ERROR("ha_apply_single_entry: persist_wal_write_replicated failed for "
"lsn=%" PRIu64 " type=%u rc=%d — not applying/acking",
e->lsn, (unsigned)e->entry_type, prc);
/* No apply, no ACK on persist failure (mirrors no-ACK-on-apply-fail). */
return prc;
}
/* Logical entity restore only. persist_wal_write_replicated (above) already
* wrote the entry and owns the on-disk CRC chain + current_lsn — so we pass
* the entry fields directly rather than fabricating a header. (bug-778:
* a synthetic crc=0 header here used to clobber p->last_crc.) */
return apply_wal_entry(p, (uint32_t)e->entry_type, e->lsn,
e->raw_payload, (uint32_t)e->raw_payload_len);
}
/* ha_send_wal_ack_reply: send WalAckReply {term, lsn, success=1} back to
* the primary over ep. Skipped when ep or c is NULL (test/offline path).
* Task 3.1 (ha-wal-replication): ACK each successfully applied WAL entry. */
static void ha_send_wal_ack_reply(ha_state_t *ha, struct endpoint_t *ep,
cluster_t *c, uint64_t lsn)
{
if (!ep || !c)
return;
wal_ack_reply_t ack;
ack.term = (uint64_t)atomic_load_explicit(&ha->term, memory_order_acquire);
ack.lsn = lsn;
ack.success = 1;
xce_reply_t reply;
memset(&reply, 0, sizeof(reply));
reply.opcode = OPCode_walAck;
reply.status = 0;
reply.data_which = RequestReply_data_walackreplymsg;
reply.data.walackreplymsg = &ack;
xce_reply(ep, c->myhost, &reply);
}
/* ha_apply_term_entry_via_library: route ONE term-carrying replicated entry
* (a live broadcast or a catch-up sub-entry) through the raft library
* (raft_on_append_entries_request via ha_consume_append_entries) so the LIBRARY
* does logOk + (term,lsn) reconcile and on_persist_entry stamps the LEADER's
* term — never the local term-0 direct-apply path (ha-term-safety task 3.4).
*
* `e` is the single entry (its term/prev_log_index/prev_log_term carry the AE
* metadata, set by the leader's broadcast/stream path). On accept (the entry's
* term is now recorded at its LSN) the caller ACKs and advances applied. Returns
* 1 if the entry was accepted (recorded), 0 otherwise (reject / not yet
* applicable — e.g. a logOk-failing forward gap that catch-up will backfill). */
static int ha_apply_term_entry_via_library(ha_state_t *ha, persist_t *p,
cluster_t *c, struct endpoint_t *ep,
wal_catchup_entry_t *e)
{
/* Wrap the single entry in an AE container whose TOP-LEVEL fields carry the
* prev tuple + term that ha_consume_append_entries reads, with entries[0]
* pointing at the entry itself (the same shape repl_send_append_entries /
* repl_broadcast produce on the wire). */
wal_catchup_entry_t *sub_arr[1] = { e };
wal_catchup_entry_t ae;
memset(&ae, 0, sizeof(ae));
ae.n_entries = 1;
ae.entries = sub_arr;
ae.term = e->term;
ae.prev_log_index = e->prev_log_index;
ae.prev_log_term = e->prev_log_term;
ae.leader_commit = e->leader_commit;
uint64_t before_term = persist_entry_term(p, e->lsn);
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/ -1);
if (rc != 0)
return 0;
uint64_t after_term = persist_entry_term(p, e->lsn);
if (after_term == e->term) {
/* Accepted (appended / conflict-truncate-then-persist) or already-have
* (idempotent re-stream): ACK + advance applied on that signal.
*
* bug-991: applied_lsn is the "last contiguously-held LSN" (the same
* semantic recovery/checkpoint/init use, and the watermark the leader's
* catch-up streamer reads via sr->applied_lsn to stream (standby_lsn,
* committed]). Advance to e->lsn (the LSN we just held), NOT e->lsn + 1:
* a +1 over-advertises one LSN, so the leader's next stream begins at
* e->lsn + 2 and SKIPS the real next-needed entry e->lsn + 1, wedging
* logOk and stalling commit. */
(void)before_term;
/* bug-1043: the library route (on_persist_entry → persist_wal_write_
* replicated) writes the entry to the WAL but does NOT apply it to the
* in-memory entity state (job/queue/worker tables). The legacy direct-
* apply path (ha_apply_single_entry) called apply_wal_entry after the
* persist; the term-safe library route dropped it. So a standby that
* caught up entries 8..N via replication held them on disk but NOT in the
* queryable job table — on promotion `joblist`/`expect-job` could not
* find a committed job (it IS on disk; walcompare passes), looking like a
* committed-job loss. Re-apply here, mirroring the legacy path. Idempotent
* on a re-stream (restore_job upserts by jobid). WAL_NOOP / WAL_CONFIG_
* MARKER carry no entity payload — apply_wal_entry no-ops on them. */
apply_wal_entry(p, (uint32_t)e->entry_type, e->lsn,
e->raw_payload, (uint32_t)e->raw_payload_len);
ha_advance_applied_lsn(ha, e->lsn);
ha_send_wal_ack_reply(ha, ep, c, e->lsn);
return 1;
}
/* logOk reject (e.g. forward gap before the prefix arrives): no ACK; the
* ACK-driven catch-up continuation re-streams from current_lsn in order. */
return 0;
}
int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p,
wal_catchup_entry_t *entry,
struct endpoint_t *ep, cluster_t *c)
{
if (!ha || !p || !entry)
return -EINVAL;
/* ── ha-term-safety task 3.2: term-carrying AppendEntries → library route ──
* A term-safe AppendEntries (one sent by repl_send_append_entries / the
* library send path) carries a non-zero entry term stamp. Route it through
* raft_on_append_entries_request so the LIBRARY performs UpdateTerm (G5),
* logOk (G2), and the (term,lsn) conflict-truncate (G3) — its on_persist_entry
* callback does the WAL write (persist-before-ACK + bug-850 guard beneath).
* This REPLACES the direct persist_wal_write_replicated apply for term-safe
* entries; the legacy term-blind broadcast (entry term 0, no prev tuple) keeps
* the direct apply below until the live send path is retired (task 3.3).
*
* Single-entry term-safe AE only (n_entries==1 with a term). The library
* computes the target index from the prev tuple, then ACK on accept. */
if (c && c->ha.raft_c == c && entry->n_entries == 1 &&
entry->entries && entry->entries[0] && entry->entries[0]->term != 0) {
ha_apply_term_entry_via_library(ha, p, c, ep, entry->entries[0]);
return 0;
}
/* ── Batch mode: entry->n_entries > 0 ─────────────────────────────────── */
if (entry->n_entries > 0) {
/* Apply each sub-entry in LSN order. Track the last successfully
* applied LSN so we can advance applied_lsn after the full batch.
* Req 5.5: applied_lsn advances only after the full batch is written.
* Req 3.3: NULL or failed entries are logged and skipped; no crash. */
uint64_t last_good_lsn = 0;
int had_good = 0;
for (int i = 0; i < entry->n_entries; i++) {
wal_catchup_entry_t *sub = entry->entries[i];
if (!sub) {
ERROR("ha_apply_wal_catchup_entry: NULL sub-entry at "
"batch index %d — skipping", i);
continue;
}
/* ha-term-safety task 3.4: a term-carrying catch-up sub-entry routes
* through the LIBRARY (logOk + (term,lsn) reconcile + on_persist_entry
* stamps the LEADER's term). Each sub-entry carries its own prev tuple
* (wal_entry_to_wire encodes prevLog{Index,Term} per element). This
* retires the term-0 direct-apply path for live/catch-up replication.
* Entries stream in ascending LSN order, so the library's contiguity
* (prev tuple) is satisfied in sequence; a not-yet-applicable entry
* (logOk reject) is left for the catch-up continuation. */
if (c && c->ha.raft_c == c && sub->term != 0) {
if (ha_apply_term_entry_via_library(ha, p, c, ep, sub)) {
last_good_lsn = sub->lsn;
had_good = 1;
}
continue;
}
int rc = ha_apply_single_entry(p, sub);
if (rc != 0) {
/* -EAGAIN = contiguity gap (bug-850): entry ahead of our tail,
* dropped quietly — not an error (catch-up will backfill). */
if (rc != -EAGAIN)
ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed "
"for batch index %d lsn=%" PRIu64 " type=%u rc=%d "
"— skipping entry",
i, sub->lsn, (unsigned)sub->entry_type, rc);
/* Task 3.2: no ACK for failed/dropped entries */
continue;
}
/* Task 3.1: ACK each successfully applied sub-entry */
ha_send_wal_ack_reply(ha, ep, c, sub->lsn);
last_good_lsn = sub->lsn;
had_good = 1;
}
/* Advance applied_lsn to last_good_lsn after the full batch (bug-991:
* last contiguously-held LSN, NOT last_good_lsn + 1 — see the
* ha_apply_term_entry_via_library comment for why the +1 wedged the
* leader's catch-up streamer). Only advance if at least one entry was
* applied successfully. */
if (had_good)
ha_advance_applied_lsn(ha, last_good_lsn);
return 0;
}
/* ── Single-entry mode (task 2.1 backward-compatible path) ────────────── */
int rc = ha_apply_single_entry(p, entry);
if (rc != 0) {
/* -EAGAIN = contiguity gap (bug-850): dropped quietly, backfilled by
* the in-order catch-up; any other rc is a real failure. */
if (rc != -EAGAIN)
ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed for "
"lsn=%" PRIu64 " type=%u rc=%d — skipping entry",
entry->lsn, (unsigned)entry->entry_type, rc);
/* Task 3.2: no ACK on apply failure */
return rc;
}
/* Advance applied_lsn atomically to the last contiguously-held LSN
* (bug-991: entry->lsn, NOT entry->lsn + 1 — see the
* ha_apply_term_entry_via_library comment for the streamer-skip wedge a +1
* caused). This is the same last-held semantic recovery/checkpoint/init use,
* and matches the (standby_lsn, committed] range the leader streams. */
ha_advance_applied_lsn(ha, entry->lsn);
/* Task 3.1: ACK the successfully applied single entry */
ha_send_wal_ack_reply(ha, ep, c, entry->lsn);
return 0;
}
/* ── ha-term-safety task 3.2: library-routed AppendEntries intake ───────────
*
* ha_consume_append_entries: route ONE incoming heartbeat (0-entry) or
* replication (1-entry) AppendEntries through the raft library
* (raft_on_append_entries_request) so the LIBRARY — not hand-rolled term-blind
* code — performs:
* - UpdateTerm first on EVERY RPC incl. heartbeat (G5, R2.6): a higher-term
* AE makes the node adopt the term + step down (via on_become_follower).
* - logOk (G2, R3.1): (prevLogIndex,prevLogTerm) match against the local log
* via on_get_entry_term — a mismatch is REJECTED (no false convergence).
* - (term,lsn) conflict-truncate (G3, R3.2/R3.3): a divergent same-LSN tail is
* truncated (on_truncate_from) then the leader's entry re-persisted
* (on_persist_entry), never touching the agreed prefix below the divergence.
*
* This REPLACES the term-blind term handling in ha_consume_heartbeat_committed
* and the committed-boundary truncate in ha_rejoin_reconcile: the library's
* (term,lsn) decision, not a watermark, drives truncation.
*
* `entry` is a decoded wal_catchup_entry_t whose TOP-LEVEL fields carry the AE
* metadata (prev_log_index, prev_log_term, leader_commit) and, for a single
* entry, the entry itself (lsn, entry_type, term, raw_payload). `leader_idx` is
* the raft peer index of the sending leader.
*
* The library callbacks own all persistence/truncation; on a REJECT (stale term
* or logOk fail) the library mutates nothing and the member keeps its current
* state to retry on a later catch-up (R3.5/R3.6 — never report false
* convergence). The loud non-promotable fence for an unreconcilable member is
* the EXISTING ha-commit-durability promotion-time fence; this intake never
* advertises convergence it did not achieve.
*
* Single-receiver-thread model (controller request handling), matching the
* raft_on_vote_request intake: no ha->lock is taken (the raft state is serial
* per the receive thread). Returns 0 (no crash) on every path; the AE result is
* reflected in the library/WAL state, and a future ACK reply (task 3.3) carries
* the success/match_index to the leader.
*/
int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry,
int leader_idx)
{
if (!c || !entry)
return -EINVAL;
int n_entries = (entry->n_entries > 0) ? 1 : 0; /* 0 = heartbeat, 1 = entry */
/* AppendEntries metadata from the top-level container. */
uint64_t prev_log_index = entry->prev_log_index;
uint64_t prev_log_term = entry->prev_log_term;
uint64_t leader_commit = entry->leader_commit;
/* The incoming AE's term: for a single entry, the entry's term stamp; for a
* heartbeat, the top-level container's term (set by the sender to its live
* term). raft_on_append_entries_request uses this for UpdateTerm (G5). */
uint64_t ae_term;
const void *blob = NULL;
size_t blob_len = 0;
uint64_t entry_term = 0;
/* For a single entry, repack the [raft_entry_blob_t{type,len}][payload]
* envelope that on_persist_entry unpacks (the shared wal_types.h format), so
* the library-driven persist writes the same bytes the leader sent. */
uint8_t stackbuf[512];
uint8_t *heapbuf = NULL;
if (n_entries == 1) {
/* Single-entry mode: the entry lives in entries[0] (batch) or on the
* top-level container (single). Prefer the sub-entry when present. */
wal_catchup_entry_t *e =
(entry->entries && entry->entries[0]) ? entry->entries[0] : entry;
uint32_t type = e->entry_type;
const void *payload = e->raw_payload;
uint32_t plen = (uint32_t)e->raw_payload_len;
entry_term = e->term;
/* bug-1067: the AppendEntries HEADER term (UpdateTerm/stale-check) is the
* leader's CURRENT term, NOT the entry's stored term. For a LIVE append
* the two are equal (the leader appends at its current term), but for a
* CATCH-UP entry written by a PRIOR leader the stored term is LOWER than
* the current leader's term — using it as the AE term made the library
* stale-reject (term < s->term) every historical entry, wedging cross-term
* catch-up forever. The streamer stamps leader_current_term = ha->term;
* fall back to the stored term only when it is unset (legacy sender). */
ae_term = (e->leader_current_term != 0) ? e->leader_current_term
: entry_term;
blob_len = sizeof(raft_entry_blob_t) + plen;
uint8_t *buf = (blob_len <= sizeof(stackbuf)) ? stackbuf
: (heapbuf = malloc(blob_len));
if (!buf)
return -ENOMEM;
raft_entry_blob_t *bh = (raft_entry_blob_t *)buf;
bh->type = type;
bh->payload_len = plen;
if (payload && plen > 0)
memcpy(buf + sizeof(*bh), payload, plen);
blob = buf;
} else {
/* Heartbeat: the top-level container's term carries the leader's live
* term (the sender stamps it). No entry to persist. */
ae_term = entry->term;
}
raft_append_result_t res;
memset(&res, 0, sizeof(res));
int rc = raft_on_append_entries_request(&c->ha.raft, leader_idx, ae_term,
prev_log_index, prev_log_term,
leader_commit, blob, blob_len,
entry_term, n_entries, &res);
if (heapbuf)
free(heapbuf);
if (rc != 0) {
/* Library-internal error (not a protocol reject — those return RAFT_OK
* with res.success==0). Log loud; the member stays available to retry. */
WARN("ha_consume_append_entries: raft_on_append_entries_request rc=%d "
"(leader_idx=%d term=%" PRIu64 " prevIdx=%" PRIu64 ")",
rc, leader_idx, ae_term, prev_log_index);
return 0; /* never crash the receive thread */
}
TRACE("ha_consume_append_entries: leader_idx=%d term=%" PRIu64
" prevIdx=%" PRIu64 " n=%d -> success=%d match=%" PRIu64,
leader_idx, ae_term, prev_log_index, n_entries,
res.success, res.match_index);
/* res.success / res.match_index are consumed by the ACK reply path (task
* 3.3). On a reject (res.success==0) the library left the log untouched —
* R3.5/R3.6: no false convergence, the member reconciles on a later AE. */
return 0;
}
/* ha_handle_snapshot_chunk — standby snapshot receive handler (Task 3.5c).
*
* Each call appends payload_bytes to snap_dir/snap.part.
* When chunk->is_last is non-zero:
* 1. Verify total bytes written matches chunk->total_size.
* 2. Atomically rename snap.part → snap-{snapshot_lsn:016x}.bin.
* 3. Load the snapshot via persist_load_snapshot().
* 4. Set ha->applied_lsn = p->checkpoint_lsn (in-memory).
*
* Mid-transfer reconnect (Req 4.4): the primary restarts every transfer at
* chunk 0, so chunk_index == 0 discards any stale snap.part left by an
* interrupted transfer before appending — no cross-chunk state needed.
* Returns 0 on success, negative errno on error.
* Requirements: 4.2, 4.3, 4.4 (ha-repl-hardening spec)
*/
int ha_handle_snapshot_chunk(ha_state_t *ha, persist_t *p,
const char *snap_dir,
snapshot_transfer_t *chunk,
const uint8_t *payload_bytes,
size_t payload_len)
{
if (!ha || !p || !snap_dir || !chunk)
return -EINVAL;
char part_path[4096 + 16];
snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir);
/* Req 4.4: a transfer always restarts from chunk 0 — discard any stale
* partial file from an interrupted previous transfer. */
if (chunk->chunk_index == 0 && unlink(part_path) != 0 && errno != ENOENT)
WARN("ha_handle_snapshot_chunk: unlink stale snap.part failed: %s",
strerror(errno));
/* Append chunk payload to snap.part */
if (payload_bytes && payload_len > 0) {
FILE *f = fopen(part_path, "ab");
if (!f) {
int err = errno;
ERROR("ha_handle_snapshot_chunk: cannot open snap.part: %s",
strerror(err));
return -err;
}
size_t written = fwrite(payload_bytes, 1, payload_len, f);
fclose(f);
if (written != payload_len) {
ERROR("ha_handle_snapshot_chunk: short write to snap.part "
"(%zu of %zu bytes)", written, payload_len);
return -EIO;
}
}
if (!chunk->is_last)
return 0;
/* isLast=true: verify size, rename, load, set applied_lsn */
/* Verify total file size equals chunk->total_size */
struct stat st;
if (stat(part_path, &st) != 0) {
int err = errno;
ERROR("ha_handle_snapshot_chunk: stat(snap.part) failed: %s",
strerror(err));
return -err;
}
if ((uint64_t)st.st_size != chunk->total_size) {
ERROR("ha_handle_snapshot_chunk: size mismatch — "
"written=%" PRIu64 " expected=%" PRIu64,
(uint64_t)st.st_size, chunk->total_size);
return -EIO;
}
/* Atomic rename snap.part → snap-{lsn:016x}.bin */
char final_path[4096 + 32];
snprintf(final_path, sizeof(final_path),
"%s/snap-%016" PRIx64 ".bin", snap_dir, chunk->snapshot_lsn);
if (rename(part_path, final_path) != 0) {
int err = errno;
ERROR("ha_handle_snapshot_chunk: rename snap.part → %s failed: %s",
final_path, strerror(err));
return -err;
}
/* Load the snapshot — sets p->checkpoint_lsn and p->current_lsn */
int rc = persist_load_snapshot(p, final_path, chunk->snapshot_lsn);
if (rc != 0) {
ERROR("ha_handle_snapshot_chunk: persist_load_snapshot failed rc=%d", rc);
return rc;
}
/* Advance applied_lsn to checkpoint_lsn (in-memory only — Req 4.2) */
atomic_store_explicit(&ha->applied_lsn, p->checkpoint_lsn,
memory_order_release);
INFO("HA: snapshot loaded lsn=%" PRIu64 " applied_lsn set",
p->checkpoint_lsn);
return 0;
}
/* ha_nop_reply_state_report — populate *sr from ha state and wire into reply.
* Reads ha->applied_lsn (atomic) and myconf->config_version_tag, stores them
* in *sr, then sets reply_out->data_which = stateReportMsg and
* reply_out->data.state_report = sr.
* Caller keeps *sr alive until xce_reply() serialises the message.
* Requirements: 1.1, 1.3, 1.4 (ha-repl-hardening spec) */
void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf,
state_report_t *sr, xce_reply_t *reply_out)
{
sr->applied_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
sr->config_tag = myconf ? myconf->config_version_tag : 0;
reply_out->data_which = RequestReply_data_stateReportMsg;
reply_out->data.state_report = sr;
}
/*
* ha_evaluate_repl_action — pure decision function for the unified heartbeat-
* driven replication protocol.
*
* Implements the decision tree from the ha-repl-hardening design:
*
* if standby_tag < committed_tag → HA_REPL_PUSH_CONFIG
* elif standby_lsn + 1 < oldest_wal_lsn → HA_REPL_SEND_SNAPSHOT
* elif standby_lsn < committed_lsn → HA_REPL_STREAM_WAL
* else (tag >= committed AND lsn >= committed) → HA_REPL_CONVERGED
*
* No side effects. All callers must use the returned action to drive the
* replication send; they must not cache the result beyond the current cycle.
*
* Requirements: 2.2, 2.3, 2.4 (ha-repl-hardening spec)
* Boundary: Primary Heartbeat Handler (ha.c)
*/
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
uint64_t standby_lsn,
uint64_t committed_tag,
uint64_t committed_lsn,
uint64_t oldest_wal_lsn)
{
if (standby_tag < committed_tag)
return HA_REPL_PUSH_CONFIG;
/* Snapshot only when the standby's NEXT-needed entry (standby_lsn + 1) has
* been garbage-collected below the oldest retained WAL entry — i.e. it can
* no longer be served by streaming. An empty standby (standby_lsn=0) on a
* fresh cluster whose oldest retained entry IS lsn 1 (post reserve-lsn-0)
* needs lsn 1, which is retained: 0 + 1 < 1 is false → STREAM, no snapshot.
* Using standby_lsn + 1 (not standby_lsn - ... ) avoids unsigned underflow
* when oldest_wal_lsn == 0 (empty WAL): 0 + 1 < 0 is false → not snapshot. */
if (standby_lsn + 1 < oldest_wal_lsn)
return HA_REPL_SEND_SNAPSHOT;
if (standby_lsn < committed_lsn)
return HA_REPL_STREAM_WAL;
return HA_REPL_CONVERGED;
}
/*
* ha_process_standby_state — update per-standby convergence tracking.
*
* Called on each heartbeat reply from a standby (CTRL_PRIMARY).
* Stores (reported_tag, reported_lsn) into the per-standby
* arrays, evaluates convergence, and logs INFO on the first
* not-converged → converged transition (Req 7.2). Regression resets the flag
* (Req 2.5).
*
* Convergence: reported_tag >= committed_tag AND reported_lsn >= committed_lsn.
*
* Requirements: 1.2, 2.2, 2.5 (ha-repl-hardening spec)
*/
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
uint64_t reported_tag, uint64_t reported_lsn,
uint64_t committed_tag, uint64_t committed_lsn)
{
if (peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES)
return;
/* Store reported values */
ha->standby_reported_tag[peer_idx] = reported_tag;
ha->standby_reported_lsn[peer_idx] = reported_lsn;
/* Evaluate convergence */
bool was_converged = ha->standby_converged[peer_idx];
bool now_converged = (reported_tag >= committed_tag) &&
(reported_lsn >= committed_lsn);
ha->standby_converged[peer_idx] = now_converged;
/* Log INFO on first transition to converged (Req 7.2) */
if (now_converged && !was_converged) {
INFO("HA: standby[%d] converged: config_tag=%" PRIu64
" applied_lsn=%" PRIu64 " (committed_tag=%" PRIu64
" committed_lsn=%" PRIu64 ")",
peer_idx, reported_tag, reported_lsn,
committed_tag, committed_lsn);
}
}
/* ───────────────────────────────────────────────────────────────────────
* raft-drive-async Task 3.2 — SnapshotSend slow-lane offload (bug-1198).
*
* The snapshot transfer (a fresh persist_checkpoint → fdatasync, then a
* chunked socket write of the snapshot file) is the one unbounded operation
* reachable from the repl-ack decision loop. Run inline it blocks the
* io_worker/drive thread for the whole transfer (and queues that peer's later
* messages behind it). This offloads it to the bounded slow lane, keyed per
* standby so at most one snapshot per peer is in flight (Req 1.5).
* ─────────────────────────────────────────────────────────────────────── */
typedef struct {
cluster_t *c;
endpoint_t *ep;
int peer_idx;
} snapshot_send_ctx_t;
/* ha_snapshot_do_send: the actual (blocking) snapshot work — shared by the
* inline path and the lane worker so both behave identically. In XCE_TESTING
* builds the send is replaced by ha_repl_send_snapshot_hook(peer_idx). */
static void ha_snapshot_do_send(cluster_t *c, endpoint_t *ep, int peer_idx)
{
#ifdef XCE_TESTING
if (ha_repl_send_snapshot_hook != NULL) {
ha_repl_send_snapshot_hook(peer_idx);
return;
}
#endif
if (c && c->persist.snap_path[0]) {
extern int ha_send_snapshot_to_standby(endpoint_t *, const char *,
uint64_t);
/* Req 4.1: create a FRESH checkpoint so the transferred snapshot
* reflects current state (also trims WAL + rotates old snaps).
* -EBUSY means another checkpoint is in flight — fall back to the
* existing checkpoint_lsn rather than skipping the cycle (at most one
* checkpoint in flight is the existing persist_checkpoint invariant). */
int ckpt_rc = persist_checkpoint(&c->persist);
if (ckpt_rc != 0 && ckpt_rc != -EBUSY)
WARN("HA repl: persist_checkpoint before snapshot send "
"failed rc=%d — sending existing checkpoint", ckpt_rc);
ha_send_snapshot_to_standby(ep, c->persist.snap_path,
c->persist.checkpoint_lsn);
}
}
/*
* snapshot_lane_fn: slow-lane worker entry. Runs the bounded blocking
* snapshot send off the drive thread, then ALWAYS clears the per-peer
* in-flight flag (success or failure) so the dedup never wedges and a later
* heartbeat can retry. Frees the ctx.
*
* ⚠ BORROWED-ENDPOINT LIFETIME HAZARD (bug-1197 family — why this path is
* gated OFF by default). `ep` here is a standby endpoint shared with the
* io_worker epoll loop, NOT one this lane owns (contrast reconnect_lane_fn,
* which owns the ep it creates). If the standby RDHUP-disconnects while this
* transfer runs, the io_worker can endpoint_fini/free `ep` concurrently → UAF.
* Today (flag OFF) the send runs inline ON the io_worker, so teardown and send
* are serialized on one thread and cannot overlap. Flipping
* g_drive_async_snapshot_offload ON is therefore only safe once Task 3.4
* establishes the endpoint-lifetime discipline for the whole pool/lane
* dispatch model (refcount or lane-serialized teardown). Until then this code
* lands DORMANT and is exercised only by unit tests with a stable endpoint. */
static void snapshot_lane_fn(void *arg)
{
snapshot_send_ctx_t *sx = (snapshot_send_ctx_t *)arg;
if (!sx) return;
cluster_t *c = sx->c;
int peer_idx = sx->peer_idx;
ha_snapshot_do_send(c, sx->ep, peer_idx);
/* Clear in-flight LAST so a concurrent heartbeat tick cannot enqueue a
* second snapshot for this peer until this one has fully finished.
* Clear the tracked LSN (Req 3.1-3.3) alongside the flag — same
* lifecycle, one more field — so rotate_snapshots never sees a stale
* in-flight LSN for a transfer that has already completed. */
atomic_store_explicit(&c->ha.snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&c->ha.snapshot_inflight[peer_idx], 0,
memory_order_release);
free(sx);
}
/*
* ha_snapshot_send_dispatch: route a SEND_SNAPSHOT decision either inline
* (flag OFF, production default — runs on the caller/drive thread, unchanged
* legacy behavior) or onto the slow lane (flag ON — non-blocking). The
* per-peer single-flight CAS on ha->snapshot_inflight[] is the authoritative
* dedup; the lane's per-key dedup is a redundant backstop. Mirrors the
* reconnect-offload terminal-path handling (Task 2.1).
*/
static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c,
endpoint_t *ep, int peer_idx)
{
/* Flag OFF (default) or no cluster handle (no lane) → inline. */
if (!ha_snapshot_offload_enabled() || c == NULL) {
ha_snapshot_do_send(c, ep, peer_idx);
return;
}
/* Per-peer single-flight: CAS 0->1. Already 1 → a snapshot to this peer
* is in flight; enqueue nothing (Req 1.5). */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&ha->snapshot_inflight[peer_idx], &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return;
}
/* Record the checkpoint position this transfer references (Req 3.1-3.3)
* at the SAME point the flag above is set — before the first byte of the
* transfer goes out — so a reader that observes snapshot_inflight[peer_idx]
* == 1 can find a meaningful LSN alongside it. rotate_snapshots (task 4.3)
* consults this to avoid deleting the checkpoint file this transfer is
* still reading from. */
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx],
c->persist.checkpoint_lsn, memory_order_release);
snapshot_send_ctx_t *sx = malloc(sizeof(*sx));
if (!sx) {
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
sx->c = c; sx->ep = ep; sx->peer_idx = peer_idx;
uint64_t key = (uint64_t)(peer_idx + 1); /* +1: key 0 disables dedup */
int srv = slowlane_submit(&c->slowlane, key, snapshot_lane_fn, sx);
if (srv == -EEXIST) {
/* Lane still holds this peer's key from a prior snapshot task that has
* already cleared snapshot_inflight (its last action) but whose key the
* lane has not yet removed. Our CAS saw 0 and set 1, but the submit is
* refused — release the flag we set so no clear is lost, and let the
* next tick retry. */
free(sx);
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure (still queued, will run). Only on a real failure tear down
* and release the guard so the next tick retries; never self-terminate. */
WARN("HA repl: slowlane_submit snapshot peer[%d] failed: %d — retry "
"next cycle", peer_idx, srv);
free(sx);
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: enqueued exactly once; snapshot_lane_fn clears the flag
* and the tracked LSN. */
}
/* ───────────────────────────────────────────────────────────────────────
* raft-drive-async Task 3.3 — requester-less ConfigPush slow-lane offload
* (bug-1198).
*
* The two REQUESTER-LESS config-push sites — post-election promotion
* (raft_cb_become_leader) and the heartbeat-driven HA_REPL_PUSH_CONFIG
* convergence case — call push_config_transport_only, which spawns one push
* thread per standby and then BLOCKS the caller in wait_all_with_timeout up to
* CONF_PUSH_TIMEOUT_MS. Run inline on the single io_worker that drives both of
* those sites, that wait stalls every heartbeat/vote for the whole timeout.
* This offloads it to the bounded slow lane. Config push is a CLUSTER-WIDE
* fan-out, so a single cluster-wide single-flight guard (configpush_inflight) +
* a single dedup key (key=1) suffice — at most one push in flight at a time.
*
* The admin-upgrade path (push_config_to_standbys, which has a waiting
* requester / majority barrier) is NOT routed here and stays unchanged.
* ─────────────────────────────────────────────────────────────────────── */
#ifdef XCE_TESTING
extern void (*ha_configpush_probe)(void);
#endif
typedef struct {
cluster_t *c;
uint8_t *bytes; /* deep copy, owned by the ctx; NULL when blen==0 */
size_t blen;
uint8_t sha[32]; /* valid only when has_sha != 0 */
int has_sha;
uint64_t new_tag;
} configpush_ctx_t;
/* ha_config_push_do_send: the actual (blocking) config-push work — shared by
* the inline path and the lane worker so both behave identically. In
* XCE_TESTING builds a non-NULL ha_configpush_probe() is invoked right around
* the send so a test can record the running thread / park the lane worker. */
static void ha_config_push_do_send(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag)
{
#ifdef XCE_TESTING
/* XCE_TESTING does not link cluster.c and uses hook shims for the real
* transport (push_config_transport_only's extern is #ifndef XCE_TESTING).
* The probe stands in for the send and records the running thread. */
if (ha_configpush_probe != NULL)
ha_configpush_probe();
(void)c; (void)bytes; (void)blen; (void)sha256; (void)new_tag;
#else
push_config_transport_only(c, bytes, blen, sha256, new_tag, NULL);
#endif
}
/*
* configpush_lane_fn: slow-lane worker entry. Runs the bounded blocking
* config push off the drive thread, then ALWAYS clears the cluster-wide
* in-flight flag (success or failure) so the dedup never wedges and a later
* promotion/heartbeat can retry. Frees the deep-copied bytes and the ctx.
*
* ⚠ BORROWED-ENDPOINT LIFETIME HAZARD (bug-1197 family — why this path is
* gated OFF by default). push_config_transport_only here reads
* c->ha.standby_endpoints and writes to those standby endpoints, which are
* shared with the io_worker epoll loop, NOT owned by this lane (contrast
* reconnect_lane_fn, which owns the ep it creates). If a standby
* RDHUP-disconnects while this push runs, the io_worker can endpoint_fini/free
* those endpoints concurrently → UAF. Today (flag OFF) the push runs inline ON
* the io_worker, so teardown and push are serialized on one thread and cannot
* overlap. Flipping g_drive_async_configpush_offload ON is therefore only safe
* once Task 3.4 establishes the endpoint-lifetime discipline for the whole
* pool/lane dispatch model. Until then this code lands DORMANT and is
* exercised only by unit tests with a stable (or absent) endpoint set.
*/
static void configpush_lane_fn(void *arg)
{
configpush_ctx_t *cx = (configpush_ctx_t *)arg;
if (!cx) return;
cluster_t *c = cx->c;
ha_config_push_do_send(c, cx->bytes, cx->blen,
cx->has_sha ? cx->sha : NULL, cx->new_tag);
/* Clear in-flight LAST so a concurrent promotion/heartbeat tick cannot
* enqueue a second config push until this one has fully finished. */
atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release);
free(cx->bytes);
free(cx);
}
/*
* ha_config_push_offloadable: route a requester-less config push either inline
* (flag OFF, production default — runs on the caller/drive thread, unchanged
* legacy behavior; the CALLER still owns `bytes` and frees them after we
* return) or onto the slow lane (flag ON — non-blocking; we take a deep copy of
* `bytes` so the caller's free is safe). The cluster-wide single-flight CAS on
* ha.configpush_inflight is the authoritative dedup; the lane's per-key dedup
* is a redundant backstop. Mirrors the snapshot-offload terminal-path handling
* (Task 3.2). Never self-terminates: on any failure it clears the guard and
* the next cycle retries.
*/
static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag)
{
/* Flag OFF (default) or no cluster handle (no lane) → inline. */
if (!ha_configpush_offload_enabled() || c == NULL) {
ha_config_push_do_send(c, bytes, blen, sha256, new_tag);
return;
}
/* Cluster-wide single-flight: CAS 0->1. Already 1 → a config push is in
* flight; enqueue nothing. */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&c->ha.configpush_inflight, &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return;
}
configpush_ctx_t *cx = malloc(sizeof(*cx));
if (!cx) {
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
cx->c = c;
cx->new_tag = new_tag;
cx->has_sha = (sha256 != NULL) ? 1 : 0;
if (cx->has_sha)
memcpy(cx->sha, sha256, 32);
if (bytes != NULL && blen > 0) {
cx->bytes = malloc(blen);
if (!cx->bytes) {
free(cx);
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
memcpy(cx->bytes, bytes, blen);
cx->blen = blen;
} else {
cx->bytes = NULL;
cx->blen = 0;
}
uint64_t key = 1; /* single cluster-wide dedup key (one config-push stream) */
int srv = slowlane_submit(&c->slowlane, key, configpush_lane_fn, cx);
if (srv == -EEXIST) {
/* Lane still holds the key from a prior config-push task that has
* already cleared configpush_inflight (its last action) but whose key
* the lane has not yet removed. Our CAS saw 0 and set 1, but the
* submit is refused — release the flag we set so no clear is lost, and
* let the next tick retry. */
free(cx->bytes);
free(cx);
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure (still queued, will run). Only on a real failure tear down
* and release the guard so the next tick retries; never self-terminate. */
WARN("HA repl: slowlane_submit configpush failed: %d — retry next cycle",
srv);
free(cx->bytes);
free(cx);
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: enqueued exactly once; configpush_lane_fn clears the flag. */
}
#ifdef XCE_TESTING
/* Test seam: directly drive ha_config_push_offloadable (the two production call
* sites are #ifndef XCE_TESTING). Returns 0 (the dispatch itself cannot fail
* fatally — failures clear the guard and retry). */
int ha_config_push_offloadable_for_test(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag)
{
ha_config_push_offloadable(c, bytes, blen, sha256, new_tag);
return 0;
}
#endif
/*
* ha_dispatch_repl_action_for_standby — execute the decision-loop action for
* one standby on the current heartbeat cycle (Task 3.3).
*
* Reads ha->standby_reported_tag[peer_idx] / standby_reported_lsn[peer_idx]
* (stored by ha_process_standby_state on the same heartbeat reply), calls
* ha_evaluate_repl_action, and acts on the result:
*
* HA_REPL_PUSH_CONFIG → push_config_transport_only(c, …) [or test hook] (Task 3.3)
* HA_REPL_SEND_SNAPSHOT→ stub (Task 3.5)
* HA_REPL_STREAM_WAL → ha_stream_wal_to_standby(c, ha, idx) (Task 3.4)
* HA_REPL_CONVERGED → no-op
*
* One action per heartbeat cycle per standby (design §Unified Heartbeat-Driven
* Protocol). Returns 0 on success, -EINVAL on bad peer_idx.
*
* Requirements: 1.4, 5.1–5.5 (ha-repl-hardening spec)
*/
int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
cluster_t *c,
endpoint_t *ep,
int peer_idx,
uint64_t committed_tag,
uint64_t committed_lsn,
uint64_t oldest_wal_lsn)
{
if (!ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES)
return -EINVAL;
uint64_t standby_tag = ha->standby_reported_tag[peer_idx];
uint64_t standby_lsn = ha->standby_reported_lsn[peer_idx];
ha_repl_action_t action = ha_evaluate_repl_action(standby_tag, standby_lsn,
committed_tag, committed_lsn,
oldest_wal_lsn);
switch (action) {
case HA_REPL_PUSH_CONFIG:
INFO("HA repl: standby[%d] tag=%" PRIu64 " < committed_tag=%" PRIu64
" → push config", peer_idx, standby_tag, committed_tag);
#ifdef XCE_TESTING
if (ha_repl_push_config_for_standby_hook != NULL) {
ha_repl_push_config_for_standby_hook(c, peer_idx);
break;
}
#endif
if (c) {
#ifndef XCE_TESTING
extern int conf_read_raw(const char *, uint8_t **, size_t *);
uint8_t *cfg_bytes = NULL;
size_t cfg_len = 0;
if (c->myconf && c->myconf->conffile &&
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
/* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON).
* The offload deep-copies cfg_bytes, so this free is still
* correct regardless of routing. */
ha_config_push_offloadable(c, cfg_bytes, cfg_len,
c->myconf->config_sha256,
c->myconf->config_version_tag);
free(cfg_bytes);
} else {
ha_config_push_offloadable(c, NULL, 0, NULL,
c->myconf ? c->myconf->config_version_tag : 0);
}
#endif
}
break;
case HA_REPL_SEND_SNAPSHOT:
INFO("HA repl: standby[%d] lsn=%" PRIu64
" < oldest_wal_lsn=%" PRIu64 " → requesting snapshot transfer",
peer_idx, standby_lsn, oldest_wal_lsn);
/* Task 3.2: route inline (flag OFF, default) or onto the slow lane
* (flag ON). The send body — fresh checkpoint + chunked transfer, or
* the XCE_TESTING hook — lives in ha_snapshot_do_send so both paths
* are identical. */
ha_snapshot_send_dispatch(ha, c, ep, peer_idx);
break;
case HA_REPL_STREAM_WAL:
/* Task 3.4: batch WAL streaming.
* Enqueues all batch OPCode_walCatchupEntry messages for the range
* [standby_lsn+1, committed_lsn] via ha_stream_wal_to_standby().
* All xce_request() calls are non-blocking (enqueue-only). */
INFO("HA repl: standby[%d] lsn=%" PRIu64
" < committed_lsn=%" PRIu64 " → stream WAL batches",
peer_idx, standby_lsn, committed_lsn);
#ifdef XCE_TESTING
if (ha_repl_stream_wal_hook != NULL) {
const char *wal_path = (c && c->persist.wal_path[0])
? c->persist.wal_path : "";
ha_repl_stream_wal_hook(ep, wal_path, standby_lsn, committed_lsn);
break;
}
#endif
if (c && c->persist.wal_path[0]) {
extern int ha_stream_wal_to_standby(endpoint_t *ep,
const char *wal_path,
const persist_t *p,
uint64_t leader_term,
uint64_t standby_lsn,
uint64_t committed_lsn);
/* bug-1028: pass &c->persist so streamed entry terms come from the
* authoritative per-LSN index (persist_entry_term), matching the
* standby's logOk + the library send path.
* bug-1067: pass the leader's CURRENT term so historical entries
* carry the AE header term (not their lower stored term). */
uint64_t leader_term =
atomic_load_explicit(&c->ha.term, memory_order_acquire);
ha_stream_wal_to_standby(ep, c->persist.wal_path, &c->persist,
leader_term, standby_lsn, committed_lsn);
}
break;
case HA_REPL_CONVERGED:
/* No action needed — standby is up to date */
break;
}
return 0;
}
/* ── Dispatch gate (req 1.3, 1.4) ──────────────────────────────────────────
*
* Called from myreq_handler() in cluster.c before the opcode switch.
* Returns 0 when the operation is permitted for the current role.
*
* Primary-only ops are any op that is not one of the always-allowed set:
* nop, reply, hello, join, shutdown, heartbeat, redirect,
* steppingDown, replEntry, replAck, replSnapshot, replCatchUpDone,
* replConfig, replConfigPush, requestVote, probeConfigTag,
* walAck, upgradeConfigAck, snapshotTransfer, walCatchupEntry.
*
* For CTRL_STANDBY the check is: op is in the primary-only set → deny.
* For CTRL_INIT / CTRL_ELECTING the allowlist is tighter: only
* OP_requestVote and OP_probeConfigTag pass through.
*/
/* ha_has_write_quorum — check whether the cluster has enough connected
* standbys to achieve majority for state-changing operations.
*
* Quorum = ceil(n_peers / 2) + 1 (self-vote included).
* Example: n_peers=3 → quorum=2 → need ≥1 connected standby.
*
* Called from ha_dispatch_gate for CTRL_PRIMARY — if the cluster loses
* majority (standby failures), write ops are rejected with -XCE_ERR_NOT_READY
* to preserve the majority-consistency invariant.
*
* Read-only queries (joblist, reslist, hostlist, etc.) remain available
* regardless of majority status — they don't mutate state. */
static bool ha_has_write_quorum(ha_state_t *ha)
{
if (!ha)
return false;
/* Single-node cluster: no standbys needed — self is the majority. */
if (ha->n_peers <= 1)
return true;
/* Count standbys able to receive new log entries. When the replicator
* back-pointer is wired (task 5.3 init), use registry truth: slots with
* ep != NULL && !in_catch_up under r->lock (Req 3.1-3.3) — catch-up
* members cannot ACK new entries, so they must not count toward write
* admission. NULL back-pointer (legacy contexts) falls back to the
* standby_endpoints count populated by raft_cb_become_leader (promoted
* vote eps) and on_standby_connected (late joiners); removed by error
* handlers. */
int connected;
if (ha->repl) {
connected = repl_active_standby_count(ha->repl);
} else {
connected = ha->standby_endpoints
? ha->standby_endpoints->count : 0;
}
/* Need at least n_peers/2 connected standbys to achieve quorum.
* Integer division: n_peers/2 for n_peers=3 → 1 standby minimum. */
return connected >= (int)(ha->n_peers / 2);
}
int ha_dispatch_gate(ha_state_t *ha, int opcode)
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
switch (role) {
case CTRL_PRIMARY:
/* ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): the config
* serve-fence is the PRODUCTION READER that makes the fence real. When
* set, the latest committed WAL_CONFIG_MARKER references config bytes that
* do not hash-match the locally-held bytes (or the bytes are absent), so
* this member MUST NOT serve under the new configuration. Refuse
* config-dependent state-changing ops (loud non-serving), but keep
* replication / read-only / status ops flowing so the correct bytes can
* arrive out-of-band and re-verify clears the fence. The member STAYS
* ALIVE (returns an error code, never exits — controller-never-self-
* terminate). Checked before the quorum gate so a fenced primary never
* serves divergent config even with quorum. */
if (config_serve_fenced(ha->raft_c)) {
switch (opcode) {
/* Replication ops — let the correct config bytes arrive + re-verify */
case OPCode_heartbeat:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_upgradeConfigAck:
case OPCode_hello:
case OPCode_walAck:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replCatchUpDone:
case OPCode_walCatchupEntry:
case OPCode_snapshotTransfer:
/* Read-only queries / status — answerable while fenced */
case OPCode_get:
case OPCode_probeConfigTag:
case OPCode_ctrlstatus:
break; /* fall through to the quorum check below */
default:
/* Config-dependent state-changing op while fenced: refuse. */
return -XCE_ERR_NOT_READY;
}
}
/* State-changing operations require majority so committed entries
* are durable across failover. Read-only queries and replication
* ops are always allowed. */
if (!ha_has_write_quorum(ha)) {
/* Quorum-transition observability backstop (Req 8.1, 8.3): record the
* loss edge if a membership-change site missed it. The edge detector
* makes this silent on every call after the first — repeated rejected
* operations do NOT produce a log storm. */
ha_log_quorum_transition(ha);
switch (opcode) {
/* Replication ops — always allowed (they build majority) */
case OPCode_heartbeat:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_upgradeConfigAck:
case OPCode_hello:
case OPCode_walAck:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replCatchUpDone:
case OPCode_walCatchupEntry:
case OPCode_snapshotTransfer:
/* Read-only queries */
case OPCode_get:
case OPCode_probeConfigTag:
/* Read-only controller status (controller-status 7.1, 7.2):
* answers the local identity/health view even without quorum —
* precisely when an operator most wants it. */
case OPCode_ctrlstatus:
/* Fact-bearing ops — reports of events that already happened
* (worker job transitions/rusage, scheduler pend reasons).
* Rejecting them loses the fact forever: the reporter never
* retries (bug-396). Admit so the RETAIN path can append and
* hold them until quorum returns (Req 3.2; umbrella 2.8 —
* facts are retained, never dropped). */
case OPCode_notify:
case OPCode_update:
case OPCode_setpreason:
case OPCode_clrpreason:
return 0;
default:
return -XCE_ERR_NOT_READY;
}
}
return 0;
case CTRL_INIT:
/* Allow election RPCs AND primary→standby replication messages.
* During the startup stagger a primary may already connect and push
* config — accepting these ensures the node doesn't miss the initial
* bootstrap and can transition to CTRL_STANDBY immediately. */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_heartbeat ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
opcode == OPCode_replConfigPush ||
opcode == OPCode_walCatchupEntry ||
opcode == OPCode_snapshotTransfer ||
/* Read-only controller status — answerable in every role
* (controller-status 7.1, 7.2). */
opcode == OPCode_ctrlstatus)
return 0;
return -XCE_ERR_NOT_READY;
case CTRL_ELECTING:
/* Allow election RPCs AND primary→standby replication messages.
* A node campaigning for leadership must still accept config, WAL,
* and heartbeats from the current primary — if it wins the election
* it needs current state; if it loses, it becomes STANDBY and must
* resume replication without having missed pushes. Only job/client
* ops are rejected (we're not ready to serve). */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
opcode == OPCode_replConfigPush ||
opcode == OPCode_walCatchupEntry ||
opcode == OPCode_snapshotTransfer ||
/* Read-only controller status — answerable in every role
* (controller-status 7.1, 7.2). */
opcode == OPCode_ctrlstatus)
return 0;
return -XCE_ERR_NOT_READY;
case CTRL_STANDBY: {
/* Allow ops that standbys legitimately handle */
switch (opcode) {
case OPCode_nop:
case OPCode_reply:
case OPCode_hello:
case OPCode_join:
case OPCode_shutdown:
case OPCode_heartbeat:
case OPCode_redirect:
case OPCode_steppingDown:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replSnapshot:
case OPCode_replCatchUpDone:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_requestVote:
case OPCode_preVoteRequest:
case OPCode_probeConfigTag:
case OPCode_walAck:
case OPCode_upgradeConfigAck:
case OPCode_snapshotTransfer:
case OPCode_walCatchupEntry:
/* Read-only controller status — a standby answers with its local
* view (it reports the primary it knows) (controller-status 7.1, 7.2). */
case OPCode_ctrlstatus:
return 0;
default:
return -XCE_ERR_STANDBY;
}
}
default:
return 0;
}
}
/* ha_config_promotion_fenced (ha-term-safety task 4.3, R7.9): a member is
* non-promotable while config serve-fenced — its latest committed config marker
* references bytes it does not hold/match, so it must not become authoritative.
* Pure atomic read via config_serve_fenced (no I/O) — safe on the election path. */
int ha_config_promotion_fenced(const ha_state_t *ha)
{
if (!ha)
return 0;
return config_serve_fenced(ha->raft_c) ? 1 : 0;
}
/* ── Redirect helpers used by join.c ───────────────────────── */
/*
* ha_should_redirect: returns 1 when this standby knows the primary's
* address and should redirect incoming join requests there.
* Returns 0 for PRIMARY role, or for STANDBY with no known primary yet.
*/
int ha_should_redirect(const ha_state_t *ha)
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
if (role == CTRL_PRIMARY)
return 0;
if (ha->primary_host[0] == '\0' || ha->primary_port == 0)
return 0;
return 1;
}
/*
* ha_no_primary_known: returns 1 when this node is a STANDBY but does not
* yet know the primary's address. Callers should reply E_XCE_NOMASTER so
* the client retries rather than treating the error as permanent.
*/
int ha_no_primary_known(const ha_state_t *ha)
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
if (role == CTRL_PRIMARY)
return 0;
return (ha->primary_host[0] == '\0' || ha->primary_port == 0) ? 1 : 0;
}
/*
* ha_register_persist_hooks: register the three persistence hooks on the
* cluster's hook system, guarded by the hooks_registered idempotency flag.
*
* ha-single-leader-role task 2.2: extracted from ha_promote_to_primary so
* BOTH roads to PRIMARY (the raft become-leader callback and the non-raft
* ha_promote_to_primary path) share one hook-registration sequence and
* cannot diverge. Safe on a zero-init cluster_t (zeroed hooksys →
* xce_add_hook returns -EINVAL harmlessly; hooks_registered still set to 1).
* Requires non-NULL c (dereferences &c->hooksys).
*/
static void ha_register_persist_hooks(cluster_t *c)
{
if (c->ha.hooks_registered)
return;
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun, &c->persist);
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun, &c->persist);
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun, &c->persist);
c->ha.hooks_registered = 1;
}
/*
* ha_primary_log_append (ha-term-safety task 3.1, R1.1/R1.2/R2.1): the SINGLE
* primary append entry point. Routes a local primary append (job-state entry or
* the own-term promotion no-op) through the raft library's raft_log_append, so
* the LIBRARY stamps the live election term (s->term) and assigns the LSN
* (s->log_len+1), then drives on_persist_entry — which (primary role) performs
* the actual WAL write via persist_wal_async. This makes the library index the
* authority for both the per-entry term (R1) and the LSN, retiring the
* hand-rolled bypass that assigned the LSN directly through persist_wal_async.
*
* Co-tracking invariant (closes the task 2.1 deferral): because raft_log_append
* is the only primary append path, every primary append advances BOTH the
* library index (s->log_len) and the persist write cursor (p->current_lsn) by
* exactly one, in lockstep — so p->current_lsn == s->log_len + 1 always holds and
* the LSN the library assigns equals the LSN on_persist_entry writes. raft_init
* (task 2.5) seeds s->log_len from the recovered durable floor, which equals
* p->current_lsn - 1 at boot, establishing the invariant before the first append.
*
* Persist-before-act: raft_log_append calls on_persist_entry BEFORE advancing
* s->log_len and propagates a non-zero persist rc without mutating the log (no
* phantom entry) — preserving persist-before-ACK and the bug-850 contiguity
* guard beneath the WAL write.
*
* Legacy fallback (non-raft road): when the raft library is not in RAFT_LEADER
* state (raft_log_append returns -EPERM) — the init-time non-raft promotion road
* (ha_promote_to_primary) and engine-less unit-test contexts — the library is
* not tracking the log, so there is no co-tracking hazard. We append locally
* through the legacy persist_wal_async path (which still records the live term
* via the persist_set_term seam set on the prior promotion) so those roads keep
* working unchanged. Once raft drives leadership in production, the -EPERM
* branch is never taken and the library is the sole LSN/term source.
*
* out_lsn (optional) receives the assigned 1-based LSN on success.
* Returns 0 on success; propagates a non-zero persist rc on append failure.
*/
int ha_primary_log_append(cluster_t *c, uint32_t type,
const void *payload, uint32_t len, uint64_t *out_lsn)
{
if (!c)
return -EINVAL;
/* Pack the in-flight adapter envelope [raft_entry_blob_t{type,len}][payload]
* — the SAME format on_persist_entry (ha_cb_persist_entry_stub) unpacks, so
* the encoder and decoder share the single wal_types.h definition. */
size_t blob_len = sizeof(raft_entry_blob_t) + len;
uint8_t stackbuf[256];
uint8_t *blob = (blob_len <= sizeof(stackbuf)) ? stackbuf : malloc(blob_len);
if (!blob)
return -ENOMEM;
raft_entry_blob_t *bh = (raft_entry_blob_t *)blob;
bh->type = type;
bh->payload_len = len;
if (payload && len > 0)
memcpy(blob + sizeof(*bh), payload, len);
uint64_t idx = 0;
int rc = raft_log_append(&c->ha.raft, blob, blob_len, &idx);
if (rc == -EPERM) {
/* Non-raft road: the library is not LEADER and is not tracking the log.
* Append locally through the legacy path (no library index to co-track).
* Stamp the live election term (ha->term) so the entry still records the
* leader's term (R1.1) even off the raft road — matching on_persist_entry's
* persist_set_term on the raft path. */
persist_set_term(&c->persist,
atomic_load_explicit(&c->ha.term, memory_order_acquire));
rc = persist_wal_async(&c->persist, type, payload, len);
if (rc == 0) {
pthread_mutex_lock(&c->persist.lock);
idx = c->persist.current_lsn - 1; /* the LSN just assigned */
pthread_mutex_unlock(&c->persist.lock);
}
}
if (blob != stackbuf)
free(blob);
if (rc == 0 && out_lsn)
*out_lsn = idx;
return rc;
}
/* ha_repl_ack_abs_peer_idx (ha-term-safety task 3.3 remediation): resolve the
* ABSOLUTE library peer index from an ACK endpoint's stored peer_idx, branching
* on endpoint type EXACTLY as the standby_endpoints scan at ha.c:6187-6189.
*
* Two endpoint types reach the walAck branch of repl_ack_decode_and_dispatch,
* and their stored peer_idx has DIFFERENT meaning:
* - Promoted vote ep (vote_ep_ctx_t, ep_is_vote != 0): peer_idx is a
* self-EXCLUDED "votes" index (relative). Map it back to the absolute index
* — the inverse of raft_cb_send_vote_request's votes_idx remap (self_idx ==
* my_priority): abs = votes_idx < self_idx ? votes_idx : votes_idx + 1.
* - Heartbeat standby ep (standby_ep_ctx_t, ctx==NULL, ep_is_vote == 0):
* peer_idx is ALREADY the absolute candidate index — pass it through.
*
* The earlier unconditional remap corrupted the heartbeat path: a standby ep's
* already-absolute index was remapped a second time, crediting the WRONG peer
* (false quorum / stuck commit) or, near the array bound, an OUT-OF-BOUNDS
* match_index[] write. This resolver is the single source of truth so the
* production call site and the unit test exercise identical logic. */
int ha_repl_ack_abs_peer_idx(ha_state_t *ha, int ep_is_vote, int peer_idx)
{
int self_idx = ha->my_priority; /* self_idx == my_priority by convention */
if (ep_is_vote)
return (peer_idx < self_idx) ? peer_idx : peer_idx + 1;
return peer_idx; /* heartbeat standby ep: already absolute */
}
/* ha-term-safety task 3.3 (R4.1/R4.5): drive the raft library on a replication
* ACK. Route the reply through raft_on_append_entries_reply (updating
* match_index[]/next_index[]) then raft_advance_commit, so the library
* recomputes the current-term-gated commit (Figure-8) and MAY fire
* on_commit_advanced — the task-2.4 callback feeding the synchronous barrier.
*
* Peer-index contract: `abs_peer_idx` is the ABSOLUTE library peer index
* (j != self_idx), the same index space the library uses for
* match_index[]/next_index[]. The CALLER resolves it per endpoint type via
* ha_repl_ack_abs_peer_idx (promoted-vote eps carry a relative votes index;
* heartbeat standby eps carry an already-absolute index) — this seam must NOT
* remap again or it would corrupt the heartbeat path.
*
* Defense-in-depth: a peer index outside [0, raft.n_peers) (and never self) is
* rejected loudly rather than passed into raft_on_append_entries_reply, whose
* match_index[] write is unchecked. This fences an OOB / wrong-peer write at the
* controller boundary (no assert inside src/lib/raft/) and self-heals — the
* caller simply skips this ACK's library accounting.
*
* The library drops a stale-term reply (reply_term != s->term) inside
* raft_on_append_entries_reply, so a stale ACK never touches match_index[] and
* never advances commit. raft_advance_commit is leader-only (-EPERM otherwise);
* on a non-leader the call is a harmless no-op. Neither library call performs
* I/O — safe on the iomgr ACK path. */
int ha_raft_on_repl_ack(cluster_t *c, int abs_peer_idx, uint64_t acked_lsn,
uint64_t reply_term)
{
if (!c)
return -EINVAL;
/* Defense-in-depth bounds/self check: the library's match_index[] write is
* unchecked, so reject a bogus index here (fail loud, never corrupt). */
int self_idx = c->ha.my_priority;
int n_peers = c->ha.raft.n_peers;
if (abs_peer_idx < 0 || abs_peer_idx >= n_peers || abs_peer_idx == self_idx) {
ERROR("ha_raft_on_repl_ack: rejecting out-of-range peer index %d "
"(n_peers=%d self_idx=%d) — skipping library ACK accounting",
abs_peer_idx, n_peers, self_idx);
return -EINVAL;
}
/* A successful WalAck reports the standby's matched index (acked_lsn). The
* library applies it only when reply_term == s->term (stale-term drop). */
(void)raft_on_append_entries_reply(&c->ha.raft, abs_peer_idx, /*success*/1,
acked_lsn, reply_term);
/* Recompute the current-term-gated commit; fires on_commit_advanced exactly
* once on a strict advance of a current-term entry (else holds). Leader-only:
* -EPERM off the leader leaves commit_index unchanged with no callback. */
(void)raft_advance_commit(&c->ha.raft);
return 0;
}
/*
* ha_arm_own_term_gate: append the own-term WAL_NOOP and arm the commit
* engine's writable_lsn gate (ha-single-leader-role task 2.2, Req 1.4/2.1/2.2).
*
* The shared promotion tail both roads to PRIMARY run. Sequence:
* 1. persist_wal_async(WAL_NOOP) — appends a header-only entry and
* broadcasts it to standbys via repl_broadcast (one call does both).
* 2. derive noop_lsn = current_lsn - 1 under p->lock (the append
* post-incremented current_lsn; persist.c:1019-1021 pattern).
* 3. commit_submit(noop_lsn, COMMIT_MODE_RETAIN) — lock-free, non-blocking,
* registration-free observability bookkeeping (records nothing).
* 4. commit_set_writable_lsn(noop_lsn) — arms the own-term gate so the
* engine withholds WAIT completion until committed_lsn >= noop_lsn.
*
* Non-blocking by construction: every call here returns without waiting for
* the no-op to commit (Req 1.4 — status honesty does not wait).
*
* Degraded-but-safe (design §Error Strategy): if the no-op append fails,
* no LSN exists, so the gate stays UNSET; promotion STILL COMPLETES. The
* universal commit barrier still requires per-entry majority of a new-term
* entry, so the first state-changing entry of this term becomes the de-facto
* own-term vehicle. We do NOT block and do NOT abort promotion.
*/
static void ha_arm_own_term_gate(cluster_t *c)
{
/* ha-term-safety task 3.1: route the own-term promotion no-op through the
* library (ha_primary_log_append → raft_log_append) so the WAL_NOOP records
* the newly established term (R1.2) and the library assigns its LSN. On the
* non-raft promotion road this falls back to the legacy append; either way
* the returned LSN is the no-op's WAL LSN. */
uint64_t noop_lsn = 0;
int rc = ha_primary_log_append(c, WAL_NOOP, NULL, 0, &noop_lsn);
if (rc < 0) {
ERROR("HA: own-term WAL_NOOP append failed: %d", rc);
WARN("HA: own-term gate left UNSET (degraded-but-safe); first "
"new-term entry becomes the own-term commit vehicle");
return;
}
/* Observability-only bookkeeping (lock-free, non-blocking, records
* nothing per entry — commit state derives from registry watermarks). */
(void)commit_submit(&c->commit, noop_lsn, COMMIT_MODE_RETAIN, 0);
/* Arm the own-term gate: completion of any WAIT is withheld until
* committed_lsn >= noop_lsn. */
commit_set_writable_lsn(&c->commit, noop_lsn);
INFO("HA: own-term noop appended lsn=%" PRIu64 ", writable_lsn gate armed",
noop_lsn);
}
/*
* ha_promote_to_primary: the NON-RAFT promotion road (cluster.c init-time,
* legacy XCE_TESTING evaluation path, tests). This path OWNS its term, so it
* calls ha_do_promote (which increments the term + sets role=PRIMARY). It
* then runs the SAME shared sequence the raft road runs: register hooks,
* start the heartbeat, and arm the own-term gate (noop + writable_lsn).
*
* Every road to PRIMARY arms the own-term gate. The noop+gate tail is
* guarded on c != NULL (callers always pass a real cluster here).
*/
int ha_promote_to_primary(cluster_t *c)
{
ha_do_promote(&c->ha); /* non-raft path owns its term increment */
/* Shared promotion sequence — must match the raft become-leader tail:
* hooks live BEFORE the no-op append so the persistence seam is active
* before any racing client write. */
ha_register_persist_hooks(c);
/* Arm periodic heartbeat to standbys — keeps their watchdog alive when
* the cluster is idle (no WAL entries or config pushes). */
ha_start_primary_heartbeat(c);
/* Own-term no-op + writable_lsn gate (Req 1.4, 2.1, 2.2). */
ha_arm_own_term_gate(c);
return 0;
}
/*
* ha_demote_to_standby: disable persistence hooks, then set role=STANDBY
* and reset the hooks_registered idempotency guard (via ha_do_demote).
*/
int ha_demote_to_standby(cluster_t *c)
{
/* Disable persistence hooks before changing role */
if (c->ha.hooks_registered) {
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun);
}
ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */
return 0;
}
/* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */
/*
* ha_wait_for_heartbeat_or_timeout: called from the startup path after
* ha_init() when the node needs to discover whether a primary already exists.
*
* Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where
* deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter
*
* The heartbeat handler (cluster.c) posts heartbeat_seen when it receives
* OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits
* OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler.
*
* Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived);
* ha_do_demote() was called (role = CTRL_STANDBY).
* Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT);
* caller is responsible for driving the election.
*/
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha)
{
/* Seed jitter PRNG on first use */
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
/* Compute timeout: election_timeout + priority_step * priority + random jitter.
* Higher-priority (lower index) nodes time out sooner, giving them a head
* start in the election before lower-priority nodes begin campaigning. */
long to_ms = (long)ha->config.election_timeout_ms;
to_ms += (long)ha->my_priority * (long)ha->config.election_priority_step_ms;
if (ha->config.election_jitter_max_ms > 0) {
to_ms += (long)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)ha->config.election_jitter_max_ms + 1));
}
/* sem_timedwait requires an absolute CLOCK_REALTIME deadline */
struct timespec deadline;
clock_gettime(CLOCK_REALTIME, &deadline);
deadline.tv_sec += to_ms / 1000;
deadline.tv_nsec += (to_ms % 1000) * 1000000L;
if (deadline.tv_nsec >= 1000000000L) {
deadline.tv_sec++;
deadline.tv_nsec -= 1000000000L;
}
int rc = sem_timedwait(&ha->heartbeat_seen, &deadline);
if (rc == 0) {
/* Heartbeat arrived before the deadline — demote to standby */
ha_do_demote(ha);
return CTRL_STANDBY;
}
/* ETIMEDOUT (or any other error): treat as election timeout */
return CTRL_ELECTING;
}
/* ── Term Durability (Req 9) ────────────────────────────────────────────────
*
* ha_term_file_t: on-disk format for ha-term.bin.
* Fields are in host byte order (single-host durability, no cross-endian migration).
* CRC32 covers all preceding bytes via zlib crc32().
*/
#define HA_TERM_MAGIC 0xDEADBEEFCAFEBABEULL
typedef struct {
uint64_t magic;
uint64_t term;
uint64_t voted_for_term;
char voted_for_name[XCE_MAXNAME_LEN];
uint32_t crc32;
} ha_term_file_t;
/*
* ha_persist_term: atomically write ha_term_file_t to {workdir}/ha-term.bin.
* Uses write-to-tmp + fsync + rename for durability (existing project pattern).
* If workdir is NULL, the persist is skipped (test mode or no workdir configured).
* Req 9.1: must be called before any vote grant or term increment is reported.
*/
void ha_persist_term(ha_state_t *ha, const char *workdir)
{
if (!workdir)
return; /* test mode: skip file I/O */
ha_term_file_t f;
memset(&f, 0, sizeof(f));
f.magic = HA_TERM_MAGIC;
f.term = atomic_load_explicit(&ha->term, memory_order_acquire);
f.voted_for_term = ha->voted_for_term;
strncpy(f.voted_for_name, ha->voted_for_name, sizeof(f.voted_for_name) - 1);
f.voted_for_name[sizeof(f.voted_for_name) - 1] = '\0';
/* CRC32 covers all bytes before the crc32 field itself */
size_t crc_len = offsetof(ha_term_file_t, crc32);
f.crc32 = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len);
/* Write to tmp then rename — atomic from filesystem perspective */
char path[1024], tmp[1040];
snprintf(path, sizeof(path), "%s/ha-term.bin", workdir);
snprintf(tmp, sizeof(tmp), "%s/ha-term.bin.tmp", workdir);
int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0)
return; /* best-effort; caller must not rely on this for correctness */
ssize_t written = write(fd, &f, sizeof(f));
if (written == (ssize_t)sizeof(f))
fsync(fd);
close(fd);
if (written == (ssize_t)sizeof(f))
rename(tmp, path);
}
/*
* ha_restore_term: read {workdir}/ha-term.bin at startup and restore
* ha->term and ha->voted_for_* into ha_state_t.
*
* On missing file (ENOENT): no-op — term stays 0. Req 9.3.
* On CRC/magic mismatch: log WARN, zero ha->term and ha->voted_for_*. Req 9.2.
* On valid file: atomically restore ha->term; copy voted_for fields. Req 9.2.
*
* Returns 0 on success (including gracefully-handled file-not-found and CRC bad),
* -errno on unexpected I/O error.
* If workdir is NULL, function is a no-op (safe for unit tests without a workdir).
*/
int ha_restore_term(ha_state_t *ha, const char *workdir)
{
if (!workdir)
return 0;
char path[1024];
snprintf(path, sizeof(path), "%s/ha-term.bin", workdir);
int fd = open(path, O_RDONLY);
if (fd < 0) {
if (errno == ENOENT)
return 0; /* first boot — no file yet, term stays 0 (Req 9.3) */
return -errno;
}
ha_term_file_t f;
memset(&f, 0, sizeof(f));
ssize_t n = read(fd, &f, sizeof(f));
close(fd);
if (n != (ssize_t)sizeof(f)) {
WARN("HA: ha-term.bin short read (%zd/%zu) — ignoring, starting at term 0",
n, sizeof(f));
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
return 0;
}
/* Verify magic */
if (f.magic != HA_TERM_MAGIC) {
WARN("HA: ha-term.bin bad magic (0x%016" PRIx64 ") — ignoring, starting at term 0",
f.magic);
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
return 0;
}
/* Verify CRC32 */
size_t crc_len = offsetof(ha_term_file_t, crc32);
uint32_t computed = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len);
if (computed != f.crc32) {
WARN("HA: ha-term.bin CRC mismatch (expected 0x%08x got 0x%08x) — ignoring, starting at term 0",
computed, f.crc32);
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
return 0;
}
/* Valid file — restore state (Req 9.2) */
atomic_store_explicit(&ha->term, f.term, memory_order_release);
ha->voted_for_term = f.voted_for_term;
strncpy(ha->voted_for_name, f.voted_for_name, sizeof(ha->voted_for_name) - 1);
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
return 0;
}
/* ── ha_heartbeat_is_fresh (Req 6.2) ────────────────────────────────────────
*
* Returns 1 if the elapsed time since the last primary message
* (ha->last_activity, written by ha_on_heartbeat_received) is strictly less
* than ha->config.heartbeat_timeout_ms (i.e. the primary is still alive).
* Returns 0 if no primary message has been seen yet (last_activity == 0) or
* the timeout has expired.
*
* Uses CLOCK_MONOTONIC seconds to avoid wall-clock skew; whole-second
* granularity is sufficient for the 10 s default timeout.
*/
int ha_heartbeat_is_fresh(ha_state_t *ha)
{
time_t last = ha->last_activity; /* time_t load is atomic on LP64 */
if (last == 0)
return 0; /* no primary message seen yet */
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
long ms = (long)(now.tv_sec - last) * 1000L;
return ms < (long)ha->config.heartbeat_timeout_ms;
}
/* ── Voter freshness floor (election safety) ───────────────────────────────
*
* Compute the voter's effective freshness floor as the per-field max of its
* local state and its learned-committed cluster state. Outdated candidates
* (below this floor) are denied so the cluster can never be rolled back below
* what it has already committed. See ha.h for full contract.
*/
void ha_vote_self_freshness(ha_state_t *ha,
uint64_t local_config_tag,
uint64_t local_applied_lsn,
uint64_t *out_tag,
uint64_t *out_lsn)
{
uint64_t learned_tag =
atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire);
uint64_t learned_lsn =
atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire);
if (out_tag)
*out_tag = (local_config_tag > learned_tag) ? local_config_tag : learned_tag;
if (out_lsn)
*out_lsn = (local_applied_lsn > learned_lsn) ? local_applied_lsn : learned_lsn;
}
/* ── Candidate advertised LSN (election safety, ha-commit-durability 4.2) ───
*
* A candidate advertises EXACTLY its durable WAL tip on the LSN axis — the
* bare `persist_durable_floor(p)`, NOT the folded
* max(durable_floor, last_known_committed_lsn) that ha_vote_self_freshness
* computes for the VOTER floor.
*
* Why bare, not folded: ha_vote_self_freshness unconditionally returns
* max(local_lsn, last_known_committed_lsn). Feeding it durable_floor would let
* a node restarted with a TRUNCATED log (durable_floor low) but a stale
* volatile learned_committed (high) advertise the stale-high value and WIN —
* exactly the truncated-candidate-wins hole this task closes (design §1a/§5a).
* A candidate may only claim a log position it can durably back.
*
* The candidate's TAG axis is unchanged: it still uses
* max(local_config_tag, learned_tag) (the config epoch is orthogonal to log
* durability — design §1a). Only the LSN axis bypasses the fold.
*
* NULL-safe (persist_durable_floor guards NULL → 0) for the test/seam paths.
*/
uint64_t ha_candidate_advertise_lsn(persist_t *p)
{
return persist_durable_floor(p);
}
/* bug-848: see ha.h. After local WAL recovery, advertise the recovered durable
* tip for catch-up so the primary streams only the missing suffix instead of
* re-streaming + overwriting consistent records. Monotonic: never lowers a
* higher applied_lsn (e.g. if catch-up already advanced it). */
void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p)
{
if (!ha || !p)
return;
uint64_t recovered = persist_durable_floor(p);
uint64_t cur = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
if (recovered > cur)
atomic_store_explicit(&ha->applied_lsn, recovered,
memory_order_release);
}
/* bug-1042: re-sync the raft library's log_len / commit_index / term from the
* recovered WAL AFTER persist_recover.
*
* Init-ordering hazard: ha_init_raft (myapp.c, before the election) sources
* raft.initial_log_len from persist_durable_floor — but on a STANDBY-REJOIN boot
* persist_recover has not run yet at that point, so the library comes up with
* log_len=0. After cluster_init_phase2 runs persist_recover (recovering e.g.
* durable_floor=7), the library still thinks its log is EMPTY. Every incoming
* AppendEntries then fails logOk (prevLogIndex>=1 against log_len=0 → reject),
* so the rejoined member is PERMANENTLY stuck as a non-converging standby and
* the primary re-streams [tip+1..committed] every heartbeat forever.
*
* Fix: after recovery, raise the library's log_len to the recovered WAL tip and
* its commit_index to the recovered durable floor (clamped to log_len, honoring
* the commit_index<=log_len invariant), and align raft.term with the persisted
* term so the per-LSN (term,index) reconcile reads consistent state. Monotonic:
* never lowers an already-higher library position (e.g. if catch-up advanced it
* before this runs). The library stores no entry bytes — on_get_entry_term reads
* the recovered per-LSN term index — so raising log_len is sufficient for logOk. */
void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p)
{
if (!ha || ha->raft_c == NULL || !p)
return;
uint64_t recovered_tip = persist_durable_floor(p);
if (recovered_tip > ha->raft.log_len)
ha->raft.log_len = recovered_tip;
/* commit_index advances to the recovered durable floor but never past the
* log tip (library invariant commit_index <= log_len), and never decreases. */
uint64_t recovered_commit = recovered_tip;
if (recovered_commit > ha->raft.log_len)
recovered_commit = ha->raft.log_len;
if (recovered_commit > ha->raft.commit_index)
ha->raft.commit_index = recovered_commit;
/* Align the library term with the persisted/restored HA term so the
* (term,lsn) reconcile and own-term gate read a consistent value after a
* rejoin (ha_restore_term already loaded ha->term before ha_init_raft). */
uint64_t ha_term = atomic_load_explicit(&ha->term, memory_order_acquire);
if (ha_term > ha->raft.term)
ha->raft.term = ha_term;
}
/* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ────────────────────────
*
* handle_request_vote: process an incoming OP_requestVote RPC.
*
* Decision order (fail-fast):
* 0. role gate (Req 6.1, 6.2):
* - CTRL_PRIMARY → deny
* - CTRL_STANDBY with fresh heartbeat → deny
* 1. in_catch_up → deny (Req 2.5)
* 2. req term < my term → deny (Req 2.2)
* 3. req term > my term → update term, clear voted_for (Req 2.3)
* 4. already voted for a DIFFERENT candidate in this term → deny (Req 2.6)
* 5. freshness gate: candidate not at least as fresh → deny (Req 2.4)
* 6. grant vote, record voted_for, persist (Req 9.1)
*
* Returns 1 = granted, 0 = denied, <0 = error.
*
* TEST-ONLY (XCE_TESTING): production vote dispatch goes through
* cluster.c → raft_on_vote_request. This function is the legacy
* entry point used by test_ha_vote_gate.c and test_ha_election.c
* to exercise role-gate logic in isolation without a full raft state.
*/
#ifdef XCE_TESTING
int handle_request_vote(ha_state_t *ha,
uint64_t term,
const char *candidate_id,
uint64_t config_tag,
uint64_t last_lsn,
uint64_t my_config_tag,
uint64_t my_last_lsn,
int in_catch_up_flag)
{
/* Req 2.3: higher term → step down BEFORE any role gate.
*
* Raft §5.1: "If RPC request or response contains term T > currentTerm:
* set currentTerm = T, convert to follower."
*
* Without this ordering, a CTRL_PRIMARY node at term=N would deny a vote
* request at term=N+1 without ever seeing the higher term, leaving two
* nodes both believing they lead → split-brain. */
uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire);
if (term > my_term) {
atomic_store_explicit(&ha->term, term, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
/* Step down from leader: a higher-term election is in progress and we
* must not block it with the role gate below. */
ctrl_role_t cur_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (cur_role == CTRL_PRIMARY) {
ha_do_demote(ha);
INFO("HA: stepping down (term %" PRIu64 " → %" PRIu64
") — higher-term election in progress", my_term, term);
}
my_term = term;
}
/* Req 2.2: stale term → deny, do not update any state */
if (term < my_term) {
return 0;
}
/* Req 6.1, 6.2: role-based deny gate.
* Evaluated AFTER the term catch-up above: if we just stepped down due to
* a higher term, role is now CTRL_STANDBY and we fall through to voting.
* PRIMARY at the current term still denies votes (protecting the live
* primary from same-term challengers). */
ctrl_role_t my_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (my_role == CTRL_PRIMARY)
return 0; /* deny — I am primary; protect against same-term challengers */
if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(ha))
return 0; /* deny — my primary is alive (active primary connection) */
/* Req 2.5: catching-up standbys deny all votes */
if (in_catch_up_flag)
return 0;
/* Protect voted_for check+set from concurrent vote requests on different vtasks */
pthread_mutex_lock(&ha->lock);
/* Req 2.6: already voted for a DIFFERENT candidate in this term → deny.
* Same-candidate re-grant is idempotent (Raft §5.2: grant iff votedFor ∈
* {Nil, candidateId}) — matches the duplicate-vote check in raft.c. */
if (ha->voted_for_term == term && ha->voted_for_name[0] != '\0' &&
strcmp(ha->voted_for_name, candidate_id) != 0) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
/* Req 2.4 + election-safety fix #2: freshness gate against the EFFECTIVE
* floor — the max of the voter's local state and its learned-committed
* cluster state. A candidate that is below what the cluster has already
* committed (config_tag or, at equal tag, LSN) would roll the cluster back
* and must be denied even when this voter's local state is colder.
* Deny if candidate has a lower config_tag, or equal config_tag with lower
* LSN; grant if strictly fresher or equally fresh. */
uint64_t eff_tag = 0, eff_lsn = 0;
ha_vote_self_freshness(ha, my_config_tag, my_last_lsn, &eff_tag, &eff_lsn);
if (config_tag < eff_tag) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
if (config_tag == eff_tag && last_lsn < eff_lsn) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
/* Grant the vote: record voted_for and persist before responding (Req 9.1) */
ha->voted_for_term = term;
strncpy(ha->voted_for_name, candidate_id, sizeof(ha->voted_for_name) - 1);
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
pthread_mutex_unlock(&ha->lock);
/* workdir is not available here — persist with NULL (test mode or caller
* wraps this function with workdir before sending the network reply).
* Production dispatch path calls ha_persist_term(ha, workdir) explicitly
* before this function writes its network reply. */
ha_persist_term(ha, NULL);
return 1;
}
#endif /* XCE_TESTING */
/* ── raft-drive-async Task 2.1: reconnect-offload feature flag (ALL builds) ──
*
* drive_async_reconnect_offload gates the peer reconnect path:
* ON (production default) → primary_heartbeat_fn submits the peer reconnect
* to the slow lane and returns immediately; the delaymgr thread NEVER
* performs the blocking connect inline (ha-async-io.md thread-role
* contract — a blocking connect on the heartbeat/election-timeout thread
* stalls liveness to healthy peers and triggers spurious elections).
* OFF → legacy inline blocking connect on the delaymgr thread; retained only
* so the TDD RED test can demonstrate the pre-change stall.
*
* Compiled in ALL builds (production reads it on every heartbeat tick); the
* setter is the test seam to flip RED↔GREEN. */
static _Atomic int g_drive_async_reconnect_offload = 1;
void ha_set_reconnect_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_reconnect_offload, on ? 1 : 0,
memory_order_release);
}
int ha_reconnect_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_reconnect_offload,
memory_order_acquire);
}
/* raft-drive-async task 3.2: SnapshotSend feature flag.
*
* DEFAULT OFF (0): the snapshot transfer runs INLINE on the caller (io_worker/
* drive) thread — byte-for-byte the legacy behavior. This task lands DORMANT,
* exactly like the Task 3.1 peer-control dispatch flag, because the offload
* captures a BORROWED standby endpoint (see snapshot_lane_fn's bug-1197 hazard
* note): moving the transfer to a lane thread while the io_worker may
* RDHUP-free that endpoint is a cross-thread UAF. Today (flag OFF) the send
* and the endpoint teardown are serialized on the single io_worker, so they
* cannot overlap.
*
* Flipping ON is the job of Task 3.4 (receiver-side integration), which
* establishes the endpoint-lifetime discipline for the pool/lane dispatch
* model and proves the transfer no longer stalls other peers' heartbeats/votes
* with the pool near saturation. Until then, bug-1198 (the repl-ack body is
* still unbounded when run inline) remains open by design.
*
* Keyed per standby (Req 1.5) so at most one snapshot per peer is in flight;
* at most one persist_checkpoint in flight (existing atomic guard). Compiled
* in ALL builds; the setter is the test seam to flip RED↔GREEN. */
static _Atomic int g_drive_async_snapshot_offload = 0;
void ha_set_snapshot_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_snapshot_offload, on ? 1 : 0,
memory_order_release);
}
int ha_snapshot_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_snapshot_offload,
memory_order_acquire);
}
/* raft-drive-async task 3.3: ConfigPush feature flag.
*
* DEFAULT OFF (0): the requester-less config push runs INLINE on the caller
* (io_worker/drive) thread — byte-for-byte the legacy behavior, i.e.
* push_config_transport_only's fan-out followed by its blocking
* wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS). This task lands DORMANT,
* exactly like the Task 3.1/3.2 flags, because the offload borrows standby
* endpoints it does not own (bug-1197 family — push_config_transport_only
* reads c->ha.standby_endpoints): moving the push to a lane thread while the
* io_worker may RDHUP-free those endpoints is a cross-thread UAF. Today (flag
* OFF) the push and the endpoint teardown are serialized on the single
* io_worker, so they cannot overlap.
*
* Flipping ON is the job of Task 3.4, which establishes the endpoint-lifetime
* discipline for the whole pool/lane dispatch model and flips this together
* with the snapshot + peer-control dispatch flags. Until then this offload is
* exercised only by unit tests with a stable (or absent) endpoint set.
*
* ONLY the requester-less call sites (post-election promotion, heartbeat-driven
* HA_REPL_PUSH_CONFIG) route through this flag; the admin-upgrade barrier path
* (push_config_to_standbys) is unaffected. Compiled in ALL builds; the setter
* is the test seam to flip RED<->GREEN. */
static _Atomic int g_drive_async_configpush_offload = 0;
void ha_set_configpush_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_configpush_offload, on ? 1 : 0,
memory_order_release);
}
int ha_configpush_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_configpush_offload,
memory_order_acquire);
}
/* raft-drive-async task 3.1: PeerCtrlDispatch feature flag.
*
* DEFAULT OFF (0): the registered peer-control handler runs its routing body
* INLINE on the iomgr io_worker thread — byte-for-byte the pre-task behavior,
* so THIS COMMIT DOES NOT CHANGE PRODUCTION BEHAVIOR. ON (1): the handler
* hands the decoded packet to the vtask pool and returns promptly. The flag
* stays OFF until the task 3.4 deadlock gate proves the dispatch path safe.
*
* Compiled in ALL builds (production reads it on every peer-control message);
* the setter is the test seam to flip RED<->GREEN. */
static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 0;
void ha_set_peerctrl_dispatch_for_test(int on)
{
atomic_store_explicit(&g_drive_async_peerctrl_dispatch_enabled, on ? 1 : 0,
memory_order_release);
}
int ha_peerctrl_dispatch_enabled(void)
{
return atomic_load_explicit(&g_drive_async_peerctrl_dispatch_enabled,
memory_order_acquire);
}
/* raft-drive-async task 3.1: test-only observation seam (plain global, NULL in
* production — zero cost, same idiom as ha_nop_req_hook). Invoked at the top
* of peer_ctrl_msg_handler_inline before any role routing. */
void (*ha_peerctrl_body_probe)(endpoint_t *ep) = NULL;
/* ── Test-only hook shims (XCE_TESTING only) ──────────────────────────────
* Type definitions (election_timeout_ctx_t), extern declarations
* (cluster_init_phase2, push_config_transport_only,
* ha_send_ctrlhello_to_all_standbys), and static forward declarations
* (ha_heartbeat_is_fresh) have been moved to the forward-declarations block
* near the top of this file so that the raft callback implementations
* (Task 6.2) can reference them.
*/
#ifdef XCE_TESTING
/*
* ha_phase2_hook: test-only seam.
* When non-NULL, ha_elect_evaluate_and_signal() calls this INSTEAD OF the real
* cluster_init_phase2() so tests can count phase2 calls without a live cluster_t.
*/
int (*ha_phase2_hook)(void *c) = NULL;
/*
* ha_send_ctrlhello_hook: test-only seam for ha_send_ctrlhello_to_all_standbys.
* When non-NULL, the post-election path calls this INSTEAD OF the real function
* (cluster.c is not linked in unit-test binary). Req 7.2.
*/
int (*ha_send_ctrlhello_hook)(void *c) = NULL;
/*
* ha_push_config_hook: test-only seam for push_config_transport_only.
* When non-NULL, the post-election path calls this INSTEAD OF the real function
* (config_repl.c's push_config_transport_only needs a real cluster). Req 7.1.
*/
int (*ha_push_config_hook)(void *c) = NULL;
/*
* ha_nop_req_hook: test-only seam for xce_nop_req in primary_heartbeat_fn.
* When non-NULL, primary_heartbeat_fn calls this INSTEAD OF xce_nop_req so
* tests can count NOP sends without a live endpoint. Req 3.2, task 2.2.
*/
int (*ha_nop_req_hook)(endpoint_t *ep) = NULL;
/*
* ha_heartbeat_req_hook: test-only seam for the HA heartbeat send in
* primary_heartbeat_fn. When non-NULL, primary_heartbeat_fn calls this INSTEAD
* OF xce_ha_heartbeat_req with the SAME (term, committed_lsn,
* committed_config_tag) it stamps onto the wire, so tests can assert the
* heartbeat carries the primary's own current committed state. Req 1.1, 3.4.
*/
int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term,
uint64_t committed_lsn,
uint64_t committed_config_tag) = NULL;
/*
* ha_reconnect_connect_hook: test-only seam for the BLOCKING half of the
* offloaded peer reconnect (raft-drive-async Task 2.1). When non-NULL, the
* slow-lane reconnect worker calls this INSTEAD OF the real bounded
* endpoint_connect + iomgr_monitor, so a test can drive the reconnect path
* deterministically (e.g. park the worker to simulate an unreachable peer).
* Returns 0 to simulate success, <0 to simulate failure. Req 2.1, 2.2, 2.5.
*/
int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx) = NULL;
#ifdef XCE_TESTING
/*
* ha_standby_evicted_count_for_test: counts how many stale same-peer outbound
* channels on_standby_connected evicted while resolving the inbound-vs-outbound
* connection-ownership race (raft-drive-async Task 2.2, Req 2.6). Test-only
* observability — never read on the production path.
*/
_Atomic int ha_standby_evicted_count_for_test = 0;
#endif
/*
* ha_repl_push_config_for_standby_hook: test-only seam for the per-standby
* config push triggered by the heartbeat-reply decision loop (Task 3.3).
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* push_config_transport_only() when the decision is HA_REPL_PUSH_CONFIG.
* Requirements: 1.4 (ha-repl-hardening)
*/
int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx) = NULL;
/*
* ha_repl_stream_wal_hook: test-only seam for ha_stream_wal_to_standby.
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* ha_stream_wal_to_standby() when the decision is HA_REPL_STREAM_WAL.
* Requirements: 5.1–5.5 (ha-repl-hardening)
*/
int (*ha_repl_stream_wal_hook)(void *ep, const char *wal_path,
uint64_t standby_lsn,
uint64_t committed_lsn) = NULL;
/*
* ha_repl_send_snapshot_hook: test-only seam for ha_send_snapshot_to_standby.
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* ha_send_snapshot_to_standby() when the decision is HA_REPL_SEND_SNAPSHOT.
* Requirements: 4.1–4.5 (ha-repl-hardening)
*/
int (*ha_repl_send_snapshot_hook)(int peer_idx) = NULL;
/*
* ha_configpush_probe: test-only observation seam for the requester-less
* config-push offload (raft-drive-async Task 3.3). When non-NULL,
* ha_config_push_do_send() invokes it right around push_config_transport_only
* (on BOTH the inline and the lane path), so a test can record the running
* thread and optionally park the lane worker in flight to prove the
* cluster-wide single-flight dedup. NULL in production (zero cost).
*/
void (*ha_configpush_probe)(void) = NULL;
#endif /* XCE_TESTING */
/* ── Election Evaluation (Req 2.1, 2.7, 2.8) ───────────────────────────────
*
* ha_elect_evaluate_and_signal: evaluate accumulated votes; set ctx->result
* and post ctx->done when quorum is reached or all peers replied.
*
* Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size.
* ctx->votes_granted starts at 1 (self-vote); ctx->n_peers is n_other_peers.
* Quorum = ceil((n_peers + 1) / 2) = (n_peers + 1 + 1) / 2 = (n_peers + 2) / 2.
*
* Yield gate (Req 2.7): if any responding peer has a higher config_tag, or
* equal config_tag with a higher LSN than the candidate → yield (CTRL_STANDBY).
* Candidate's own tag/lsn is stored in ctx->votes[0].my_tag / my_lsn.
*
* Fatal gate (Req 2.8): if candidate has config_tag=0 and any peer has tag>0
* → CTRL_STANDBY (never PRIMARY with no config when peer has config).
*
* Returns CTRL_PRIMARY or CTRL_STANDBY; also sets ctx->result.
*/
ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx)
{
/* Guard: called from handle_request_vote_reply under ctx->lock; called from
* unit-test path (peers==NULL) without threads — done_posted prevents double
* execution if two threads both hit quorum simultaneously. */
if (ctx->done_posted)
return (ctx->result == 1) ? CTRL_PRIMARY : CTRL_STANDBY;
ctrl_role_t role_result;
/* Fix 2 (CRITICAL): zero-peer guard — single-node shortcircuit.
* n_peers==0 means no other nodes; votes[] flexible array has 0 entries,
* so accessing votes[0] would be out-of-bounds. Self-vote (votes_granted=1)
* always satisfies quorum=1. No fresher-peer or fatal-gate checks apply. */
if (ctx->n_peers == 0) {
ctx->result = 1;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_PRIMARY;
goto apply_role;
}
{
int total_nodes = ctx->n_peers + 1; /* include self */
int quorum = (total_nodes + 1) / 2; /* ceil(total/2) */
/* Req 2.8: fatal gate — candidate has no config but a peer does */
uint64_t my_tag = ctx->votes[0].my_tag;
uint64_t my_lsn = ctx->votes[0].my_lsn;
if (my_tag == 0 && ctx->max_resp_tag > 0) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
/* Check quorum */
if (ctx->votes_granted < quorum) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
/* Req 2.7: yield gate — fresher responding peer */
if (ctx->max_resp_tag > my_tag) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
if (ctx->max_resp_tag == my_tag && ctx->max_resp_lsn > my_lsn) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
/* Quorum met and no fresher peer → PRIMARY */
ctx->result = 1;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_PRIMARY;
}
apply_role:
/* ── Task 4.1: apply role and call cluster_init_phase2 ─────────────────
*
* After determining the election result, update ha->role and (on first boot)
* call cluster_init_phase2. This runs once per process lifetime — the
* phase2_done flag prevents re-entry on subsequent elections (RDHUP failover).
*
* The cluster_t pointer is carried in ctx->c (NULL in unit tests).
*
* Cancel the election timeout watchdog if we won (it would be a no-op now
* but the explicit cancel avoids a spurious second evaluate call).
*/
{
cluster_t *c = ctx->c;
int is_first_boot = !ha->phase2_done;
if (role_result == CTRL_PRIMARY) {
/* Cancel election timeout watchdog (quorum already reached) */
if (ctx->timeout_ctx) {
election_timeout_ctx_t *etc =
(election_timeout_ctx_t *)ctx->timeout_ctx;
atomic_store(&etc->cancelled, 1);
}
/* First boot: call cluster_init_phase2 to load config+data (Req 9.2) */
if (is_first_boot) {
#ifdef XCE_TESTING
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
}
/* ha-single-leader-role task 3.1 (Req 1.1, 1.5): enter CTRL_PRIMARY
* DIRECTLY — symmetric with raft_cb_become_leader's rewrite. There
* is no transitional COMMITTING state and no committing-timeout
* watchdog; new-leader safety comes from the own-term WAL_NOOP +
* writable_lsn gate armed below (ha_arm_own_term_gate), not from a
* fourth role value. The legacy XCE_TESTING election path OWNS its
* term (no raft library in test mode); it historically did NOT
* increment the term in this branch, so we preserve that — no
* ha_do_promote, no new term store (research.md §4 finding 2). */
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
memory_order_release);
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64
", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire),
is_first_boot);
/* Push config to standbys with lower config_tag (Req 7.1).
* Task 3.3: requester-less promotion push → offloadable (dormant
* flag OFF = inline, unchanged; flag ON = slow lane). */
#ifdef XCE_TESTING
if (ha_push_config_hook != NULL)
ha_push_config_hook(c);
#else
if (c != NULL)
ha_config_push_offloadable(c, NULL, 0, NULL, 0);
#endif
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up (Req 7.2) */
#ifdef XCE_TESTING
if (ha_send_ctrlhello_hook != NULL)
ha_send_ctrlhello_hook(c);
#else
if (c != NULL)
ha_send_ctrlhello_to_all_standbys(c);
#endif
/* ha-single-leader-role task 3.1: route promotion side effects
* through the SAME shared sequence raft_cb_become_leader runs so the
* test-driven (legacy) and raft-driven promotion paths cannot diverge
* (design §Legacy Path Symmetry). Guarded on a real cluster_t — the
* helpers dereference &c->hooksys / &c->persist / &c->commit, and the
* unit-test evaluation path passes c == NULL. Identical role / hook /
* heartbeat / no-watchdog / own-term-gate state results either road. */
if (c) {
/* Register the persistence hooks BEFORE the own-term no-op append,
* so the persistence seam is live before any racing client write.
* Idempotent via the hooks_registered guard. */
ha_register_persist_hooks(c);
/* Periodic heartbeat to standbys (idempotency guard prevents
* double-arming if already started). */
ha_start_primary_heartbeat(c);
/* Own-term WAL_NOOP + writable_lsn gate (Req 1.4, 2.1, 2.2) —
* non-blocking; append failure degrades safe (gate unset,
* promotion still completes). */
ha_arm_own_term_gate(c);
}
} else {
/* STANDBY result */
/* Cancel election timeout watchdog — it is now a no-op but explicit
* cancellation matches spec wording and avoids a spurious lock-acquire
* if the timeout fires after STANDBY is decided. */
if (ctx->timeout_ctx) {
election_timeout_ctx_t *etc =
(election_timeout_ctx_t *)ctx->timeout_ctx;
atomic_store(&etc->cancelled, 1);
}
/* First boot: call cluster_init_phase2 so structures are ready;
* config arrives via primary's replication push (Req 9.2, 7.3) */
if (is_first_boot) {
#ifdef XCE_TESTING
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
if (ha_phase2_hook != NULL)
ha_phase2_hook(c);
#else
if (c != NULL)
cluster_init_phase2(c);
#endif
ha->phase2_done = 1;
}
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
atomic_load_explicit(&ha->term, memory_order_acquire),
is_first_boot);
/* Primary connects outbound and pushes config via ctrlhello.
* Standby is fully passive here — no fetch_config_from_primary. */
}
}
return role_result;
}
/* ── Vote Reply Handler (Req 2.2–2.6) ──────────────────────────────────────
*
* handle_request_vote_reply: process a reply to an OP_requestVote RPC.
*
* Called by each per-peer vote-request thread once the peer replies.
* Updates ctx->votes[peer_idx] with the reply fields, tracks the freshest
* responding peer in ctx->max_resp_tag/max_resp_lsn, increments counters,
* and evaluates whether to signal ctx->done.
*
* Quorum = ceil((n_peers + 1) / 2) = (n_peers + 2) / 2
* (n_peers = number of other peers, not counting self; votes_granted starts at 1)
*
* Posts ctx->done via ha_elect_evaluate_and_signal when:
* - votes_granted reaches quorum (fast path — no need to wait for all replies), OR
* - n_replied reaches n_peers (all peers accounted for, win or lose)
*/
/*
* ha_record_vote_reply: record a decoded vote reply into ctx->votes[peer_idx]
* WITHOUT signalling the legacy promotion path (bug-828).
*
* Records the granting flag, peer config_tag, and peer LSN under ctx->lock with
* the same idempotency guard handle_request_vote_reply uses, and updates the
* vote tally + freshest-responder tracking. It does NOT call
* ha_elect_evaluate_and_signal — so it never posts ctx->done and never runs the
* legacy apply_role promotion side-effects.
*
* This is the seam the PRODUCTION vote_reply_dispatch path calls so the granting
* voter's advertised LSN reaches ha_compute_safe_committed_lsn (via ctx->votes[])
* while the raft library remains the SOLE production promotion driver
* (raft_on_vote_reply → on_become_leader). Routing the record through here (not
* handle_request_vote_reply) avoids a second, racing promotion signal.
*
* Returns non-zero if this reply newly reached the legacy quorum/all-replied
* threshold (so handle_request_vote_reply knows to signal); 0 otherwise.
* Must be called WITHOUT ctx->lock held (it takes the lock internally).
*/
static int ha_record_vote_reply_locked(ha_elect_ctx_t *ctx,
int peer_idx,
int vote_granted,
uint64_t peer_config_tag,
uint64_t peer_lsn)
{
/* Guard against out-of-range index */
if (peer_idx < 0 || peer_idx >= ctx->n_peers)
return 0;
pthread_mutex_lock(&ctx->lock);
ha_vote_entry_t *entry = &ctx->votes[peer_idx];
/* Idempotency guard: ignore duplicate replies */
if (entry->replied) {
pthread_mutex_unlock(&ctx->lock);
return 0;
}
/* Record reply fields */
entry->replied = 1;
entry->vote_granted = vote_granted ? 1 : 0;
entry->peer_tag = peer_config_tag;
entry->peer_lsn = peer_lsn;
/* Update counters */
if (vote_granted)
ctx->votes_granted++;
ctx->n_replied++;
/* Track the freshest responding peer for the yield gate (Req 2.7) */
if (peer_config_tag > ctx->max_resp_tag) {
ctx->max_resp_tag = peer_config_tag;
ctx->max_resp_lsn = peer_lsn;
} else if (peer_config_tag == ctx->max_resp_tag && peer_lsn > ctx->max_resp_lsn) {
ctx->max_resp_lsn = peer_lsn;
}
/* Compute whether the legacy quorum/all-replied threshold is now reached.
* Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. */
int total_nodes = ctx->n_peers + 1;
int quorum = (total_nodes + 1) / 2;
int reached = (ctx->votes_granted >= quorum || ctx->n_replied >= ctx->n_peers);
pthread_mutex_unlock(&ctx->lock);
return reached;
}
/* Public record-only seam (bug-828): records the reply but never signals the
* legacy promotion path. Used by the production vote_reply_dispatch path. */
void ha_record_vote_reply(ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t term,
int vote_granted,
uint64_t peer_config_tag,
uint64_t peer_lsn)
{
(void)ha; /* fencing: caller bumps ha->term before calling here */
(void)term;
if (!ctx)
return;
(void)ha_record_vote_reply_locked(ctx, peer_idx, vote_granted,
peer_config_tag, peer_lsn);
}
void handle_request_vote_reply(ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t term,
int vote_granted,
uint64_t peer_config_tag,
uint64_t peer_lsn)
{
(void)term; /* fencing: callers bump ha->term if peer's term > ours, before calling here */
if (peer_idx < 0 || peer_idx >= ctx->n_peers)
return;
/* Record the reply (shared with the production record-only seam) and learn
* whether the legacy quorum/all-replied threshold is now reached. */
int reached = ha_record_vote_reply_locked(ctx, peer_idx, vote_granted,
peer_config_tag, peer_lsn);
/* Decide whether to signal the waiting thread.
* ha_elect_evaluate_and_signal takes ctx->lock; its done_posted guard
* prevents double sem_post if two threads reach quorum simultaneously.
* Skip if the index was out of range / a duplicate reply (reached==0 and
* nothing recorded) — the duplicate case is already idempotent in
* ha_record_vote_reply_locked, which returns 0 without re-counting. */
if (reached) {
pthread_mutex_lock(&ctx->lock);
ha_elect_evaluate_and_signal(ha, ctx);
pthread_mutex_unlock(&ctx->lock);
}
}
/* ── Task 2.2: iomgr-native vote send path ──────────────────────────────────
*
* vote_ep_ctx_t: per-peer context stored in ep->data for each outgoing vote
* endpoint. on_vote_connected fires after the TLS handshake completes (when
* endpoint_t.ssl_handshake_done transitions to 1 in io_handler).
*/
/* vote_ep_ctx_t: typedef moved to Task 6.2 forward-declaration block above */
/*
* peer_ctrl_msg_handler / vote_reply_decode_and_dispatch /
* repl_ack_decode_and_dispatch: forward declarations.
* peer_ctrl_msg_handler is the single iomgr handler set in on_vote_connected
* and never reassigned — it routes by ha->role at dispatch time.
*/
static int peer_ctrl_msg_handler(void *ep_raw);
static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt);
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx,
io_packet_t *pkt);
/* ha-prevote task 5.1: preVoteReply decode/route; defined alongside the
* real-vote decoder, feeds the later-defined pre_vote_reply_dispatch. */
static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx,
io_packet_t *pkt);
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted);
#ifdef XCE_TESTING
/*
* xce_vote_request_hook: test-only seam.
*
* When non-NULL, on_vote_connected() calls this INSTEAD OF the real
* xce_request() so tests can capture the call without requiring a live
* iomgr or SSL endpoint. The hook receives the fully built
* xce_request_t and may inspect all fields.
*
* Pattern mirrors xce_retry_send_hook in request.c.
*/
int (*xce_vote_request_hook)(endpoint_t *ep, xce_request_t *req) = NULL;
#endif /* XCE_TESTING */
/*
* on_vote_connected: iomgr callback fired after TLS handshake completes on an
* outgoing vote endpoint.
*
* Steps:
* 1. Cast ep->data to vote_ep_ctx_t.
* 2. Build request_vote_req_t from ctx fields.
* 3. Build xce_request_t{opcode=OPCode_requestVote, body.requestvotemsg=&vrq}.
* 4. Call xce_vote_request_hook (test seam, XCE_TESTING only) — then
* call xce_request() to queue the packet on the iomgr send path.
* 5. Set ep->msg_handler = vote_reply_msg_handler so the next inbound
* packet is decoded as a RequestVoteReply.
*
* Req 2.3: send vote after TLS handshake.
* Req 1.2: vote RPC via iomgr path (xce_request enqueues to endpoint_enq).
*/
static void on_vote_connected(endpoint_t *ep)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
/* Build the RequestVoteRequest payload */
request_vote_req_t vrq;
memset(&vrq, 0, sizeof(vrq));
vrq.term = vctx->my_term;
vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
vrq.config_tag = vctx->my_tag;
vrq.last_log_lsn = vctx->my_lsn;
/* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */
vrq.last_log_term = vctx->my_last_log_term;
vrq.last_log_index = vctx->my_last_log_index;
/* Wrap in xce_request_t */
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_requestVote;
req.body_which = Request_body_requestvotemsg;
req.body.requestvotemsg = &vrq;
#ifdef XCE_TESTING
if (xce_vote_request_hook != NULL)
xce_vote_request_hook(ep, &req);
else
xce_request(ep, &req);
#else
xce_request(ep, &req);
#endif
/* Set state-aware dispatcher — routes by ha->role at message-arrival time.
* Never reassigned: ELECTING → vote reply; PRIMARY → repl ack. */
ep->msg_handler = peer_ctrl_msg_handler;
}
/*
* on_pre_vote_connected: iomgr callback fired after the TLS handshake completes
* on an outgoing PreVote endpoint (ha-prevote task 4.1).
*
* Mirrors on_vote_connected but emits OPCode_preVoteRequest (reusing the
* RequestVoteRequest payload struct + union arm, per the wire schema in task
* 1.1). prospective_term (current+1) rides in the term field; nothing is
* persisted. The reply-routing handler for preVoteReply is wired in task 5.1;
* here we set the same state-aware dispatcher used by the real-vote path.
*/
static void on_pre_vote_connected(endpoint_t *ep)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
request_vote_req_t vrq;
memset(&vrq, 0, sizeof(vrq));
vrq.term = vctx->my_term; /* prospective term = current + 1 */
vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
vrq.config_tag = vctx->my_tag;
vrq.last_log_lsn = vctx->my_lsn;
/* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */
vrq.last_log_term = vctx->my_last_log_term;
vrq.last_log_index = vctx->my_last_log_index;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_preVoteRequest;
req.body_which = Request_body_requestvotemsg;
req.body.requestvotemsg = &vrq;
#ifdef XCE_TESTING
if (xce_vote_request_hook != NULL)
xce_vote_request_hook(ep, &req);
else
xce_request(ep, &req);
#else
xce_request(ep, &req);
#endif
ep->msg_handler = peer_ctrl_msg_handler;
}
/*
* xce_vote_reply_hook: test-only seam for vote_reply_msg_handler.
*
* Forward-declared here (before vote_reply_msg_handler) so the handler body
* can reference it. Defined only under XCE_TESTING; guarded below.
*
* When non-NULL, vote_reply_msg_handler() calls this INSTEAD OF the real
* handle_request_vote_reply() so tests can capture decoded fields without
* a live iomgr. Receives the decoded scalar fields — no capnp pointers.
*
* Pattern mirrors xce_vote_request_hook used in task 2.2.
*/
#ifdef XCE_TESTING
int (*xce_vote_reply_hook)(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx,
uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn) = NULL;
#endif /* XCE_TESTING */
/*
* vote_reply_dispatch: internal helper — delegates to the raft library or
* the test hook (when XCE_TESTING and hook is non-NULL).
*
* Production path: raft_on_vote_reply owns quorum tracking and fires
* on_become_leader/on_become_follower via the raft callbacks.
* Test path: xce_vote_reply_hook captures decoded fields without real iomgr.
*/
static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx,
uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn)
{
#ifdef XCE_TESTING
if (xce_vote_reply_hook != NULL) {
xce_vote_reply_hook(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn);
return;
}
#endif
/* bug-828 (PRIMARY fix): route the decoded reply into the election context
* BEFORE driving the raft library. Without this, the granting voter's
* advertised peer_lsn was discarded — ctx->votes[idx].vote_granted stayed 0,
* so a clean failover's ha_compute_safe_committed_lsn (read in
* raft_cb_become_leader / on_become_leader) saw only the self-vote
* (n_granted=1) and truncated committed entries down to self_lsn.
*
* Record-only: ha_record_vote_reply records ctx->votes[idx] under ctx->lock
* but does NOT post ctx->done / run the legacy apply_role promotion path.
* raft_on_vote_reply remains the SOLE production promotion driver (it owns
* the quorum tally and fires on_become_leader) — so there is no double
* promotion. `idx` is vctx->peer_idx, the local ctx->votes[] index (set in
* raft_cb_send_vote_request via the absolute→local mapping), which is exactly
* the index ha_compute_safe_committed_lsn and the test hook use. */
ha_record_vote_reply(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn);
raft_on_vote_reply(&ha->raft, idx, vote_granted, term);
}
/*
* vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch
* to vote_reply_dispatch. Factored from vote_reply_msg_handler so that
* peer_ctrl_msg_handler can invoke the same logic without code duplication.
* Packet ownership (raft-drive-async task 3.1): the caller
* (peer_ctrl_msg_handler) has ALREADY taken `pkt` off ep->fsm_in.pkt and
* cleared ep->fsm_in.pkt to NULL — `pkt` is passed in explicitly and this
* function frees it on every path. Clearing in the caller (not here) is what
* makes the off-iomgr dispatch path race-free: the body never touches
* ep->fsm_in.pkt, so a concurrent io_worker read on the same endpoint cannot
* collide with the in-flight packet. vote_reply_dispatch can still reach
* raft_cb_become_follower → endpoint_fini(ep) (loser path) safely because
* ep->fsm_in.pkt is already NULL.
*/
static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt)
{
ha_state_t *ha = vctx->ha;
ha_elect_ctx_t *ctx = vctx->ctx;
int idx = vctx->peer_idx;
int ok = 0;
uint64_t vr_term = 0;
int vr_granted = 0;
uint64_t vr_tag = 0;
uint64_t vr_lsn = 0;
if (!pkt || !pkt->data || pkt->sz == 0)
goto done;
{
struct capn capn_ctx;
struct Request outer_rq;
Request_ptr outer_rp;
struct RequestReply rep_s;
struct RequestVoteReply vr_raw;
int capn_ok;
capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
(size_t)pkt->sz, 0);
if (capn_ok != 0)
goto done;
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
read_Request(&outer_rq, outer_rp);
if (outer_rq.body_which != Request_body_replymsg ||
!outer_rq.body.replymsg.p.data) {
capn_free(&capn_ctx);
goto done;
}
read_RequestReply(&rep_s, outer_rq.body.replymsg);
if (rep_s.data_which != RequestReply_data_requestvotereplymsg) {
capn_free(&capn_ctx);
goto done;
}
memset(&vr_raw, 0, sizeof(vr_raw));
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
capn_free(&capn_ctx);
vr_term = (uint64_t)vr_raw.term;
vr_granted = vr_raw.voteGranted ? 1 : 0;
vr_tag = (uint64_t)vr_raw.peerConfigTag;
vr_lsn = (uint64_t)vr_raw.peerLastLogLsn;
ok = 1;
}
done:
if (pkt) { free_packet(pkt); free(pkt); }
if (ok) {
uint64_t cur = atomic_load_explicit(&ha->term, memory_order_acquire);
while (vr_term > cur) {
if (atomic_compare_exchange_weak_explicit(&ha->term, &cur, vr_term,
memory_order_acq_rel,
memory_order_acquire))
break;
}
vote_reply_dispatch(ha, ctx, idx, vr_term, vr_granted, vr_tag, vr_lsn);
} else {
vote_reply_dispatch(ha, ctx, idx,
atomic_load_explicit(&ha->term, memory_order_acquire),
0, 0, 0);
}
return 0;
}
/*
* pre_vote_reply_decode_and_dispatch: decode a preVoteReply packet on an
* outbound PreVote endpoint and feed it to the PreVote tally (ha-prevote task
* 5.1). Mirrors vote_reply_decode_and_dispatch exactly — the PreVote reply
* reuses the identical RequestVoteReply payload / union arm — but routes to
* pre_vote_reply_dispatch (the raft-library PreVote tally) instead of the
* real-vote tally, and deliberately performs NO term adoption (a PreVote reply
* never changes our term; Req 1.2). vctx->peer_idx is the ABSOLUTE cluster
* index that raft_on_pre_vote_reply expects (set in ha_send_pre_vote_request).
* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly
* (already taken off ep->fsm_in.pkt by peer_ctrl_msg_handler); this function
* frees it on every path and never touches ep->fsm_in.pkt.
*/
static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx,
io_packet_t *pkt)
{
ha_state_t *ha = vctx->ha;
cluster_t *c = vctx->c;
int idx = vctx->peer_idx; /* ABSOLUTE cluster index */
int ok = 0;
int vr_granted = 0;
if (!pkt || !pkt->data || pkt->sz == 0)
goto done;
{
struct capn capn_ctx;
struct Request outer_rq;
Request_ptr outer_rp;
struct RequestReply rep_s;
struct RequestVoteReply vr_raw;
int capn_ok;
capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
(size_t)pkt->sz, 0);
if (capn_ok != 0)
goto done;
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
read_Request(&outer_rq, outer_rp);
if (outer_rq.body_which != Request_body_replymsg ||
!outer_rq.body.replymsg.p.data) {
capn_free(&capn_ctx);
goto done;
}
read_RequestReply(&rep_s, outer_rq.body.replymsg);
if (rep_s.data_which != RequestReply_data_requestvotereplymsg) {
capn_free(&capn_ctx);
goto done;
}
memset(&vr_raw, 0, sizeof(vr_raw));
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
capn_free(&capn_ctx);
vr_granted = vr_raw.voteGranted ? 1 : 0;
ok = 1;
}
done:
if (pkt) { free_packet(pkt); free(pkt); }
/* A malformed / wrong-body reply counts as a non-grant (best-effort
* delivery, per design Event Contract). No term adoption either way. */
pre_vote_reply_dispatch(ha, c, idx, ok ? vr_granted : 0);
return 0;
}
/*
* repl_ack_decode_and_dispatch: decode an inbound packet on a promoted vote
* endpoint while role == CTRL_PRIMARY.
*
* - OPCode_replCatchUpDone → INFO no-op (ack-counting removed in task 4.1)
* - OPCode_reply with stateReportMsg → ha_process_standby_state (Task 3.1)
* - OPCode_reply with walackreplymsg → repl_on_ack (Task 4.1)
* - OPCode_reply with replconfigpushreplymsg → repl_on_config_ack
* (barrier-rollout task 5.2 r1 / ha-config-bootstrap task 8.1)
* - OPCode_nop or OPCode_reply with any other body → silently ignored
* - anything else → WARN + drop
*/
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx,
io_packet_t *pkt)
{
ha_state_t *ha = vctx->ha;
cluster_t *c = vctx->c;
/* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in
* explicitly (peer_ctrl_msg_handler already cleared ep->fsm_in.pkt); this
* function frees it and never touches ep->fsm_in.pkt.
*
* raft-drive-async tasks 3.2/3.3 (FUTURE): the long ops reachable from the
* stateReport branch below — ha_dispatch_repl_action_for_standby's
* snapshot send / config push — are where the SlowLane offload lands. For
* task 3.1 the body only relocates onto the pool and still runs bounded
* decode/route work inline in the vtask entry. */
if (!pkt || !pkt->data || pkt->sz == 0) {
if (pkt) { free_packet(pkt); free(pkt); }
return 0;
}
struct capn capn_ctx;
struct Request outer_rq;
Request_ptr outer_rp;
int capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
(size_t)pkt->sz, 0);
free_packet(pkt);
free(pkt);
if (capn_ok != 0)
return 0;
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
read_Request(&outer_rq, outer_rp);
int opcode = (int)outer_rq.opcode;
if (opcode == OPCode_replCatchUpDone) {
capn_free(&capn_ctx);
/* ha-single-leader-role task 4.1: the force-promotion ack-counting
* machinery has been deleted — promotion is driven solely by the
* heartbeat-driven decision loop and per-entry majority ack. Standbys
* still send OPCode_replCatchUpDone and the dispatch gate still admits
* it; it is now an INFO-logged no-op. */
(void)ha;
(void)c;
INFO("HA: replCatchUpDone received (no-op; ack-counting machinery removed in task 4.1)");
} else if (opcode == OPCode_reply &&
outer_rq.body_which == Request_body_replymsg &&
outer_rq.body.replymsg.p.data) {
/* Heartbeat NOP reply — decode inner RequestReply for stateReport */
struct RequestReply rep_s;
memset(&rep_s, 0, sizeof(rep_s));
read_RequestReply(&rep_s, outer_rq.body.replymsg);
if (rep_s.data_which == RequestReply_data_stateReportMsg) {
/* Decode stateReport and call convergence tracker (Task 3.1).
* read_StateReport reads scalar fields directly from the capnp
* pointer without heap allocation — safe to call before capn_free. */
struct StateReport sr_raw;
memset(&sr_raw, 0, sizeof(sr_raw));
read_StateReport(&sr_raw, rep_s.data.stateReportMsg);
state_report_t sr;
sr.applied_lsn = (uint64_t)sr_raw.appliedLsn;
sr.config_tag = (uint64_t)sr_raw.configTag;
capn_free(&capn_ctx);
/* peer_idx from standby_ep_ctx_t (layout-compatible with vote_ep_ctx_t) */
int peer_idx = vctx->peer_idx;
uint64_t committed_lsn = 0;
uint64_t committed_tag = 0;
uint64_t last_written = 0; /* latest WAL LSN — may be > committed */
#ifndef XCE_TESTING
if (c) {
committed_lsn = atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
committed_tag = atomic_load_explicit(&ha->committed_config_tag,
memory_order_acquire);
/* current_lsn is the next LSN to write; last written = current - 1.
* Read without lock — monotonic, approximate value is sufficient
* for the replication decision. */
last_written = (c->persist.current_lsn > 0)
? c->persist.current_lsn - 1 : 0;
}
#else
/* In test mode, c may be NULL; caller sets ha fields directly */
committed_lsn = atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire);
committed_tag = atomic_load_explicit(&ha->committed_config_tag,
memory_order_acquire);
(void)c;
#endif
ha_process_standby_state(ha, peer_idx,
sr.config_tag, sr.applied_lsn,
committed_tag, committed_lsn);
/* Derive oldest_wal_lsn from the WAL directory.
* The oldest retained segment's start LSN is the boundary —
* standbys below it need a snapshot (Task 3.5b). */
uint64_t oldest_wal_lsn = 0;
#ifndef XCE_TESTING
if (c && c->persist.wal_path[0]) {
extern uint64_t ha_compute_oldest_wal_lsn(const char *wal_path);
oldest_wal_lsn = ha_compute_oldest_wal_lsn(c->persist.wal_path);
}
#endif
/* Replication target: use max(committed_lsn, last_written) so
* standbys can catch up even when committed_lsn is stuck at 0
* (no replicator-registered standbys to advance it). */
uint64_t target_lsn = committed_lsn;
if (last_written > target_lsn)
target_lsn = last_written;
ha_dispatch_repl_action_for_standby(ha, c, ep, peer_idx,
committed_tag, target_lsn,
oldest_wal_lsn);
} else if (rep_s.data_which == RequestReply_data_walackreplymsg) {
/* Task 4.1: WAL ACK reply from standby — extract lsn + success,
* call repl_on_ack to update acked_lsn and signal majority waiter. */
struct WalAckReply war_raw;
memset(&war_raw, 0, sizeof(war_raw));
read_WalAckReply(&war_raw, rep_s.data.walackreplymsg);
uint64_t acked_lsn = (uint64_t)war_raw.lsn;
uint64_t reply_term = (uint64_t)war_raw.term;
int success = war_raw.success ? 1 : 0;
capn_free(&capn_ctx);
if (success && c) {
repl_on_ack(&c->persist.repl, ep, acked_lsn);
TRACE("repl_ack_decode_and_dispatch: WAL ACK from standby"
" lsn=%lu success=1", (unsigned long)acked_lsn);
/* ha-term-safety task 3.3 (R4.1/R4.5): route the ACK through the
* raft library so it updates match_index[]/next_index[] and
* recomputes the current-term-gated commit (raft_advance_commit).
* This fires on_commit_advanced (task 2.4) ONLY when a quorum
* agrees on a CURRENT-TERM entry — making the library the live
* commit source alongside the durable barrier; a stale-term reply
* is dropped inside the library. Driven on the same iomgr ACK
* path; neither library call performs I/O.
*
* Resolve the ABSOLUTE library peer index per endpoint type FIRST
* (remediation): this branch fires for BOTH promoted-vote eps
* (vctx->ctx != NULL, peer_idx is a relative votes index) AND
* heartbeat standby eps (vctx->ctx == NULL, peer_idx is already
* absolute) — see the authoritative scan at ha.c:6187-6189. A
* blind remap would double-map the heartbeat path onto the wrong
* (or out-of-range) match_index[] slot. */
int ack_abs_idx = ha_repl_ack_abs_peer_idx(
&c->ha, /*ep_is_vote*/ vctx->ctx != NULL, vctx->peer_idx);
ha_raft_on_repl_ack(c, ack_abs_idx, acked_lsn, reply_term);
/* ha-convergence-recovery task 4.2 (D6, Req 5.1/5.2/7.1):
* ACK-driven catch-up — continue streaming to a standby still
* behind target, or re-include it at parity. Shares the exact
* decision the unit test drives via ha_walack_continuation_for_test.
* Scheduling is O(1) on iomgr; all WAL file I/O runs in the
* deferred continuation task. */
ha_walack_on_ack(c, ep, vctx->peer_idx, acked_lsn);
} else if (!success) {
WARN("repl_ack_decode_and_dispatch: WAL NACK from standby"
" lsn=%lu success=0 — ignoring", (unsigned long)acked_lsn);
}
} else if (rep_s.data_which == RequestReply_data_replconfigpushreplymsg) {
/* Config-push ACK from standby (ha-commit-barrier-rollout task
* 5.2 remediation r1; completes the unlanded half of
* ha-config-bootstrap task 8.1): the standby replies to
* OP_replConfigPush with replconfigpushreplymsg carrying
* {accepted, versionTag} (config_repl.c:744-752). Decode the
* acked tag and apply it to the replicator so the configAck
* barrier (repl_wait_majority_ack) can observe the covering
* ACK — without this branch every multi-member config upgrade
* would time out. read_ReplConfigPushReply reads scalars only
* — safe to call before capn_free. */
struct ReplConfigPushReply rcp_raw;
memset(&rcp_raw, 0, sizeof(rcp_raw));
read_ReplConfigPushReply(&rcp_raw, rep_s.data.replconfigpushreplymsg);
uint64_t acked_tag = (uint64_t)rcp_raw.versionTag;
int accepted = rcp_raw.accepted ? 1 : 0;
capn_free(&capn_ctx);
if (accepted && c) {
repl_on_config_ack(&c->persist.repl, ep, acked_tag);
TRACE("repl_ack_decode_and_dispatch: config ACK from standby"
" tag=%016" PRIx64 " accepted=1", acked_tag);
} else if (!accepted) {
WARN("repl_ack_decode_and_dispatch: config push rejected by"
" standby (tag=%016" PRIx64 ") — ignoring", acked_tag);
}
} else {
capn_free(&capn_ctx);
}
} else if (opcode == OPCode_nop) {
/* Plain NOP with no body — silently ignore */
capn_free(&capn_ctx);
} else {
capn_free(&capn_ctx);
WARN("repl_ack_decode_and_dispatch: unexpected opcode %d on promoted ep",
opcode);
}
return 0;
}
/*
* peer_ctrl_msg_handler_inline: the state-aware peer-control routing body.
* Routes by ha->role at dispatch time — no explicit handler reassignment needed.
* CTRL_ELECTING → vote_reply_decode_and_dispatch
* CTRL_PRIMARY → repl_ack_decode_and_dispatch
* CTRL_STANDBY + PreVote → pre_vote_reply_decode_and_dispatch (task 5.1)
* any other role → drop packet (WARN)
*
* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly —
* the caller (peer_ctrl_msg_handler) has already taken it off ep->fsm_in.pkt and
* cleared that slot to NULL. This body and the routers it calls free `pkt` on
* every path and NEVER touch ep->fsm_in.pkt, which is what lets this body run
* off the iomgr thread (on the vtask pool) without colliding with a concurrent
* io_worker read on the same endpoint.
*
* This body is run either INLINE on the io_worker thread (dispatch flag OFF —
* the production default; identical to the pre-task behavior) or on a vtask
* pool worker (flag ON; via peer_ctrl_vtask_entry).
*
* PreVote note (ha-prevote task 5.1): a preVoteReply arrives on the candidate's
* OUTBOUND PreVote endpoint while the candidate's role is still CTRL_STANDBY
* (a PreVote round performs no role change — Req 1.1). Without this branch the
* reply would hit the default case and be dropped, stranding the round until it
* times out. We gate on raft.pre_vote_in_progress so a STANDBY node with no
* active round still drops unexpected traffic.
*/
static int peer_ctrl_msg_handler_inline(endpoint_t *ep, io_packet_t *pkt)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
ha_state_t *ha = vctx->ha;
/* raft-drive-async task 3.1: test-only observation seam — fires before any
* routing so a test can count/timestamp/park the body. NULL in production. */
if (ha_peerctrl_body_probe)
ha_peerctrl_body_probe(ep);
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
/* PreVote replies land while CTRL_STANDBY with a round in progress. */
if (role == CTRL_STANDBY && ha->raft.pre_vote_in_progress)
return pre_vote_reply_decode_and_dispatch(vctx, pkt);
switch (role) {
case CTRL_ELECTING:
return vote_reply_decode_and_dispatch(vctx, pkt);
case CTRL_PRIMARY:
return repl_ack_decode_and_dispatch(ep, vctx, pkt);
default:
if (pkt) { free_packet(pkt); free(pkt); }
WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role);
return 0;
}
}
/*
* peer_ctrl_vtask_entry (raft-drive-async task 3.1): vtask pool entry that runs
* the peer-control routing body off the iomgr thread. Extracts (ep, pkt) from
* the VTASK_IO task data — the SAME io_task_data_t shape the lib vtask path
* uses — takes ownership of pkt out of the task data (so vtask_fini, which only
* frees the data struct, cannot reach it) and hands it to the body, which frees
* it. No double-free: pkt is freed exactly once inside the body; run_task then
* frees the task + data struct. endpoint_unblock_tasks (per-endpoint
* serialization, Req 4.1/4.2) is driven by run_task after this returns.
*/
static int peer_ctrl_vtask_entry(vtask_t *task)
{
io_task_data_t *iod = &task->data->u.iodata;
endpoint_t *ep = iod->ep;
io_packet_t *pkt = iod->pkt;
iod->pkt = NULL; /* body owns pkt now; nothing else may reference it */
if (ep == NULL) {
if (pkt) { free_packet(pkt); free(pkt); }
return -EINVAL;
}
return peer_ctrl_msg_handler_inline(ep, pkt);
}
/*
* peer_ctrl_classify (raft-drive-async task 3.1): assign the LivenessAdmission
* task class for an inbound peer-control message. This handler decodes vote
* replies, PreVote replies and WAL/config ACKs — all RECEIPT decode + commit
* eval, which is bounded liveness work. Per the design's deadlock-freedom
* argument (R4.5), ack/commit-advance decode MUST be VTASK_LIVENESS so it is
* NEVER admission-gated: a quorum of acks can always find a reserved worker to
* advance committed_lsn even when the pool is saturated by *_BLOCK tasks.
*
* (The vote-GRANT term fsync — VTASK_LIVE_BLOCK, R4.6 — is handled on the
* INBOUND RequestVote handler, not on these reply/ack endpoints; and the long
* config-push/snapshot ops are offloaded to the SlowLane in tasks 3.2/3.3. So
* everything this handler dispatches is liveness work today.)
*
* ┌─ HIGH FLAG-ON HAZARD (raft-drive-async finding-2; bug-1198) ─────────────┐
* │ LIVENESS is correct ONLY once the snapshot send runs OFF the pool. The │
* │ CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT → │
* │ persist_checkpoint (a BLOCKING fdatasync) in │
* │ repl_ack_decode_and_dispatch → ha_dispatch_repl_action_for_standby. │
* │ Running that blocking op while tagged VTASK_LIVENESS (never gated) breaks │
* │ the deadlock-freedom reservation — a burst of snapshot sends could occupy │
* │ every liveness worker and starve the acks that advance committed_lsn. │
* │ │
* │ Task 3.2 has LANDED the slow-lane offload (ha_snapshot_send_dispatch), │
* │ but it is DORMANT: g_drive_async_snapshot_offload defaults OFF (the send │
* │ still runs INLINE) because the offload borrows a standby endpoint whose │
* │ lifetime is not yet pool-safe (bug-1197). So bug-1198 is NOT yet closed │
* │ in the default build. Task 3.4 is the single gate that flips BOTH │
* │ g_drive_async_snapshot_offload AND g_drive_async_peerctrl_dispatch ON │
* │ together (offload bounds the body → LIVENESS correct) and proves the │
* │ deadlock-freedom + endpoint-lifetime properties under load. │
* │ DO NOT flip g_drive_async_peerctrl_dispatch_enabled ON until task 3.4's │
* │ gate is green WITH the snapshot offload also ON. │
* └──────────────────────────────────────────────────────────────────────────┘
*/
static vtask_class_t peer_ctrl_classify(ha_state_t *ha, io_packet_t *pkt)
{
(void)ha;
(void)pkt;
return VTASK_LIVENESS;
}
/* ── bug-1197: deferred vote-ep teardown ──────────────────────────────────────
*
* When g_drive_async_peerctrl_dispatch_enabled is ON, raft_cb_become_follower
* runs on a pool worker and must endpoint_fini() every loser vote-ep. But
* another pool worker can be mid-vtask (or have a queued vtask) on one of those
* same eps reading ep->data (a vote_ep_ctx_t) — freeing ep->data underneath it
* is a UAF. Instead of freeing inline we enqueue a teardown vtask BOUND to the
* ep (VTASK_IO): the pool's per-ep busy serialization guarantees it runs after
* any in-flight or earlier-queued same-ep vtask completes. We also mark the ep
* tearing_down + iomgr_unmonitor it BEFORE enqueue so the io_worker drops any
* late same-batch event instead of dispatching a new vtask onto a doomed ep. */
#ifdef XCE_TESTING
/* test-only ordering probe: fired (with the real ep) at the top of the teardown
* vtask body, before endpoint_fini, so a test can assert teardown ran strictly
* after the last live same-ep use. NULL in production. */
void (*ha_vep_teardown_probe)(endpoint_t *) = NULL;
/* test-only drain probe: fired in peer_ctrl_msg_handler AFTER the tearing_down
* check passes but BEFORE the pool enqueue (inside the dispatch_inflight
* bracket), so a test can park a dispatch in the drained window and prove
* endpoint_mark_tearing_down blocks until the enqueue completes. NULL in
* production. */
void (*ha_peerctrl_predispatch_probe)(endpoint_t *) = NULL;
#endif
/* Runs on a pool worker, bound VTASK_IO to ep, strictly AFTER any in-flight or
* queued same-ep peer-control vtask (per-ep busy FIFO). On the loser path
* ectx->votes[i].ep is the sole reference to ep, so this owns the free. */
static int vote_ep_teardown_vtask_entry(vtask_t *task)
{
io_task_data_t *iod = &task->data->u.iodata;
endpoint_t *ep = iod->ep;
if (ep != NULL) {
#ifdef XCE_TESTING
if (ha_vep_teardown_probe)
ha_vep_teardown_probe(ep);
#endif
/* bug-1298: drop the creator/owner reference. We DON'T fini/free here
* and DON'T NULL iod->ep: this vtask still holds the per-vtask reference
* (taken in vtask_enq), so the ep stays alive through run_task's post-body
* endpoint_unblock_tasks; run_task then drops the per-vtask ref, and that
* last endpoint_unref performs the fini+free. No UAF, no NULL hack. */
endpoint_unref(ep);
}
return 0;
}
/* Build + enqueue the teardown vtask. Caller has ALREADY
* endpoint_mark_tearing_down(ep) + iomgr_unmonitor(ep). Controllers never
* self-terminate: on any alloc/enqueue failure we fail loud (WARN) and tear the
* ep down inline — with the pool unwired or the task rejected there is no other
* worker that could be racing this ep, so an inline endpoint_fini+free is safe. */
static void vote_ep_teardown_enqueue(ha_state_t *ha, endpoint_t *ep)
{
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = calloc(1, sizeof(*task));
if (pool == NULL || data == NULL || task == NULL) {
WARN("bug-1197: vote-ep teardown enqueue unavailable (pool=%p data=%p "
"task=%p) — tearing down inline (stay alive)",
(void *)pool, (void *)data, (void *)task);
free(data);
free(task);
endpoint_unref(ep); /* bug-1298: no vtask enqueued — drop owner ref → free */
return;
}
data->vtype = VTASK_IO;
data->u.iodata.ep = ep; /* drives per-ep serialization in bind_vtask */
data->u.iodata.pkt = NULL;
vtask_init(task, (vtask_func_t)vote_ep_teardown_vtask_entry, data);
/* VTASK_LIVENESS is never admission-gated, so the teardown can always be
* enqueued even when the pool is saturated by blocking tasks. */
if (vtask_enq_classified(pool, task, VTASK_LIVENESS) < 0) {
WARN("bug-1197: vote-ep teardown enqueue failed — tearing down inline "
"(stay alive)");
free(data);
free(task);
/* bug-1298: vtask_enq already dropped the per-vtask ref it took on the
* failed enqueue, so refcnt is back to the owner ref — drop it → free. */
endpoint_unref(ep);
}
}
/* The flag-ON loser-path teardown for one vote-ep: mark tearing_down, stop epoll
* wakeups, then defer the free onto the ep's per-ep FIFO. ORDERING (bug-1197):
* - endpoint_mark_tearing_down sets the flag under ep->lock; the io_worker's
* late-event gate (peer_ctrl_msg_handler, ON branch) reads it under ep->lock
* via endpoint_is_tearing_down, so any dispatch DECISION taken after the
* mark observes it and drops — nothing new is enqueued for this ep.
* - iomgr_unmonitor stops FUTURE epoll events (an event already in the current
* batch is handled by the tearing_down gate above).
* - the teardown vtask is itself VTASK_IO bound to ep, so the pool's per-ep
* busy serialization runs it strictly after any in-flight / earlier-queued
* same-ep vtask completes.
* NOTE: we cannot hold ep->lock ACROSS the dispatch enqueue (vtask_enq_classified
* → dataq_enq takes the taskq lock, while the pool dispatcher takes ep->lock
* UNDER the taskq lock in vtask_eligible → endpoint_is_task_blocked — an AB/BA
* inversion). The check is therefore done under ep->lock and the enqueue
* outside it; see the io_worker gate for the residual-window note. */
static void vote_ep_teardown_async(ha_state_t *ha, endpoint_t *ep)
{
endpoint_mark_tearing_down(ep);
if (ep->iomgr != NULL)
iomgr_unmonitor(ep->iomgr, ep);
vote_ep_teardown_enqueue(ha, ep);
}
#ifdef XCE_TESTING
/* ha_vote_ep_teardown_for_test: drive the flag-ON per-ep async teardown branch
* for a single ep without staging a full election/become_follower. */
void ha_vote_ep_teardown_for_test(ha_state_t *ha, endpoint_t *ep)
{
vote_ep_teardown_async(ha, ep);
}
#endif
/*
* peer_ctrl_dispatch_to_pool (raft-drive-async task 3.1): hand the decoded
* peer-control packet to the vtask pool and return promptly. Per-endpoint
* arrival-order serialization is automatic — bind_vtask calls
* endpoint_block_tasks(ep) and the dispatcher won't bind a second task for the
* same endpoint until this one completes (Req 4.1/4.2).
*
* Controllers NEVER self-terminate (CLAUDE.md): on any failure to enqueue
* (pool unwired, OOM, or admission reject) we fail loud and run the body INLINE
* rather than drop a consensus message — staying alive and correct.
*
* ┌─ CRITICAL FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197) ──────────┐
* │ Enabling this dispatch (g_drive_async_peerctrl_dispatch_enabled == 1) │
* │ introduces an endpoint use-after-free that does NOT exist while the body │
* │ runs inline. raft_cb_become_follower (ha.c, see its endpoint_fini loop) │
* │ runs on a POOL worker and calls endpoint_fini() on ALL peer vote-eps — │
* │ freeing each ep->data (vctx) via FREEUP. Meanwhile the single io_worker │
* │ thread can be concurrently inside peer_ctrl_msg_handler_inline(ep_j, ...) │
* │ for a same-batch queued event on another ep, reading ep_j->data → races a │
* │ freed vctx (null-deref / UAF). Safe TODAY only because the flag is OFF and │
* │ the io_worker processes peer-control sequentially. │
* │ FIX REQUIRED BEFORE task 3.4 FLIPS THE FLAG ON: every pool-reachable │
* │ teardown that frees a peer ep (raft_cb_become_follower's endpoint_fini │
* │ loop) MUST iomgr_unmonitor(ep) BEFORE endpoint_fini(ep) so the io_worker │
* │ can never deliver a new event on a freed ep->data. Do NOT enable dispatch │
* │ until that ordering is in place. │
* └──────────────────────────────────────────────────────────────────────────┘
*/
static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
ha_state_t *ha = vctx->ha;
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
if (pool == NULL) {
WARN("peer_ctrl_dispatch_to_pool: vtask pool not wired — running "
"inline (stay alive, never drop a peer-control message)");
return peer_ctrl_msg_handler_inline(ep, pkt);
}
vtask_data_t *data = (vtask_data_t *)calloc(1, sizeof(*data));
vtask_t *task = (vtask_t *)calloc(1, sizeof(*task));
if (data == NULL || task == NULL) {
WARN("peer_ctrl_dispatch_to_pool: OOM allocating vtask — running "
"inline (stay alive)");
free(data);
free(task);
return peer_ctrl_msg_handler_inline(ep, pkt);
}
data->vtype = VTASK_IO;
data->u.iodata.ep = ep; /* drives per-endpoint serialization in bind_vtask */
data->u.iodata.pkt = pkt; /* ownership transferred into the task */
vtask_init(task, (vtask_func_t)peer_ctrl_vtask_entry, data);
/* raft-drive-async task 3.1: the task is the SOLE owner of pkt (the shim
* already cleared ep->fsm_in.pkt). Mark it so taskq_item_killer frees pkt
* if this task is destroyed while still queued (never executed) at
* vtaskmgr_fini — without this the pkt would leak on shutdown. The
* executed path frees pkt inside peer_ctrl_vtask_entry (and NULLs
* iod->pkt), so there is no double-free. */
task->owns_io_pkt = 1;
vtask_class_t cls = peer_ctrl_classify(ha, pkt);
int rc = vtask_enq_classified(pool, task, cls);
if (rc < 0) {
/* Task NOT enqueued — we still own task/data/pkt. LIVENESS is never
* gated so this should not happen; if it does, fail loud and run inline
* (never drop). Free the wrapper structs only — pkt is freed by the
* inline body below. */
WARN("peer_ctrl_dispatch_to_pool: vtask_enq_classified failed (rc=%d) "
"— running inline (stay alive)", rc);
free(data);
free(task);
return peer_ctrl_msg_handler_inline(ep, pkt);
}
return 0; /* returns promptly; the body runs on a pool worker */
}
/*
* peer_ctrl_msg_handler: the registered iomgr msg_handler for peer-control
* endpoints (set in on_vote_connected / on_pre_vote_connected / on_standby_
* connected). Runs on the io_worker thread.
*
* raft-drive-async task 3.1: takes ownership of the inbound packet off
* ep->fsm_in.pkt and clears that slot IMMEDIATELY (so a subsequent io_worker
* read on this endpoint allocates a fresh packet and cannot collide with the
* one we are about to process), then either runs the routing body inline
* (dispatch flag OFF — production default, byte-for-byte the prior behavior) or
* hands it to the vtask pool and returns promptly (flag ON).
*/
static int peer_ctrl_msg_handler(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
io_packet_t *pkt = ep->fsm_in.pkt;
ep->fsm_in.pkt = NULL; /* take ownership before any routing/dispatch */
if (!ha_peerctrl_dispatch_enabled())
return peer_ctrl_msg_handler_inline(ep, pkt);
/* bug-1197 late-event gate (ON path only): if this ep is being torn down,
* a same-batch event may still be delivered by the io_worker after the
* teardown thread called iomgr_unmonitor. Dropping it here keeps the doomed
* ep off the vtask pool.
*
* The window between the tearing_down check and the pool enqueue is closed
* by ep->dispatch_inflight: we increment it (seq_cst) BEFORE reading the
* flag and decrement it AFTER the enqueue completes. endpoint_mark_tearing_
* down sets the flag (under ep->lock) and then DRAINS this counter to 0
* before returning. So once the teardown caller's mark returns, every
* dispatch that read the flag as false has already finished its enqueue;
* because the teardown vtask is enqueued only AFTER mark returns, that
* peer-control vtask is strictly FIFO-before the teardown vtask (and gates
* on the same ep busy bit). Any dispatch starting after the flag is set
* reads it true here and drops. No same-ep vtask is ever enqueued after the
* teardown vtask — the window is fully closed. We must NOT hold ep->lock
* across the enqueue (it would invert with the taskq lock, see
* vote_ep_teardown_async); the counter is the lock-free handshake instead. */
atomic_fetch_add_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
if (endpoint_is_tearing_down(ep)) {
if (pkt) { free_packet(pkt); free(pkt); }
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1,
memory_order_seq_cst);
return 0;
}
#ifdef XCE_TESTING
/* test-only seam: fires AFTER the flag-check passed but BEFORE the enqueue,
* so a test can park a dispatch in exactly the drained window and prove
* endpoint_mark_tearing_down blocks until the enqueue completes. */
if (ha_peerctrl_predispatch_probe)
ha_peerctrl_predispatch_probe(ep);
#endif
/* NOTE (drain liveness): this call runs while dispatch_inflight is held, so
* a concurrent endpoint_mark_tearing_down on this ep spins until it returns.
* peer_ctrl_dispatch_to_pool only ever enqueues (non-blocking) or, on OOM,
* runs peer_ctrl_msg_handler_inline — which must also stay non-blocking
* (the vtask-handler no-block invariant). Keep it bounded or the drain is. */
int rc = peer_ctrl_dispatch_to_pool(ep, pkt);
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
return rc;
}
#ifdef XCE_TESTING
/*
* vote_reply_build_pkt_for_test: test helper that encodes a RequestVoteReply
* into a newly allocated io_packet_t that vote_reply_msg_handler_for_test can decode.
*
* Wire format mirrors the real peer reply:
* Request{opcode=reply, body.replymsg=RequestReply{
* data_which=requestvotereplymsg,
* data.requestvotereplymsg=RequestVoteReply{term, voteGranted, ...}
* }}
*
* Returns a freshly allocated io_packet_t* on success, NULL on failure.
* Handler takes ownership: it calls free_packet()+free() when done.
* Tests set ep->fsm_in.pkt to the returned pointer before calling the handler.
*/
io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn)
{
struct capn cs;
struct capn_segment *seg;
struct RequestVoteReply vr_capn;
RequestVoteReply_ptr vr_ptr;
struct RequestReply rr_capn;
RequestReply_ptr rr_ptr;
struct Request rq_capn;
Request_ptr rq_ptr;
io_packet_t *pkt;
int rc;
capn_init_malloc(&cs);
seg = capn_root(&cs).seg;
/* RequestVoteReply — all scalar fields */
memset(&vr_capn, 0, sizeof(vr_capn));
vr_capn.term = term;
vr_capn.voteGranted = vote_granted ? 1 : 0;
vr_capn.peerConfigTag = peer_tag;
vr_capn.peerLastLogLsn = peer_lsn;
vr_ptr = new_RequestVoteReply(seg);
write_RequestVoteReply(&vr_capn, vr_ptr);
/* RequestReply envelope */
memset(&rr_capn, 0, sizeof(rr_capn));
rr_capn.seq = 1;
rr_capn.opcode = OPCode_requestVote;
rr_capn.status = 0;
rr_capn.data_which = RequestReply_data_requestvotereplymsg;
rr_capn.data.requestvotereplymsg = vr_ptr;
rr_ptr = new_RequestReply(seg);
write_RequestReply(&rr_capn, rr_ptr);
/* Outer Request{opcode=reply} */
memset(&rq_capn, 0, sizeof(rq_capn));
rq_capn.opcode = OPCode_reply;
rq_capn.seq = 1;
rq_capn.body_which = Request_body_replymsg;
rq_capn.body.replymsg = rr_ptr;
rq_ptr = new_Request(seg);
write_Request(&rq_capn, rq_ptr);
capn_setp(capn_root(&cs), 0, rq_ptr.p);
pkt = calloc(1, sizeof(*pkt));
if (!pkt) { capn_free(&cs); return NULL; }
pkt->sz = capn_size(&cs);
rc = alloc_packet(pkt);
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
capn_free(&cs);
return pkt;
}
/*
* vote_reply_msg_handler_for_test: test entry point for vote-reply decode path.
* Temporarily forces role to CTRL_ELECTING so peer_ctrl_msg_handler routes to
* vote_reply_decode_and_dispatch, then restores the original role.
* Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
*/
int vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
ha_state_t *ha = vctx ? vctx->ha : NULL;
int32_t saved_role = ha ? atomic_load_explicit(&ha->role, memory_order_acquire) : 0;
if (ha)
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
ep->fsm_in.pkt = pkt;
int rc = peer_ctrl_msg_handler(ep);
if (ha)
atomic_store_explicit(&ha->role, saved_role, memory_order_release);
return rc;
}
/*
* vote_ep_ctx_build_for_test: test helper that allocates a vote_ep_ctx_t,
* fills it from the supplied parameters, and wires up ep->data and
* ep->on_connected so the test can drive on_vote_connected indirectly.
*
* Returns ep (same pointer passed in) for call-chain convenience.
* Caller is responsible for free(ep->data) after use.
*/
endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t my_term,
uint64_t my_tag,
uint64_t my_lsn,
const char *my_name)
{
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
if (!vctx) return ep; /* test will fail when on_connected is NULL */
vctx->ha = ha;
vctx->c = NULL;
vctx->ctx = ctx;
vctx->peer_idx = peer_idx;
vctx->my_term = my_term;
vctx->my_tag = my_tag;
vctx->my_lsn = my_lsn;
vctx->my_name = my_name;
ep->data = vctx;
ep->on_connected = on_vote_connected;
return ep;
}
/*
* election_timeout_fn_for_test: test helper that exercises the election timeout
* callback logic with an explicit cancelled_val instead of an atomic field.
* Allows testing the cancelled-gate without setting up a real delaymgr.
*
* Returns 0 in all cases (mirrors the real callback's return value).
*/
int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
int cancelled_val)
{
if (cancelled_val)
return 0;
pthread_mutex_lock(&ctx->lock);
ha_elect_evaluate_and_signal(ha, ctx);
pthread_mutex_unlock(&ctx->lock);
return 0;
}
/*
* repl_catchup_done_pkt_for_test: build a minimal io_packet_t encoding
* Request{opcode=replCatchUpDone, body_which=nodata}.
* Used by peer_ctrl_msg_handler tests to verify the repl-ack dispatch path.
*/
io_packet_t *repl_catchup_done_pkt_for_test(void)
{
struct capn cs;
struct capn_segment *seg;
struct Request rq_capn;
Request_ptr rq_ptr;
io_packet_t *pkt;
int rc;
capn_init_malloc(&cs);
seg = capn_root(&cs).seg;
memset(&rq_capn, 0, sizeof(rq_capn));
rq_capn.opcode = OPCode_replCatchUpDone;
rq_capn.seq = 1;
rq_capn.body_which = Request_body_nomsg;
rq_ptr = new_Request(seg);
write_Request(&rq_capn, rq_ptr);
capn_setp(capn_root(&cs), 0, rq_ptr.p);
pkt = calloc(1, sizeof(*pkt));
if (!pkt) { capn_free(&cs); return NULL; }
pkt->sz = capn_size(&cs);
rc = alloc_packet(pkt);
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
capn_free(&cs);
return pkt;
}
/*
* vote_ep_err_handler_for_test: call the static vote_ep_err_handler on ep.
* Allows tests to invoke the err_handler without going through iomgr/endpoint machinery.
*/
int vote_ep_err_handler_for_test(endpoint_t *ep)
{
return vote_ep_err_handler((void *)ep);
}
/*
* peer_ctrl_msg_handler_for_test: expose peer_ctrl_msg_handler for tests.
* Sets ep->fsm_in.pkt then invokes the handler. Packet ownership transferred.
*/
int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
{
ep->fsm_in.pkt = pkt;
return peer_ctrl_msg_handler(ep);
}
/*
* pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL
* decode/route path (peer_ctrl_msg_handler) while the candidate is CTRL_STANDBY
* with a PreVote round in progress — the exact production condition under which
* a PreVote reply arrives. Unlike vote_reply_msg_handler_for_test it does NOT
* force the role: it proves the STANDBY+pre_vote_in_progress branch added in
* task 5.1 actually routes the reply to the PreVote tally (instead of dropping
* it). Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
*/
int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
{
ep->fsm_in.pkt = pkt;
return peer_ctrl_msg_handler(ep);
}
/*
* vote_ep_ctx_set_cluster_for_test: set the cluster_t pointer inside the
* vote_ep_ctx_t stored at ep->data. Tests that need the promoted-endpoint
* dispatch path to reach ha_promote_to_primary(c) with a live cluster pointer
* must call this after vote_ep_ctx_build_for_test (which sets vctx->c = NULL
* by default).
* Requirements: task 7.3 (ha-peer-connection-reuse).
*/
void vote_ep_ctx_set_cluster_for_test(endpoint_t *ep, cluster_t *c)
{
if (!ep || !ep->data) return;
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
vctx->c = c;
}
/*
* raft_cb_become_leader_for_test: drive the leader promotion path without
* a live cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress
* production-only calls (phase2, push_config, ctrlhello).
*/
void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term)
{
ha->raft_c = NULL;
raft_cb_become_leader((void *)ha, term);
}
#ifdef XCE_TESTING
/* ha_arm_promotion_fence_timer_for_test (ha-commit-durability task 4.4):
* expose the file-static bounded-window arm path so the durability suite can
* drive the exact deferral wired into raft_cb_become_leader without staging a
* full election. Production never references this symbol (XCE_TESTING only). */
void ha_arm_promotion_fence_timer_for_test(cluster_t *c, bool truncated,
uint64_t durable_floor,
uint64_t target_hwm)
{
ha_arm_promotion_fence_timer(&c->ha, c, truncated, durable_floor,
target_hwm);
}
#endif
/*
* raft_cb_become_follower_for_test: drive the follower path without a live
* cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production
* calls (phase2, fetch_config); ha_arm_heartbeat_timeout_vtask is a no-op
* when c=NULL so no watchdog thread is spawned.
*/
void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term)
{
ha->raft_c = NULL;
raft_cb_become_follower((void *)ha, term, -1);
}
/*
* raft_cb_become_follower_with_cluster_for_test (ha-convergence-recovery
* task 2.1): drive the follower path with a LIVE cluster pointer so the
* cluster-dependent demotion-convergence branch (task 2.2) is exercised
* against a real persist/hooksys/delaymgr fixture. Sets ha->raft_c = c so
* the become-follower callback sees a real cluster (unlike the NULL-cluster
* shim above). raft_cb_become_follower is file-static, so this seam must
* live here in ha.c.
*/
void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term)
{
c->ha.raft_c = c;
raft_cb_become_follower((void *)&c->ha, term, -1);
}
/*
* raft_cb_become_leader_with_cluster_for_test (fix-promotion-standby-registration):
* drive the leader promotion path with a LIVE cluster pointer so the
* cluster-dependent standby-registration branch (repl_add_standby for each
* surviving vote endpoint whose votes[i].peer is set) is exercised against a
* real persist/repl/commit/delaymgr fixture. Sets ha->raft_c = c so the
* become-leader callback sees a real cluster (unlike the NULL-cluster shim
* raft_cb_become_leader_for_test above, which suppresses the c-dependent
* registration block entirely). raft_cb_become_leader is file-static, so this
* seam must live here in ha.c. Symmetric with
* raft_cb_become_follower_with_cluster_for_test.
*/
void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, uint64_t term)
{
c->ha.raft_c = c;
raft_cb_become_leader((void *)&c->ha, term);
}
#endif /* XCE_TESTING */
/* ── Task 2.4: vote_ep_err_handler ─────────────────────────────────────────
*
* Called by iomgr when the outgoing vote endpoint encounters an error
* (connect failure, SSL error, remote close before reply).
* Counts the peer as denied: grant=0, tag=0, lsn=0.
*/
static int vote_ep_err_handler(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
if (!vctx) return 0;
ha_state_t *ha = vctx->ha;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_PRIMARY) {
/* Promoted connection dropped — remove from standby_endpoints */
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ha->standby_endpoints->count; i++) {
if (zarray_get(ha->standby_endpoints, i) == ep) {
zarray_del(ha->standby_endpoints, i);
break;
}
}
pthread_mutex_unlock(&ha->lock);
}
/* Task 2.2: deregister from WAL replicator.
* Req 1.2: when a standby disconnects, the HA layer shall deregister
* the endpoint from the replicator via repl_remove_standby.
* vctx->c may be NULL in test mode — guard accordingly. */
if (vctx->c)
repl_remove_standby(&vctx->c->persist.repl, ep);
/* Quorum-transition observability (Req 8.1): a standby just dropped — the
* write-quorum count changed. Record the loss edge once (silent if
* quorum is still held after the removal). */
ha_log_quorum_transition(ha);
} else {
/* Election phase: count as denied vote */
handle_request_vote_reply(ha, vctx->ctx, vctx->peer_idx,
vctx->my_term,
0, /* vote_granted = false */
0, 0);
}
/* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data.
* vctx is freed in ha_election_cleanup() when all per-peer contexts release. */
return 0;
}
/* ── ha_init_raft: one-time raft state machine initialization ────────────────
*
* Called once at startup after ha_restore_term() to wire the raft state
* machine with the correct cluster size and callbacks. Subsequent elections
* use raft_start_election() / raft_on_leader_lost() on this same state.
*
* n_peers: total candidates_count from cluster config (includes self).
*/
void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers)
{
/* Map persisted voted_for_name back to a peer index for initial_voted_for.
* This lets raft_check_and_start_election (RDHUP path) see the prior vote
* without re-reading the file. */
int restored_voted_for = -1;
if (ha->voted_for_name[0] != '\0' && c && c->myconf) {
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
for (int i = 0; i < pcfg->candidates_count; i++) {
if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) {
restored_voted_for = i;
break;
}
}
}
/* ha-term-safety task 2.5 (R1.5, R2.1): restore the durable log/commit floor
* from the recovered WAL so the library boots at the recovered tip — not an
* empty log at LSN 0. The library never reads storage; the caller restores
* initial_log_len / initial_commit_index here (same discipline as
* initial_term / initial_voted_for above).
*
* initial_log_len = recovered durable last-log index
* (persist_durable_floor: re-derived from the WAL tip
* by persist_recover; 0 on empty first boot).
* initial_commit_index = recovered committed LSN (committed_lsn). Recovery
* re-derives the durable tip but the post-election
* safe-commit point governs the committed position;
* we pass whatever committed prefix is established at
* init (0 on a fresh boot). Clamp to log_len to honor
* the library invariant commit_index <= log_len.
*
* The recovered last-log TERM is surfaced on demand via on_get_entry_term(
* log_len) (the library's logOk freshness gate reads it) — there is no
* separate initial last-log-term field. */
uint64_t initial_log_len = c ? persist_durable_floor(&c->persist) : 0;
uint64_t initial_commit_index = c
? atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire)
: 0;
if (initial_commit_index > initial_log_len)
initial_commit_index = initial_log_len;
raft_config_t rcfg;
memset(&rcfg, 0, sizeof(rcfg));
rcfg.n_peers = n_peers;
rcfg.self_idx = ha->my_priority;
rcfg.election_timeout_ms = ha->config.election_timeout_ms;
rcfg.my_priority = ha->my_priority;
rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire);
rcfg.initial_voted_for = restored_voted_for;
rcfg.initial_log_len = initial_log_len;
rcfg.initial_commit_index = initial_commit_index;
rcfg.cbs = &s_real_cbs;
rcfg.ctx = ha;
raft_init(&ha->raft, &rcfg);
ha->n_peers = n_peers;
ha->raft_c = c;
INFO("HA: raft initialized (n_peers=%d, self=%d, term=%" PRIu64
", log_len=%" PRIu64 ", commit_index=%" PRIu64 ")",
n_peers, ha->my_priority,
atomic_load_explicit(&ha->term, memory_order_acquire),
initial_log_len, initial_commit_index);
}
/* ── ha-prevote task 4.1: candidate-side PreVote orchestration ─────────────
*
* The PreVote round is a side-effect-free majority probe interposed BEFORE the
* destructive truncate + CTRL_ELECTING + real-election sequence. During the
* round the node stays CTRL_STANDBY and mutates no persistent state (term /
* voted_for / role). Only on a WIN does it enter the unchanged real-election
* path; on a LOSS / round-timeout it stays STANDBY, re-arms the liveness
* watchdog, and logs campaign abandonment.
*
* This task implements the round machinery + the WON/LOST branches. The
* failover triggers (ha_on_primary_ctrl_fd_rdhup / liveness_poll_fn /
* ha_on_heartbeat_received) are rewired to call this in task 4.2 — until then
* the round is INERT in production.
*/
/*
* prevote_round_ctx_t: delaymgr context for the PreVote round timeout.
* cancelled is set when the round is decided (WON or LOST) before the timeout
* fires, so a late timeout task is a no-op.
*/
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
} prevote_round_ctx_t;
/* Active round-timeout context. _Atomic so arm (vtask) vs timeout-fire
* (delaymgr) vs cancel-on-decision is well-defined. */
static prevote_round_ctx_t * _Atomic s_prevote_round_ctx = NULL;
#ifdef XCE_TESTING
/*
* ha_prevote_rearm_liveness_hook: test-only seam. When non-NULL, the LOST /
* timeout branch calls this INSTEAD OF ha_arm_heartbeat_timeout_vtask so a unit
* test can count re-arms without a live delaymgr (Req 5.4).
*/
int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c) = NULL;
#endif
/* ha_prevote_rearm_liveness: re-arm the standby liveness watchdog after a lost
* round so a later genuine failure still triggers a new PreVote round (the
* watchdog does not re-arm itself once it has fired). */
static void ha_prevote_rearm_liveness(ha_state_t *ha, cluster_t *c)
{
#ifdef XCE_TESTING
if (ha_prevote_rearm_liveness_hook != NULL) {
ha_prevote_rearm_liveness_hook(ha, c);
return;
}
#endif
ha_arm_heartbeat_timeout_vtask(ha, c);
}
/* ha_prevote_cancel_round_timeout: mark the active round-timeout context
* cancelled (the delaymgr task frees it on fire) and clear the static pointer
* so a later decision/timeout sees no active round. */
static void ha_prevote_cancel_round_timeout(void)
{
prevote_round_ctx_t *old = atomic_load_explicit(&s_prevote_round_ctx,
memory_order_acquire);
if (old) {
atomic_store_explicit(&old->cancelled, 1, memory_order_release);
atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release);
}
}
/*
* ha_enter_real_election: the WON-branch sequence — truncate uncommitted WAL
* above the committed boundary, then enter the UNCHANGED real-election path
* (ha_start_election_async transitions to CTRL_ELECTING and delegates the term
* bump + self-vote to raft_start_election). Factored here so both the PreVote
* WON branch (task 4.1) and the rewired trigger (task 4.2) can call it; this
* task does NOT change ha_on_primary_ctrl_fd_rdhup's own behaviour.
*
* c == NULL (unit-test mode): skip truncate + the live election; just transition
* the role to CTRL_ELECTING so the WON transition is observable.
*/
static void ha_enter_real_election(ha_state_t *ha, cluster_t *c)
{
if (!c) {
/* Test mode: the real election needs iomgr/delaymgr; just mark the
* destructive transition so the WON branch is observable. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
memory_order_release);
return;
}
/* bug-828 final slice (Leader Completeness): do NOT pre-truncate the WAL
* here. The earlier code truncated above ha->last_known_committed_lsn
* BEFORE the vote — but that watermark is the LOCALLY-LEARNED commit point,
* which LAGS the true cluster commit point (the primary commits an entry on
* a durable majority, then dies before a heartbeat carries that committed
* LSN to this standby). persist_truncate_wal_above clamps durable_lsn down
* to the cutoff, so a standby that had DURABLY persisted committed entries
* [committed+1 .. tip] would discard them and then advertise the stale-low
* cutoff as its durable floor. Every voter doing the same made
* safe-committed-from-votes collapse to the stale watermark, and the
* AUTHORITATIVE post-election truncate (raft_cb_become_leader,
* persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale
* prefix — losing committed jobs on a clean failover (bug-828/795).
*
* Raft model: a candidate does NOT truncate its log before campaigning. It
* keeps its full durable log and advertises its real durable tip
* (ha_candidate_advertise_lsn == persist_durable_floor), so voters holding
* more committed entries deny a colder candidate and the winner has every
* committed entry (Leader Completeness). Divergent UNCOMMITTED tails are
* removed ONLY by the authoritative post-election safe-commit truncate
* (above the majority-held point) and by the library's per-entry (term,lsn)
* reconcile (on_truncate_from) during subsequent AppendEntries — never by a
* stale-watermark pre-truncate that can erase committed state. */
/* Candidate freshness for the RequestVote payload (ha-commit-durability
* task 4.2). The TAG axis uses the same config-epoch fold the voter side
* computes (ha_vote_self_freshness), but the LSN axis advertises the BARE
* durable WAL tip — a candidate may only claim a log position it can
* durably back, so a truncated-restart node cannot over-advertise a stale
* volatile learned-committed value and win (design §1a/§5a). */
uint64_t cand_tag = 0, cand_lsn = 0;
if (c->myconf) {
uint64_t local_tag = c->myconf->config_version_tag;
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL);
cand_lsn = ha_candidate_advertise_lsn(&c->persist);
}
int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
const char *my_name = (c->myconf) ? c->myconf->name : "";
ha->raft_c = c;
#ifdef XCE_TESTING
/* Test seam: the WON-branch real election is the same step the pre-rewire
* rdhup body fired, so the existing rdhup-integration tests intercept it
* here (capturing my_tag/my_lsn / counting election starts) — they now
* drive it through the PreVote→WON path (ha-prevote task 4.2).
*
* Mark the role CTRL_ELECTING before invoking the hook: the real
* ha_start_election_async transitions to CTRL_ELECTING as its first step,
* so a test that intercepts the election must still observe the destructive
* role transition the WON branch performs (the spy hook does not start a
* real election that would set it). */
if (ha_rdhup_election_hook != NULL) {
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
memory_order_release);
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
int np = 0;
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
if (strcmp(cfg->candidates[i].name,
c->myhost ? c->myhost->name : "") != 0)
peer_ptrs[np++] = &cfg->candidates[i];
}
ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn,
my_name);
}
return;
}
#endif
/* ha_start_election_async transitions to CTRL_ELECTING and drives the
* UNCHANGED raft_start_election (term++ + self-vote). Peer connections are
* driven by raft_cb_send_vote_request; the peers[] arg is unused. */
ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name);
}
/*
* ha_start_prevote_async: start a PreVote round (ha-prevote task 4.1).
*
* Snapshots self tag/lsn into the round metadata carrier (ha->elect_ctx, read
* by raft_cb_send_pre_vote_request), calls raft_begin_pre_vote (self-grant +
* per-peer send via the callback), and arms a round-timeout bounded by
* election_timeout_ms (Req 5.4 — no unbounded delay). The node stays
* CTRL_STANDBY; NOTHING persistent is mutated (no term/voted_for/role change).
*
* If raft_begin_pre_vote returns RAFT_PRE_VOTE_WON immediately (single-node
* cluster: self-grant is already a majority), the WON branch runs at once.
*
* c == NULL (unit-test mode): the send callback returns early and the round
* timeout is a no-op; the round state lives entirely in raft_state_t so the
* reply-driven branches are still fully exercisable via the test seam.
*/
void ha_start_prevote_async(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
ha->raft_c = c;
/* Snapshot self freshness for the PreVote request payload (same floor as a
* real vote). Stored in the metadata carrier ha->elect_ctx so
* raft_cb_send_pre_vote_request can read my_tag/my_lsn. */
uint64_t cand_tag = 0, cand_lsn = 0;
int n_peers = ha->n_peers > 0 ? ha->n_peers : ha->raft.n_peers;
if (c && c->myconf) {
uint64_t local_tag = c->myconf->config_version_tag;
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
/* Task 4.2: TAG via the config-epoch fold, LSN as the bare durable WAL
* tip (PreVote must not over-advertise either — design §1a). */
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL);
cand_lsn = ha_candidate_advertise_lsn(&c->persist);
n_peers = c->myconf->peers_cfg.candidates_count;
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
ha_elect_ctx_t *mc = calloc(1, sizeof(*mc) +
(size_t)n_other * sizeof(mc->votes[0]));
if (mc) {
mc->n_peers = n_other;
mc->result = -1;
mc->c = c;
mc->self_tag = cand_tag;
mc->self_lsn = cand_lsn;
pthread_mutex_init(&mc->lock, NULL);
sem_init(&mc->done, 0, 0);
if (n_other > 0) {
mc->votes[0].my_tag = cand_tag;
mc->votes[0].my_lsn = cand_lsn;
}
/* Do NOT free a prior ctx (live vote endpoints may hold a bare
* pointer to it — same UAF caution as the rdhup path). */
atomic_store_explicit(&ha->elect_ctx, mc, memory_order_release);
}
ha->n_peers = n_peers;
}
INFO("HA: starting PreVote round (prospective term=%" PRIu64 ", n_peers=%d) "
"— staying CTRL_STANDBY, no persist",
ha->raft.term + 1, n_peers);
/* raft_begin_pre_vote: self-grant + per-peer send; mutates only the
* pre_vote_* tally in raft_state_t (no term/voted_for/role change). */
int rc = raft_begin_pre_vote(&ha->raft);
/* Arm the round timeout (bounded by election_timeout_ms). Production only;
* the round state is in raft_state_t so tests exercise the branches via the
* reply seam without the timer. */
ha_prevote_cancel_round_timeout();
if (c && c->delaymgr.running) {
prevote_round_ctx_t *tctx = calloc(1, sizeof(*tctx));
if (tctx) {
atomic_init(&tctx->cancelled, 0);
tctx->ha = ha;
tctx->c = c;
atomic_store_explicit(&s_prevote_round_ctx, tctx,
memory_order_release);
extern int ha_prevote_round_timeout_fn(void *arg);
int trc = delay_task_ms(&c->delaymgr,
(int)ha->config.election_timeout_ms, 1,
(delay_task_entry_fn_t)ha_prevote_round_timeout_fn,
tctx);
if (trc < 0) {
WARN("HA: ha_start_prevote_async: round-timeout schedule failed: %d",
trc);
atomic_store_explicit(&s_prevote_round_ctx, NULL,
memory_order_release);
free(tctx);
}
}
}
/* Single-node: the self-grant is already a majority — win immediately. */
if (rc == RAFT_PRE_VOTE_WON) {
ha_prevote_cancel_round_timeout();
INFO("HA: PreVote won immediately (single-node) — entering real election");
ha_enter_real_election(ha, c);
}
}
/*
* pre_vote_reply_dispatch: feed a PreVote reply into the raft tally and branch
* on its outcome (ha-prevote task 4.1). peer_idx is the ABSOLUTE cluster index
* of the replying peer (the same index raft_begin_pre_vote sent to).
*
* RAFT_PRE_VOTE_WON → cancel round timeout, run the real-election sequence
* (truncate + CTRL_ELECTING + ha_start_election_async).
* RAFT_PRE_VOTE_LOST → stay CTRL_STANDBY, re-arm liveness, log abandonment.
* RAFT_PRE_VOTE_PENDING → no-op (await more replies).
*
* Mirrors vote_reply_dispatch → raft_on_vote_reply, but the raft library makes
* NO role/term change here — the controller acts on the returned status.
*
* Production caller (task 5.1): pre_vote_reply_decode_and_dispatch, reached from
* peer_ctrl_msg_handler when a preVoteReply arrives on the outbound PreVote
* endpoint. Also driven by the XCE_TESTING reply seam.
*/
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted)
{
int outcome = raft_on_pre_vote_reply(&ha->raft, peer_idx, granted);
if (outcome == RAFT_PRE_VOTE_WON) {
ha_prevote_cancel_round_timeout();
INFO("HA: PreVote round WON — entering real election (term will be %"
PRIu64 ")", ha->raft.term + 1);
ha_enter_real_election(ha, c);
return;
}
if (outcome == RAFT_PRE_VOTE_LOST) {
ha_prevote_cancel_round_timeout();
/* Req 7.3: log campaign abandonment distinctly from a genuine failover
* so operators can tell a suppressed disruptive election apart. */
INFO("HA: PreVote round LOST — abandoning campaign, staying CTRL_STANDBY "
"(no term increment); re-arming liveness watchdog");
/* Stay CTRL_STANDBY — no persist, no role change (Req 1.4). */
ha_prevote_rearm_liveness(ha, c);
return;
}
/* RAFT_PRE_VOTE_PENDING — await more replies, do nothing. */
}
/*
* ha_prevote_round_timeout_fn: delaymgr task body for the PreVote round timeout.
* A timeout that fires before a decision is treated as a LOSS (Req 5.4).
*/
int ha_prevote_round_timeout_fn(void *arg)
{
prevote_round_ctx_t *ctx = (prevote_round_ctx_t *)arg;
if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) {
#ifndef XCE_TESTING
free(ctx);
#endif
return 0;
}
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
/* Only act if the round is still in progress (not already decided). */
if (ha->raft.pre_vote_in_progress) {
ha->raft.pre_vote_in_progress = 0;
atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release);
WARN("HA: PreVote round timed out before a decision — treating as LOSS, "
"staying CTRL_STANDBY; re-arming liveness watchdog (Req 5.4)");
ha_prevote_rearm_liveness(ha, c);
}
#ifndef XCE_TESTING
free(ctx);
#endif
return 0;
}
#ifdef XCE_TESTING
/*
* pre_vote_reply_dispatch_for_test: drive the real pre_vote_reply_dispatch with
* decoded (peer_idx, granted) fields, bypassing iomgr (ha-prevote task 4.1).
*/
void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted)
{
pre_vote_reply_dispatch(ha, c, peer_idx, granted);
}
/*
* ha_prevote_round_timeout_fire_for_test: simulate the round timeout firing
* before a decision, driving the LOST branch (Req 5.4).
*/
void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c)
{
prevote_round_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = ha;
ctx.c = c;
atomic_init(&ctx.cancelled, 0);
ha_prevote_round_timeout_fn(&ctx);
}
#endif /* XCE_TESTING */
/* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ────
*
* Non-blocking election initiator. Sets up the raft state machine and
* delegates all election mechanics to raft_start_election().
*
* raft_start_election() handles:
* - term increment and self-vote (via on_persist callback)
* - single-node shortcircuit (fires on_become_leader immediately)
* - multi-node RPC fan-out (via on_send_vote_request callback per peer)
* - election timeout scheduling (via on_schedule_timeout callback)
*
* ha_elect_ctx_t is still allocated here because raft_cb_send_vote_request
* reads ha->elect_ctx to retrieve my_tag/my_lsn for the capnp VoteRequest
* payload. Vote counting and quorum decisions are now owned by the raft
* library (raft_on_vote_reply fires on_become_leader/follower callbacks).
*
* Requirements: 1.3, 2.1 (shortcircuit), 2.4 (async vote send), 9.2
*/
void ha_start_election_async(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name)
{
(void)peers; /* peer connections driven by raft_cb_send_vote_request */
(void)my_name; /* self-vote name stored in on_persist via raft_start_election */
/* Step 0: raft is initialized once at startup by ha_init_raft(); nothing
* to do here. ha->raft_c and ha->n_peers are already set. */
/* Step 1: transition to CTRL_ELECTING before firing RPCs.
* raft_start_election fires on_become_leader / on_send_vote_request below;
* the role must already be ELECTING so ha_dispatch_gate() admits vote RPCs. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
INFO("HA FSM: → CTRL_ELECTING (async, n_peers=%d)", n_peers);
/* Step 2: pre-allocate ha_elect_ctx_t so raft_cb_send_vote_request can
* read my_tag/my_lsn from it when building the capnp VoteRequest payload.
* Vote counting is now owned by the raft library; this ctx is only a
* metadata carrier for tag/lsn. For multi-node elections it is kept alive
* until all per-peer vote_ep_ctx_t endpoints finish (iomgr lifetime).
* For single-node, raft_start_election fires on_become_leader immediately
* so no ctx reads occur after return.
*
* n_peers = total cluster size (including self), matching raft convention.
* standby_converged[] is indexed by votes_idx (0 .. n_peers-2), so
* convergence loops iterate i < n_peers - 1 (skip self). */
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
/* Store total cluster size (including self) — matches raft_config_t.n_peers */
ha->n_peers = n_peers;
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
(size_t)n_other * sizeof(ctx->votes[0]));
if (!ctx) {
WARN("ha_start_election_async: calloc ctx failed");
return;
}
ctx->n_peers = n_other;
ctx->votes_granted = 1; /* self-vote placeholder (raft tracks actual count) */
ctx->n_replied = 0;
ctx->max_resp_tag = 0;
ctx->max_resp_lsn = 0;
ctx->result = -1;
ctx->done_posted = 0;
ctx->c = c;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
/* Store candidate's own tag/lsn unconditionally in self_lsn/self_tag (task 6.3):
* ha_compute_safe_committed_lsn uses these for the majority computation,
* including the single-node case where votes[] is empty. */
ctx->self_lsn = my_lsn;
ctx->self_tag = my_config_tag;
/* Also store in votes[0] for raft_cb_send_vote_request (multi-node only) */
if (n_other > 0) {
ctx->votes[0].my_tag = my_config_tag;
ctx->votes[0].my_lsn = my_lsn;
}
/* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx */
atomic_store_explicit(&ha->elect_ctx, ctx, memory_order_release);
/* Step 4: delegate election to raft library.
*
* A node with no valid local config (my_config_tag == 0) must not start an
* election — any peer with a real config tag will beat it in the freshness
* gate and the node would waste a term increment. It stays in CTRL_ELECTING,
* accepts incoming vote requests (granting to fresher peers), and will
* become STANDBY after the winner sends it a heartbeat/ctrlhello.
* n_peers and raft state are already initialised above so RDHUP retries work.
*
* raft_start_election when allowed:
* - increments ha->raft.term and records self-vote (fires on_persist)
* - single-node: fires on_become_leader → raft_cb_become_leader
* - multi-node: fires on_send_vote_request per peer → raft_cb_send_vote_request
* then fires on_schedule_timeout → raft_cb_schedule_timeout
*
* No direct ha->term++ or ha->voted_for_* writes happen in this function
* after this call; the raft library owns those via on_persist. */
/* Req 4.3 / ha-config-bootstrap Req 2.8:
* A node with no local config (my_config_tag == 0) must not start an
* election if any peer has a valid config — it cannot win (freshness gate
* denies it) and it would waste a term increment. It stays in CTRL_ELECTING,
* accepts incoming vote requests (granting to fresher peers), and becomes
* STANDBY after the winner sends a heartbeat.
*
* Exception: if ALL peers also have config_tag=0 (cluster cold-start with no
* configured node), someone must win — allow election normally.
*
* my_config_tag comes from myconf->config_version_tag set by load_primary_config;
* n_peers and the peer list from --peers are always available via the parameters.
* We check if any peer returned a non-zero tag via the xce_probe_config_tag path
* already called in myapp_run before ha_start_election_async. */
if (my_config_tag == 0 && n_peers > 1) {
INFO("ha_start_election_async: no local config (tag=0, n_peers=%d) — "
"participating as voter only, not starting own election", n_peers);
/* Transition to CTRL_STANDBY so myapp_run schedules fetch_config_from_primary.
* The primary will push config via ctrlhello once it becomes CTRL_PRIMARY. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
/* Free ctx — no vote sends will happen from this node this round. */
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
return;
}
raft_start_election(&ha->raft);
/* For single-node, raft_start_election fires on_become_leader synchronously.
* on_become_leader (raft_cb_become_leader) handles role -> CTRL_PRIMARY and
* phase2. Free the ctx here since no raft callbacks
* will reference it after raft_start_election returns for single-node. */
if (n_peers == 1) {
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
return;
}
INFO("ha_start_election_async: returning (n_peers=%d, raft election started)",
n_peers);
/* Returns immediately — election result delivered via raft on_become_leader/follower */
}
#ifdef XCE_TESTING
/*
* ha_rdhup_election_hook: test-only seam for ha_start_election_async in the
* RDHUP failover path. When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls
* this instead of the real ha_start_election_async(). Req 6.3, 6.5, 9.3.
*/
void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name) = NULL;
#endif
/* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ───────────────────
*
* Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback),
* 6.4 (no heartbeat within timeout → elect), 6.8 (role flip)
* 10.5 (INFO logs on FSM transitions)
*/
/*
* ha_begin_failover: the single failover entry point (ha-prevote task 4.2).
*
* Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and
* the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route
* here AFTER their connection-teardown bookkeeping and the config-less
* voter-only guard. It starts a side-effect-free PreVote round and returns;
* it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those
* destructive steps are deferred to PreVote success (ha_enter_real_election,
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
* and term untouched (Req 1.1, 4.3, 4.4).
*/
/* ha_election_stagger_ms: see ha.h. Same formula as the startup stagger
* (myapp.c) — priority baseline plus fresh random jitter, re-randomized each
* call so repeated split-vote collisions converge (Raft §5.2; bug-606). */
uint32_t ha_election_stagger_ms(ha_state_t *ha)
{
if (!ha) return 0;
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
uint32_t step = ha->config.election_priority_step_ms;
uint32_t jmax = ha->config.election_jitter_max_ms;
uint32_t delay = (uint32_t)ha->my_priority * step;
if (jmax > 0)
delay += (uint32_t)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)jmax + 1));
return delay;
}
/* ha_jittered_election_delay: see ha.h. Canonical Raft randomized election
* timeout — [base, 2*base], re-drawn each campaign — applied to the per-campaign
* watchdog so a constant cold-connection grant RTT cannot resonate just past a
* fixed deadline forever (task 5.7). Independent of election_jitter_max_ms and
* never below the configured floor. */
uint32_t ha_jittered_election_delay(ha_state_t *ha, uint32_t base)
{
if (!ha || base == 0)
return base;
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
uint32_t extra = (uint32_t)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)base + 1));
return base + extra;
}
/* failover_stagger_ctx_t: carries ha/c into the delaymgr-scheduled PreVote start.
* Freed by ha_failover_stagger_fn (one-shot). */
typedef struct {
ha_state_t *ha;
cluster_t *c;
} failover_stagger_ctx_t;
/* ha_failover_stagger_fn: delaymgr body that starts the PreVote round after the
* per-node stagger (bug-606). Guard: if a primary connected during the stagger
* (we are a STANDBY with a fresh heartbeat), the cluster already has a leader —
* skip the PreVote entirely instead of needlessly probing a live primary. */
static int ha_failover_stagger_fn(void *arg)
{
failover_stagger_ctx_t *ctx = (failover_stagger_ctx_t *)arg;
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
INFO("HA: failover stagger elapsed but a primary is alive — skipping "
"PreVote (bug-606)");
free(ctx);
return 0;
}
ha_start_prevote_async(ha, c);
free(ctx);
return 0;
}
/* CLOCK_MONOTONIC milliseconds — for the RDHUP failover debounce (5.10b). */
static uint64_t ha_mono_ms(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L);
}
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
/* Stamp the failover-trigger time so a rapid RDHUP cadence from a dead
* primary's fd cannot re-trigger a fresh failover on every event (5.10b
* debounce in ha_on_primary_ctrl_fd_rdhup). Any trigger source (RDHUP or the
* deferred heartbeat-timeout stagger) records here. */
atomic_store_explicit(&ha->last_failover_ms, ha_mono_ms(), memory_order_release);
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
"PreVote wins) — staying CTRL_STANDBY");
/* Stagger the PreVote start per node so two survivors that lost the primary
* simultaneously do not campaign in lock-step and split the vote forever
* (Raft §5.2; bug-606). Production only: in unit-test mode (c == NULL) or
* with no running delaymgr, start synchronously so the existing failover /
* PreVote tests observe the unchanged immediate behaviour. */
uint32_t stagger = ha_election_stagger_ms(ha);
if (c && c->delaymgr.running && stagger > 0) {
failover_stagger_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (sctx) {
sctx->ha = ha;
sctx->c = c;
INFO("HA: staggering PreVote start by %ums (priority=%d) to avoid "
"split vote (bug-606)", stagger, ha->my_priority);
int rc = delay_task_ms(&c->delaymgr, (int)stagger, 1,
ha_failover_stagger_fn, sctx);
if (rc >= 0)
return;
/* Scheduling failed — fall through to a synchronous start. */
WARN("HA: ha_begin_failover: stagger schedule failed (%d) — "
"starting PreVote immediately", rc);
free(sctx);
}
}
ha_start_prevote_async(ha, c);
}
void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
/* raft-ha-core-safety task 5.6: do NOT restart failover while an election is
* already in flight. A won PreVote enters the real election (CTRL_ELECTING),
* or a PreVote round may be mid-decision (pre_vote_in_progress). The dead
* primary's control fd keeps RDHUP'ing on a deterministic ~15s cadence — if
* each RDHUP re-triggers a fresh ha_begin_failover it PREEMPTS the in-progress
* real election before it can complete, bumping the term every cycle and
* keeping two surviving candidates dueling in lockstep (split vote, unbounded
* term escalation; research.md §"Task 5.5 fix VERIFIED"). Drop the RDHUP:
* close the dead control fd and let the in-progress round finish or time out —
* its election timeout is jittered, which desynchronizes the duel so one
* survivor wins. A node NOT currently electing falls through to the normal
* 5.2 suppress / genuine-loss failover path below. */
{
ctrl_role_t cur = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (cur == CTRL_ELECTING || ha->raft.pre_vote_in_progress) {
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
INFO("HA: primary_ctrl_fd RDHUP but an election is already in progress "
"(role=%d, prevote=%d) — dropping the RDHUP, not restarting "
"failover (task 5.6: don't preempt the in-progress election / "
"churn the term)", (int)cur, ha->raft.pre_vote_in_progress);
return;
}
}
/* raft-ha-core-safety 5.10b: RDHUP failover DEBOUNCE. The 5.6 guard above
* only catches an RDHUP while the role is INSTANTANEOUSLY CTRL_ELECTING /
* pre_vote_in_progress. But a dead primary's fd RDHUPs faster than an
* election settles, and each RDHUP that lands in the gap AFTER a real
* election loses (role briefly back at CTRL_STANDDBY before the next attempt)
* re-triggers a fresh failover → a term bump per RDHUP (observed: 48
* begin_failovers, terms 6→24 in one round; research.md §"5.10b DEEPER").
* Suppress an RDHUP-triggered failover for up to 2× the election timeout
* after the last trigger — long enough to cover the jittered [T,2T] election
* watchdog (5.7) so the election's OWN retry runs without the external RDHUP
* cadence piling on extra term bumps. A genuinely-dead primary is still
* failed over — at most once per window, not once per RDHUP. First trigger
* (last_failover_ms == 0) is never debounced. */
{
uint64_t now_ms = ha_mono_ms();
uint64_t last = atomic_load_explicit(&ha->last_failover_ms,
memory_order_acquire);
uint64_t cooldown_ms = 2u * (uint64_t)ha->config.election_timeout_ms;
if (last != 0 && now_ms >= last && (now_ms - last) < cooldown_ms) {
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
INFO("HA: primary_ctrl_fd RDHUP within failover cooldown (%lums since "
"last trigger, cooldown=%lums) — debouncing, not restarting "
"failover (5.10b: a sustained RDHUP cadence must not ladder the "
"term; the in-flight election's own jittered retry handles it)",
(unsigned long)(now_ms - last), (unsigned long)cooldown_ms);
return;
}
}
/* Cancel the liveness poll task first (Req 5.4): prevents a concurrent
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
ha_disarm_heartbeat_timeout_vtask(ha);
/* Close and null the primary control fd. Connection-teardown bookkeeping
* stays at the trigger regardless of the PreVote outcome (ha-prevote task
* 4.2). */
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
/* raft-ha-core-safety task 5.2: SPURIOUS-FAILOVER SUPPRESSION.
*
* A TRANSIENT control-channel RDHUP while the current-term primary is still
* alive (a heartbeat arrived within heartbeat_timeout_ms) must NOT launch an
* unwinnable PreVote failover. The live stall (research.md §"Phase 4/5"): a
* recovered standby whose channel to the live primary flaps treats each
* RDHUP as "primary died" → PreVote → peers see the primary alive and deny →
* round times out → reconnect → repeat ~15s, so the node prevote-spins
* forever instead of rejoining. Mirror the guard ha_failover_stagger_fn
* already applies on the DEFERRED path (a flap = the primary re-established
* the channel, so ha_heartbeat_is_fresh is the right liveness signal —
* the same one handle_request_vote uses to deny same-term challengers).
*
* Stay CTRL_STANDBY and await the primary's reconnect (handle_ctrlhello);
* re-arm the liveness watchdog so a GENUINE subsequent loss is still caught
* (the watchdog resets last_activity to now, granting a fresh grace window —
* if the primary truly never heartbeats again it fires a real failover after
* heartbeat_timeout_ms). GENUINE loss (stale heartbeat) falls through to the
* failover path below unchanged; Election Safety is unaffected (no term bump,
* no role change here). */
ctrl_role_t rdhup_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (rdhup_role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
INFO("HA: primary_ctrl_fd RDHUP but the primary is alive (fresh "
"heartbeat) — suppressing PreVote failover, staying CTRL_STANDBY "
"and awaiting reconnect (spurious-failover suppression, "
"raft-ha-core-safety 5.2)");
if (c)
ha_arm_heartbeat_timeout_vtask(ha, c);
return;
}
/* ha-prevote task 4.2: the failover trigger NO LONGER transitions to
* CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and
* STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is
* deferred to PreVote success (ha_enter_real_election). The node only ever
* inflates its term / unseats a healthy primary if a PreVote majority is
* first obtained (Req 1.1, 4.3, 4.4). */
INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying "
"CTRL_STANDBY until PreVote wins) (Req 1.1)");
if (!c) {
/* Unit-test mode: no persist / no live transport. Still begin the
* round so the deferred-destructive behaviour is observable (the round
* state lives in raft_state_t; ha_enter_real_election with c==NULL only
* flips the role on a WIN). */
ha_begin_failover(ha, c);
return;
}
/* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening
* Req 10.5): a config-less node (config_version_tag == 0) must NEVER start
* an RDHUP re-election while a configured peer exists. Two such nodes would
* grant each other (the freshness gate only denies a LOWER tag; tag0==tag0
* with equal lsn is granted), forming a majority and electing a config-less
* primary over the node that actually holds the config — which then pushes
* an empty config and breaks replication.
*
* This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a
* config-less node can never win a PreVote (the freshness gate denies it),
* so there is no point starting a round — and starting one would needlessly
* probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote
* still GRANTS incoming vote requests to the fresher configured peer
* regardless of role, and this node transitions/fetches config when the
* configured winner connects and pushes config via ctrlhello.
*
* Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false,
* so a lone node still self-elects (its PreVote trivially wins). */
uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0;
int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
if (my_config_tag == 0 && n_peers_guard > 1) {
INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — "
"voter only, not starting PreVote/RDHUP re-election; awaiting "
"configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
return;
}
/* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async
* (it snapshots self freshness into ha->elect_ctx so the PreVote request
* payload — and, on a WIN, the deferred real election — advertise the
* effective freshness floor, NOT the stale persist.committed_lsn: see the
* bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */
ha_begin_failover(ha, c);
}
/* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */
/*
* liveness_poll_ctx_t: context for the delaymgr-based standby liveness task.
*
* Requirements: 1.2, 1.4, 1.5, 1.6, 5.4
*
* Replaces hb_vtask_t + hb_timeout_thread. No pthread, no semaphore.
* The poll fn reads ha->last_activity (written by ha_on_heartbeat_received)
* and fires ha_on_primary_ctrl_fd_rdhup if elapsed >= heartbeat_timeout_ms.
*
* cancelled is _Atomic so that writes from ha_disarm_heartbeat_timeout_vtask
* (main/vtask context) are visible to liveness_poll_fn (delaymgr worker thread)
* without a data race.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
_Atomic int cancelled;
} liveness_poll_ctx_t;
/* Static pointer to the active context. _Atomic so concurrent arm/disarm
* vs poll-fn execution is well-defined. Access pattern:
* arm — allocates and stores (single writer, vtask context)
* disarm — sets cancelled, frees (single writer, vtask context)
* poll_fn — reads via passed arg only; never dereferences s_liveness_ctx
*/
static liveness_poll_ctx_t * _Atomic s_liveness_ctx = NULL;
/*
* liveness_poll_fn: delaymgr task replacing hb_timeout_thread.
*
* Checks whether the primary has been silent for >= heartbeat_timeout_ms.
* If so, calls ha_on_primary_ctrl_fd_rdhup and does not re-arm.
* If not, re-arms itself for the next poll interval.
* If cancelled, returns immediately without any action.
*
* Also skips if role != CTRL_STANDBY (Req 5.4): prevents duplicate failover
* if EPOLLRDHUP already transitioned the role to CTRL_ELECTING.
*/
static int liveness_poll_fn(void *arg)
{
liveness_poll_ctx_t *ctx = (liveness_poll_ctx_t *)arg;
/* Check cancelled flag first — set by ha_arm_heartbeat_timeout_vtask when
* re-arming replaces this ctx. When cancelled, free the ctx here (the
* re-armer cannot safely free it because this task may already be running
* on a delaymgr thread). Do NOT re-arm.
*
* In XCE_TESTING builds callers pass stack-allocated ctxs, so we must
* not free in that path. Production builds always heap-allocate via
* ha_arm_heartbeat_timeout_vtask. */
if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) {
#ifndef XCE_TESTING
free(ctx);
#endif
return 0;
}
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
/* Skip if not standby — EPOLLRDHUP may have already changed the role */
if (atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY)
return 0;
/* Compute elapsed time since last primary message */
struct timespec now_ts;
clock_gettime(CLOCK_MONOTONIC, &now_ts);
time_t now = now_ts.tv_sec;
time_t last = ha->last_activity; /* time_t write is atomic on LP64 */
time_t elapsed = (last > 0) ? (now - last) : 0;
uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000;
if (last > 0 && (uint32_t)elapsed >= timeout_s) {
/* Silence exceeded timeout — trigger failover */
WARN("HA: liveness poll: no primary activity for %lus (timeout=%us) → failover",
(unsigned long)elapsed, timeout_s);
ha_on_primary_ctrl_fd_rdhup(ha, c);
/* Do NOT re-arm — failover handles re-election */
return 0;
}
/* Still fresh — re-arm for next poll interval */
if (c && c->delaymgr.running) {
uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2;
if (poll_ms < 100) poll_ms = 100;
delay_task_ms(&c->delaymgr, (int)poll_ms, 1,
(delay_task_entry_fn_t)liveness_poll_fn, ctx);
}
return 0;
}
#ifdef XCE_TESTING
/*
* liveness_poll_fn_test_entry: test-only entry point that calls liveness_poll_fn
* directly with a pre-constructed ctx, bypassing delaymgr.
* Only compiled when XCE_TESTING is defined.
*/
void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx)
{
liveness_poll_fn(ctx);
}
/*
* primary_heartbeat_fn_for_test: call primary_heartbeat_fn directly with c.
* Bypasses delaymgr scheduling so tests can drive the idle-NOP logic without
* a live timer. Only compiled when XCE_TESTING is defined. (Task 2.2)
*/
int primary_heartbeat_fn_for_test(cluster_t *c)
{
return primary_heartbeat_fn(c);
}
#endif
void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
/* Disarm any existing poll ctx by setting cancelled=1.
* Do NOT free it here — the delaymgr task may already be running on
* another thread and will free the ctx itself when it sees cancelled=1.
* Clearing s_liveness_ctx prevents a second re-arm attempt from operating
* on the stale pointer. */
liveness_poll_ctx_t *old = atomic_load_explicit(&s_liveness_ctx,
memory_order_acquire);
if (old) {
atomic_store_explicit(&old->cancelled, 1, memory_order_release);
atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release);
}
liveness_poll_ctx_t *ctx = calloc(1, sizeof(*ctx));
if (!ctx) return;
ctx->ha = ha;
ctx->c = c;
atomic_init(&ctx->cancelled, 0);
/* Initialise last_activity to now so the first poll computes elapsed from
* the arm time rather than from zero. */
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
ha->last_activity = ts.tv_sec;
atomic_store_explicit(&s_liveness_ctx, ctx, memory_order_release);
uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2;
if (poll_ms < 100) poll_ms = 100;
INFO("HA: standby liveness poll armed (timeout=%ums, poll=%ums)",
ha->config.heartbeat_timeout_ms, poll_ms);
if (c && c->delaymgr.running) {
int rc = delay_task_ms(&c->delaymgr, (int)poll_ms, 1,
(delay_task_entry_fn_t)liveness_poll_fn, ctx);
if (rc < 0)
WARN("HA: ha_arm_heartbeat_timeout_vtask: delay_task_ms failed: %d", rc);
}
}
/* ha_discover_primary: if CTRL_ELECTING, step down to STANDBY on receiving
* a valid primary message. A campaigning node that sees a live primary must
* defer — continuing the election risks split-brain (bug-353). */
void ha_discover_primary(ha_state_t *ha, cluster_t *c)
{
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role != CTRL_ELECTING)
return;
uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire);
INFO("HA: discovered live primary while ELECTING (term=%" PRIu64 ") — stepping down",
term);
raft_cb_become_follower(ha, term, -1);
}
void ha_on_heartbeat_received(ha_state_t *ha)
{
if (!ha) return;
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
ha->last_activity = ts.tv_sec;
tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec);
/* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader
* is alive — a PreVote round started by a transient blip must be ABANDONED.
* Cancel the in-progress round (clear the raft round state + the controller
* round-timeout handle) and re-attach as a standby. No persistent term /
* voted_for / role change happens here: the node never left CTRL_STANDBY
* during the round, so re-attaching is a no-op on role/term — exactly the
* non-disruptive rejoin the spec requires. */
if (ha->raft.pre_vote_in_progress) {
ha->raft.pre_vote_in_progress = 0;
ha->raft.pre_votes_granted = 0;
ha_prevote_cancel_round_timeout();
INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round "
"and re-attaching as CTRL_STANDBY (leader alive, no term change) "
"(Req 4.2)");
}
}
/*
* ha_observe_heartbeat_committed: reconcile-free monotonic learn of the
* primary's committed position from a heartbeat (ha-heartbeat-committed-
* propagation, Req 2.1, 2.2, 2.3, 3.1, 3.3).
*
* Single-receiver-thread path (controller request handling), so a relaxed
* compare-then-store is sufficient. The config tag is written BEFORE the LSN
* (both release-ordered) so a concurrent acquire-load reader that observes the
* advanced LSN never pairs it with a stale tag.
*
* Advisory only: NO ha_rejoin_reconcile / NO WAL truncation / NO change to any
* produced/persisted committed, role, term, or applied state. This is the
* explicit distinction from cluster_apply_ctrlhello_reply (which truncates).
*/
void ha_observe_heartbeat_committed(cluster_t *c, uint64_t committed_lsn,
uint64_t committed_config_tag)
{
if (!c) return;
ha_state_t *ha = &c->ha;
uint64_t cur = atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire);
/* Monotonic no-op: a zero (absent/liveness-only) or non-advancing LSN must
* leave the learned position — and its config tag — untouched (Req 2.2). */
if (committed_lsn == 0 || committed_lsn <= cur)
return;
/* Tag moves with the LSN, never independently regressed (Req 2.3): publish
* the tag first so the LSN's release store carries the matching tag. */
atomic_store_explicit(&ha->last_known_committed_config_tag,
committed_config_tag, memory_order_release);
atomic_store_explicit(&ha->last_known_committed_lsn, committed_lsn,
memory_order_release);
/* Reconcile-free: nothing else is touched (Req 3.1, 3.3). */
tlog(TLOG_DEBUG3,
"HA: learned committed_lsn=%" PRIu64 " config_tag=%" PRIu64
" from heartbeat (was %" PRIu64 ")",
committed_lsn, committed_config_tag, cur);
}
/*
* ha_consume_heartbeat_committed: standby OP_nop consume branch — the
* non-primary gate (Req 3.4) + the stale-term guard (Req 3.2), then the
* monotonic observe (Req 1.2). Reconcile-free; updates nothing on a NULL body,
* a failed gate, or a zero committed value (Req 4.2). Pure additive helper —
* the caller runs the unchanged liveness/reply path first (Req 4.1).
*/
void ha_consume_heartbeat_committed(cluster_t *c, const heartbeat_msg_t *hb)
{
if (!c || !hb)
return;
/* (3.4) A primary uses its own authoritative committed — never an inbound
* heartbeat value. Only non-primary nodes consume. */
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&c->ha.role, memory_order_acquire);
if (role == CTRL_PRIMARY)
return;
/* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat through the raft
* library as a 0-entry AppendEntries so the LIBRARY performs UpdateTerm — a
* heartbeat at a strictly-higher term makes this node adopt the term and step
* down. This REPLACES the prior hand-rolled term-guard that only DROPPED a
* stale heartbeat (hb->term < myterm) and never adopted a higher term.
*
* The heartbeat carries no entries and no prev tuple, so we present the
* node's own log tip as (prevLogIndex,prevLogTerm) — logOk holds (already-
* have at the tip) for an equal/higher-term heartbeat, and UpdateTerm runs
* first regardless. raft_init must have wired the raft state (production
* boot); guard for the pre-init placeholder where raft_c is unset. */
if (c->ha.raft_c == c) {
uint64_t my_log_len = c->ha.raft.log_len;
uint64_t my_tip_term = (my_log_len > 0)
? persist_entry_term(&c->persist, my_log_len) : 0;
wal_catchup_entry_t hbae;
memset(&hbae, 0, sizeof(hbae));
hbae.n_entries = 0; /* heartbeat: zero entries */
hbae.term = hb->term; /* leader's live term (UpdateTerm) */
hbae.prev_log_index = my_log_len; /* already-have at our tip */
hbae.prev_log_term = my_tip_term;
hbae.leader_commit = hb->committed_lsn;
(void)ha_consume_append_entries(c, &hbae, /*leader_idx*/-1);
}
/* (3.2) Stale-term guard for the committed LEARN below: after UpdateTerm a
* heartbeat from a primary whose term is behind ours is stale/partitioned —
* do not let its committed inflate the vote-freshness floor. Re-read our term
* (it may have advanced via the library UpdateTerm above). */
uint64_t myterm = atomic_load_explicit(&c->ha.term, memory_order_acquire);
if (hb->term < myterm)
return;
/* (1.2) Monotonic, reconcile-free learn of the committed position. Zero/
* non-advancing LSN no-ops inside the observe (Req 4.2). */
ha_observe_heartbeat_committed(c, hb->committed_lsn,
hb->committed_config_tag);
}
void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)
{
(void)ha;
liveness_poll_ctx_t *ctx = atomic_load_explicit(&s_liveness_ctx,
memory_order_acquire);
if (!ctx) return;
atomic_store_explicit(&ctx->cancelled, 1, memory_order_release);
atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release);
}
/* ── Primary heartbeat sender + late-joiner outbound connect ─────────────── */
/*
* standby_ep_ctx_t: per-peer context for outbound standby connections opened
* by the primary during the heartbeat task.
*
* Layout deliberately matches vote_ep_ctx_t for the first 4 fields so that
* peer_ctrl_msg_handler (which casts ep->data to vote_ep_ctx_t*) can safely
* read ha, c, and peer_idx. ctx is always NULL here (no active election).
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
ha_elect_ctx_t *ctx; /* always NULL — not a vote endpoint */
int peer_idx; /* index into c->myconf->peers_cfg.candidates[] */
/* Task 1.1 (ha-heartbeat-liveness): last time a push/nop was sent to this
* standby (CLOCK_MONOTONIC seconds). Initialized to 0 at allocation;
* written by on_standby_connected and primary_heartbeat_fn (task 2.2). */
time_t last_push_time;
} standby_ep_ctx_t;
#ifdef XCE_TESTING
/*
* standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data,
* and return ep. last_push_time is initialised to 0 (calloc).
* Caller must free(ep->data) after the test. (Task 2.2)
*/
endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
cluster_t *c,
int peer_idx)
{
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (!sctx) return NULL;
sctx->ha = ha;
sctx->c = c;
sctx->ctx = NULL;
sctx->peer_idx = peer_idx;
/* last_push_time = 0 from calloc — tests set it explicitly */
ep->data = sctx;
return ep;
}
/*
* on_standby_connected_for_test: call on_standby_connected directly.
* ep->data must be a valid standby_ep_ctx_t (use standby_ep_ctx_build_for_test).
* xce_request will fail on a null fd — tests only check last_push_time. (Task 2.2)
*/
int on_standby_connected_for_test(endpoint_t *ep)
{
on_standby_connected(ep);
return 0;
}
/*
* standby_ep_err_handler_primary_for_test: expose the outbound-standby error
* handler so a test can tear down a heap endpoint the production way (remove
* from standby_endpoints + replicator, free ep->data) before reclaiming the
* endpoint itself. (raft-drive-async Task 2.2)
*/
int standby_ep_err_handler_primary_for_test(endpoint_t *ep)
{
return standby_ep_err_handler_primary(ep);
}
#endif /* XCE_TESTING */
/*
* on_standby_connected: iomgr on_connected callback for outbound standby
* endpoints created by the heartbeat task.
*
* Sends OPCode_hello (CtrlHelloRequest) so the receiving standby runs
* handle_ctrlhello and triggers WAL catch-up + config push. Registers the
* endpoint in ha->standby_endpoints so future heartbeats reach it.
*/
static void on_standby_connected(endpoint_t *ep)
{
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
ha_state_t *ha = sctx->ha;
cluster_t *c = sctx->c;
/*
* Register in standby_endpoints, resolving the inbound-vs-outbound
* connection-ownership race (raft-drive-async Task 2.2, Req 2.6/4.4;
* ha-async-io.md connection ownership).
*
* The hazard: BOTH raft_cb_become_leader (promotes surviving vote eps) and
* this path (late-joiner / reconnect) zarray_store into standby_endpoints
* keyed only by ep-POINTER identity. So while an outbound reconnect to
* peer X is in flight, a DIFFERENT ep for the same peer X (a just-promoted
* vote-ep, or a stale prior channel whose RDHUP has not yet been processed)
* may already be registered. Leaving both would create two live outbound
* state-push channels to one peer — heartbeats/WAL fan out to a dead/stale
* duplicate (lost/reordered messages, Req 2.6) and the older endpoint is
* orphaned in the replicator.
*
* Resolution (deterministic, per-peer): the freshly-connected channel wins.
* Under ha->lock collect any DIFFERENT ep with the same peer_idx and drop
* it from the array; then register `ep`. The actual teardown of the stale
* ep runs AFTER releasing ha->lock because standby_ep_err_handler_primary
* re-takes ha->lock (and touches the replicator) — avoiding self-deadlock
* and holding the lock across no I/O (R4.4: iomgr/registration state mutated
* only via the lock-guarded interface, never racing the event loop).
*/
endpoint_t *stale_same_peer = NULL;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
int already = 0;
for (int i = 0; i < ha->standby_endpoints->count; i++) {
endpoint_t *cur = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
if (cur == ep) { already = 1; continue; }
/* Different ep, same peer → stale duplicate to evict. */
standby_ep_ctx_t *cur_sctx =
cur ? (standby_ep_ctx_t *)cur->data : NULL;
if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx) {
stale_same_peer = cur;
zarray_del(ha->standby_endpoints, i);
/* count shrank by one; the surviving scan continues but we
* only ever expect a single same-peer duplicate. */
break;
}
}
if (!already)
zarray_store(ha->standby_endpoints, ep);
pthread_mutex_unlock(&ha->lock);
}
/* Tear down the evicted stale channel OUTSIDE ha->lock. Mirror the library
* disconnect path (handle_error → iomgr_unmonitor → err_handler) so we
* never free an endpoint whose fd is still in the epoll set (R4.4: mutate
* iomgr/registration state ONLY via the lock-guarded interface, never
* racing the event loop). err-handler removes it from standby_endpoints +
* the replicator and frees ep->data; we then reclaim the endpoint itself.
* Never self-terminate — this is routine ownership reconciliation. */
if (stale_same_peer) {
WARN("HA: peer[%d] reconnected — evicting stale duplicate outbound "
"channel to keep exactly one live connection (Req 2.6)",
sctx->peer_idx);
if (stale_same_peer->iomgr)
iomgr_unmonitor((iomgr_t *)stale_same_peer->iomgr, stale_same_peer);
(void)standby_ep_err_handler_primary(stale_same_peer);
/* err-handler set ep->data = NULL and freed sctx; bug-1298: drop the
* owner ref — endpoint_unref reclaims outq/ssl/packets and frees the
* calloc'd struct when the last reference goes (deferred if a pool vtask
* still references this just-unmonitored ep). */
endpoint_unref(stale_same_peer);
#ifdef XCE_TESTING
atomic_fetch_add(&ha_standby_evicted_count_for_test, 1);
#endif
}
/* Reset failure counter so the next disconnect restarts logging from attempt 1. */
ha->peer_connect_fail_count[sctx->peer_idx] = 0;
/* Record last_push_time before any send so the first heartbeat tick after
* connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */
{
struct timespec _ts;
clock_gettime(CLOCK_MONOTONIC, &_ts);
sctx->last_push_time = _ts.tv_sec;
}
/* Send ctrlhello — triggers handle_ctrlhello on the standby side */
uint64_t lsn = atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
uint64_t tag = c->myconf ? c->myconf->config_version_tag : 0;
char *name = c->myconf ? c->myconf->name : "";
ctrl_hello_req_t hello = { .name = name, .key = NULL,
.applied_lsn = lsn, .config_tag = tag };
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_hello;
req.body_which = Request_body_ctrlhellomsg;
req.body.ctrlhellomsg = &hello;
int rc = xce_request(ep, &req);
if (rc < 0)
WARN("HA: ctrlhello to peer[%d] failed: %d", sctx->peer_idx, rc);
else
INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent",
sctx->peer_idx);
/* Task 2.1: register late-joiner with WAL replicator.
* Req 1.1: primary must register standby with repl_add_standby when
* establishing the standby control channel (late-joiner outbound connect).
* Host/port from candidates[peer_idx] — the peer we connected to. */
if (c && c->myconf &&
sctx->peer_idx >= 0 &&
sctx->peer_idx < c->myconf->peers_cfg.candidates_count) {
const xce_candidate_t *peer =
&c->myconf->peers_cfg.candidates[sctx->peer_idx];
int rc = repl_add_standby(&c->persist.repl, ep,
peer->hostname, (uint32_t)peer->port);
if (rc >= 0)
INFO("HA: repl registered late-joiner standby peer[%d] (slot %d, %s:%u)",
sctx->peer_idx, rc, peer->hostname, (uint32_t)peer->port);
else
WARN("HA: repl_add_standby late-joiner peer[%d] failed: %d",
sctx->peer_idx, rc);
}
/* Quorum-transition observability (Req 8.2): a late-joiner standby connected —
* the write-quorum count changed. Record the restore edge once (silent if
* quorum was already held). */
ha_log_quorum_transition(ha);
/* Set state-aware dispatcher for subsequent messages on this endpoint */
ep->msg_handler = peer_ctrl_msg_handler;
}
/*
* standby_ep_err_handler_primary: error handler for outbound standby
* endpoints created by the heartbeat task. Removes from standby_endpoints
* and frees the context — the next heartbeat tick will attempt reconnect.
*/
static int standby_ep_err_handler_primary(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
if (!sctx) return 0;
ha_state_t *ha = sctx->ha;
cluster_t *c = sctx->c;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ha->standby_endpoints->count; i++) {
if (zarray_get(ha->standby_endpoints, i) == ep) {
zarray_del(ha->standby_endpoints, i);
break;
}
}
pthread_mutex_unlock(&ha->lock);
}
/* Task 2.2: deregister late-joiner standby from WAL replicator before
* freeing the context. Req 1.2: RDHUP or connection error triggers
* repl_remove_standby so the replicator stops sending WAL entries to it. */
if (c)
repl_remove_standby(&c->persist.repl, ep);
free(sctx);
ep->data = NULL;
return 0;
}
/*
* reconnect_ctx_t: owns the prepared endpoint across the in-flight reconnect
* window (raft-drive-async Task 2.1). Built on the delaymgr tick by the
* non-blocking prep (ha_connect_prep_peer), consumed once on a slow-lane
* worker by reconnect_lane_fn, then freed. The ctx keeps `ep`/`sctx` valid
* for the whole window so the lane worker never touches freed memory while the
* delaymgr thread races ahead to the next tick (Req 2.3 lifetime).
*/
typedef struct {
cluster_t *c;
int peer_idx; /* ABSOLUTE candidate index */
endpoint_t *ep; /* prepared, not yet connected */
standby_ep_ctx_t *sctx; /* == ep->data */
int should_log;
} reconnect_ctx_t;
/*
* ha_connect_prep_peer: the NON-BLOCKING half of the outbound peer connect.
* Allocates the endpoint context, initialises the endpoint, sets the SSL
* client state and the on-connected / err handlers — all of which are
* non-blocking and safe to run on the delaymgr tick. Does NOT perform the
* blocking connect (that is ha_connect_finish_peer, run on the slow lane).
*
* Returns a heap reconnect_ctx_t* (caller owns) on success, or NULL on an
* allocation failure (nothing leaked).
*/
static reconnect_ctx_t *ha_connect_prep_peer(cluster_t *c, int peer_idx)
{
ha_state_t *ha = &c->ha;
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (!sctx) return NULL;
sctx->ha = ha;
sctx->c = c;
sctx->peer_idx = peer_idx;
endpoint_t *ep = endpoint_get();
if (!ep) { free(sctx); return NULL; }
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
endpoint_set_peer(ep, c->myhost);
ep->data = sctx;
ep->iomgr = &c->iomgr;
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
if (ep->ssl)
SSL_set_connect_state(ep->ssl);
}
endpoint_set_on_connected(ep, on_standby_connected);
endpoint_set_err_handler(ep, standby_ep_err_handler_primary);
int n = ++ha->peer_connect_fail_count[peer_idx];
int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0);
reconnect_ctx_t *rc_ctx = calloc(1, sizeof(*rc_ctx));
if (!rc_ctx) {
ep->data = NULL;
endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (not monitored) */
free(sctx);
return NULL;
}
rc_ctx->c = c;
rc_ctx->peer_idx = peer_idx;
rc_ctx->ep = ep;
rc_ctx->sctx = sctx;
rc_ctx->should_log = should_log;
return rc_ctx;
}
/*
* ha_connect_finish_peer: the BLOCKING half of the outbound peer connect —
* the bounded connect + lock-guarded iomgr registration. MUST run on a
* slow-lane worker, NEVER on the delaymgr tick (Req 2.1, 2.2; ha-async-io.md
* thread-role contract).
*
* On success the endpoint is connecting and registered with iomgr; on_connected
* (on_standby_connected) fires later from the iomgr loop. On failure the
* endpoint is torn down here and the controller simply retries on the next
* heartbeat tick (controllers never self-terminate).
*
* Returns 0 on a successful connect+register, <0 otherwise.
*
* NOTE (Task 2.2 seam): this implements the happy-path connect+register +
* teardown-on-failure. The inbound-vs-outbound connection-ownership race
* resolution, the no-lost/no-reordered-message guarantee on success, and the
* richer failure/retry hardening are owned by Task 2.2 — they layer on top of
* this happy path without changing this function's contract (return 0/<0,
* caller clears the per-peer in-flight flag).
*/
static int ha_connect_finish_peer(reconnect_ctx_t *rc_ctx)
{
cluster_t *c = rc_ctx->c;
endpoint_t *ep = rc_ctx->ep;
standby_ep_ctx_t *sctx = rc_ctx->sctx;
int peer_idx = rc_ctx->peer_idx;
const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[peer_idx];
#ifdef XCE_TESTING
if (ha_reconnect_connect_hook) {
/* Deterministic test path: the hook stands in for the real bounded
* connect + register so a test can park the worker (unreachable peer)
* or return a synthetic success/failure without a live socket. On
* the test path we do NOT touch the real endpoint fd. */
int hrc = ha_reconnect_connect_hook(c, peer_idx);
/* The hook stands in for the real connect+iomgr_monitor: it NEVER
* registers the synthetic endpoint with iomgr, so the prepped ep/sctx
* are unowned on BOTH success and failure here — reclaim them in either
* case (test-only path; production reclaims via iomgr/err-handler).
* Reverting this to free only on failure re-introduces the 2.1
* GREEN-test endpoint leak masked by detect_leaks=0. */
ep->data = NULL;
endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (test path) */
free(sctx);
return hrc;
}
#endif
if (rc_ctx->should_log)
INFO("HA: peer[%d] (%s) not connected — initiating outbound connect "
"(attempt %d)", peer_idx, peer->hostname,
c->ha.peer_connect_fail_count[peer_idx]);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) {
if (rc_ctx->should_log)
WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
peer_idx, peer->hostname, peer->port, rc);
/* bug-1298: ep->data cleared (sctx freed separately below) then drop the
* owner ref — endpoint_unref does fini+free of the calloc'd struct. */
ep->data = NULL;
endpoint_unref(ep);
free(sctx);
return rc;
}
rc = iomgr_monitor(ep); /* lock-guarded epoll registration (Req 4.4) */
if (rc < 0) {
ep->data = NULL;
endpoint_unref(ep); /* bug-1298: fini+free (deferred if a vtask refs it) */
free(sctx);
return rc;
}
INFO("HA: connecting outbound to peer[%d] (%s:%u)",
peer_idx, peer->hostname, peer->port);
return 0;
}
/*
* reconnect_lane_fn: the slow-lane worker entry (raft-drive-async Task 2.1).
* Runs the bounded blocking connect off the delaymgr thread, then ALWAYS
* clears the per-peer in-flight flag (success or failure) so the dedup never
* wedges and the next heartbeat tick can retry an unreachable peer (Req 2.5,
* 2.4 — never self-terminates). Frees the reconnect ctx.
*/
static void reconnect_lane_fn(void *arg)
{
reconnect_ctx_t *rc_ctx = (reconnect_ctx_t *)arg;
if (!rc_ctx) return;
cluster_t *c = rc_ctx->c;
int peer_idx = rc_ctx->peer_idx;
(void)ha_connect_finish_peer(rc_ctx);
/* Clear in-flight LAST, after the endpoint is fully connected/registered
* or torn down — so a concurrent heartbeat tick cannot enqueue a second
* reconnect that races this one's endpoint setup (Req 2.5). */
atomic_store_explicit(&c->ha.reconnect_inflight[peer_idx], 0,
memory_order_release);
free(rc_ctx);
}
/*
* ha_connect_to_peer: open a new outbound TLS connection to a peer candidate
* and register it as a standby channel. Called from primary_heartbeat_fn for
* peers that are absent from standby_endpoints.
*
* Offload flag ON (default, Task 2.1): prepares the endpoint (non-blocking)
* and submits the bounded connect to the slow lane with a per-peer dedup key,
* then returns immediately — the delaymgr thread NEVER blocks on connect.
*
* Offload flag OFF (legacy / TDD RED only): runs the blocking connect inline
* on the calling (delaymgr) thread. Retained solely so the RED test can show
* the pre-change stall; production runs with the flag ON.
*/
static void ha_connect_to_peer(cluster_t *c, int peer_idx)
{
ha_state_t *ha = &c->ha;
if (!ha_reconnect_offload_enabled()) {
/* ── Legacy inline path (flag OFF) — blocks the caller. ── */
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
if (!rc_ctx) return;
(void)ha_connect_finish_peer(rc_ctx);
free(rc_ctx);
return;
}
/* ── Offload path (flag ON) — non-blocking on the delaymgr thread. ──
*
* Per-peer single-flight: CAS the in-flight flag 0->1. If it was already
* 1, a reconnect to this peer is already in flight on the lane — enqueue
* nothing (Req 2.5). This guard is the authoritative dedup; the lane's
* own per-key dedup is a redundant backstop. */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&ha->reconnect_inflight[peer_idx], &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return; /* already reconnecting this peer */
}
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
if (!rc_ctx) {
/* prep failed (alloc) — release the in-flight guard so the next tick
* retries; controller stays alive (Req 2.4). */
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* Submit the blocking connect to the slow lane. Dedup key is the
* per-peer absolute index (+1 so it is non-zero; key 0 disables dedup). */
uint64_t key = (uint64_t)(peer_idx + 1);
int srv = slowlane_submit(&c->slowlane, key, reconnect_lane_fn, rc_ctx);
if (srv == -EEXIST) {
/* Lane still holds this peer's dedup key from a PRIOR reconnect task.
* This is the narrow window where that task already cleared our
* reconnect_inflight flag (its last action) but the lane has not yet
* removed its key. Our CAS therefore succeeded (flag was 0) even
* though the lane refuses the submit. We MUST release the flag we just
* set — otherwise no task owns its clear and dedup wedges permanently.
* Tear down our duplicate prep; the next tick retries cleanly. */
rc_ctx->ep->data = NULL;
endpoint_unref(rc_ctx->ep); /* bug-1298: fini+free the calloc'd struct */
free(rc_ctx->sctx);
free(rc_ctx);
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure: the task is still queued and will run, so we keep the flag
* set and let reconnect_lane_fn clear it. Only on a true failure do
* we tear down and release the guard so the next tick retries. */
WARN("HA: slowlane_submit reconnect peer[%d] failed: %d — retry next tick",
peer_idx, srv);
rc_ctx->ep->data = NULL;
endpoint_unref(rc_ctx->ep); /* bug-1298: fini+free the calloc'd struct */
free(rc_ctx->sctx);
free(rc_ctx);
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: the task is enqueued and will run exactly once on a lane
* worker; reconnect_lane_fn clears the in-flight flag on completion. */
}
/*
* primary_heartbeat_fn: self-perpetuating one-shot delaymgr task.
*
* On each tick:
* 1. Sends OPCode_nop to all connected standbys (keeps watchdog alive).
* 2. Scans the peers list; for any peer NOT in standby_endpoints, opens a
* fresh outbound TLS connection so late joiners are bootstrapped without
* the standby ever initiating an outbound connect.
*
* Stops naturally on demotion: the role check prevents re-arming.
*/
static int primary_heartbeat_fn(void *arg)
{
cluster_t *c = (cluster_t *)arg;
ha_state_t *ha = &c->ha;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role != CTRL_PRIMARY)
return 0; /* demoted — stop sending NOPs */
/* 1. Heartbeat all connected standbys — idle-triggered NOP (Req 3.1-3.4).
*
* For each standby ep, compute elapsed time since the last outbound push.
* Send NOP only when elapsed >= heartbeat_interval_ms so that WAL entries,
* config pushes, and ctrlhello suppress redundant NOPs (Req 3.4).
*/
{
struct timespec _hb_ts;
clock_gettime(CLOCK_MONOTONIC, &_hb_ts);
time_t now = _hb_ts.tv_sec;
int interval_s = (int)(ha->config.heartbeat_interval_ms / 1000);
if (interval_s <= 0)
interval_s = HA_DEFAULT_HEARTBEAT_INTERVAL_MS / 1000;
if (ha->standby_endpoints) {
/* Liveness eviction threshold (wall clock, matches ep->ts_last_in). */
time_t now_wall = time(NULL);
uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000;
if (timeout_s == 0)
timeout_s = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS / 1000;
/* Stamp the primary's OWN current committed position + term onto
* every heartbeat this tick (Req 1.1, 3.4). Read atomically here,
* once per tick, so every standby sees a consistent snapshot. Use
* the primary's authoritative state — NOT the standby-side
* last_known_* fields. */
uint64_t hb_term =
atomic_load_explicit(&ha->term, memory_order_acquire);
uint64_t hb_committed_lsn =
atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
uint64_t hb_committed_config_tag =
atomic_load_explicit(&ha->committed_config_tag,
memory_order_acquire);
/* Endpoints evicted this tick; their repl deregistration runs AFTER
* ha->lock is released (repl lock-ordering — mirrors the err handlers). */
endpoint_t *evicted[MAX_HA_CANDIDATES];
int n_evicted = 0;
pthread_mutex_lock(&ha->lock);
int n = ha->standby_endpoints->count;
for (int i = 0; i < n; i++) {
endpoint_t *ep = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
if (!ep) continue;
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
if (!sctx) continue;
/* Liveness eviction (symmetric with the standby's liveness poll):
* a standby that was responsive at least once (ts_last_in > 0) but
* has sent nothing for longer than heartbeat_timeout is dead or was
* restarted. A wedged/dead endpoint left in the array makes the
* reconnect scan below treat the peer as still-connected, so the
* restarted node never receives a fresh ctrlhello and loops
* elections forever (terms climbing unbounded). Evict it here; the
* reconnect scan re-establishes a fresh channel on this/next tick.
* Removal from standby_endpoints is the load-bearing action — the
* endpoint object itself is reclaimed by its err handler when the
* dead connection finally surfaces an epoll event (bounded). */
if (ep->ts_last_in > 0 &&
(now_wall - ep->ts_last_in) > (time_t)timeout_s) {
WARN("HA: standby[%d] silent for %lds (timeout=%us) — evicting "
"dead endpoint, will reconnect",
i, (long)(now_wall - ep->ts_last_in), timeout_s);
zarray_del(ha->standby_endpoints, i);
if (n_evicted < MAX_HA_CANDIDATES)
evicted[n_evicted++] = ep;
i--; n--;
continue;
}
time_t elapsed = now - sctx->last_push_time;
if (elapsed >= (time_t)interval_s) {
/* Heartbeat carries the primary's own committed + term
* (Req 1.1, 3.4). */
#ifdef XCE_TESTING
int rc;
if (ha_heartbeat_req_hook)
rc = ha_heartbeat_req_hook(ep, hb_term,
hb_committed_lsn,
hb_committed_config_tag);
else if (ha_nop_req_hook)
rc = ha_nop_req_hook(ep);
else
rc = xce_ha_heartbeat_req(ep, hb_term,
hb_committed_lsn,
hb_committed_config_tag);
#else
int rc = xce_ha_heartbeat_req(ep, hb_term,
hb_committed_lsn,
hb_committed_config_tag);
#endif
if (rc < 0)
WARN("HA: heartbeat nop to standby[%d] failed: %d", i, rc);
else
sctx->last_push_time = now;
}
}
pthread_mutex_unlock(&ha->lock);
for (int k = 0; k < n_evicted; k++)
repl_remove_standby(&c->persist.repl, evicted[k]);
}
}
/* 2. Reconnect to any peer missing from standby_endpoints */
if (c->myconf) {
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
for (int i = 0; i < pcfg->candidates_count; i++) {
if (i == ha->my_priority) continue; /* skip self */
/* Check if this peer already has a standby endpoint.
*
* Two endpoint types can be in standby_endpoints:
* - Promoted vote eps: ep->data = vote_ep_ctx_t, peer_idx is
* votes_idx (relative, self excluded). Convert absolute i to
* votes_idx using the same formula as raft_cb_send_vote_request.
* - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL),
* peer_idx is absolute candidate index. Compare directly with i.
*/
int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1;
int found = 0;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int j = 0; j < ha->standby_endpoints->count; j++) {
endpoint_t *sep = (endpoint_t *)zarray_get(
ha->standby_endpoints, j);
if (!sep || !sep->data) continue;
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)sep->data;
int match_idx = (sctx->ctx != NULL)
? votes_idx_for_i /* promoted vote ep: relative */
: i; /* standby ep: absolute */
if (sctx->peer_idx == match_idx) { found = 1; break; }
}
pthread_mutex_unlock(&ha->lock);
}
if (!found)
ha_connect_to_peer(c, i);
}
}
/* runcount=-1 (infinite repeat): the delaymgr re-enqueues this task
* automatically. The role check at entry stops work on demotion. */
return 0;
}
void ha_start_primary_heartbeat(cluster_t *c)
{
if (!c) return;
ha_state_t *ha = &c->ha;
int interval_ms = (int)ha->config.heartbeat_interval_ms;
if (interval_ms <= 0)
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
/* Idempotency: only arm once per process lifetime. With runcount=-1 the
* delaymgr re-enqueues the task automatically; duplicate tasks would stack
* on every re-election. */
{
static _Atomic int s_armed = 0;
int expected = 0;
if (!atomic_compare_exchange_strong(&s_armed, &expected, 1))
return; /* already armed — task is still running */
}
/* Guard: skip if delaymgr is not running (e.g. unit tests with zeroed cluster_t).
* In production the delaymgr is always started before any promotion. */
if (!c->delaymgr.running) {
INFO("HA: heartbeat skipped (delaymgr not running)");
return;
}
/* runcount=-1 (infinite): delaymgr re-enqueues automatically.
* The role check in primary_heartbeat_fn no-ops on demotion. */
int rc = delay_task_ms(&c->delaymgr, interval_ms, -1,
(delay_task_entry_fn_t)primary_heartbeat_fn, c);
if (rc < 0)
WARN("ha_start_primary_heartbeat: delay_task_ms failed: %d", rc);
else
INFO("HA: primary heartbeat armed (interval=%dms, rc=%d)", interval_ms, rc);
}
Reference in a new issue View git blame Copy permalink