Rongsong Shen
bd02064ebc
push_config_transport_only spawns one thread per standby then joins them with a CONF_PUSH_TIMEOUT_MS bounded wait — blocking the caller. On the requester-less drive-thread paths (promotion + heartbeat convergence) that wait stalls the io_worker. Route those through ha_config_push_offloadable, gated by g_drive_async_configpush_offload (default OFF → inline, byte-for-byte legacy; ON → slow lane, non-blocking). Flag ON deep-copies the config bytes + sha into a lane ctx (caller keeps owning and freeing its own buffer); a single cluster-wide single-flight guard (ha_state_t.configpush_inflight scalar CAS + lane key=1) bounds it to one config-push fan-out in flight; the in-flight flag is cleared on every terminal path (two alloc failures, -EEXIST, hard submit error, lane completion). Send body factored into ha_config_push_do_send. Three requester-less drive-thread call sites converted: raft_cb_become_leader (promotion), the HA_REPL_PUSH_CONFIG heartbeat case, and the direct-CTRL_PRIMARY FSM promotion branch. The admin-upgrade barrier (push_config_to_standbys, has a waiting requester) and join.c are untouched. Lands DORMANT like tasks 3.1/3.2: the lane reads c->ha.standby_endpoints and writes those borrowed standby endpoints, racing io_worker RDHUP teardown if run off-thread (bug-1197 family). Flag OFF keeps push+teardown serialized on the io_worker. Task 3.4 flips this ON together with the snapshot and peerctrl flags after establishing endpoint-lifetime discipline. Tests (controller suite EXIT=0): test_configpush_offload_runs_on_lane, test_configpush_inline_when_flag_off, test_configpush_offload_dedups. Adversarial cpp-review: no CRITICAL/HIGH/MEDIUM findings. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01S6fDCtXkkEgupu7rUah1y2
7629 lines
339 KiB
C
7629 lines
339 KiB
C
#define _GNU_SOURCE
|
||
#include "ha.h"
|
||
|
||
#include <arpa/inet.h>
|
||
#include <dirent.h>
|
||
#include <errno.h>
|
||
#include <fcntl.h>
|
||
#include <inttypes.h>
|
||
#include <netdb.h>
|
||
#include <poll.h>
|
||
#include <pthread.h>
|
||
#include <signal.h>
|
||
#include <stdatomic.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
#include <sys/epoll.h>
|
||
#include <sys/socket.h>
|
||
#include <time.h>
|
||
#include <unistd.h>
|
||
#include <zlib.h>
|
||
|
||
#include <openssl/ssl.h>
|
||
#include <openssl/err.h>
|
||
|
||
/* Circular-include safety: cluster_def.h includes ha.h (include-guarded),
|
||
* and ha.h only forward-declares cluster_t — no cycle. */
|
||
#include "xce/cluster_def.h"
|
||
#include "xce/errnox.h"
|
||
#include "controller.capnp.h"
|
||
#include "persist.h"
|
||
#include "request.h"
|
||
#include "utils.h"
|
||
|
||
/* ── HA dispatch gate error codes (defined here; declared extern in errnox.h) ── */
|
||
int XCE_ERR_NOT_READY = E_XCE_NOT_READY;
|
||
int XCE_ERR_STANDBY = E_XCE_STANDBY;
|
||
|
||
/* No-op stubs removed: s_real_cbs (below) is used exclusively since task 6.2. */
|
||
|
||
/* ── Forward declarations needed by real raft callbacks (Task 6.2) ────────
|
||
* These static functions and types are defined later in this file; declared
|
||
* here so the callback implementations can reference them. */
|
||
|
||
/* vote_ep_ctx_t: per-peer context for outgoing vote endpoints (full def below) */
|
||
typedef struct {
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
ha_elect_ctx_t *ctx;
|
||
int peer_idx;
|
||
uint64_t my_term;
|
||
uint64_t my_tag;
|
||
uint64_t my_lsn;
|
||
/* ha-term-safety task 1.2: candidate's own Raft (lastLogTerm, lastLogIndex)
|
||
* advertised in its RequestVote/PreVote so a voter's library logOk gate can
|
||
* compare. my_last_log_index == candidate WAL log_len; my_last_log_term ==
|
||
* term of that last entry (recovered/live WAL term). */
|
||
uint64_t my_last_log_term;
|
||
uint64_t my_last_log_index;
|
||
const char *my_name;
|
||
} vote_ep_ctx_t;
|
||
|
||
/* election_timeout_ctx_t: delaymgr callback context for raft election timeout */
|
||
typedef struct {
|
||
_Atomic int cancelled;
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
ha_elect_ctx_t *ctx;
|
||
} election_timeout_ctx_t;
|
||
|
||
/* cluster_init_phase2 / push_config_transport_only /
|
||
* ha_send_ctrlhello_to_all_standbys: defined in cluster.c / config_repl.c;
|
||
* only referenced in production builds.
|
||
* (XCE_TESTING uses hook shims and does not link cluster.c.)
|
||
* ha.c uses push_config_transport_only ONLY (task 5.2 remediation r1):
|
||
* every ha.c push site is reachable from peer_ctrl_msg_handler / io_worker
|
||
* (vote replies, heartbeat replies), where the push_config_to_standbys
|
||
* barrier wait would block the single iomgr epoll thread — and the config
|
||
* ACK it waits for would arrive on that same blocked thread (structural
|
||
* deadlock). Convergence is owned by the OP_replConfig anti-entropy pull
|
||
* and the heartbeat decision loop, not by these best-effort pushes. */
|
||
#ifndef XCE_TESTING
|
||
extern int cluster_init_phase2(cluster_t *c);
|
||
extern void ha_send_ctrlhello_to_all_standbys(cluster_t *c);
|
||
extern int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
|
||
size_t blen, const uint8_t sha256[32],
|
||
uint64_t new_tag, int *n_out);
|
||
#endif
|
||
|
||
/* raft-drive-async Task 3.3: requester-less config-push offload router (defined
|
||
* below, near the snapshot helpers). Forward-declared because
|
||
* raft_cb_become_leader (earlier) is one of its two call sites. */
|
||
static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes,
|
||
size_t blen, const uint8_t sha256[32],
|
||
uint64_t new_tag);
|
||
|
||
/* config_repl.c is linked in BOTH production and XCE_TESTING builds.
|
||
* ha-term-safety task 4.3 (remediation r1): the commit-advance hook activates a
|
||
* newly-committed config marker (Finding B); ha_dispatch_gate / the pre-promotion
|
||
* gate read the serve-fence (Finding C). */
|
||
extern int config_activate_committed(cluster_t *c);
|
||
extern int config_serve_fenced(const cluster_t *c);
|
||
/* task 4.3 (remediation r2, Finding D): the commit-advance path schedules an
|
||
* off-iomgr retry that re-activates a newly-committed config marker and clears a
|
||
* stale serve-fence (non-blocking ordq_put only). */
|
||
extern int config_schedule_activate_retry(cluster_t *c);
|
||
|
||
static void on_vote_connected(endpoint_t *ep);
|
||
static void on_pre_vote_connected(endpoint_t *ep);
|
||
static int vote_ep_err_handler(void *ep_raw);
|
||
/* ha-single-leader-role task 2.2: shared promotion seam — every road to
|
||
* PRIMARY registers the persist hooks and arms the own-term gate. */
|
||
static void ha_register_persist_hooks(cluster_t *c);
|
||
static void ha_arm_own_term_gate(cluster_t *c);
|
||
int ha_heartbeat_is_fresh(ha_state_t *ha); /* exposed for PreVote denial-reason logging (cluster.c) */
|
||
static int primary_heartbeat_fn(void *arg);
|
||
static void on_standby_connected(endpoint_t *ep);
|
||
static int standby_ep_err_handler_primary(void *ep_raw);
|
||
|
||
/* ── Real raft callback implementations (Task 6.2) ─────────────
|
||
*
|
||
* These are registered as s_real_cbs in ha_init, replacing the no-op stubs.
|
||
* They are wired into the active election path in task 6.3.
|
||
*
|
||
* All callbacks receive ctx = ha_state_t*.
|
||
* ha->raft_c is set by ha_start_election_async before any raft_* calls.
|
||
* When ha->raft_c is NULL (unit test mode), callbacks that require cluster_t
|
||
* are no-ops.
|
||
*/
|
||
|
||
/*
|
||
* raft_cb_send_vote_request: encode and send a capnp VoteRequest to peer.
|
||
*
|
||
* peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[].
|
||
* Uses the existing iomgr vote-send path (on_vote_connected / vote_ep_ctx_t).
|
||
* Reads the current election context from ha->elect_ctx (set before raft_start_election).
|
||
* Returns 0 on success; non-zero if no cluster context or alloc failure.
|
||
*/
|
||
static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
if (!c || !c->myconf) return 1; /* test mode */
|
||
|
||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||
if (peer_idx < 0 || peer_idx >= pcfg->candidates_count)
|
||
return 1;
|
||
|
||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||
&ha->elect_ctx, memory_order_acquire);
|
||
if (!ectx) return 1; /* no election in progress */
|
||
|
||
xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx];
|
||
|
||
/* Compute the local index of peer_idx within ectx->votes[]:
|
||
* votes[] has n_peers - 1 = n_other entries for peers excluding self.
|
||
* Map: votes_idx = peer_idx if peer_idx < self_idx, else peer_idx - 1. */
|
||
int self_idx = ha->my_priority; /* self_idx == my_priority by convention */
|
||
int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1;
|
||
if (votes_idx < 0 || votes_idx >= ectx->n_peers)
|
||
return 1;
|
||
|
||
/* Allocate per-peer context */
|
||
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
|
||
if (!vctx) {
|
||
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
|
||
return 1;
|
||
}
|
||
vctx->ha = ha;
|
||
vctx->c = c;
|
||
vctx->ctx = ectx;
|
||
vctx->peer_idx = votes_idx;
|
||
vctx->my_term = term;
|
||
/* tag and lsn come from votes[0] which holds our own values */
|
||
vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0;
|
||
vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0;
|
||
/* ha-term-safety task 1.2: advertise our Raft (lastLogTerm, lastLogIndex).
|
||
* lastLogIndex == our WAL log length (the library's log_len == WAL tip LSN);
|
||
* lastLogTerm == the recovered/live WAL term that stamps that last entry.
|
||
* NOTE: sourced directly from persist (c->persist.term / raft.log_len), NOT
|
||
* via on_get_entry_term — that callback seam is task 2.x and is unwired here
|
||
* (see CONCERNS). The formal persist_entry_term accessor is task 1.3. */
|
||
vctx->my_last_log_index = ha->raft.log_len;
|
||
vctx->my_last_log_term = c ? c->persist.term : 0;
|
||
vctx->my_name = c->myconf ? c->myconf->name : "";
|
||
|
||
/* Get a fresh endpoint */
|
||
endpoint_t *ep = endpoint_get();
|
||
if (!ep) {
|
||
free(vctx);
|
||
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
|
||
return 1;
|
||
}
|
||
|
||
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
|
||
if (c) endpoint_set_peer(ep, c->myhost);
|
||
ep->data = vctx;
|
||
ep->iomgr = &c->iomgr;
|
||
|
||
if (ha->client_ssl_ctx) {
|
||
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
|
||
ep->ssl = SSL_new(client_ctx);
|
||
if (ep->ssl)
|
||
SSL_set_connect_state(ep->ssl);
|
||
}
|
||
|
||
endpoint_set_on_connected(ep, on_vote_connected);
|
||
endpoint_set_err_handler(ep, vote_ep_err_handler);
|
||
|
||
int rc = endpoint_connect(ep, peer->hostname, peer->port,
|
||
DEFAULT_CONNECT_TIMEOUT_MS);
|
||
if (rc < 0) {
|
||
endpoint_fini(ep);
|
||
if (!c)
|
||
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
|
||
return 1;
|
||
}
|
||
|
||
rc = iomgr_monitor(ep);
|
||
if (rc < 0) {
|
||
endpoint_fini(ep); /* frees ep->data (vctx) via FREEUP */
|
||
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
|
||
return 1;
|
||
}
|
||
|
||
/* Track endpoint for promotion/close after election result.
|
||
* Also record the candidate pointer: raft_cb_become_leader uses
|
||
* votes[i].peer->hostname / ->port to register the surviving vote
|
||
* endpoint with the WAL replicator (repl_add_standby). Without this,
|
||
* the promotion registration block (guarded by votes[i].peer != NULL)
|
||
* is skipped — the standbys never enter c->persist.repl.standbys[],
|
||
* commit/quorum see zero standbys, and every write fails NOT_READY.
|
||
* `peer` points into c->myconf->peers_cfg.candidates[] (persistent;
|
||
* outlives the election — repl_add_standby copies host/port).
|
||
* (fix-promotion-standby-registration, Req 1.1) */
|
||
ectx->votes[votes_idx].ep = ep;
|
||
ectx->votes[votes_idx].peer = peer;
|
||
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* raft_cb_send_pre_vote_request: encode and send a capnp PreVote request to a
|
||
* peer (ha-prevote task 4.1).
|
||
*
|
||
* Mirrors raft_cb_send_vote_request exactly — same iomgr connect path, same
|
||
* vote_ep_ctx_t carrier, same per-peer endpoint tracking — but the connected
|
||
* handler (on_pre_vote_connected) emits OPCode_preVoteRequest instead of
|
||
* OPCode_requestVote. prospective_term is the candidate's CURRENT+1 term; it
|
||
* is NEVER persisted (PreVote is a side-effect-free straw poll).
|
||
*
|
||
* peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[].
|
||
* Returns 0 on success; non-zero in test mode (no cluster) or on alloc failure.
|
||
*
|
||
* Transport-side reply routing for preVoteReply is wired separately in task 5.1;
|
||
* this callback only emits the request.
|
||
*/
|
||
static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx,
|
||
uint64_t prospective_term)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
if (!c || !c->myconf) return 1; /* test mode — caller owns transport */
|
||
|
||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||
if (peer_idx < 0 || peer_idx >= pcfg->candidates_count)
|
||
return 1;
|
||
|
||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||
&ha->elect_ctx, memory_order_acquire);
|
||
if (!ectx) return 1; /* no round metadata carrier */
|
||
|
||
xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx];
|
||
|
||
/* Map absolute peer_idx → votes[] local index (same convention as the
|
||
* real-vote send callback). */
|
||
int self_idx = ha->my_priority;
|
||
int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1;
|
||
if (votes_idx < 0 || votes_idx >= ectx->n_peers)
|
||
return 1;
|
||
|
||
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
|
||
if (!vctx) return 1;
|
||
vctx->ha = ha;
|
||
vctx->c = c;
|
||
vctx->ctx = ectx;
|
||
/* Carry the ABSOLUTE peer index: the PreVote tally in raft_state_t is keyed
|
||
* by absolute cluster index (raft_begin_pre_vote loops over i in
|
||
* [0, n_peers) and raft_on_pre_vote_reply dedups by that same index). */
|
||
vctx->peer_idx = peer_idx;
|
||
vctx->my_term = prospective_term;
|
||
vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0;
|
||
vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0;
|
||
/* ha-term-safety task 1.2: advertise our Raft (lastLogTerm, lastLogIndex)
|
||
* — identical sourcing to the real-vote send path above. */
|
||
vctx->my_last_log_index = ha->raft.log_len;
|
||
vctx->my_last_log_term = c ? c->persist.term : 0;
|
||
vctx->my_name = c->myconf ? c->myconf->name : "";
|
||
|
||
endpoint_t *ep = endpoint_get();
|
||
if (!ep) { free(vctx); return 1; }
|
||
|
||
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
|
||
if (c) endpoint_set_peer(ep, c->myhost);
|
||
ep->data = vctx;
|
||
ep->iomgr = &c->iomgr;
|
||
|
||
if (ha->client_ssl_ctx) {
|
||
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
|
||
ep->ssl = SSL_new(client_ctx);
|
||
if (ep->ssl)
|
||
SSL_set_connect_state(ep->ssl);
|
||
}
|
||
|
||
endpoint_set_on_connected(ep, on_pre_vote_connected);
|
||
endpoint_set_err_handler(ep, vote_ep_err_handler);
|
||
|
||
int rc = endpoint_connect(ep, peer->hostname, peer->port,
|
||
DEFAULT_CONNECT_TIMEOUT_MS);
|
||
if (rc < 0) { endpoint_fini(ep); return 1; }
|
||
|
||
rc = iomgr_monitor(ep);
|
||
if (rc < 0) { endpoint_fini(ep); return 1; }
|
||
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* raft_cb_vote_eligible: freshness gate delegated from raft library.
|
||
*
|
||
* from_idx is the absolute candidate index of the requester.
|
||
* Applies the same checks as handle_request_vote:
|
||
* - role gate: PRIMARY → deny
|
||
* - fresh heartbeat on STANDBY → deny
|
||
* - in_catch_up → deny
|
||
* - freshness: candidate config_tag/lsn must be ≥ ours
|
||
* Returns 1 to grant, 0 to deny.
|
||
*/
|
||
static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term)
|
||
{
|
||
(void)from_idx; (void)term;
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
|
||
/* Role gate */
|
||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
if (role == CTRL_PRIMARY)
|
||
return 0;
|
||
|
||
/* Fresh heartbeat on STANDBY WITH a live primary connection → primary is
|
||
* alive, deny. BOTH conditions are required (mirrors the handle_request_vote
|
||
* gate at ~ha.c:2205). Without the primary_ctrl_fd>=0 guard a node whose
|
||
* primary has DIED keeps a falsely-"fresh" status: ha_heartbeat_is_fresh
|
||
* derives from last_activity, which ha_arm_heartbeat_timeout_vtask re-arms to
|
||
* now on every failover retry — so the node would deny every legitimate
|
||
* higher-term PreVote for minutes, preventing convergence (bug-660).
|
||
* primary_ctrl_fd is -1 after the primary RDHUPs / the liveness watchdog
|
||
* fires / on a fresh boot, so a dead-or-absent primary correctly does NOT
|
||
* deny. */
|
||
if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
|
||
ha_heartbeat_is_fresh(ha))
|
||
return 0;
|
||
|
||
/* In catch-up → deny */
|
||
if (atomic_load_explicit(&ha->in_catch_up, memory_order_acquire))
|
||
return 0;
|
||
|
||
/* Freshness gate requires knowing the candidate's config_tag/lsn.
|
||
* The raft library does not pass these; they are carried in the capnp
|
||
* RequestVoteRequest fields. For the adapter, freshness is checked in
|
||
* handle_request_vote (called from the iomgr decode path in task 6.3).
|
||
* Here we return 1 (eligible) so the raft library proceeds; the full
|
||
* freshness gate is enforced by the caller before invoking raft_on_vote_request.
|
||
* This is safe because the caller (task 6.3 dispatcher) already applies
|
||
* the freshness check from handle_request_vote before delegating here. */
|
||
return 1;
|
||
}
|
||
|
||
/* raft_election_timeout_fn: delaymgr callback for raft election timeout.
|
||
* Fires raft_on_election_timeout after delay_ms if not cancelled.
|
||
* Owns the election_timeout_ctx_t — frees it on exit. */
|
||
static int raft_election_timeout_fn(void *arg)
|
||
{
|
||
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
|
||
ha_state_t *ha = t->ha;
|
||
|
||
/* Clear the stored pointer before checking cancelled so on_cancel_timeout
|
||
* is a safe no-op if it races with this callback. */
|
||
ha->raft_election_timeout_ctx = NULL;
|
||
|
||
if (atomic_load(&t->cancelled)) {
|
||
free(t);
|
||
return 0;
|
||
}
|
||
raft_on_election_timeout(&ha->raft);
|
||
free(t);
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* raft_cb_schedule_timeout: schedule election timeout via delaymgr.
|
||
*
|
||
* Creates an election_timeout_ctx_t and calls delay_task_ms.
|
||
* The pointer is stored in ha->raft_election_timeout_ctx for cancellation.
|
||
*/
|
||
static void raft_cb_schedule_timeout(void *ctx, uint32_t delay_ms)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
if (!c) return; /* test mode */
|
||
|
||
election_timeout_ctx_t *t = calloc(1, sizeof(*t));
|
||
if (!t) return;
|
||
|
||
atomic_init(&t->cancelled, 0);
|
||
t->ha = ha;
|
||
t->c = c;
|
||
t->ctx = NULL; /* raft path does not use ha_elect_ctx_t */
|
||
|
||
int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1,
|
||
raft_election_timeout_fn, t);
|
||
if (rc < 0) {
|
||
WARN("raft_cb_schedule_timeout: delay_task_ms failed: %d", rc);
|
||
free(t);
|
||
return;
|
||
}
|
||
|
||
ha->raft_election_timeout_ctx = t;
|
||
}
|
||
|
||
/*
|
||
* raft_cb_cancel_timeout: cancel the pending election timeout.
|
||
* Sets cancelled=1 on the stored election_timeout_ctx_t.
|
||
*/
|
||
static void raft_cb_cancel_timeout(void *ctx)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
election_timeout_ctx_t *t =
|
||
(election_timeout_ctx_t *)ha->raft_election_timeout_ctx;
|
||
if (t)
|
||
atomic_store(&t->cancelled, 1);
|
||
ha->raft_election_timeout_ctx = NULL;
|
||
}
|
||
|
||
/* raft_rdhup_delay_fn: delaymgr callback for RDHUP-triggered delayed election.
|
||
* Owns its ctx (election_timeout_ctx_t) — frees it on exit.
|
||
* Calls raft_check_and_start_election to start election only if not already voted. */
|
||
static int raft_rdhup_delay_fn(void *arg)
|
||
{
|
||
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
|
||
ha_state_t *ha = t->ha;
|
||
if (!atomic_load(&t->cancelled))
|
||
raft_check_and_start_election(&ha->raft);
|
||
free(t);
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* raft_cb_start_election_after_delay: schedule RDHUP retry timer.
|
||
* After delay_ms, calls raft_check_and_start_election to start election
|
||
* only if this node hasn't already voted for another candidate.
|
||
*/
|
||
static void raft_cb_start_election_after_delay(void *ctx, uint32_t delay_ms)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
if (!c) return; /* test mode */
|
||
|
||
/* Priority 0 → delay_ms == 0: highest-priority node starts immediately. */
|
||
if (delay_ms == 0) {
|
||
raft_check_and_start_election(&ha->raft);
|
||
return;
|
||
}
|
||
|
||
election_timeout_ctx_t *t = calloc(1, sizeof(*t));
|
||
if (!t) return;
|
||
|
||
atomic_init(&t->cancelled, 0);
|
||
t->ha = ha;
|
||
t->c = c;
|
||
t->ctx = NULL;
|
||
|
||
int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1,
|
||
raft_rdhup_delay_fn, t);
|
||
if (rc < 0) {
|
||
WARN("raft_cb_start_election_after_delay: delay_task_ms failed: %d", rc);
|
||
free(t);
|
||
}
|
||
}
|
||
|
||
/* ── Task 4.1 (ha-repl-hardening): safe commit point computation ─────────────
|
||
*
|
||
* ha_compute_safe_committed_lsn: determine the safe committed LSN and config
|
||
* tag after winning an election, using the vote replies collected in ctx.
|
||
*
|
||
* Algorithm (design §Flow 0: Post-Election — Determine Safe Commit Point):
|
||
* 1. Collect LSNs from all granted voters:
|
||
* - Self always granted: self_lsn = ctx->self_lsn (stored unconditionally)
|
||
* - Each peer with vote_granted=1: lsn = votes[i].peer_lsn
|
||
* 2. Sort granted LSNs descending.
|
||
* 3. safe_committed_lsn = sorted[floor(n_granted / 2)]
|
||
* (the majority-threshold element — the highest LSN held by a majority)
|
||
* 4. Same logic for config tags (ctx->self_tag / votes[i].peer_tag).
|
||
*
|
||
* Single-node (n_peers == 0): votes[] is empty; safe_committed_lsn = ctx->self_lsn.
|
||
* This prevents WAL truncation to 0 on re-election in a single-node cluster.
|
||
*
|
||
* Requirements: 2.3 (ha-repl-hardening spec)
|
||
* Boundary: Controller FSM (ha.c)
|
||
*/
|
||
static int _cmp_u64_desc(const void *a, const void *b)
|
||
{
|
||
uint64_t ua = *(const uint64_t *)a;
|
||
uint64_t ub = *(const uint64_t *)b;
|
||
if (ua > ub) return -1;
|
||
if (ua < ub) return 1;
|
||
return 0;
|
||
}
|
||
|
||
void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
|
||
uint64_t *safe_lsn_out,
|
||
uint64_t *safe_tag_out)
|
||
{
|
||
if (!ctx || !safe_lsn_out || !safe_tag_out) return;
|
||
|
||
/* Single-node: no votes[] entries — self is the only majority.
|
||
* Return self_lsn so WAL is not truncated to 0 on re-election. */
|
||
if (ctx->n_peers == 0) {
|
||
*safe_lsn_out = ctx->self_lsn;
|
||
*safe_tag_out = ctx->self_tag;
|
||
return;
|
||
}
|
||
|
||
/* Collect granted voter LSNs and tags.
|
||
* Maximum granted voters = n_peers + 1 (self + all other peers). */
|
||
int max_voters = ctx->n_peers + 1;
|
||
uint64_t *lsns = calloc((size_t)max_voters, sizeof(uint64_t));
|
||
uint64_t *tags = calloc((size_t)max_voters, sizeof(uint64_t));
|
||
if (!lsns || !tags) {
|
||
free(lsns);
|
||
free(tags);
|
||
*safe_lsn_out = 0;
|
||
*safe_tag_out = 0;
|
||
return;
|
||
}
|
||
|
||
int n_granted = 0;
|
||
|
||
/* Self always granted: self's lsn/tag from the dedicated self_lsn/self_tag fields */
|
||
lsns[n_granted] = ctx->self_lsn;
|
||
tags[n_granted] = ctx->self_tag;
|
||
n_granted++;
|
||
|
||
/* Each peer with vote_granted=1 */
|
||
for (int i = 0; i < ctx->n_peers; i++) {
|
||
if (ctx->votes[i].vote_granted) {
|
||
lsns[n_granted] = ctx->votes[i].peer_lsn;
|
||
tags[n_granted] = ctx->votes[i].peer_tag;
|
||
n_granted++;
|
||
}
|
||
}
|
||
|
||
/* Sort descending */
|
||
qsort(lsns, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc);
|
||
qsort(tags, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc);
|
||
|
||
/* Majority-threshold element: index = floor(n_granted / 2)
|
||
* For n_granted=3: index=1 (2nd element, 0-based) — majority=2 out of 3.
|
||
* For n_granted=1: index=0 (self only). */
|
||
int idx = n_granted / 2;
|
||
*safe_lsn_out = lsns[idx];
|
||
*safe_tag_out = tags[idx];
|
||
|
||
free(lsns);
|
||
free(tags);
|
||
|
||
INFO("HA: safe_committed_lsn=%" PRIu64 " safe_config_tag=%" PRIu64
|
||
" (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
|
||
}
|
||
|
||
/* ── bug-1060: never truncate the new leader's own durable log ───────────────
|
||
*
|
||
* The post-election WAL truncation cutoff must never fall below the leader's own
|
||
* durable tip, or it destroys entries committed by a prior leader (which this
|
||
* node had acked as a standby, and so are committed) — a Raft §5.4 Leader
|
||
* Completeness violation that diverges WALs and deadlocks the next election.
|
||
* See ha.h for the full rationale. */
|
||
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
|
||
{
|
||
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
|
||
}
|
||
|
||
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
|
||
*
|
||
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
|
||
* applied LAST so deadlock-freedom (>=1 guaranteed-free liveness worker) holds
|
||
* even when the n_vcpus ceiling would otherwise drive the value to 0 (e.g.
|
||
* n_vcpus == 1). See ha.h for the full rationale; the caller WARNs on the
|
||
* under-provisioned (n_vcpus < n_peers) clamp — this helper never logs. */
|
||
int drive_async_liveness_reserved(int n_peers, int n_vcpus)
|
||
{
|
||
int desired = n_peers - 1; /* worst-case concurrent liveness work */
|
||
int ceiling = n_vcpus - 1; /* keep >=1 commit (REQ_BLOCK) slot */
|
||
|
||
int reserved = desired < ceiling ? desired : ceiling; /* min(desired, ceiling) */
|
||
if (reserved < 1) /* floor applied LAST — always wins */
|
||
reserved = 1;
|
||
return reserved;
|
||
}
|
||
|
||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
|
||
*
|
||
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
|
||
* Policy Decision Table inputs from the promoting member's recovery/durability
|
||
* state and delegate to persist_recover_decision so the 3.2/3.3 table is the
|
||
* single source of truth (DRY). "Behind the HWM" is durable_floor < safe_hwm;
|
||
* a truncated log is treated as behind regardless of its volatile position
|
||
* (design.md note). A hollow node (truncated OR behind) can never be ELIGIBLE
|
||
* until it catches up — independent of quorum size (Req 2.5).
|
||
*/
|
||
recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
|
||
uint64_t durable_floor,
|
||
uint64_t safe_hwm,
|
||
bool peer_reachable,
|
||
bool catch_up_succeeded)
|
||
{
|
||
recover_policy_inputs_t in = {
|
||
.verdict = recovery_truncated ? RECOVER_TRUNCATED
|
||
: RECOVER_CLEAN,
|
||
.durable_below_hwm = (durable_floor < safe_hwm),
|
||
.healthy_peer_reachable = peer_reachable,
|
||
.catch_up_succeeded = catch_up_succeeded,
|
||
.durable_floor_lsn = durable_floor,
|
||
.cluster_hwm_lsn = safe_hwm,
|
||
};
|
||
return persist_recover_decision(&in);
|
||
}
|
||
|
||
/* ha_evaluate_promotion_fence: live gate over the existing in_catch_up fence
|
||
* (Req 2.2, 2.6). Reuses ha->in_catch_up — does NOT invent a parallel fence.
|
||
* - ELIGIBLE → clear the fence, return ELIGIBLE (a member that caught up past
|
||
* the committed boundary regains eligibility, Req 2.6).
|
||
* - FENCE → set the fence, return FENCE (refuse promotion; catch up from a
|
||
* healthy peer).
|
||
* - FENCE_UNRECOVERABLE → run the 3.3 loud-fence action (logs LOUDLY + sets
|
||
* the sticky non-promotable flag) and ALSO set the in_catch_up
|
||
* fence. The member STAYS ALIVE, non-promotable, retrying
|
||
* catch-up — it NEVER self-exits (Req 2.4).
|
||
*/
|
||
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
|
||
bool recovery_truncated,
|
||
uint64_t durable_floor,
|
||
uint64_t safe_hwm,
|
||
bool peer_reachable,
|
||
bool catch_up_succeeded)
|
||
{
|
||
recover_decision_t d = ha_promotion_gate_decision(
|
||
recovery_truncated, durable_floor, safe_hwm, peer_reachable,
|
||
catch_up_succeeded);
|
||
|
||
switch (d) {
|
||
case RECOVER_DECISION_ELIGIBLE:
|
||
/* Release the fence: catch-up brought the member to a
|
||
* committed-consistent state at/above the cluster boundary (Req 2.6). */
|
||
if (ha != NULL)
|
||
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
|
||
break;
|
||
case RECOVER_DECISION_FENCE:
|
||
/* Refuse promotion: extend the catch-up fence so the member catches up
|
||
* from a healthy peer before it can lead (Req 2.2). */
|
||
if (ha != NULL)
|
||
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
|
||
INFO("HA: promotion FENCED — recovery_truncated=%d durable_floor=%"
|
||
PRIu64 " < cluster HWM=%" PRIu64 " (catch up before leading; "
|
||
"ha-commit-durability Req 2.2)",
|
||
recovery_truncated, durable_floor, safe_hwm);
|
||
break;
|
||
case RECOVER_DECISION_FENCE_UNRECOVERABLE: {
|
||
/* Unrecoverable: route through the 3.3 loud-fence action — logs LOUDLY
|
||
* and sets the sticky non-promotable flag. Also hold the in_catch_up
|
||
* fence so the member refuses promotion. The process STAYS ALIVE and
|
||
* keeps retrying catch-up; it NEVER self-exits (Req 2.4). */
|
||
recover_policy_inputs_t in = {
|
||
.verdict = recovery_truncated ? RECOVER_TRUNCATED
|
||
: RECOVER_CLEAN,
|
||
.durable_below_hwm = (durable_floor < safe_hwm),
|
||
.healthy_peer_reachable = peer_reachable,
|
||
.catch_up_succeeded = catch_up_succeeded,
|
||
.durable_floor_lsn = durable_floor,
|
||
.cluster_hwm_lsn = safe_hwm,
|
||
};
|
||
persist_t *p = (ha != NULL && ha->raft_c != NULL) ? &ha->raft_c->persist
|
||
: NULL;
|
||
persist_recover_loud_fence_if_unrecoverable(p, &in);
|
||
if (ha != NULL)
|
||
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
|
||
break;
|
||
}
|
||
}
|
||
return d;
|
||
}
|
||
|
||
/* ── ha-commit-durability task 4.4: bounded-window catch-up re-alert ─────────
|
||
*
|
||
* (never-self-exit rework 2026-06-18.) A member that wins an election while
|
||
* recovery-truncated / behind the HWM with no peer reachable AT THE PROMOTION
|
||
* INSTANT is FENCED (in_catch_up set, non-promotable) and STAYS ALIVE, retrying
|
||
* catch-up indefinitely. This one-shot timer exists ONLY for operator
|
||
* VISIBILITY: when it fires, if the member is STILL unrecoverable (still
|
||
* truncated/behind AND no peer AND no progress) it RE-EMITS the loud,
|
||
* operator-visible alert via persist_recover_loud_fence_if_unrecoverable (which
|
||
* also keeps the sticky non-promotable flag set) — it NEVER terminates the
|
||
* process. If the member has caught up (not truncated AND durable_floor >=
|
||
* HWM) the timer RELEASES the fence (Req 2.6). If progress is possible (peer
|
||
* back, or durable floor advanced) it leaves the member fenced under the
|
||
* existing catch-up machinery.
|
||
*
|
||
* Earlier this timer DEFERRED-then-ABORTED. The abort is GONE: a controller
|
||
* MUST NEVER self-terminate as a designed response to an operational condition
|
||
* (user invariant, 2026-06-18). The task-5.2 soak quorum collapse was caused
|
||
* by that self-abort; staying alive + retrying self-heals when a peer returns.
|
||
*
|
||
* The window is a settable seam so the timer test runs fast. Default =
|
||
* DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */
|
||
static _Atomic uint32_t s_promotion_fence_window_ms =
|
||
DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
|
||
|
||
uint32_t ha_promotion_fence_window_ms(void)
|
||
{
|
||
uint32_t w = atomic_load_explicit(&s_promotion_fence_window_ms,
|
||
memory_order_acquire);
|
||
return w ? w : DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
void ha_set_promotion_fence_window_ms(uint32_t ms)
|
||
{
|
||
atomic_store_explicit(&s_promotion_fence_window_ms, ms,
|
||
memory_order_release);
|
||
}
|
||
#endif
|
||
|
||
/* promotion_fence_ctx_t: delaymgr context for the bounded catch-up window.
|
||
* Captures the cluster committed high-water mark (target_hwm) and the durable
|
||
* floor AT SCHEDULE TIME (baseline_durable_floor) so the timer can detect
|
||
* catch-up progress (durable_floor advanced toward the HWM). `cancelled` is set
|
||
* by ha_cancel_promotion_fence_timer (recovery / shutdown) so a fired-but-
|
||
* cancelled callback is a safe no-op. */
|
||
typedef struct {
|
||
_Atomic int cancelled;
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
bool was_truncated; /* recovery-truncated at schedule time */
|
||
uint64_t target_hwm; /* cluster committed high-water mark */
|
||
uint64_t baseline_durable_floor; /* durable floor at schedule time */
|
||
} promotion_fence_ctx_t;
|
||
|
||
/* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer.
|
||
* Idempotent; safe to call when no timer is armed. Used when the member catches
|
||
* up (fence released) or when the controller is shutting down — so neither a
|
||
* caught-up nor a stopped member ever reaches the deferred FATAL. */
|
||
void ha_cancel_promotion_fence_timer(ha_state_t *ha)
|
||
{
|
||
if (!ha) return;
|
||
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)atomic_exchange_explicit(
|
||
&ha->promotion_fence_ctx, NULL, memory_order_acq_rel);
|
||
if (t)
|
||
atomic_store_explicit(&t->cancelled, 1, memory_order_release);
|
||
}
|
||
|
||
/* Forward decl: the timeout callback re-arms itself for periodic re-alert. */
|
||
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
|
||
bool truncated, uint64_t durable_floor,
|
||
uint64_t target_hwm);
|
||
|
||
/* promotion_fence_timeout_fn: the bounded-window re-alert timer callback. Owns
|
||
* its ctx — frees it on exit. Re-evaluates the LIVE recovery/durability state.
|
||
* It NEVER terminates the process (never-self-exit invariant, 2026-06-18):
|
||
* - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
|
||
* in_catch_up fence + clear the sticky unrecoverable flag; eligible again
|
||
* (Req 2.6).
|
||
* - a healthy peer is reachable now, OR catch-up is making progress → leave
|
||
* the member fenced under the existing catch-up machinery (it will release
|
||
* via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not
|
||
* re-arm the re-alert while progress is happening.
|
||
* - STILL unrecoverable (truncated/behind AND no peer AND no progress) →
|
||
* RE-EMIT the loud operator alert via persist_recover_loud_fence_if_-
|
||
* unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the
|
||
* timer so the alert keeps repeating until a peer returns. The process
|
||
* STAYS ALIVE and keeps retrying — it NEVER aborts. */
|
||
static int promotion_fence_timeout_fn(void *arg)
|
||
{
|
||
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg;
|
||
ha_state_t *ha = t->ha;
|
||
cluster_t *c = t->c;
|
||
|
||
/* Clear the stored handle before checking cancelled so a concurrent
|
||
* ha_cancel_promotion_fence_timer is a safe no-op (matches the
|
||
* raft_election_timeout_fn convention). */
|
||
promotion_fence_ctx_t *cur = (promotion_fence_ctx_t *)atomic_load_explicit(
|
||
&ha->promotion_fence_ctx, memory_order_acquire);
|
||
if (cur == t)
|
||
atomic_store_explicit(&ha->promotion_fence_ctx, NULL,
|
||
memory_order_release);
|
||
|
||
if (atomic_load_explicit(&t->cancelled, memory_order_acquire)) {
|
||
free(t);
|
||
return 0;
|
||
}
|
||
|
||
if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */
|
||
|
||
/* Re-read the LIVE state after the window. */
|
||
bool truncated = persist_recovery_truncated(&c->persist);
|
||
uint64_t durable_floor = persist_durable_floor(&c->persist);
|
||
bool below_hwm = durable_floor < t->target_hwm;
|
||
bool made_progress = durable_floor > t->baseline_durable_floor;
|
||
bool peer_reachable = (ha->repl != NULL)
|
||
? (repl_active_standby_count(ha->repl) > 0)
|
||
: false;
|
||
|
||
/* Caught up: not truncated AND durable position reached the boundary →
|
||
* release the fence + clear the sticky flag; the member is eligible on its
|
||
* next election (Req 2.6). */
|
||
if (!truncated && !below_hwm) {
|
||
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
|
||
persist_clear_recovery_unrecoverable(&c->persist);
|
||
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
|
||
PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)",
|
||
durable_floor, t->target_hwm);
|
||
free(t);
|
||
return 0;
|
||
}
|
||
|
||
/* Still hollow, but a peer is reachable or catch-up advanced the durable
|
||
* floor: progress is possible. Leave the member fenced (in_catch_up stays
|
||
* set) under the existing catch-up machinery; do not re-arm the re-alert
|
||
* while progress is happening. */
|
||
if (peer_reachable || made_progress) {
|
||
INFO("HA: promotion fence catch-up window elapsed but progress is "
|
||
"possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
|
||
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, catching up "
|
||
"(ha-commit-durability Req 2.2)",
|
||
peer_reachable, durable_floor, t->baseline_durable_floor,
|
||
t->target_hwm);
|
||
free(t);
|
||
return 0;
|
||
}
|
||
|
||
/* STILL unrecoverable: truncated/behind AND no healthy peer AND no catch-up
|
||
* progress. RE-EMIT the loud operator alert and keep the member fenced +
|
||
* non-promotable + ALIVE (Req 2.4) — NEVER terminate. Re-arm the re-alert
|
||
* timer so the operator keeps hearing about it until a peer returns. */
|
||
uint64_t target_hwm = t->target_hwm; /* capture before free (UAF guard) */
|
||
{
|
||
recover_policy_inputs_t in = {
|
||
.verdict = truncated ? RECOVER_TRUNCATED
|
||
: RECOVER_CLEAN,
|
||
.durable_below_hwm = below_hwm,
|
||
.healthy_peer_reachable = false,
|
||
.catch_up_succeeded = false,
|
||
.durable_floor_lsn = durable_floor,
|
||
.cluster_hwm_lsn = target_hwm,
|
||
};
|
||
/* Loud, operator-visible alert + sticky non-promotable flag. STAYS
|
||
* ALIVE: no abort/exit anywhere on this path. */
|
||
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
|
||
}
|
||
|
||
free(t);
|
||
|
||
/* Re-arm so the loud alert repeats periodically while the member remains
|
||
* unrecoverable. The member keeps retrying catch-up; the next firing
|
||
* re-evaluates and releases the fence once a peer returns and it catches up. */
|
||
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, target_hwm);
|
||
return 0;
|
||
}
|
||
|
||
/* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule
|
||
* the one-shot bounded-window RE-ALERT timer. Cancels any previously-armed
|
||
* timer first so a re-evaluation never leaks or double-fires. On a failure to
|
||
* allocate/schedule the timer it emits the loud unrecoverable fence ONCE (stays
|
||
* alive, non-promotable) — it NEVER aborts (never-self-exit invariant). */
|
||
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
|
||
bool truncated, uint64_t durable_floor,
|
||
uint64_t target_hwm)
|
||
{
|
||
/* Fence first (Req 2.2): refuse promotion while hollow. */
|
||
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
|
||
|
||
/* Cancel a stale timer (e.g. a prior promotion attempt). */
|
||
ha_cancel_promotion_fence_timer(ha);
|
||
|
||
promotion_fence_ctx_t *t = calloc(1, sizeof(*t));
|
||
if (!t)
|
||
goto loud_fence_no_timer;
|
||
|
||
atomic_init(&t->cancelled, 0);
|
||
t->ha = ha;
|
||
t->c = c;
|
||
t->was_truncated = truncated;
|
||
t->target_hwm = target_hwm;
|
||
t->baseline_durable_floor = durable_floor;
|
||
|
||
uint32_t window_ms = ha_promotion_fence_window_ms();
|
||
int rc = delay_task_ms(&c->delaymgr, (int)window_ms, 1,
|
||
promotion_fence_timeout_fn, t);
|
||
if (rc < 0) {
|
||
WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc);
|
||
free(t);
|
||
goto loud_fence_no_timer;
|
||
}
|
||
|
||
atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release);
|
||
INFO("HA: promotion FENCED + catch-up re-alert window armed (%u ms) — "
|
||
"recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64
|
||
"; staying ALIVE + non-promotable, retrying catch-up "
|
||
"(ha-commit-durability Req 2.2/2.4)",
|
||
window_ms, truncated, durable_floor, target_hwm);
|
||
return;
|
||
|
||
loud_fence_no_timer:
|
||
{
|
||
/* Could not arm the re-alert timer — still emit the loud unrecoverable
|
||
* fence once so the operator sees it; the member stays fenced
|
||
* (in_catch_up set above) + non-promotable + ALIVE. NO abort. */
|
||
recover_policy_inputs_t in = {
|
||
.verdict = truncated ? RECOVER_TRUNCATED
|
||
: RECOVER_CLEAN,
|
||
.durable_below_hwm = (durable_floor < target_hwm),
|
||
.healthy_peer_reachable = false,
|
||
.catch_up_succeeded = false,
|
||
.durable_floor_lsn = durable_floor,
|
||
.cluster_hwm_lsn = target_hwm,
|
||
};
|
||
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
|
||
}
|
||
}
|
||
|
||
/*
|
||
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
|
||
*
|
||
* Equivalent of ha_elect_evaluate_and_signal CTRL_PRIMARY path:
|
||
* - enter CTRL_PRIMARY (no transitional state)
|
||
* - call cluster_init_phase2 on first boot
|
||
* - push config, send ctrlhello
|
||
*
|
||
* ha->raft_c must be non-NULL (production path only; test mode uses noop stubs).
|
||
*/
|
||
static void raft_cb_become_leader(void *ctx, uint64_t term)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
|
||
/* Cancel any pending election timeout */
|
||
raft_cb_cancel_timeout(ctx);
|
||
|
||
/* Sync ha->term from raft term (raft owns term increments) */
|
||
atomic_store_explicit(&ha->term, term, memory_order_release);
|
||
|
||
/* The cluster committed high-water mark for this promotion — derived from
|
||
* the voter replies (NOT a stale ctrlhello). The task-4.3 pre-promotion
|
||
* eligibility gate (below) compares the local CLAMPED durable position
|
||
* against it. Lifted to function scope so the gate can read it after the
|
||
* truncate-clamp. */
|
||
uint64_t promotion_safe_lsn = 0;
|
||
|
||
/* Task 4.1 (ha-repl-hardening): compute safe_committed_lsn from vote replies.
|
||
* Uses voter LSNs collected during election (ha->elect_ctx->votes[]) to find
|
||
* the highest LSN held by a majority without an extra round-trip.
|
||
* Design §Flow 0: Post-Election — Determine Safe Commit Point. */
|
||
{
|
||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||
&ha->elect_ctx, memory_order_acquire);
|
||
if (ectx) {
|
||
uint64_t safe_lsn = 0;
|
||
uint64_t safe_tag = 0;
|
||
ha_compute_safe_committed_lsn(ectx, &safe_lsn, &safe_tag);
|
||
promotion_safe_lsn = safe_lsn;
|
||
/* Store results for use in WAL truncation and post-election commit.
|
||
* safe_lsn is the highest LSN on a majority; entries above it
|
||
* may not have been committed on all nodes and must be truncated.
|
||
*
|
||
* bug-1060 (Leader Completeness, Raft §5.4): the truncation cutoff is
|
||
* clamped to the leader's OWN durable tip so it never destroys entries
|
||
* this node holds durably — those were acked by this node as a standby
|
||
* and so were committed by the prior leader. Truncating to the bare
|
||
* majority element (which can sit BELOW the leader's durable tip when
|
||
* a slower peer is in the granting majority) diverged WALs and dead-
|
||
* locked the next election. With the clamp, persist_truncate_wal_above
|
||
* is a no-op on the leader's durable log; safe_lsn only governs removal
|
||
* of a non-durable divergent tail strictly above the durable tip. */
|
||
if (c) {
|
||
uint64_t durable_tip = persist_durable_floor(&c->persist);
|
||
uint64_t trunc_floor =
|
||
ha_promotion_truncate_floor(safe_lsn, durable_tip);
|
||
persist_truncate_wal_above(&c->persist, trunc_floor);
|
||
}
|
||
/* Task 4.2 (ha-repl-hardening): set committed_lsn = safe_lsn.
|
||
* persist_truncate_wal_above already sets c->persist.committed_lsn
|
||
* in the production path (c != NULL). Also update
|
||
* ha->last_known_committed_lsn so the test-mode decision loop
|
||
* (XCE_TESTING path in standby heartbeat reply handler) and any
|
||
* code reading ha->last_known_committed_lsn see the safe commit
|
||
* point immediately after promotion.
|
||
* Design §Flow 0 step 5: committed_lsn = safe_committed_lsn.
|
||
* Req 2.3 (ha-repl-hardening). */
|
||
atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn,
|
||
memory_order_release);
|
||
/* Update committed config tag baseline for the post-election decision loop */
|
||
atomic_store_explicit(&ha->committed_config_tag, safe_tag,
|
||
memory_order_release);
|
||
}
|
||
}
|
||
|
||
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────
|
||
* Refuse promotion BEFORE the node becomes authoritative CTRL_PRIMARY if it
|
||
* carries the recovery-truncated flag (task 3.1 — a structurally corrupt log
|
||
* must never lead) OR its CLAMPED durable position is below the cluster
|
||
* committed high-water mark (promotion_safe_lsn, from the voter replies).
|
||
* The durable position is read AFTER persist_truncate_wal_above above, so it
|
||
* reflects the 2.2 truncate-clamp (a node whose durable tip was clamped down
|
||
* is correctly seen as behind). A hollow node (truncated OR behind) is
|
||
* refused regardless of quorum size — combined with the 4.2 freshness floor
|
||
* it can neither win the vote nor self-promote (Req 2.5). On refusal we set
|
||
* the in_catch_up fence and return WITHOUT becoming primary; the member
|
||
* catches up from a healthy peer and a later promotion attempt passes the
|
||
* gate (Req 2.2/2.6). Only the production path (real persist) is gated; the
|
||
* c==NULL unit shim has no persist and skips it.
|
||
*
|
||
* Belt-and-suspenders (never-self-exit rework 2026-06-18): a member that has
|
||
* been marked sticky-unrecoverable (persist_recovery_unrecoverable) must
|
||
* NEVER become primary while that flag is set — it stays alive, fenced, and
|
||
* keeps retrying catch-up. The flag clears automatically on catch-up. */
|
||
if (c != NULL) {
|
||
bool truncated = persist_recovery_truncated(&c->persist);
|
||
uint64_t durable_floor = persist_durable_floor(&c->persist);
|
||
bool unrecoverable = persist_recovery_unrecoverable(&c->persist);
|
||
|
||
/* A healthy peer is reachable for catch-up iff some voter granted with a
|
||
* live endpoint survives this election (these become the standby
|
||
* channels promoted below). */
|
||
bool peer_reachable = false;
|
||
{
|
||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||
&ha->elect_ctx, memory_order_acquire);
|
||
if (ectx) {
|
||
for (int i = 0; i < ectx->n_peers; i++) {
|
||
if (ectx->votes[i].vote_granted && ectx->votes[i].ep != NULL) {
|
||
peer_reachable = true;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
|
||
* a hollow member never silently proceeds to leadership. We evaluate
|
||
* the PURE 3.2/3.3 decision here. A FENCE or an UNRECOVERABLE
|
||
* (truncated/behind + no peer right now) both fence the member
|
||
* (in_catch_up) and arm the catch-up re-alert timer; the member STAYS
|
||
* ALIVE and keeps retrying — it NEVER self-exits (never-self-exit
|
||
* invariant, 2026-06-18; this is what fixes the task-5.2 soak collapse
|
||
* where a self-aborting recovering winner repeatedly dropped quorum). */
|
||
recover_decision_t gate = ha_promotion_gate_decision(
|
||
truncated, durable_floor, promotion_safe_lsn,
|
||
peer_reachable, /*catch_up_succeeded=*/false);
|
||
|
||
if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable) {
|
||
/* ELIGIBLE and not sticky-unrecoverable: release the fence (a
|
||
* caught-up member regains eligibility, Req 2.6), clear the sticky
|
||
* flag if set, and cancel any pending re-alert timer. Proceed to
|
||
* authoritative promotion below. */
|
||
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
|
||
persist_clear_recovery_unrecoverable(&c->persist);
|
||
ha_cancel_promotion_fence_timer(ha);
|
||
} else {
|
||
/* FENCE, UNRECOVERABLE, or sticky-unrecoverable: refuse promotion —
|
||
* do NOT become CTRL_PRIMARY. Fence the member and arm the catch-up
|
||
* re-alert timer. The member STAYS ALIVE, non-promotable, retrying
|
||
* catch-up (Req 2.2/2.4). NEVER self-exit, never fence silently. */
|
||
WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d) — "
|
||
"recovery_truncated=%d durable_floor=%" PRIu64
|
||
" cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + arming "
|
||
"catch-up re-alert window, staying ALIVE "
|
||
"(ha-commit-durability Req 2.2/2.4)",
|
||
(int)gate, unrecoverable, truncated, durable_floor,
|
||
promotion_safe_lsn, peer_reachable);
|
||
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
|
||
promotion_safe_lsn);
|
||
return;
|
||
}
|
||
}
|
||
|
||
/* ── ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): config
|
||
* serve-fence promotion gate ──────────────────────────────────────────────
|
||
* A member whose latest committed config marker references bytes it does not
|
||
* hold (config_serve_fenced) MUST NOT become authoritative CTRL_PRIMARY — it
|
||
* cannot serve under the committed configuration. Refuse promotion and STAY
|
||
* ALIVE (controller-never-self-terminate): the member remains non-promotable
|
||
* and keeps retrying; once the correct bytes arrive out-of-band and re-verify
|
||
* clears the fence, a later promotion attempt passes this gate. The check is a
|
||
* pure atomic read (no I/O) — safe on the election path. */
|
||
if (config_serve_fenced(c)) {
|
||
WARN("HA: promotion REFUSED — config serve-FENCED (committed config "
|
||
"marker references bytes not present/matching locally); member "
|
||
"stays ALIVE, non-promotable, awaiting out-of-band config bytes "
|
||
"(ha-term-safety R7.9)");
|
||
return; /* do NOT become CTRL_PRIMARY; never self-exit */
|
||
}
|
||
|
||
int is_first_boot = !ha->phase2_done;
|
||
|
||
/* First boot: call cluster_init_phase2 */
|
||
if (is_first_boot) {
|
||
#ifdef XCE_TESTING
|
||
if (ha_phase2_hook != NULL)
|
||
ha_phase2_hook(c);
|
||
#else
|
||
if (c != NULL)
|
||
cluster_init_phase2(c);
|
||
#endif
|
||
ha->phase2_done = 1;
|
||
}
|
||
|
||
/* ha-single-leader-role (Req 1.1): enter CTRL_PRIMARY directly — there is
|
||
* no transitional COMMITTING state. New-leader safety comes from the
|
||
* own-term commit rule (the WAL_NOOP + writable_lsn gate armed below),
|
||
* not from a fourth role value. The raft-owned term was already synced
|
||
* at entry (~line 463), so we do NOT call ha_do_promote here (it would
|
||
* double-increment the term, research.md §4 finding 2). This store sits
|
||
* OUTSIDE the c != NULL guard so the c==NULL test shim
|
||
* (raft_cb_become_leader_for_test) reaches it. */
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
|
||
memory_order_release);
|
||
INFO("HA FSM (raft): → CTRL_PRIMARY (term=%" PRIu64
|
||
", first_boot=%d)", term, is_first_boot);
|
||
|
||
/* Req 2.1 (ha-repl-hardening): re-initialize per-standby convergence
|
||
* tracking for THIS primary term. Stale converged flags from a
|
||
* previous term (demote → re-elect cycle) must not satisfy the
|
||
* majority-convergence gate before standbys re-report their state. */
|
||
memset(ha->standby_reported_tag, 0, sizeof(ha->standby_reported_tag));
|
||
memset(ha->standby_reported_lsn, 0, sizeof(ha->standby_reported_lsn));
|
||
memset(ha->standby_converged, 0, sizeof(ha->standby_converged));
|
||
|
||
/* Promote surviving vote endpoints to standby replication channels.
|
||
* Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to
|
||
* ha->standby_endpoints so push_config and ctrlhello reach them. */
|
||
{
|
||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||
&ha->elect_ctx, memory_order_acquire);
|
||
if (ectx && ha->standby_endpoints) {
|
||
pthread_mutex_lock(&ha->lock);
|
||
for (int i = 0; i < ectx->n_peers; i++) {
|
||
if (ectx->votes[i].ep != NULL) {
|
||
zarray_store(ha->standby_endpoints, ectx->votes[i].ep);
|
||
INFO("HA: promoted vote ep[%d] → standby channel", i);
|
||
/* Task 2.1: register promoted ep with WAL replicator.
|
||
* Req 1.1: primary must register standby with repl_add_standby
|
||
* when establishing the standby control channel.
|
||
* peer->hostname / peer->port from the vote entry's peer pointer. */
|
||
if (c && ectx->votes[i].peer != NULL) {
|
||
int rc = repl_add_standby(&c->persist.repl,
|
||
ectx->votes[i].ep,
|
||
ectx->votes[i].peer->hostname,
|
||
(uint32_t)ectx->votes[i].peer->port);
|
||
if (rc >= 0)
|
||
INFO("HA: repl registered standby ep[%d] (slot %d, %s:%u)",
|
||
i, rc,
|
||
ectx->votes[i].peer->hostname,
|
||
ectx->votes[i].peer->port);
|
||
else
|
||
WARN("HA: repl_add_standby ep[%d] failed: %d", i, rc);
|
||
}
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&ha->lock);
|
||
}
|
||
}
|
||
|
||
/* Quorum-transition observability (Req 8.1, 8.2): promotion registered the
|
||
* surviving vote endpoints as standbys — the write-quorum count just changed.
|
||
* Record any loss/restore edge once (no-op when quorum is held at promotion). */
|
||
ha_log_quorum_transition(ha);
|
||
|
||
if (c) {
|
||
/* Push config to standbys — read raw bytes from disk so standbys
|
||
* can save the file. NULL bytes are passed only in test mode. */
|
||
#ifdef XCE_TESTING
|
||
if (ha_push_config_hook != NULL)
|
||
ha_push_config_hook(c);
|
||
#else
|
||
{
|
||
uint8_t *cfg_bytes = NULL;
|
||
size_t cfg_len = 0;
|
||
extern int conf_read_raw(const char *, uint8_t **, size_t *);
|
||
if (c->myconf && c->myconf->conffile &&
|
||
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
|
||
/* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON).
|
||
* The offload deep-copies cfg_bytes, so this free is still
|
||
* correct regardless of routing. */
|
||
ha_config_push_offloadable(c, cfg_bytes, cfg_len,
|
||
c->myconf->config_sha256,
|
||
c->myconf->config_version_tag);
|
||
free(cfg_bytes);
|
||
} else {
|
||
ha_config_push_offloadable(c, NULL, 0, NULL,
|
||
c->myconf ? c->myconf->config_version_tag : 0);
|
||
}
|
||
}
|
||
#endif
|
||
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up */
|
||
#ifdef XCE_TESTING
|
||
if (ha_send_ctrlhello_hook != NULL)
|
||
ha_send_ctrlhello_hook(c);
|
||
#else
|
||
ha_send_ctrlhello_to_all_standbys(c);
|
||
#endif
|
||
/* ha-single-leader-role task 2.2: register the persistence hooks FIRST,
|
||
* before the own-term no-op append, so the persistence seam is live
|
||
* before any racing client write can reach the cluster. Idempotent
|
||
* via the hooks_registered guard. */
|
||
ha_register_persist_hooks(c);
|
||
|
||
/* Start primary heartbeat immediately so standbys receive NOP and their
|
||
* liveness poll does not fire. ha_start_primary_heartbeat is also
|
||
* called in ha_promote_to_primary; the idempotency guard (role check
|
||
* at fn top) prevents double-arming. */
|
||
ha_start_primary_heartbeat(c);
|
||
|
||
/* ha-single-leader-role task 2.2 (Req 1.4, 2.1, 2.2): append the
|
||
* own-term WAL_NOOP and arm the commit engine's writable_lsn gate.
|
||
* This is the shared promotion tail both roads to PRIMARY run.
|
||
* Non-blocking: promotion never waits for the no-op to commit. */
|
||
ha_arm_own_term_gate(c);
|
||
}
|
||
}
|
||
|
||
/*
|
||
* raft_cb_become_follower: node lost election or saw higher term — enter STANDBY.
|
||
*
|
||
* Syncs ha->term from the raft-reported term, sets role to CTRL_STANDBY,
|
||
* calls cluster_init_phase2 on first boot (standbys also need it to
|
||
* receive config replication push from the new primary).
|
||
*/
|
||
static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
|
||
{
|
||
(void)leader_idx;
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
|
||
/* Cancel any pending election timeout */
|
||
raft_cb_cancel_timeout(ctx);
|
||
|
||
/* Sync ha->term */
|
||
atomic_store_explicit(&ha->term, term, memory_order_release);
|
||
|
||
int is_first_boot = !ha->phase2_done;
|
||
|
||
/* First boot: call cluster_init_phase2 so structures are ready */
|
||
if (is_first_boot) {
|
||
#ifdef XCE_TESTING
|
||
if (ha_phase2_hook != NULL)
|
||
ha_phase2_hook(c);
|
||
#else
|
||
if (c != NULL)
|
||
cluster_init_phase2(c);
|
||
#endif
|
||
ha->phase2_done = 1;
|
||
}
|
||
|
||
/* Close all outgoing vote endpoints (loser path).
|
||
* The primary will push config and ctrlhello via promoted connections;
|
||
* the standby waits passively on its listener for the inbound connection.
|
||
*
|
||
* FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197): when
|
||
* g_drive_async_peerctrl_dispatch_enabled is ON this loop runs on a POOL
|
||
* worker and frees ep->data while the io_worker may still deliver a queued
|
||
* peer-control event on another ep → UAF. Before task 3.4 flips the flag
|
||
* ON, each endpoint_fini below MUST be preceded by iomgr_unmonitor(ep). See
|
||
* the full hazard note at peer_ctrl_dispatch_to_pool. Safe today (flag OFF). */
|
||
{
|
||
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
|
||
&ha->elect_ctx, memory_order_acquire);
|
||
if (ectx) {
|
||
for (int i = 0; i < ectx->n_peers; i++) {
|
||
if (ectx->votes[i].ep != NULL) {
|
||
INFO("HA: closing vote ep[%d] (loser path)", i);
|
||
endpoint_fini(ectx->votes[i].ep);
|
||
ectx->votes[i].ep = NULL;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
/* ha-convergence-recovery Req 1.1-1.3, 1.5: demotion convergence.
|
||
*
|
||
* Read the PRIOR role BEFORE the CTRL_STANDBY store below. If this node was
|
||
* the PRIMARY, it may hold an uncommitted, potentially divergent WAL suffix
|
||
* and provisional in-memory state. Synchronously disable the three
|
||
* persistence hooks HERE (before any further event can be persisted — Req
|
||
* 1.1, reusing ha_demote_to_standby's disable sequence), then schedule a
|
||
* one-shot delaymgr task that truncates above committed_lsn and self-restarts
|
||
* (Req 1.2/1.3/1.5) off this callback thread (D1: no file I/O or exec on the
|
||
* iomgr/raft callback thread).
|
||
*
|
||
* Non-PRIMARY demotions (STANDBY/INIT/ELECTING) take the unchanged path —
|
||
* they have no registered hooks and no provisional suffix to discard, so
|
||
* their behavior is byte-equivalent to before this spec. */
|
||
ctrl_role_t prior_role = (ctrl_role_t)atomic_load_explicit(
|
||
&ha->role, memory_order_acquire);
|
||
|
||
if (prior_role == CTRL_PRIMARY && c != NULL) {
|
||
/* Req 1.1: disable persistence hooks SYNCHRONOUSLY, before the role
|
||
* store and before scheduling, so no provisional entry can be produced
|
||
* after the higher term was observed. Mirrors ha_demote_to_standby
|
||
* (ha.c) — disable each hook, then clear the idempotency guard. */
|
||
if (ha->hooks_registered) {
|
||
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||
(hook_fn_t)persist_job_hook_fun);
|
||
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
|
||
(hook_fn_t)persist_queue_hook_fun);
|
||
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
(hook_fn_t)persist_worker_hook_fun);
|
||
ha->hooks_registered = 0;
|
||
}
|
||
}
|
||
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
|
||
memory_order_release);
|
||
INFO("HA FSM (raft): → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
|
||
term, is_first_boot);
|
||
|
||
/* Req 1.2/1.3/1.5: schedule the deferred truncate-above-committed-LSN +
|
||
* self-restart now that hooks are disabled and the role is STANDBY. Only
|
||
* the was-PRIMARY edge converges; all other edges skip this entirely. */
|
||
if (prior_role == CTRL_PRIMARY && c != NULL) {
|
||
WARN("HA: PRIMARY demoted (term=%" PRIu64 ") — scheduling demotion "
|
||
"convergence (truncate above committed_lsn + self-restart)", term);
|
||
(void)ha_schedule_demotion_convergence(c);
|
||
}
|
||
|
||
/* Arm heartbeat watchdog — fires ha_on_primary_ctrl_fd_rdhup if primary
|
||
* sends no ctrlhello within election_timeout_ms. (This also resets
|
||
* last_activity = now, granting a full grace window before the first poll.) */
|
||
ha_arm_heartbeat_timeout_vtask(ha, c);
|
||
}
|
||
|
||
/*
|
||
* raft_cb_persist: persist term + voted_for to disk before granting vote.
|
||
*
|
||
* Maps raft integer voted_for index to a candidate name using cluster config.
|
||
* Updates ha->term, ha->voted_for_term, ha->voted_for_name then calls
|
||
* ha_persist_term(ha, workdir).
|
||
*
|
||
* voted_for == -1 means "no vote"; voted_for >= 0 is the candidate index.
|
||
* Returns 0 on success, non-zero on error (causes raft library to deny vote).
|
||
*/
|
||
static int raft_cb_persist(void *ctx, uint64_t term, int voted_for)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
cluster_t *c = ha->raft_c;
|
||
/* Store ha-term.bin under {workdir}/{myname}/data/ alongside WAL/snapshot. */
|
||
char data_dir[1024] = {0};
|
||
if (c && c->myconf && c->myconf->workdir && c->myconf->name)
|
||
snprintf(data_dir, sizeof(data_dir), "%s/%s/data",
|
||
c->myconf->workdir, c->myconf->name);
|
||
const char *workdir = data_dir[0] ? data_dir : NULL;
|
||
|
||
/* Update ha->term */
|
||
atomic_store_explicit(&ha->term, term, memory_order_release);
|
||
|
||
/* Map voted_for index to name */
|
||
pthread_mutex_lock(&ha->lock);
|
||
|
||
if (voted_for == -1) {
|
||
ha->voted_for_term = 0;
|
||
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
|
||
} else {
|
||
ha->voted_for_term = term;
|
||
/* Look up candidate name by index */
|
||
if (c && c->myconf &&
|
||
voted_for < c->myconf->peers_cfg.candidates_count) {
|
||
const char *name = c->myconf->peers_cfg.candidates[voted_for].name;
|
||
strncpy(ha->voted_for_name, name, sizeof(ha->voted_for_name) - 1);
|
||
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
|
||
} else {
|
||
/* Test mode or out-of-range: use "self" placeholder */
|
||
if (voted_for == ha->my_priority) {
|
||
strncpy(ha->voted_for_name, "self",
|
||
sizeof(ha->voted_for_name) - 1);
|
||
} else {
|
||
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
|
||
}
|
||
}
|
||
}
|
||
|
||
pthread_mutex_unlock(&ha->lock);
|
||
|
||
ha_persist_term(ha, workdir);
|
||
return 0;
|
||
}
|
||
|
||
/* raft_cb_trace: emit raft state-machine events at DEBUG3 level.
|
||
* Active when controller is started with -vvv; suppressed otherwise. */
|
||
static void raft_cb_trace(void *ctx, const char *event, const char *detail)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
tlog(TLOG_DEBUG3, "RAFT[%s] term=%" PRIu64 " event=%s%s%s",
|
||
ha->my_priority == 0 ? "p0" :
|
||
ha->my_priority == 1 ? "p1" : "p2",
|
||
atomic_load_explicit(&ha->term, memory_order_relaxed),
|
||
event,
|
||
detail ? " " : "", detail ? detail : "");
|
||
}
|
||
|
||
/* ── ha-term-safety: MINIMAL log/commit callback placeholders ────────────────
|
||
*
|
||
* TODO(ha-term-safety task 2.x): REPLACE these stubs with the real bodies.
|
||
* The raft library's raft_init() now REQUIRES all five non-NULL, and
|
||
* raft_on_vote_request() internally calls on_get_entry_term for its logOk gate.
|
||
* These placeholders exist ONLY to restore the controller build (tasks 1.1/1.2);
|
||
* they perform NO persistence, NO replication, and NO truncation. See CONCERNS.
|
||
*
|
||
* on_persist_entry → task 2.1 (durable WAL append/overwrite) — IMPLEMENTED
|
||
* on_get_entry_term → task 2.2 (WAL entry-term lookup; persist_entry_term, 1.3) — IMPLEMENTED
|
||
* on_send_append_entries→ task 2.3 (AppendEntries transmit) — IMPLEMENTED
|
||
* on_truncate_from → task 2.2 (conflict truncation) — IMPLEMENTED
|
||
* on_commit_advanced → task 2.4 (commit-barrier trigger / commit source) — IMPLEMENTED
|
||
*
|
||
* on_get_entry_term returns 0 (== "absent") for now: with the log-replication
|
||
* path unwired, the library's local last-log term reads as 0, so the logOk gate
|
||
* grants on lastLogTerm >= 0 — i.e. it does NOT yet tighten the existing
|
||
* config_tag/last_log_lsn freshness gate already enforced in cluster.c. The real
|
||
* accessor (task 2.2/1.3) makes the gate effective. */
|
||
/* on_persist_entry (ha-term-safety task 2.1, R1.1–R1.4): durably append/overwrite
|
||
* the log entry at 1-based `index` (== WAL LSN) stamping the LEADER-supplied
|
||
* `term` into hdr.term, then return the persist rc.
|
||
*
|
||
* Persist-before-act: this is called BEFORE the library reports the append/accept
|
||
* (raft_log_append / raft_on_append_entries_request). On a non-zero (failure) rc
|
||
* the library does NOT advance its log and a standby does NOT ACK (the existing
|
||
* persist-before-ACK + bug-850 contiguity guard sit beneath the WAL write and are
|
||
* preserved — we do not bypass them). We never false-ACK on failure.
|
||
*
|
||
* `entry` is the adapter envelope (raft_entry_blob_t prefix + payload) packed by
|
||
* the append/replication driver (tasks 3.1/3.2); the library holds no entry bytes
|
||
* and treats it as opaque. We decode (type, payload, payload_len) from it for the
|
||
* existing persist write path.
|
||
*
|
||
* Path selection by role:
|
||
* - PRIMARY own-append: persist_wal_async — assigns the contiguous next LSN
|
||
* (== index here, since the library's index == log_len+1 == current_lsn) and
|
||
* defers fsync to the group-commit flusher (the steady-state append path).
|
||
* - STANDBY replicated apply: persist_wal_write_replicated — writes at the GIVEN
|
||
* lsn=index with the contiguity guard + inline persist-before-ACK fsync.
|
||
*
|
||
* NOTE (deferred to 2.3/3.2): the leader's per-entry term is carried on the repl
|
||
* envelope by task 2.3/3.2; until then the standby path stamps the term we set
|
||
* here via persist_set_term — which IS the supplied `term` argument — so a test
|
||
* that drives this callback already records the leader term. The envelope wiring
|
||
* makes the live replication path source the same value. */
|
||
static int ha_cb_persist_entry_stub(void *ctx, uint64_t index, uint64_t term,
|
||
const void *entry, size_t len)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
if (!ha || !ha->raft_c)
|
||
return -EINVAL; /* test mode without a cluster: cannot persist */
|
||
|
||
persist_t *p = &ha->raft_c->persist;
|
||
|
||
/* Decode the adapter entry envelope: a fixed raft_entry_blob_t prefix
|
||
* carrying the WAL type + payload_len, followed by the payload bytes. */
|
||
uint32_t type = WAL_NOOP;
|
||
const void *payload = NULL;
|
||
uint32_t plen = 0;
|
||
if (entry && len >= sizeof(raft_entry_blob_t)) {
|
||
const raft_entry_blob_t *blob = (const raft_entry_blob_t *)entry;
|
||
type = blob->type;
|
||
plen = blob->payload_len;
|
||
/* Reject a malformed envelope whose declared payload exceeds the buffer
|
||
* — fail loud rather than read past the blob. */
|
||
if ((size_t)plen + sizeof(*blob) > len)
|
||
return -EINVAL;
|
||
if (plen > 0)
|
||
payload = (const uint8_t *)entry + sizeof(*blob);
|
||
} else if (entry && len > 0) {
|
||
/* Defensive: an un-enveloped blob is treated as malformed. */
|
||
return -EINVAL;
|
||
}
|
||
|
||
/* Stamp the live election term so the WAL header records the LEADER's term
|
||
* for this entry (R1.1/R1.2/R1.4). */
|
||
persist_set_term(p, term);
|
||
|
||
/* Persist at LSN=index through the existing write path (persist-before-ACK
|
||
* and the bug-850 contiguity guard are preserved beneath these). */
|
||
if (ha_is_primary(ha)) {
|
||
/* Own-append: persist_wal_async assigns the contiguous next LSN, which
|
||
* equals `index` for the leader (library index == current_lsn). */
|
||
return persist_wal_async(p, type, payload, plen);
|
||
}
|
||
|
||
return persist_wal_write_replicated(p, index, type, payload, plen);
|
||
}
|
||
|
||
/* on_get_entry_term (ha-term-safety task 2.2, R1.5/R3.1): return the recorded
|
||
* Raft term for the entry at 1-based LSN `index`, sourced from the WAL term
|
||
* index via persist_entry_term. Returns 0 when the LSN is absent (no entry).
|
||
* The library calls this for its logOk / (term,lsn) reconcile gate.
|
||
*
|
||
* Legacy-0 caveat (from the task 1.3 review MEDIUM): persist_entry_term maps a
|
||
* present legacy-0 entry (hdr.term == 0, written before this feature) to
|
||
* WAL_LEGACY_TERM (== 1). In a mixed pre/post-feature cluster where a
|
||
* post-feature leader is genuinely in term 1, a (term,lsn) EQUALITY check
|
||
* cannot tell a legacy-0 entry (mapped to 1) apart from a real term-1 entry by
|
||
* term alone. This is safe because the Raft log-matching invariant (same term
|
||
* + same index ⇒ same entry, enforced across the prefix) plus the bounded
|
||
* one-time upgrade window (every entry written after upgrade carries a real
|
||
* term) mean the only collision is at the legacy/term-1 boundary, where the
|
||
* prefixes already agree. We note it explicitly rather than special-casing it. */
|
||
static uint64_t ha_cb_get_entry_term_stub(void *ctx, uint64_t index)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
if (!ha || !ha->raft_c)
|
||
return 0; /* test mode without a cluster: no log → absent */
|
||
return persist_entry_term(&ha->raft_c->persist, index);
|
||
}
|
||
|
||
/* ha_read_wal_entry_at: source ONE local WAL entry at LSN `lsn` for the
|
||
* AppendEntries send path (ha-term-safety task 2.3). The library holds no entry
|
||
* bytes, so the leader reads its own WAL segment files (seg-%016llx.wal under
|
||
* wal_path) and returns the entry's type + payload. The per-LSN term is NOT read
|
||
* from the on-disk header here — the caller takes it from persist_entry_term
|
||
* (the authoritative in-memory term index), per the library contract that the
|
||
* wire entry's term == the term recorded for that LSN. `payload` is malloc'd and
|
||
* owned by the caller (freed after the send). Returns 0 on success, negative on
|
||
* absence/read error. */
|
||
static int ha_read_wal_entry_at(const char *wal_path, uint64_t lsn,
|
||
uint8_t *type_out, uint8_t **payload_out,
|
||
uint32_t *plen_out)
|
||
{
|
||
*type_out = 0;
|
||
*payload_out = NULL;
|
||
*plen_out = 0;
|
||
|
||
DIR *d = opendir(wal_path);
|
||
if (!d)
|
||
return -errno ? -errno : -EIO;
|
||
|
||
/* Enumerate segment start-LSNs, descending, so we open the latest segment
|
||
* whose start LSN is <= the target first. */
|
||
uint64_t seg_lsns[4096];
|
||
int nseg = 0;
|
||
struct dirent *de;
|
||
while ((de = readdir(d)) != NULL &&
|
||
nseg < (int)(sizeof(seg_lsns) / sizeof(*seg_lsns))) {
|
||
uint64_t sl = 0;
|
||
if (sscanf(de->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
|
||
seg_lsns[nseg++] = sl;
|
||
}
|
||
closedir(d);
|
||
if (nseg == 0)
|
||
return -ENOENT;
|
||
|
||
/* insertion sort ascending */
|
||
for (int i = 1; i < nseg; i++) {
|
||
uint64_t key = seg_lsns[i];
|
||
int j = i - 1;
|
||
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
|
||
seg_lsns[j + 1] = key;
|
||
}
|
||
|
||
int rc = -ENOENT;
|
||
for (int si = 0; si < nseg; si++) {
|
||
/* Skip a segment whose successor already starts past the target. */
|
||
if (si + 1 < nseg && seg_lsns[si + 1] <= lsn)
|
||
continue;
|
||
if (seg_lsns[si] > lsn)
|
||
break;
|
||
|
||
char seg_path[4096];
|
||
snprintf(seg_path, sizeof(seg_path),
|
||
"%s/seg-%016" PRIx64 ".wal", wal_path, seg_lsns[si]);
|
||
int fd = open(seg_path, O_RDONLY);
|
||
if (fd < 0)
|
||
continue;
|
||
|
||
uint32_t off = 0;
|
||
for (;;) {
|
||
wal_entry_hdr_t hdr;
|
||
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
|
||
if (n < (ssize_t)sizeof(hdr)) break;
|
||
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
|
||
|
||
if (hdr.lsn == lsn) {
|
||
*type_out = (uint8_t)hdr.type;
|
||
if (hdr.payload_len > 0) {
|
||
uint8_t *pbuf = malloc(hdr.payload_len);
|
||
if (!pbuf) { rc = -ENOMEM; }
|
||
else {
|
||
ssize_t pr = pread(fd, pbuf, hdr.payload_len,
|
||
(off_t)(off + sizeof(hdr)));
|
||
if (pr == (ssize_t)hdr.payload_len) {
|
||
*payload_out = pbuf;
|
||
*plen_out = hdr.payload_len;
|
||
rc = 0;
|
||
} else {
|
||
free(pbuf);
|
||
rc = -EIO;
|
||
}
|
||
}
|
||
} else {
|
||
rc = 0; /* zero-payload entry (e.g. WAL_NOOP) */
|
||
}
|
||
close(fd);
|
||
return rc;
|
||
}
|
||
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
|
||
}
|
||
close(fd);
|
||
}
|
||
return rc;
|
||
}
|
||
|
||
/* on_send_append_entries (ha-term-safety task 2.3, R1.3/R3.1): transmit ONE
|
||
* AppendEntries to the standby at slot `peer_idx`, carrying the prev tuple
|
||
* (prev_log_index, prev_log_term) + leader_commit, and — for n_entries==1 — the
|
||
* single entry at LSN prev_log_index+1 sourced from this leader's OWN WAL (the
|
||
* library holds no entry bytes). The entry's term is taken from
|
||
* persist_entry_term (the authoritative per-LSN term index), so the standby
|
||
* persists the LEADER's recorded term, not a substitute. n_entries==0 emits a
|
||
* heartbeat (zero entries) carrying only the prev tuple + leader_commit.
|
||
*
|
||
* Non-blocking (ha-async-io): delegates to repl_send_append_entries which
|
||
* enqueues via the async repl transport — never blocks in the callback.
|
||
* peer_idx is the library's raft peer index, mapped 1:1 to the repl standby
|
||
* registry slot (the same index match_index/next_index use). */
|
||
static int ha_cb_send_append_entries_stub(void *ctx, int peer_idx,
|
||
uint64_t prev_log_index,
|
||
uint64_t prev_log_term,
|
||
uint64_t leader_commit, int n_entries)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
if (!ha || !ha->raft_c)
|
||
return -EINVAL; /* test mode without a cluster: cannot send */
|
||
|
||
persist_t *p = &ha->raft_c->persist;
|
||
replicator_t *r = ha->repl ? ha->repl : &p->repl;
|
||
|
||
if (n_entries == 0) {
|
||
/* Heartbeat: prev tuple + leader_commit, no entry. */
|
||
return repl_send_append_entries(r, peer_idx, prev_log_index,
|
||
prev_log_term, leader_commit,
|
||
/*n_entries*/0,
|
||
/*entry_lsn*/0, /*type*/0, /*term*/0,
|
||
NULL, 0);
|
||
}
|
||
|
||
/* n_entries == 1: source the entry at prev_log_index+1 from the local WAL. */
|
||
uint64_t entry_lsn = prev_log_index + 1;
|
||
uint64_t entry_term = persist_entry_term(p, entry_lsn);
|
||
|
||
uint8_t type = 0;
|
||
uint8_t *payload = NULL;
|
||
uint32_t plen = 0;
|
||
int rc = ha_read_wal_entry_at(p->wal_path, entry_lsn, &type, &payload, &plen);
|
||
if (rc != 0) {
|
||
WARN("on_send_append_entries: cannot source local WAL entry at lsn=%lu "
|
||
"(rc=%d) — not sending to peer %d",
|
||
(unsigned long)entry_lsn, rc, peer_idx);
|
||
return rc;
|
||
}
|
||
|
||
rc = repl_send_append_entries(r, peer_idx, prev_log_index, prev_log_term,
|
||
leader_commit, /*n_entries*/1,
|
||
entry_lsn, type, entry_term, payload, plen);
|
||
free(payload);
|
||
return rc;
|
||
}
|
||
|
||
/* on_truncate_from (ha-term-safety task 2.2, R1.8/R3.2): delete the local log
|
||
* entries at LSN `index..end` because they conflict with the leader's log at a
|
||
* (term,lsn) the library detected. persist_truncate_wal_above(p, cutoff) removes
|
||
* every entry with LSN > cutoff, so to drop LSN >= index we pass cutoff =
|
||
* index - 1. This REPLACES the committed-boundary truncate heuristic on the
|
||
* library-driven path: the library, not a watermark, decides the cut point.
|
||
*
|
||
* Returns the truncate rc. A non-zero rc propagates so the library/leader
|
||
* retries; the member stays non-promotable per R3.5 (that fence is existing
|
||
* behavior and is not touched here). persist_truncate_wal_above also shrinks
|
||
* the per-LSN term index for the removed LSNs, so a later on_get_entry_term on
|
||
* a truncated LSN reads back 0 (absent). */
|
||
static int ha_cb_truncate_from_stub(void *ctx, uint64_t index)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
if (!ha || !ha->raft_c)
|
||
return -EINVAL; /* test mode without a cluster: cannot truncate */
|
||
if (index == 0)
|
||
return -EINVAL; /* LSN 0 is reserved — never an entry to truncate */
|
||
return persist_truncate_wal_above(&ha->raft_c->persist, index - 1);
|
||
}
|
||
|
||
/* on_commit_advanced (ha-term-safety task 2.4, R4.1/R4.5/R5.4): feed the raft
|
||
* library's current-term-gated `commit_index` into the existing synchronous
|
||
* commit barrier as the commit SOURCE. The library fired this ONLY on a strict
|
||
* advance of its Figure-8 current-term commit index (raft.c), so the barrier
|
||
* now advances committed_lsn only when BOTH a durable majority agrees AND the
|
||
* library has admitted the entry — the term-blind safe-commit
|
||
* (ha_compute_safe_committed_lsn) is no longer the steady-state commit source.
|
||
*
|
||
* This callback only TRIGGERS the barrier (records the ceiling + re-evaluates);
|
||
* it performs NO durability itself. The barrier mechanism — majority self-cap at
|
||
* the durable floor, persist-before-ACK, monotonic committed_lsn — is unchanged
|
||
* (ha-commit-durability). The own-term writable_lsn gate is RETAINED as a
|
||
* complementary WAIT-completion predicate (it never caps committed_lsn, so it
|
||
* cannot deadlock against this source; see ha_arm_own_term_gate).
|
||
*
|
||
* Non-blocking (ha-async-io): commit_set_raft_commit_index takes only ce->lock
|
||
* and never performs I/O — safe on the iomgr ACK path that drove
|
||
* raft_advance_commit. */
|
||
static void ha_cb_commit_advanced_stub(void *ctx, uint64_t commit_index)
|
||
{
|
||
ha_state_t *ha = (ha_state_t *)ctx;
|
||
if (!ha || !ha->raft_c)
|
||
return; /* test mode without a cluster: no barrier to trigger */
|
||
commit_set_raft_commit_index(&ha->raft_c->commit, commit_index);
|
||
|
||
/* ha-term-safety task 4.3 (remediation r1, Finding B): a config marker may
|
||
* have just entered the committed prefix. Activation is NOT performed HERE:
|
||
* this callback runs on the iomgr ACK path, where the async-I/O invariant
|
||
* forbids the disk reads config_activate_committed performs
|
||
* (scan_latest_committed_marker pread + load_primary_config). The committed
|
||
* marker is instead consumed off the iomgr thread by the call sites that make
|
||
* §7.10 true in production:
|
||
* - boot/recovery (myapp.c step 7): activates the latest committed marker,
|
||
* hash-verified, after persist recovery.
|
||
* - handle_repl_config_push: after the out-of-band bytes are saved, gates
|
||
* activation on the committed marker (verify-before-activate).
|
||
*
|
||
* ha-term-safety task 4.3 (remediation r2, Finding D / R7.9): the two sites
|
||
* above are NOT sufficient when the BYTES arrive (and are saved) BEFORE the
|
||
* NEW marker commits — the config MARKER (WAL) and the config BYTES (out-of-
|
||
* band push) travel on INDEPENDENT transports. In that ordering the push path
|
||
* hash-checks the freshly-saved NEW bytes against the still-OLDER committed
|
||
* marker, MISMATCHES, and sets the serve-fence; when the NEW marker later
|
||
* commits HERE, nothing re-runs activation (this callback can't do disk I/O,
|
||
* and the primary's config-behind re-push net does not fire because the push
|
||
* already advanced this member's advertised config_version_tag) — so the
|
||
* standby would stay PERMANENTLY fenced. Close the gap: while serve-fenced,
|
||
* schedule an OFF-IOMGR delaymgr task (config_schedule_activate_retry, NON-
|
||
* BLOCKING ordq_put only) that re-runs config_activate_committed on a thread
|
||
* allowed to block on disk; once the NEW marker is in the committed prefix and
|
||
* the saved bytes verify, the retry clears the fence and activates. No-op once
|
||
* unfenced (no busy-spin); a still-genuine mismatch reschedules on the next
|
||
* commit advance. */
|
||
if (config_serve_fenced(ha->raft_c))
|
||
config_schedule_activate_retry(ha->raft_c);
|
||
}
|
||
|
||
static const raft_callbacks_t s_real_cbs = {
|
||
.on_send_vote_request = raft_cb_send_vote_request,
|
||
.on_send_pre_vote_request = raft_cb_send_pre_vote_request,
|
||
.on_vote_eligible = raft_cb_vote_eligible,
|
||
.on_become_leader = raft_cb_become_leader,
|
||
.on_become_follower = raft_cb_become_follower,
|
||
.on_persist = raft_cb_persist,
|
||
.on_schedule_timeout = raft_cb_schedule_timeout,
|
||
.on_cancel_timeout = raft_cb_cancel_timeout,
|
||
.on_start_election_after_delay = raft_cb_start_election_after_delay,
|
||
/* ha-term-safety: MINIMAL placeholders to satisfy raft_init's non-NULL
|
||
* requirement — REPLACE in task 2.x (see stubs + CONCERNS). */
|
||
.on_persist_entry = ha_cb_persist_entry_stub,
|
||
.on_get_entry_term = ha_cb_get_entry_term_stub,
|
||
.on_send_append_entries = ha_cb_send_append_entries_stub,
|
||
.on_truncate_from = ha_cb_truncate_from_stub,
|
||
.on_commit_advanced = ha_cb_commit_advanced_stub,
|
||
.on_trace = raft_cb_trace,
|
||
};
|
||
|
||
/* ── PRNG seeding ───────────────────────────────────────────── */
|
||
|
||
void ha_seed_jitter(uint64_t *seed, int my_priority)
|
||
{
|
||
uint8_t buf[8] = {0};
|
||
int fd = open("/dev/urandom", O_RDONLY);
|
||
if (fd >= 0) {
|
||
ssize_t n = 0;
|
||
while (n < 8) {
|
||
ssize_t r = read(fd, buf + n, (size_t)(8 - n));
|
||
if (r <= 0) break;
|
||
n += r;
|
||
}
|
||
close(fd);
|
||
if (n == 8) {
|
||
memcpy(seed, buf, 8);
|
||
if (*seed != 0) return;
|
||
}
|
||
}
|
||
struct timespec ts;
|
||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||
*seed = (uint64_t)ts.tv_nsec
|
||
^ ((uint64_t)my_priority << 56)
|
||
^ ((uint64_t)getpid() << 16);
|
||
}
|
||
|
||
int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority)
|
||
{
|
||
if (!ha || !cfg)
|
||
return -EINVAL;
|
||
|
||
/* Zero first so ha_fini() is safe even if validation fails below */
|
||
memset(ha, 0, sizeof(*ha));
|
||
|
||
if (cfg->heartbeat_interval_ms < 2000)
|
||
return -EINVAL;
|
||
if (cfg->heartbeat_timeout_ms < cfg->heartbeat_interval_ms)
|
||
return -EINVAL;
|
||
if (cfg->election_timeout_ms < 500)
|
||
return -EINVAL;
|
||
ha->config = *cfg;
|
||
ha->my_priority = my_priority;
|
||
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
|
||
atomic_store_explicit(&ha->applied_lsn, 0ULL, memory_order_release);
|
||
atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed);
|
||
atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed);
|
||
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed);
|
||
/* failover_requested / watchdog_stop deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */
|
||
|
||
pthread_mutex_init(&ha->lock, NULL);
|
||
|
||
ha->ssl_ctx = NULL;
|
||
ha->jitter_seed = 0;
|
||
ha->standby_endpoints = NULL;
|
||
ha->primary_ctrl_fd = -1;
|
||
|
||
/* election-iomgr-refactor Task 2.1: explicit zero-init for documentation clarity
|
||
* (memset above already zeroes these, but being explicit is better documentation) */
|
||
ha->client_ssl_ctx = NULL;
|
||
ha->phase2_done = 0;
|
||
ha->n_peers = 0;
|
||
|
||
/* ha-heartbeat-liveness Task 1.1: zero-init liveness timestamp.
|
||
* Written by ha_on_heartbeat_received (task 2.1); read by liveness_poll_fn. */
|
||
ha->last_activity = 0;
|
||
|
||
/* ha-repl-hardening Task 1.4: per-standby convergence tracking arrays.
|
||
* memset(ha, 0, sizeof(*ha)) above already zeroes these arrays; the explicit
|
||
* note documents the invariant: on promotion to CTRL_PRIMARY every
|
||
* standby starts as not-converged (tag=0, lsn=0, converged=false). */
|
||
|
||
/* heartbeat_seen starts at 0: no heartbeat seen yet.
|
||
* ha_wait_for_heartbeat_or_timeout() calls sem_timedwait on this. */
|
||
sem_init(&ha->heartbeat_seen, 0, 0);
|
||
|
||
/* ── Task 6.1/6.2: Initialize embedded raft_state_t ─────────────────────
|
||
*
|
||
* Wire real callbacks (s_real_cbs) so the state machine is ready when
|
||
* ha_start_election_async re-initialises raft with the correct n_peers /
|
||
* self_idx / initial_term after ha_restore_term().
|
||
*
|
||
* n_peers = 1 and self_idx = 0 are safe placeholders here; they are
|
||
* overwritten by ha_raft_reinit() (called from ha_start_election_async)
|
||
* which runs after ha_restore_term() and knows the real cluster size.
|
||
*
|
||
* initial_term = 0 / initial_voted_for = -1 are first-boot defaults;
|
||
* ha_raft_reinit() restores the persisted values before any election fires.
|
||
*
|
||
* ha->raft_c is NULL until ha_start_election_async sets it.
|
||
*/
|
||
ha->raft_c = NULL;
|
||
ha->raft_election_timeout_ctx = NULL;
|
||
|
||
raft_config_t raft_cfg;
|
||
memset(&raft_cfg, 0, sizeof(raft_cfg));
|
||
raft_cfg.n_peers = 1; /* placeholder; overwritten by ha_raft_reinit */
|
||
raft_cfg.self_idx = my_priority; /* self_idx == priority by convention */
|
||
raft_cfg.election_timeout_ms = cfg->election_timeout_ms;
|
||
raft_cfg.my_priority = my_priority;
|
||
raft_cfg.initial_term = 0; /* ha_restore_term() updates ha->term after ha_init */
|
||
raft_cfg.initial_voted_for = -1; /* no vote on fresh init */
|
||
raft_cfg.cbs = &s_real_cbs;
|
||
raft_cfg.ctx = ha;
|
||
|
||
raft_init(&ha->raft, &raft_cfg);
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* SIGUSR1 handler / ha_install_sigusr1() deleted — failover_requested replaced by
|
||
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
|
||
|
||
void ha_do_promote(ha_state_t *ha)
|
||
{
|
||
/* Monotonically increasing fencing token — standbys reject entries from lower terms */
|
||
atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel);
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
||
/* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */
|
||
ha->last_received_crc = WAL_CRC_NONE;
|
||
/* Req 10.5: log FSM transition to PRIMARY */
|
||
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")",
|
||
atomic_load_explicit(&ha->term, memory_order_acquire));
|
||
}
|
||
|
||
void ha_do_demote(ha_state_t *ha)
|
||
{
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||
/* Reset so the next ha_promote_to_primary() re-registers hooks idempotently */
|
||
ha->hooks_registered = 0;
|
||
/* Hook disable requires hooksys_t — done by ha_demote_to_standby(cluster_t*) */
|
||
/* Req 10.5: log FSM transition to STANDBY */
|
||
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ")",
|
||
atomic_load_explicit(&ha->term, memory_order_acquire));
|
||
}
|
||
|
||
/* probe_thread() / wait_any_probe() deleted — only called by am_i_primary() (election-iomgr-refactor) */
|
||
|
||
/* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */
|
||
|
||
/*
|
||
* ha_is_primary: lock-free role check.
|
||
*/
|
||
int ha_is_primary(const ha_state_t *ha)
|
||
{
|
||
return (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire)
|
||
== CTRL_PRIMARY;
|
||
}
|
||
|
||
/* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */
|
||
/* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */
|
||
|
||
/* ── P3-6: Test-only failover shim ────────────────────────────── */
|
||
|
||
#ifdef XCE_TESTING
|
||
uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha)
|
||
{
|
||
ha_do_demote(ha);
|
||
ha_do_promote(ha);
|
||
return atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
}
|
||
|
||
/* ha-term-safety task 2.1: drive the WIRED on_persist_entry callback so a unit
|
||
* test exercises the real adapter body (term stamp + WAL write) without
|
||
* raft_log_append (task 3.1). Exposed only under XCE_TESTING. */
|
||
int ha_cb_persist_entry_for_test(ha_state_t *ha, uint64_t index, uint64_t term,
|
||
const void *entry, size_t len)
|
||
{
|
||
return s_real_cbs.on_persist_entry(ha, index, term, entry, len);
|
||
}
|
||
|
||
/* ha-term-safety task 2.2: drive the WIRED on_get_entry_term callback so a unit
|
||
* test exercises the real entry-term accessor. Exposed only under XCE_TESTING. */
|
||
uint64_t ha_cb_get_entry_term_for_test(ha_state_t *ha, uint64_t index)
|
||
{
|
||
return s_real_cbs.on_get_entry_term(ha, index);
|
||
}
|
||
|
||
/* ha-term-safety task 2.2: drive the WIRED on_truncate_from callback so a unit
|
||
* test exercises the real conflict truncation. Exposed only under XCE_TESTING. */
|
||
int ha_cb_truncate_from_for_test(ha_state_t *ha, uint64_t index)
|
||
{
|
||
return s_real_cbs.on_truncate_from(ha, index);
|
||
}
|
||
|
||
/* ha-term-safety task 2.3: drive the WIRED on_send_append_entries callback so a
|
||
* unit test exercises the real AppendEntries send body (peer→ep map, prev tuple
|
||
* + leader_commit carry, local-WAL entry sourcing). Pair with
|
||
* repl_send_ae_capture_hook to assert envelope content. Exposed only under
|
||
* XCE_TESTING. */
|
||
int ha_cb_send_append_entries_for_test(ha_state_t *ha, int peer_idx,
|
||
uint64_t prev_log_index,
|
||
uint64_t prev_log_term,
|
||
uint64_t leader_commit, int n_entries)
|
||
{
|
||
return s_real_cbs.on_send_append_entries(ha, peer_idx, prev_log_index,
|
||
prev_log_term, leader_commit,
|
||
n_entries);
|
||
}
|
||
|
||
/* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback so a unit
|
||
* test exercises the real commit-source trigger (feed the library commit_index
|
||
* into the cluster's commit barrier). Exposed only under XCE_TESTING. */
|
||
void ha_cb_commit_advanced_for_test(ha_state_t *ha, uint64_t commit_index)
|
||
{
|
||
s_real_cbs.on_commit_advanced(ha, commit_index);
|
||
}
|
||
|
||
/* ha-term-safety task 3.1: drive the (static) own-term gate arm — the WAL_NOOP
|
||
* append now routed through ha_primary_log_append — so a unit test can assert
|
||
* the promotion no-op records the live term via the library. Exposed only under
|
||
* XCE_TESTING. */
|
||
void ha_arm_own_term_gate_for_test(cluster_t *c)
|
||
{
|
||
ha_arm_own_term_gate(c);
|
||
}
|
||
#endif
|
||
|
||
/* ha_check_heartbeat_nocluster() / ha_watchdog_thread() / ha_start_watchdog() /
|
||
* ha_stop_watchdog() deleted — watchdog pthread replaced by
|
||
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
|
||
|
||
void ha_fini(ha_state_t *ha)
|
||
{
|
||
if (!ha)
|
||
return;
|
||
|
||
/* ha-commit-durability task 4.4: cancel any pending bounded-window FATAL
|
||
* timer so a stopped member never reaches the deferred FATAL. The delaymgr
|
||
* task ctx is freed by promotion_fence_timeout_fn when it fires; cancelling
|
||
* makes that firing a no-op (it observes cancelled=1 and frees). */
|
||
ha_cancel_promotion_fence_timer(ha);
|
||
|
||
pthread_mutex_destroy(&ha->lock);
|
||
sem_destroy(&ha->heartbeat_seen);
|
||
|
||
/* Fix ASAN leak: free the SSL context if one was allocated */
|
||
if (ha->ssl_ctx) {
|
||
SSL_CTX_free((SSL_CTX *)ha->ssl_ctx);
|
||
ha->ssl_ctx = NULL;
|
||
}
|
||
}
|
||
|
||
/* ── Task 2.1/2.2: WAL catch-up entry application (standby side) ─────────────
|
||
*
|
||
* ha_apply_wal_catchup_entry: apply WAL catch-up entries received from the
|
||
* primary to local persistent storage and advance applied_lsn on success.
|
||
*
|
||
* applied_lsn semantic (bug-991): the LAST CONTIGUOUSLY-HELD LSN — the same
|
||
* meaning recovery (= durable_floor), checkpoint (= checkpoint_lsn), and init
|
||
* (= 0) assign it, and the watermark the standby advertises to the leader
|
||
* (sr->applied_lsn) so the leader's catch-up streamer streams the half-open
|
||
* range (applied_lsn, committed]. NEVER lsn+1 (next-to-apply): a +1 makes the
|
||
* standby advertise one LSN too high, so the leader skips the real next-needed
|
||
* entry and the standby's library logOk wedges forever (no WalAck → commit stall).
|
||
*
|
||
* Two modes (Req 5.3, 5.5):
|
||
*
|
||
* Batch mode (entry->n_entries > 0):
|
||
* Iterate entry->entries[0..n_entries-1] in order. Apply each sub-entry
|
||
* individually via apply_wal_entry(). NULL sub-entries are treated as
|
||
* corrupt and skipped (Req 3.3). After ALL sub-entries are processed,
|
||
* advance applied_lsn to the last-successfully-applied lsn so the next
|
||
* heartbeat reply reflects post-batch state (Req 5.3).
|
||
*
|
||
* Single-entry mode (entry->n_entries == 0):
|
||
* Backward-compatible path from task 2.1. Apply entry->lsn/entry_type
|
||
* directly and advance applied_lsn to lsn on success.
|
||
*
|
||
* In both modes:
|
||
* - applied_lsn is advanced atomically and monotonically.
|
||
* - On error for any individual entry: log ERROR, skip, continue (Req 3.3).
|
||
* - applied_lsn is in-memory only; not persisted to disk (design §Durability).
|
||
* - Standby stays in CTRL_STANDBY throughout (Req 3.4).
|
||
*
|
||
* Returns 0 on success (even if some batch entries were skipped due to errors).
|
||
* Returns -EINVAL if ha, p, or entry is NULL.
|
||
*/
|
||
|
||
/* ha_advance_applied_lsn: atomically advance ha->applied_lsn to new_lsn
|
||
* if new_lsn > current value (monotonic). */
|
||
static void ha_advance_applied_lsn(ha_state_t *ha, uint64_t new_lsn)
|
||
{
|
||
uint64_t old_lsn = (uint64_t)atomic_load_explicit(&ha->applied_lsn,
|
||
memory_order_acquire);
|
||
while (new_lsn > old_lsn) {
|
||
if (atomic_compare_exchange_weak_explicit(
|
||
&ha->applied_lsn, &old_lsn, new_lsn,
|
||
memory_order_acq_rel, memory_order_acquire))
|
||
break;
|
||
}
|
||
}
|
||
|
||
/* ha_apply_single_entry: apply one decoded wal_catchup_entry_t sub-entry.
|
||
* Payload bytes (raw_payload/raw_payload_len, extracted by xce_decode)
|
||
* are passed through to apply_wal_entry so entity state is restored —
|
||
* not just the LSN chain (task 7.3).
|
||
* Returns 0 on success, non-zero on error (caller logs and skips). */
|
||
static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e)
|
||
{
|
||
/* Persist-before-apply-before-ACK: durably write the replicated entry to
|
||
* this standby's own WAL BEFORE applying it in memory (and before the
|
||
* caller ACKs). The ACK feeds the primary's commit majority, so it must
|
||
* imply durability — a standby must not ACK an entry it could lose on
|
||
* restart (Raft log[i] includes standbys; ha-async-io.md).
|
||
*
|
||
* persist_wal_write_replicated does NOT assign a new lsn (uses e->lsn) and
|
||
* does NOT re-broadcast; it fsyncs before returning and is idempotent on
|
||
* re-stream. WAL_CHECKPOINT is never streamed, but it would be rejected
|
||
* by both this primitive and apply_wal_entry. */
|
||
int prc = persist_wal_write_replicated(p, e->lsn, (uint32_t)e->entry_type,
|
||
e->raw_payload,
|
||
(uint32_t)e->raw_payload_len);
|
||
if (prc == -EAGAIN) {
|
||
/* Contiguity guard (bug-850): this entry is ahead of our log tail
|
||
* (a forward gap) — e.g. a live broadcast that arrived before the
|
||
* in-order catch-up of the prefix. NOT an error: drop it quietly and do
|
||
* NOT ack, so the catch-up continuation re-streams from current_lsn and
|
||
* the log fills in order. */
|
||
TRACE("ha_apply_single_entry: lsn=%" PRIu64 " ahead of log tail — "
|
||
"dropped (will arrive via in-order catch-up)", e->lsn);
|
||
return prc;
|
||
}
|
||
if (prc != 0) {
|
||
ERROR("ha_apply_single_entry: persist_wal_write_replicated failed for "
|
||
"lsn=%" PRIu64 " type=%u rc=%d — not applying/acking",
|
||
e->lsn, (unsigned)e->entry_type, prc);
|
||
/* No apply, no ACK on persist failure (mirrors no-ACK-on-apply-fail). */
|
||
return prc;
|
||
}
|
||
|
||
/* Logical entity restore only. persist_wal_write_replicated (above) already
|
||
* wrote the entry and owns the on-disk CRC chain + current_lsn — so we pass
|
||
* the entry fields directly rather than fabricating a header. (bug-778:
|
||
* a synthetic crc=0 header here used to clobber p->last_crc.) */
|
||
return apply_wal_entry(p, (uint32_t)e->entry_type, e->lsn,
|
||
e->raw_payload, (uint32_t)e->raw_payload_len);
|
||
}
|
||
|
||
/* ha_send_wal_ack_reply: send WalAckReply {term, lsn, success=1} back to
|
||
* the primary over ep. Skipped when ep or c is NULL (test/offline path).
|
||
* Task 3.1 (ha-wal-replication): ACK each successfully applied WAL entry. */
|
||
static void ha_send_wal_ack_reply(ha_state_t *ha, struct endpoint_t *ep,
|
||
cluster_t *c, uint64_t lsn)
|
||
{
|
||
if (!ep || !c)
|
||
return;
|
||
|
||
wal_ack_reply_t ack;
|
||
ack.term = (uint64_t)atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
ack.lsn = lsn;
|
||
ack.success = 1;
|
||
|
||
xce_reply_t reply;
|
||
memset(&reply, 0, sizeof(reply));
|
||
reply.opcode = OPCode_walAck;
|
||
reply.status = 0;
|
||
reply.data_which = RequestReply_data_walackreplymsg;
|
||
reply.data.walackreplymsg = &ack;
|
||
|
||
xce_reply(ep, c->myhost, &reply);
|
||
}
|
||
|
||
/* ha_apply_term_entry_via_library: route ONE term-carrying replicated entry
|
||
* (a live broadcast or a catch-up sub-entry) through the raft library
|
||
* (raft_on_append_entries_request via ha_consume_append_entries) so the LIBRARY
|
||
* does logOk + (term,lsn) reconcile and on_persist_entry stamps the LEADER's
|
||
* term — never the local term-0 direct-apply path (ha-term-safety task 3.4).
|
||
*
|
||
* `e` is the single entry (its term/prev_log_index/prev_log_term carry the AE
|
||
* metadata, set by the leader's broadcast/stream path). On accept (the entry's
|
||
* term is now recorded at its LSN) the caller ACKs and advances applied. Returns
|
||
* 1 if the entry was accepted (recorded), 0 otherwise (reject / not yet
|
||
* applicable — e.g. a logOk-failing forward gap that catch-up will backfill). */
|
||
static int ha_apply_term_entry_via_library(ha_state_t *ha, persist_t *p,
|
||
cluster_t *c, struct endpoint_t *ep,
|
||
wal_catchup_entry_t *e)
|
||
{
|
||
/* Wrap the single entry in an AE container whose TOP-LEVEL fields carry the
|
||
* prev tuple + term that ha_consume_append_entries reads, with entries[0]
|
||
* pointing at the entry itself (the same shape repl_send_append_entries /
|
||
* repl_broadcast produce on the wire). */
|
||
wal_catchup_entry_t *sub_arr[1] = { e };
|
||
wal_catchup_entry_t ae;
|
||
memset(&ae, 0, sizeof(ae));
|
||
ae.n_entries = 1;
|
||
ae.entries = sub_arr;
|
||
ae.term = e->term;
|
||
ae.prev_log_index = e->prev_log_index;
|
||
ae.prev_log_term = e->prev_log_term;
|
||
ae.leader_commit = e->leader_commit;
|
||
|
||
uint64_t before_term = persist_entry_term(p, e->lsn);
|
||
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/ -1);
|
||
if (rc != 0)
|
||
return 0;
|
||
|
||
uint64_t after_term = persist_entry_term(p, e->lsn);
|
||
if (after_term == e->term) {
|
||
/* Accepted (appended / conflict-truncate-then-persist) or already-have
|
||
* (idempotent re-stream): ACK + advance applied on that signal.
|
||
*
|
||
* bug-991: applied_lsn is the "last contiguously-held LSN" (the same
|
||
* semantic recovery/checkpoint/init use, and the watermark the leader's
|
||
* catch-up streamer reads via sr->applied_lsn to stream (standby_lsn,
|
||
* committed]). Advance to e->lsn (the LSN we just held), NOT e->lsn + 1:
|
||
* a +1 over-advertises one LSN, so the leader's next stream begins at
|
||
* e->lsn + 2 and SKIPS the real next-needed entry e->lsn + 1, wedging
|
||
* logOk and stalling commit. */
|
||
(void)before_term;
|
||
|
||
/* bug-1043: the library route (on_persist_entry → persist_wal_write_
|
||
* replicated) writes the entry to the WAL but does NOT apply it to the
|
||
* in-memory entity state (job/queue/worker tables). The legacy direct-
|
||
* apply path (ha_apply_single_entry) called apply_wal_entry after the
|
||
* persist; the term-safe library route dropped it. So a standby that
|
||
* caught up entries 8..N via replication held them on disk but NOT in the
|
||
* queryable job table — on promotion `joblist`/`expect-job` could not
|
||
* find a committed job (it IS on disk; walcompare passes), looking like a
|
||
* committed-job loss. Re-apply here, mirroring the legacy path. Idempotent
|
||
* on a re-stream (restore_job upserts by jobid). WAL_NOOP / WAL_CONFIG_
|
||
* MARKER carry no entity payload — apply_wal_entry no-ops on them. */
|
||
apply_wal_entry(p, (uint32_t)e->entry_type, e->lsn,
|
||
e->raw_payload, (uint32_t)e->raw_payload_len);
|
||
|
||
ha_advance_applied_lsn(ha, e->lsn);
|
||
ha_send_wal_ack_reply(ha, ep, c, e->lsn);
|
||
return 1;
|
||
}
|
||
/* logOk reject (e.g. forward gap before the prefix arrives): no ACK; the
|
||
* ACK-driven catch-up continuation re-streams from current_lsn in order. */
|
||
return 0;
|
||
}
|
||
|
||
int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p,
|
||
wal_catchup_entry_t *entry,
|
||
struct endpoint_t *ep, cluster_t *c)
|
||
{
|
||
if (!ha || !p || !entry)
|
||
return -EINVAL;
|
||
|
||
/* ── ha-term-safety task 3.2: term-carrying AppendEntries → library route ──
|
||
* A term-safe AppendEntries (one sent by repl_send_append_entries / the
|
||
* library send path) carries a non-zero entry term stamp. Route it through
|
||
* raft_on_append_entries_request so the LIBRARY performs UpdateTerm (G5),
|
||
* logOk (G2), and the (term,lsn) conflict-truncate (G3) — its on_persist_entry
|
||
* callback does the WAL write (persist-before-ACK + bug-850 guard beneath).
|
||
* This REPLACES the direct persist_wal_write_replicated apply for term-safe
|
||
* entries; the legacy term-blind broadcast (entry term 0, no prev tuple) keeps
|
||
* the direct apply below until the live send path is retired (task 3.3).
|
||
*
|
||
* Single-entry term-safe AE only (n_entries==1 with a term). The library
|
||
* computes the target index from the prev tuple, then ACK on accept. */
|
||
if (c && c->ha.raft_c == c && entry->n_entries == 1 &&
|
||
entry->entries && entry->entries[0] && entry->entries[0]->term != 0) {
|
||
ha_apply_term_entry_via_library(ha, p, c, ep, entry->entries[0]);
|
||
return 0;
|
||
}
|
||
|
||
/* ── Batch mode: entry->n_entries > 0 ─────────────────────────────────── */
|
||
if (entry->n_entries > 0) {
|
||
/* Apply each sub-entry in LSN order. Track the last successfully
|
||
* applied LSN so we can advance applied_lsn after the full batch.
|
||
* Req 5.5: applied_lsn advances only after the full batch is written.
|
||
* Req 3.3: NULL or failed entries are logged and skipped; no crash. */
|
||
uint64_t last_good_lsn = 0;
|
||
int had_good = 0;
|
||
|
||
for (int i = 0; i < entry->n_entries; i++) {
|
||
wal_catchup_entry_t *sub = entry->entries[i];
|
||
if (!sub) {
|
||
ERROR("ha_apply_wal_catchup_entry: NULL sub-entry at "
|
||
"batch index %d — skipping", i);
|
||
continue;
|
||
}
|
||
|
||
/* ha-term-safety task 3.4: a term-carrying catch-up sub-entry routes
|
||
* through the LIBRARY (logOk + (term,lsn) reconcile + on_persist_entry
|
||
* stamps the LEADER's term). Each sub-entry carries its own prev tuple
|
||
* (wal_entry_to_wire encodes prevLog{Index,Term} per element). This
|
||
* retires the term-0 direct-apply path for live/catch-up replication.
|
||
* Entries stream in ascending LSN order, so the library's contiguity
|
||
* (prev tuple) is satisfied in sequence; a not-yet-applicable entry
|
||
* (logOk reject) is left for the catch-up continuation. */
|
||
if (c && c->ha.raft_c == c && sub->term != 0) {
|
||
if (ha_apply_term_entry_via_library(ha, p, c, ep, sub)) {
|
||
last_good_lsn = sub->lsn;
|
||
had_good = 1;
|
||
}
|
||
continue;
|
||
}
|
||
|
||
int rc = ha_apply_single_entry(p, sub);
|
||
if (rc != 0) {
|
||
/* -EAGAIN = contiguity gap (bug-850): entry ahead of our tail,
|
||
* dropped quietly — not an error (catch-up will backfill). */
|
||
if (rc != -EAGAIN)
|
||
ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed "
|
||
"for batch index %d lsn=%" PRIu64 " type=%u rc=%d "
|
||
"— skipping entry",
|
||
i, sub->lsn, (unsigned)sub->entry_type, rc);
|
||
/* Task 3.2: no ACK for failed/dropped entries */
|
||
continue;
|
||
}
|
||
|
||
/* Task 3.1: ACK each successfully applied sub-entry */
|
||
ha_send_wal_ack_reply(ha, ep, c, sub->lsn);
|
||
|
||
last_good_lsn = sub->lsn;
|
||
had_good = 1;
|
||
}
|
||
|
||
/* Advance applied_lsn to last_good_lsn after the full batch (bug-991:
|
||
* last contiguously-held LSN, NOT last_good_lsn + 1 — see the
|
||
* ha_apply_term_entry_via_library comment for why the +1 wedged the
|
||
* leader's catch-up streamer). Only advance if at least one entry was
|
||
* applied successfully. */
|
||
if (had_good)
|
||
ha_advance_applied_lsn(ha, last_good_lsn);
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* ── Single-entry mode (task 2.1 backward-compatible path) ────────────── */
|
||
int rc = ha_apply_single_entry(p, entry);
|
||
if (rc != 0) {
|
||
/* -EAGAIN = contiguity gap (bug-850): dropped quietly, backfilled by
|
||
* the in-order catch-up; any other rc is a real failure. */
|
||
if (rc != -EAGAIN)
|
||
ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed for "
|
||
"lsn=%" PRIu64 " type=%u rc=%d — skipping entry",
|
||
entry->lsn, (unsigned)entry->entry_type, rc);
|
||
/* Task 3.2: no ACK on apply failure */
|
||
return rc;
|
||
}
|
||
|
||
/* Advance applied_lsn atomically to the last contiguously-held LSN
|
||
* (bug-991: entry->lsn, NOT entry->lsn + 1 — see the
|
||
* ha_apply_term_entry_via_library comment for the streamer-skip wedge a +1
|
||
* caused). This is the same last-held semantic recovery/checkpoint/init use,
|
||
* and matches the (standby_lsn, committed] range the leader streams. */
|
||
ha_advance_applied_lsn(ha, entry->lsn);
|
||
|
||
/* Task 3.1: ACK the successfully applied single entry */
|
||
ha_send_wal_ack_reply(ha, ep, c, entry->lsn);
|
||
return 0;
|
||
}
|
||
|
||
/* ── ha-term-safety task 3.2: library-routed AppendEntries intake ───────────
|
||
*
|
||
* ha_consume_append_entries: route ONE incoming heartbeat (0-entry) or
|
||
* replication (1-entry) AppendEntries through the raft library
|
||
* (raft_on_append_entries_request) so the LIBRARY — not hand-rolled term-blind
|
||
* code — performs:
|
||
* - UpdateTerm first on EVERY RPC incl. heartbeat (G5, R2.6): a higher-term
|
||
* AE makes the node adopt the term + step down (via on_become_follower).
|
||
* - logOk (G2, R3.1): (prevLogIndex,prevLogTerm) match against the local log
|
||
* via on_get_entry_term — a mismatch is REJECTED (no false convergence).
|
||
* - (term,lsn) conflict-truncate (G3, R3.2/R3.3): a divergent same-LSN tail is
|
||
* truncated (on_truncate_from) then the leader's entry re-persisted
|
||
* (on_persist_entry), never touching the agreed prefix below the divergence.
|
||
*
|
||
* This REPLACES the term-blind term handling in ha_consume_heartbeat_committed
|
||
* and the committed-boundary truncate in ha_rejoin_reconcile: the library's
|
||
* (term,lsn) decision, not a watermark, drives truncation.
|
||
*
|
||
* `entry` is a decoded wal_catchup_entry_t whose TOP-LEVEL fields carry the AE
|
||
* metadata (prev_log_index, prev_log_term, leader_commit) and, for a single
|
||
* entry, the entry itself (lsn, entry_type, term, raw_payload). `leader_idx` is
|
||
* the raft peer index of the sending leader.
|
||
*
|
||
* The library callbacks own all persistence/truncation; on a REJECT (stale term
|
||
* or logOk fail) the library mutates nothing and the member keeps its current
|
||
* state to retry on a later catch-up (R3.5/R3.6 — never report false
|
||
* convergence). The loud non-promotable fence for an unreconcilable member is
|
||
* the EXISTING ha-commit-durability promotion-time fence; this intake never
|
||
* advertises convergence it did not achieve.
|
||
*
|
||
* Single-receiver-thread model (controller request handling), matching the
|
||
* raft_on_vote_request intake: no ha->lock is taken (the raft state is serial
|
||
* per the receive thread). Returns 0 (no crash) on every path; the AE result is
|
||
* reflected in the library/WAL state, and a future ACK reply (task 3.3) carries
|
||
* the success/match_index to the leader.
|
||
*/
|
||
int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry,
|
||
int leader_idx)
|
||
{
|
||
if (!c || !entry)
|
||
return -EINVAL;
|
||
|
||
int n_entries = (entry->n_entries > 0) ? 1 : 0; /* 0 = heartbeat, 1 = entry */
|
||
|
||
/* AppendEntries metadata from the top-level container. */
|
||
uint64_t prev_log_index = entry->prev_log_index;
|
||
uint64_t prev_log_term = entry->prev_log_term;
|
||
uint64_t leader_commit = entry->leader_commit;
|
||
|
||
/* The incoming AE's term: for a single entry, the entry's term stamp; for a
|
||
* heartbeat, the top-level container's term (set by the sender to its live
|
||
* term). raft_on_append_entries_request uses this for UpdateTerm (G5). */
|
||
uint64_t ae_term;
|
||
const void *blob = NULL;
|
||
size_t blob_len = 0;
|
||
uint64_t entry_term = 0;
|
||
|
||
/* For a single entry, repack the [raft_entry_blob_t{type,len}][payload]
|
||
* envelope that on_persist_entry unpacks (the shared wal_types.h format), so
|
||
* the library-driven persist writes the same bytes the leader sent. */
|
||
uint8_t stackbuf[512];
|
||
uint8_t *heapbuf = NULL;
|
||
|
||
if (n_entries == 1) {
|
||
/* Single-entry mode: the entry lives in entries[0] (batch) or on the
|
||
* top-level container (single). Prefer the sub-entry when present. */
|
||
wal_catchup_entry_t *e =
|
||
(entry->entries && entry->entries[0]) ? entry->entries[0] : entry;
|
||
uint32_t type = e->entry_type;
|
||
const void *payload = e->raw_payload;
|
||
uint32_t plen = (uint32_t)e->raw_payload_len;
|
||
entry_term = e->term;
|
||
/* bug-1067: the AppendEntries HEADER term (UpdateTerm/stale-check) is the
|
||
* leader's CURRENT term, NOT the entry's stored term. For a LIVE append
|
||
* the two are equal (the leader appends at its current term), but for a
|
||
* CATCH-UP entry written by a PRIOR leader the stored term is LOWER than
|
||
* the current leader's term — using it as the AE term made the library
|
||
* stale-reject (term < s->term) every historical entry, wedging cross-term
|
||
* catch-up forever. The streamer stamps leader_current_term = ha->term;
|
||
* fall back to the stored term only when it is unset (legacy sender). */
|
||
ae_term = (e->leader_current_term != 0) ? e->leader_current_term
|
||
: entry_term;
|
||
|
||
blob_len = sizeof(raft_entry_blob_t) + plen;
|
||
uint8_t *buf = (blob_len <= sizeof(stackbuf)) ? stackbuf
|
||
: (heapbuf = malloc(blob_len));
|
||
if (!buf)
|
||
return -ENOMEM;
|
||
raft_entry_blob_t *bh = (raft_entry_blob_t *)buf;
|
||
bh->type = type;
|
||
bh->payload_len = plen;
|
||
if (payload && plen > 0)
|
||
memcpy(buf + sizeof(*bh), payload, plen);
|
||
blob = buf;
|
||
} else {
|
||
/* Heartbeat: the top-level container's term carries the leader's live
|
||
* term (the sender stamps it). No entry to persist. */
|
||
ae_term = entry->term;
|
||
}
|
||
|
||
raft_append_result_t res;
|
||
memset(&res, 0, sizeof(res));
|
||
int rc = raft_on_append_entries_request(&c->ha.raft, leader_idx, ae_term,
|
||
prev_log_index, prev_log_term,
|
||
leader_commit, blob, blob_len,
|
||
entry_term, n_entries, &res);
|
||
|
||
if (heapbuf)
|
||
free(heapbuf);
|
||
|
||
if (rc != 0) {
|
||
/* Library-internal error (not a protocol reject — those return RAFT_OK
|
||
* with res.success==0). Log loud; the member stays available to retry. */
|
||
WARN("ha_consume_append_entries: raft_on_append_entries_request rc=%d "
|
||
"(leader_idx=%d term=%" PRIu64 " prevIdx=%" PRIu64 ")",
|
||
rc, leader_idx, ae_term, prev_log_index);
|
||
return 0; /* never crash the receive thread */
|
||
}
|
||
|
||
TRACE("ha_consume_append_entries: leader_idx=%d term=%" PRIu64
|
||
" prevIdx=%" PRIu64 " n=%d -> success=%d match=%" PRIu64,
|
||
leader_idx, ae_term, prev_log_index, n_entries,
|
||
res.success, res.match_index);
|
||
|
||
/* res.success / res.match_index are consumed by the ACK reply path (task
|
||
* 3.3). On a reject (res.success==0) the library left the log untouched —
|
||
* R3.5/R3.6: no false convergence, the member reconciles on a later AE. */
|
||
return 0;
|
||
}
|
||
|
||
/* ha_handle_snapshot_chunk — standby snapshot receive handler (Task 3.5c).
|
||
*
|
||
* Each call appends payload_bytes to snap_dir/snap.part.
|
||
* When chunk->is_last is non-zero:
|
||
* 1. Verify total bytes written matches chunk->total_size.
|
||
* 2. Atomically rename snap.part → snap-{snapshot_lsn:016x}.bin.
|
||
* 3. Load the snapshot via persist_load_snapshot().
|
||
* 4. Set ha->applied_lsn = p->checkpoint_lsn (in-memory).
|
||
*
|
||
* Mid-transfer reconnect (Req 4.4): the primary restarts every transfer at
|
||
* chunk 0, so chunk_index == 0 discards any stale snap.part left by an
|
||
* interrupted transfer before appending — no cross-chunk state needed.
|
||
* Returns 0 on success, negative errno on error.
|
||
* Requirements: 4.2, 4.3, 4.4 (ha-repl-hardening spec)
|
||
*/
|
||
int ha_handle_snapshot_chunk(ha_state_t *ha, persist_t *p,
|
||
const char *snap_dir,
|
||
snapshot_transfer_t *chunk,
|
||
const uint8_t *payload_bytes,
|
||
size_t payload_len)
|
||
{
|
||
if (!ha || !p || !snap_dir || !chunk)
|
||
return -EINVAL;
|
||
|
||
char part_path[4096 + 16];
|
||
snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir);
|
||
|
||
/* Req 4.4: a transfer always restarts from chunk 0 — discard any stale
|
||
* partial file from an interrupted previous transfer. */
|
||
if (chunk->chunk_index == 0 && unlink(part_path) != 0 && errno != ENOENT)
|
||
WARN("ha_handle_snapshot_chunk: unlink stale snap.part failed: %s",
|
||
strerror(errno));
|
||
|
||
/* Append chunk payload to snap.part */
|
||
if (payload_bytes && payload_len > 0) {
|
||
FILE *f = fopen(part_path, "ab");
|
||
if (!f) {
|
||
int err = errno;
|
||
ERROR("ha_handle_snapshot_chunk: cannot open snap.part: %s",
|
||
strerror(err));
|
||
return -err;
|
||
}
|
||
size_t written = fwrite(payload_bytes, 1, payload_len, f);
|
||
fclose(f);
|
||
if (written != payload_len) {
|
||
ERROR("ha_handle_snapshot_chunk: short write to snap.part "
|
||
"(%zu of %zu bytes)", written, payload_len);
|
||
return -EIO;
|
||
}
|
||
}
|
||
|
||
if (!chunk->is_last)
|
||
return 0;
|
||
|
||
/* isLast=true: verify size, rename, load, set applied_lsn */
|
||
|
||
/* Verify total file size equals chunk->total_size */
|
||
struct stat st;
|
||
if (stat(part_path, &st) != 0) {
|
||
int err = errno;
|
||
ERROR("ha_handle_snapshot_chunk: stat(snap.part) failed: %s",
|
||
strerror(err));
|
||
return -err;
|
||
}
|
||
if ((uint64_t)st.st_size != chunk->total_size) {
|
||
ERROR("ha_handle_snapshot_chunk: size mismatch — "
|
||
"written=%" PRIu64 " expected=%" PRIu64,
|
||
(uint64_t)st.st_size, chunk->total_size);
|
||
return -EIO;
|
||
}
|
||
|
||
/* Atomic rename snap.part → snap-{lsn:016x}.bin */
|
||
char final_path[4096 + 32];
|
||
snprintf(final_path, sizeof(final_path),
|
||
"%s/snap-%016" PRIx64 ".bin", snap_dir, chunk->snapshot_lsn);
|
||
if (rename(part_path, final_path) != 0) {
|
||
int err = errno;
|
||
ERROR("ha_handle_snapshot_chunk: rename snap.part → %s failed: %s",
|
||
final_path, strerror(err));
|
||
return -err;
|
||
}
|
||
|
||
/* Load the snapshot — sets p->checkpoint_lsn and p->current_lsn */
|
||
int rc = persist_load_snapshot(p, final_path, chunk->snapshot_lsn);
|
||
if (rc != 0) {
|
||
ERROR("ha_handle_snapshot_chunk: persist_load_snapshot failed rc=%d", rc);
|
||
return rc;
|
||
}
|
||
|
||
/* Advance applied_lsn to checkpoint_lsn (in-memory only — Req 4.2) */
|
||
atomic_store_explicit(&ha->applied_lsn, p->checkpoint_lsn,
|
||
memory_order_release);
|
||
INFO("HA: snapshot loaded lsn=%" PRIu64 " applied_lsn set",
|
||
p->checkpoint_lsn);
|
||
return 0;
|
||
}
|
||
|
||
/* ha_nop_reply_state_report — populate *sr from ha state and wire into reply.
|
||
* Reads ha->applied_lsn (atomic) and myconf->config_version_tag, stores them
|
||
* in *sr, then sets reply_out->data_which = stateReportMsg and
|
||
* reply_out->data.state_report = sr.
|
||
* Caller keeps *sr alive until xce_reply() serialises the message.
|
||
* Requirements: 1.1, 1.3, 1.4 (ha-repl-hardening spec) */
|
||
void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf,
|
||
state_report_t *sr, xce_reply_t *reply_out)
|
||
{
|
||
sr->applied_lsn = atomic_load_explicit(&ha->applied_lsn,
|
||
memory_order_acquire);
|
||
sr->config_tag = myconf ? myconf->config_version_tag : 0;
|
||
reply_out->data_which = RequestReply_data_stateReportMsg;
|
||
reply_out->data.state_report = sr;
|
||
}
|
||
|
||
/*
|
||
* ha_evaluate_repl_action — pure decision function for the unified heartbeat-
|
||
* driven replication protocol.
|
||
*
|
||
* Implements the decision tree from the ha-repl-hardening design:
|
||
*
|
||
* if standby_tag < committed_tag → HA_REPL_PUSH_CONFIG
|
||
* elif standby_lsn + 1 < oldest_wal_lsn → HA_REPL_SEND_SNAPSHOT
|
||
* elif standby_lsn < committed_lsn → HA_REPL_STREAM_WAL
|
||
* else (tag >= committed AND lsn >= committed) → HA_REPL_CONVERGED
|
||
*
|
||
* No side effects. All callers must use the returned action to drive the
|
||
* replication send; they must not cache the result beyond the current cycle.
|
||
*
|
||
* Requirements: 2.2, 2.3, 2.4 (ha-repl-hardening spec)
|
||
* Boundary: Primary Heartbeat Handler (ha.c)
|
||
*/
|
||
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
|
||
uint64_t standby_lsn,
|
||
uint64_t committed_tag,
|
||
uint64_t committed_lsn,
|
||
uint64_t oldest_wal_lsn)
|
||
{
|
||
if (standby_tag < committed_tag)
|
||
return HA_REPL_PUSH_CONFIG;
|
||
|
||
/* Snapshot only when the standby's NEXT-needed entry (standby_lsn + 1) has
|
||
* been garbage-collected below the oldest retained WAL entry — i.e. it can
|
||
* no longer be served by streaming. An empty standby (standby_lsn=0) on a
|
||
* fresh cluster whose oldest retained entry IS lsn 1 (post reserve-lsn-0)
|
||
* needs lsn 1, which is retained: 0 + 1 < 1 is false → STREAM, no snapshot.
|
||
* Using standby_lsn + 1 (not standby_lsn - ... ) avoids unsigned underflow
|
||
* when oldest_wal_lsn == 0 (empty WAL): 0 + 1 < 0 is false → not snapshot. */
|
||
if (standby_lsn + 1 < oldest_wal_lsn)
|
||
return HA_REPL_SEND_SNAPSHOT;
|
||
|
||
if (standby_lsn < committed_lsn)
|
||
return HA_REPL_STREAM_WAL;
|
||
|
||
return HA_REPL_CONVERGED;
|
||
}
|
||
|
||
/*
|
||
* ha_process_standby_state — update per-standby convergence tracking.
|
||
*
|
||
* Called on each heartbeat reply from a standby (CTRL_PRIMARY).
|
||
* Stores (reported_tag, reported_lsn) into the per-standby
|
||
* arrays, evaluates convergence, and logs INFO on the first
|
||
* not-converged → converged transition (Req 7.2). Regression resets the flag
|
||
* (Req 2.5).
|
||
*
|
||
* Convergence: reported_tag >= committed_tag AND reported_lsn >= committed_lsn.
|
||
*
|
||
* Requirements: 1.2, 2.2, 2.5 (ha-repl-hardening spec)
|
||
*/
|
||
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
|
||
uint64_t reported_tag, uint64_t reported_lsn,
|
||
uint64_t committed_tag, uint64_t committed_lsn)
|
||
{
|
||
if (peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES)
|
||
return;
|
||
|
||
/* Store reported values */
|
||
ha->standby_reported_tag[peer_idx] = reported_tag;
|
||
ha->standby_reported_lsn[peer_idx] = reported_lsn;
|
||
|
||
/* Evaluate convergence */
|
||
bool was_converged = ha->standby_converged[peer_idx];
|
||
bool now_converged = (reported_tag >= committed_tag) &&
|
||
(reported_lsn >= committed_lsn);
|
||
ha->standby_converged[peer_idx] = now_converged;
|
||
|
||
/* Log INFO on first transition to converged (Req 7.2) */
|
||
if (now_converged && !was_converged) {
|
||
INFO("HA: standby[%d] converged: config_tag=%" PRIu64
|
||
" applied_lsn=%" PRIu64 " (committed_tag=%" PRIu64
|
||
" committed_lsn=%" PRIu64 ")",
|
||
peer_idx, reported_tag, reported_lsn,
|
||
committed_tag, committed_lsn);
|
||
}
|
||
}
|
||
|
||
/* ───────────────────────────────────────────────────────────────────────
|
||
* raft-drive-async Task 3.2 — SnapshotSend slow-lane offload (bug-1198).
|
||
*
|
||
* The snapshot transfer (a fresh persist_checkpoint → fdatasync, then a
|
||
* chunked socket write of the snapshot file) is the one unbounded operation
|
||
* reachable from the repl-ack decision loop. Run inline it blocks the
|
||
* io_worker/drive thread for the whole transfer (and queues that peer's later
|
||
* messages behind it). This offloads it to the bounded slow lane, keyed per
|
||
* standby so at most one snapshot per peer is in flight (Req 1.5).
|
||
* ─────────────────────────────────────────────────────────────────────── */
|
||
|
||
typedef struct {
|
||
cluster_t *c;
|
||
endpoint_t *ep;
|
||
int peer_idx;
|
||
} snapshot_send_ctx_t;
|
||
|
||
/* ha_snapshot_do_send: the actual (blocking) snapshot work — shared by the
|
||
* inline path and the lane worker so both behave identically. In XCE_TESTING
|
||
* builds the send is replaced by ha_repl_send_snapshot_hook(peer_idx). */
|
||
static void ha_snapshot_do_send(cluster_t *c, endpoint_t *ep, int peer_idx)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
if (ha_repl_send_snapshot_hook != NULL) {
|
||
ha_repl_send_snapshot_hook(peer_idx);
|
||
return;
|
||
}
|
||
#endif
|
||
if (c && c->persist.snap_path[0]) {
|
||
extern int ha_send_snapshot_to_standby(endpoint_t *, const char *,
|
||
uint64_t);
|
||
/* Req 4.1: create a FRESH checkpoint so the transferred snapshot
|
||
* reflects current state (also trims WAL + rotates old snaps).
|
||
* -EBUSY means another checkpoint is in flight — fall back to the
|
||
* existing checkpoint_lsn rather than skipping the cycle (at most one
|
||
* checkpoint in flight is the existing persist_checkpoint invariant). */
|
||
int ckpt_rc = persist_checkpoint(&c->persist);
|
||
if (ckpt_rc != 0 && ckpt_rc != -EBUSY)
|
||
WARN("HA repl: persist_checkpoint before snapshot send "
|
||
"failed rc=%d — sending existing checkpoint", ckpt_rc);
|
||
ha_send_snapshot_to_standby(ep, c->persist.snap_path,
|
||
c->persist.checkpoint_lsn);
|
||
}
|
||
}
|
||
|
||
/*
|
||
* snapshot_lane_fn: slow-lane worker entry. Runs the bounded blocking
|
||
* snapshot send off the drive thread, then ALWAYS clears the per-peer
|
||
* in-flight flag (success or failure) so the dedup never wedges and a later
|
||
* heartbeat can retry. Frees the ctx.
|
||
*
|
||
* ⚠ BORROWED-ENDPOINT LIFETIME HAZARD (bug-1197 family — why this path is
|
||
* gated OFF by default). `ep` here is a standby endpoint shared with the
|
||
* io_worker epoll loop, NOT one this lane owns (contrast reconnect_lane_fn,
|
||
* which owns the ep it creates). If the standby RDHUP-disconnects while this
|
||
* transfer runs, the io_worker can endpoint_fini/free `ep` concurrently → UAF.
|
||
* Today (flag OFF) the send runs inline ON the io_worker, so teardown and send
|
||
* are serialized on one thread and cannot overlap. Flipping
|
||
* g_drive_async_snapshot_offload ON is therefore only safe once Task 3.4
|
||
* establishes the endpoint-lifetime discipline for the whole pool/lane
|
||
* dispatch model (refcount or lane-serialized teardown). Until then this code
|
||
* lands DORMANT and is exercised only by unit tests with a stable endpoint. */
|
||
static void snapshot_lane_fn(void *arg)
|
||
{
|
||
snapshot_send_ctx_t *sx = (snapshot_send_ctx_t *)arg;
|
||
if (!sx) return;
|
||
cluster_t *c = sx->c;
|
||
int peer_idx = sx->peer_idx;
|
||
|
||
ha_snapshot_do_send(c, sx->ep, peer_idx);
|
||
|
||
/* Clear in-flight LAST so a concurrent heartbeat tick cannot enqueue a
|
||
* second snapshot for this peer until this one has fully finished. */
|
||
atomic_store_explicit(&c->ha.snapshot_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
free(sx);
|
||
}
|
||
|
||
/*
|
||
* ha_snapshot_send_dispatch: route a SEND_SNAPSHOT decision either inline
|
||
* (flag OFF, production default — runs on the caller/drive thread, unchanged
|
||
* legacy behavior) or onto the slow lane (flag ON — non-blocking). The
|
||
* per-peer single-flight CAS on ha->snapshot_inflight[] is the authoritative
|
||
* dedup; the lane's per-key dedup is a redundant backstop. Mirrors the
|
||
* reconnect-offload terminal-path handling (Task 2.1).
|
||
*/
|
||
static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c,
|
||
endpoint_t *ep, int peer_idx)
|
||
{
|
||
/* Flag OFF (default) or no cluster handle (no lane) → inline. */
|
||
if (!ha_snapshot_offload_enabled() || c == NULL) {
|
||
ha_snapshot_do_send(c, ep, peer_idx);
|
||
return;
|
||
}
|
||
|
||
/* Per-peer single-flight: CAS 0->1. Already 1 → a snapshot to this peer
|
||
* is in flight; enqueue nothing (Req 1.5). */
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&ha->snapshot_inflight[peer_idx], &expected, 1,
|
||
memory_order_acq_rel, memory_order_acquire)) {
|
||
return;
|
||
}
|
||
|
||
snapshot_send_ctx_t *sx = malloc(sizeof(*sx));
|
||
if (!sx) {
|
||
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
sx->c = c; sx->ep = ep; sx->peer_idx = peer_idx;
|
||
|
||
uint64_t key = (uint64_t)(peer_idx + 1); /* +1: key 0 disables dedup */
|
||
int srv = slowlane_submit(&c->slowlane, key, snapshot_lane_fn, sx);
|
||
if (srv == -EEXIST) {
|
||
/* Lane still holds this peer's key from a prior snapshot task that has
|
||
* already cleared snapshot_inflight (its last action) but whose key the
|
||
* lane has not yet removed. Our CAS saw 0 and set 1, but the submit is
|
||
* refused — release the flag we set so no clear is lost, and let the
|
||
* next tick retry. */
|
||
free(sx);
|
||
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
if (srv < 0 && srv != -EAGAIN) {
|
||
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
|
||
* failure (still queued, will run). Only on a real failure tear down
|
||
* and release the guard so the next tick retries; never self-terminate. */
|
||
WARN("HA repl: slowlane_submit snapshot peer[%d] failed: %d — retry "
|
||
"next cycle", peer_idx, srv);
|
||
free(sx);
|
||
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
/* 0 or -EAGAIN: enqueued exactly once; snapshot_lane_fn clears the flag. */
|
||
}
|
||
|
||
/* ───────────────────────────────────────────────────────────────────────
|
||
* raft-drive-async Task 3.3 — requester-less ConfigPush slow-lane offload
|
||
* (bug-1198).
|
||
*
|
||
* The two REQUESTER-LESS config-push sites — post-election promotion
|
||
* (raft_cb_become_leader) and the heartbeat-driven HA_REPL_PUSH_CONFIG
|
||
* convergence case — call push_config_transport_only, which spawns one push
|
||
* thread per standby and then BLOCKS the caller in wait_all_with_timeout up to
|
||
* CONF_PUSH_TIMEOUT_MS. Run inline on the single io_worker that drives both of
|
||
* those sites, that wait stalls every heartbeat/vote for the whole timeout.
|
||
* This offloads it to the bounded slow lane. Config push is a CLUSTER-WIDE
|
||
* fan-out, so a single cluster-wide single-flight guard (configpush_inflight) +
|
||
* a single dedup key (key=1) suffice — at most one push in flight at a time.
|
||
*
|
||
* The admin-upgrade path (push_config_to_standbys, which has a waiting
|
||
* requester / majority barrier) is NOT routed here and stays unchanged.
|
||
* ─────────────────────────────────────────────────────────────────────── */
|
||
|
||
#ifdef XCE_TESTING
|
||
extern void (*ha_configpush_probe)(void);
|
||
#endif
|
||
|
||
typedef struct {
|
||
cluster_t *c;
|
||
uint8_t *bytes; /* deep copy, owned by the ctx; NULL when blen==0 */
|
||
size_t blen;
|
||
uint8_t sha[32]; /* valid only when has_sha != 0 */
|
||
int has_sha;
|
||
uint64_t new_tag;
|
||
} configpush_ctx_t;
|
||
|
||
/* ha_config_push_do_send: the actual (blocking) config-push work — shared by
|
||
* the inline path and the lane worker so both behave identically. In
|
||
* XCE_TESTING builds a non-NULL ha_configpush_probe() is invoked right around
|
||
* the send so a test can record the running thread / park the lane worker. */
|
||
static void ha_config_push_do_send(cluster_t *c, const uint8_t *bytes,
|
||
size_t blen, const uint8_t sha256[32],
|
||
uint64_t new_tag)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
/* XCE_TESTING does not link cluster.c and uses hook shims for the real
|
||
* transport (push_config_transport_only's extern is #ifndef XCE_TESTING).
|
||
* The probe stands in for the send and records the running thread. */
|
||
if (ha_configpush_probe != NULL)
|
||
ha_configpush_probe();
|
||
(void)c; (void)bytes; (void)blen; (void)sha256; (void)new_tag;
|
||
#else
|
||
push_config_transport_only(c, bytes, blen, sha256, new_tag, NULL);
|
||
#endif
|
||
}
|
||
|
||
/*
|
||
* configpush_lane_fn: slow-lane worker entry. Runs the bounded blocking
|
||
* config push off the drive thread, then ALWAYS clears the cluster-wide
|
||
* in-flight flag (success or failure) so the dedup never wedges and a later
|
||
* promotion/heartbeat can retry. Frees the deep-copied bytes and the ctx.
|
||
*
|
||
* ⚠ BORROWED-ENDPOINT LIFETIME HAZARD (bug-1197 family — why this path is
|
||
* gated OFF by default). push_config_transport_only here reads
|
||
* c->ha.standby_endpoints and writes to those standby endpoints, which are
|
||
* shared with the io_worker epoll loop, NOT owned by this lane (contrast
|
||
* reconnect_lane_fn, which owns the ep it creates). If a standby
|
||
* RDHUP-disconnects while this push runs, the io_worker can endpoint_fini/free
|
||
* those endpoints concurrently → UAF. Today (flag OFF) the push runs inline ON
|
||
* the io_worker, so teardown and push are serialized on one thread and cannot
|
||
* overlap. Flipping g_drive_async_configpush_offload ON is therefore only safe
|
||
* once Task 3.4 establishes the endpoint-lifetime discipline for the whole
|
||
* pool/lane dispatch model. Until then this code lands DORMANT and is
|
||
* exercised only by unit tests with a stable (or absent) endpoint set.
|
||
*/
|
||
static void configpush_lane_fn(void *arg)
|
||
{
|
||
configpush_ctx_t *cx = (configpush_ctx_t *)arg;
|
||
if (!cx) return;
|
||
cluster_t *c = cx->c;
|
||
|
||
ha_config_push_do_send(c, cx->bytes, cx->blen,
|
||
cx->has_sha ? cx->sha : NULL, cx->new_tag);
|
||
|
||
/* Clear in-flight LAST so a concurrent promotion/heartbeat tick cannot
|
||
* enqueue a second config push until this one has fully finished. */
|
||
atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release);
|
||
free(cx->bytes);
|
||
free(cx);
|
||
}
|
||
|
||
/*
|
||
* ha_config_push_offloadable: route a requester-less config push either inline
|
||
* (flag OFF, production default — runs on the caller/drive thread, unchanged
|
||
* legacy behavior; the CALLER still owns `bytes` and frees them after we
|
||
* return) or onto the slow lane (flag ON — non-blocking; we take a deep copy of
|
||
* `bytes` so the caller's free is safe). The cluster-wide single-flight CAS on
|
||
* ha.configpush_inflight is the authoritative dedup; the lane's per-key dedup
|
||
* is a redundant backstop. Mirrors the snapshot-offload terminal-path handling
|
||
* (Task 3.2). Never self-terminates: on any failure it clears the guard and
|
||
* the next cycle retries.
|
||
*/
|
||
static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes,
|
||
size_t blen, const uint8_t sha256[32],
|
||
uint64_t new_tag)
|
||
{
|
||
/* Flag OFF (default) or no cluster handle (no lane) → inline. */
|
||
if (!ha_configpush_offload_enabled() || c == NULL) {
|
||
ha_config_push_do_send(c, bytes, blen, sha256, new_tag);
|
||
return;
|
||
}
|
||
|
||
/* Cluster-wide single-flight: CAS 0->1. Already 1 → a config push is in
|
||
* flight; enqueue nothing. */
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&c->ha.configpush_inflight, &expected, 1,
|
||
memory_order_acq_rel, memory_order_acquire)) {
|
||
return;
|
||
}
|
||
|
||
configpush_ctx_t *cx = malloc(sizeof(*cx));
|
||
if (!cx) {
|
||
atomic_store_explicit(&c->ha.configpush_inflight, 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
cx->c = c;
|
||
cx->new_tag = new_tag;
|
||
cx->has_sha = (sha256 != NULL) ? 1 : 0;
|
||
if (cx->has_sha)
|
||
memcpy(cx->sha, sha256, 32);
|
||
if (bytes != NULL && blen > 0) {
|
||
cx->bytes = malloc(blen);
|
||
if (!cx->bytes) {
|
||
free(cx);
|
||
atomic_store_explicit(&c->ha.configpush_inflight, 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
memcpy(cx->bytes, bytes, blen);
|
||
cx->blen = blen;
|
||
} else {
|
||
cx->bytes = NULL;
|
||
cx->blen = 0;
|
||
}
|
||
|
||
uint64_t key = 1; /* single cluster-wide dedup key (one config-push stream) */
|
||
int srv = slowlane_submit(&c->slowlane, key, configpush_lane_fn, cx);
|
||
if (srv == -EEXIST) {
|
||
/* Lane still holds the key from a prior config-push task that has
|
||
* already cleared configpush_inflight (its last action) but whose key
|
||
* the lane has not yet removed. Our CAS saw 0 and set 1, but the
|
||
* submit is refused — release the flag we set so no clear is lost, and
|
||
* let the next tick retry. */
|
||
free(cx->bytes);
|
||
free(cx);
|
||
atomic_store_explicit(&c->ha.configpush_inflight, 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
if (srv < 0 && srv != -EAGAIN) {
|
||
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
|
||
* failure (still queued, will run). Only on a real failure tear down
|
||
* and release the guard so the next tick retries; never self-terminate. */
|
||
WARN("HA repl: slowlane_submit configpush failed: %d — retry next cycle",
|
||
srv);
|
||
free(cx->bytes);
|
||
free(cx);
|
||
atomic_store_explicit(&c->ha.configpush_inflight, 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
/* 0 or -EAGAIN: enqueued exactly once; configpush_lane_fn clears the flag. */
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Test seam: directly drive ha_config_push_offloadable (the two production call
|
||
* sites are #ifndef XCE_TESTING). Returns 0 (the dispatch itself cannot fail
|
||
* fatally — failures clear the guard and retry). */
|
||
int ha_config_push_offloadable_for_test(cluster_t *c, const uint8_t *bytes,
|
||
size_t blen, const uint8_t sha256[32],
|
||
uint64_t new_tag)
|
||
{
|
||
ha_config_push_offloadable(c, bytes, blen, sha256, new_tag);
|
||
return 0;
|
||
}
|
||
#endif
|
||
|
||
/*
|
||
* ha_dispatch_repl_action_for_standby — execute the decision-loop action for
|
||
* one standby on the current heartbeat cycle (Task 3.3).
|
||
*
|
||
* Reads ha->standby_reported_tag[peer_idx] / standby_reported_lsn[peer_idx]
|
||
* (stored by ha_process_standby_state on the same heartbeat reply), calls
|
||
* ha_evaluate_repl_action, and acts on the result:
|
||
*
|
||
* HA_REPL_PUSH_CONFIG → push_config_transport_only(c, …) [or test hook] (Task 3.3)
|
||
* HA_REPL_SEND_SNAPSHOT→ stub (Task 3.5)
|
||
* HA_REPL_STREAM_WAL → ha_stream_wal_to_standby(c, ha, idx) (Task 3.4)
|
||
* HA_REPL_CONVERGED → no-op
|
||
*
|
||
* One action per heartbeat cycle per standby (design §Unified Heartbeat-Driven
|
||
* Protocol). Returns 0 on success, -EINVAL on bad peer_idx.
|
||
*
|
||
* Requirements: 1.4, 5.1–5.5 (ha-repl-hardening spec)
|
||
*/
|
||
int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
|
||
cluster_t *c,
|
||
endpoint_t *ep,
|
||
int peer_idx,
|
||
uint64_t committed_tag,
|
||
uint64_t committed_lsn,
|
||
uint64_t oldest_wal_lsn)
|
||
{
|
||
if (!ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES)
|
||
return -EINVAL;
|
||
|
||
uint64_t standby_tag = ha->standby_reported_tag[peer_idx];
|
||
uint64_t standby_lsn = ha->standby_reported_lsn[peer_idx];
|
||
|
||
ha_repl_action_t action = ha_evaluate_repl_action(standby_tag, standby_lsn,
|
||
committed_tag, committed_lsn,
|
||
oldest_wal_lsn);
|
||
|
||
switch (action) {
|
||
case HA_REPL_PUSH_CONFIG:
|
||
INFO("HA repl: standby[%d] tag=%" PRIu64 " < committed_tag=%" PRIu64
|
||
" → push config", peer_idx, standby_tag, committed_tag);
|
||
#ifdef XCE_TESTING
|
||
if (ha_repl_push_config_for_standby_hook != NULL) {
|
||
ha_repl_push_config_for_standby_hook(c, peer_idx);
|
||
break;
|
||
}
|
||
#endif
|
||
if (c) {
|
||
#ifndef XCE_TESTING
|
||
extern int conf_read_raw(const char *, uint8_t **, size_t *);
|
||
uint8_t *cfg_bytes = NULL;
|
||
size_t cfg_len = 0;
|
||
if (c->myconf && c->myconf->conffile &&
|
||
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
|
||
/* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON).
|
||
* The offload deep-copies cfg_bytes, so this free is still
|
||
* correct regardless of routing. */
|
||
ha_config_push_offloadable(c, cfg_bytes, cfg_len,
|
||
c->myconf->config_sha256,
|
||
c->myconf->config_version_tag);
|
||
free(cfg_bytes);
|
||
} else {
|
||
ha_config_push_offloadable(c, NULL, 0, NULL,
|
||
c->myconf ? c->myconf->config_version_tag : 0);
|
||
}
|
||
#endif
|
||
}
|
||
break;
|
||
|
||
case HA_REPL_SEND_SNAPSHOT:
|
||
INFO("HA repl: standby[%d] lsn=%" PRIu64
|
||
" < oldest_wal_lsn=%" PRIu64 " → requesting snapshot transfer",
|
||
peer_idx, standby_lsn, oldest_wal_lsn);
|
||
/* Task 3.2: route inline (flag OFF, default) or onto the slow lane
|
||
* (flag ON). The send body — fresh checkpoint + chunked transfer, or
|
||
* the XCE_TESTING hook — lives in ha_snapshot_do_send so both paths
|
||
* are identical. */
|
||
ha_snapshot_send_dispatch(ha, c, ep, peer_idx);
|
||
break;
|
||
|
||
case HA_REPL_STREAM_WAL:
|
||
/* Task 3.4: batch WAL streaming.
|
||
* Enqueues all batch OPCode_walCatchupEntry messages for the range
|
||
* [standby_lsn+1, committed_lsn] via ha_stream_wal_to_standby().
|
||
* All xce_request() calls are non-blocking (enqueue-only). */
|
||
INFO("HA repl: standby[%d] lsn=%" PRIu64
|
||
" < committed_lsn=%" PRIu64 " → stream WAL batches",
|
||
peer_idx, standby_lsn, committed_lsn);
|
||
#ifdef XCE_TESTING
|
||
if (ha_repl_stream_wal_hook != NULL) {
|
||
const char *wal_path = (c && c->persist.wal_path[0])
|
||
? c->persist.wal_path : "";
|
||
ha_repl_stream_wal_hook(ep, wal_path, standby_lsn, committed_lsn);
|
||
break;
|
||
}
|
||
#endif
|
||
if (c && c->persist.wal_path[0]) {
|
||
extern int ha_stream_wal_to_standby(endpoint_t *ep,
|
||
const char *wal_path,
|
||
const persist_t *p,
|
||
uint64_t leader_term,
|
||
uint64_t standby_lsn,
|
||
uint64_t committed_lsn);
|
||
/* bug-1028: pass &c->persist so streamed entry terms come from the
|
||
* authoritative per-LSN index (persist_entry_term), matching the
|
||
* standby's logOk + the library send path.
|
||
* bug-1067: pass the leader's CURRENT term so historical entries
|
||
* carry the AE header term (not their lower stored term). */
|
||
uint64_t leader_term =
|
||
atomic_load_explicit(&c->ha.term, memory_order_acquire);
|
||
ha_stream_wal_to_standby(ep, c->persist.wal_path, &c->persist,
|
||
leader_term, standby_lsn, committed_lsn);
|
||
}
|
||
break;
|
||
|
||
case HA_REPL_CONVERGED:
|
||
/* No action needed — standby is up to date */
|
||
break;
|
||
}
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* ── Dispatch gate (req 1.3, 1.4) ──────────────────────────────────────────
|
||
*
|
||
* Called from myreq_handler() in cluster.c before the opcode switch.
|
||
* Returns 0 when the operation is permitted for the current role.
|
||
*
|
||
* Primary-only ops are any op that is not one of the always-allowed set:
|
||
* nop, reply, hello, join, shutdown, heartbeat, redirect,
|
||
* steppingDown, replEntry, replAck, replSnapshot, replCatchUpDone,
|
||
* replConfig, replConfigPush, requestVote, probeConfigTag,
|
||
* walAck, upgradeConfigAck, snapshotTransfer, walCatchupEntry.
|
||
*
|
||
* For CTRL_STANDBY the check is: op is in the primary-only set → deny.
|
||
* For CTRL_INIT / CTRL_ELECTING the allowlist is tighter: only
|
||
* OP_requestVote and OP_probeConfigTag pass through.
|
||
*/
|
||
/* ha_has_write_quorum — check whether the cluster has enough connected
|
||
* standbys to achieve majority for state-changing operations.
|
||
*
|
||
* Quorum = ceil(n_peers / 2) + 1 (self-vote included).
|
||
* Example: n_peers=3 → quorum=2 → need ≥1 connected standby.
|
||
*
|
||
* Called from ha_dispatch_gate for CTRL_PRIMARY — if the cluster loses
|
||
* majority (standby failures), write ops are rejected with -XCE_ERR_NOT_READY
|
||
* to preserve the majority-consistency invariant.
|
||
*
|
||
* Read-only queries (joblist, reslist, hostlist, etc.) remain available
|
||
* regardless of majority status — they don't mutate state. */
|
||
static bool ha_has_write_quorum(ha_state_t *ha)
|
||
{
|
||
if (!ha)
|
||
return false;
|
||
|
||
/* Single-node cluster: no standbys needed — self is the majority. */
|
||
if (ha->n_peers <= 1)
|
||
return true;
|
||
|
||
/* Count standbys able to receive new log entries. When the replicator
|
||
* back-pointer is wired (task 5.3 init), use registry truth: slots with
|
||
* ep != NULL && !in_catch_up under r->lock (Req 3.1-3.3) — catch-up
|
||
* members cannot ACK new entries, so they must not count toward write
|
||
* admission. NULL back-pointer (legacy contexts) falls back to the
|
||
* standby_endpoints count populated by raft_cb_become_leader (promoted
|
||
* vote eps) and on_standby_connected (late joiners); removed by error
|
||
* handlers. */
|
||
int connected;
|
||
if (ha->repl) {
|
||
connected = repl_active_standby_count(ha->repl);
|
||
} else {
|
||
connected = ha->standby_endpoints
|
||
? ha->standby_endpoints->count : 0;
|
||
}
|
||
|
||
/* Need at least n_peers/2 connected standbys to achieve quorum.
|
||
* Integer division: n_peers/2 for n_peers=3 → 1 standby minimum. */
|
||
return connected >= (int)(ha->n_peers / 2);
|
||
}
|
||
|
||
int ha_dispatch_gate(ha_state_t *ha, int opcode)
|
||
{
|
||
ctrl_role_t role =
|
||
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
|
||
|
||
switch (role) {
|
||
case CTRL_PRIMARY:
|
||
/* ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): the config
|
||
* serve-fence is the PRODUCTION READER that makes the fence real. When
|
||
* set, the latest committed WAL_CONFIG_MARKER references config bytes that
|
||
* do not hash-match the locally-held bytes (or the bytes are absent), so
|
||
* this member MUST NOT serve under the new configuration. Refuse
|
||
* config-dependent state-changing ops (loud non-serving), but keep
|
||
* replication / read-only / status ops flowing so the correct bytes can
|
||
* arrive out-of-band and re-verify clears the fence. The member STAYS
|
||
* ALIVE (returns an error code, never exits — controller-never-self-
|
||
* terminate). Checked before the quorum gate so a fenced primary never
|
||
* serves divergent config even with quorum. */
|
||
if (config_serve_fenced(ha->raft_c)) {
|
||
switch (opcode) {
|
||
/* Replication ops — let the correct config bytes arrive + re-verify */
|
||
case OPCode_heartbeat:
|
||
case OPCode_replConfig:
|
||
case OPCode_replConfigPush:
|
||
case OPCode_upgradeConfigAck:
|
||
case OPCode_hello:
|
||
case OPCode_walAck:
|
||
case OPCode_replEntry:
|
||
case OPCode_replAck:
|
||
case OPCode_replCatchUpDone:
|
||
case OPCode_walCatchupEntry:
|
||
case OPCode_snapshotTransfer:
|
||
/* Read-only queries / status — answerable while fenced */
|
||
case OPCode_get:
|
||
case OPCode_probeConfigTag:
|
||
case OPCode_ctrlstatus:
|
||
break; /* fall through to the quorum check below */
|
||
default:
|
||
/* Config-dependent state-changing op while fenced: refuse. */
|
||
return -XCE_ERR_NOT_READY;
|
||
}
|
||
}
|
||
/* State-changing operations require majority so committed entries
|
||
* are durable across failover. Read-only queries and replication
|
||
* ops are always allowed. */
|
||
if (!ha_has_write_quorum(ha)) {
|
||
/* Quorum-transition observability backstop (Req 8.1, 8.3): record the
|
||
* loss edge if a membership-change site missed it. The edge detector
|
||
* makes this silent on every call after the first — repeated rejected
|
||
* operations do NOT produce a log storm. */
|
||
ha_log_quorum_transition(ha);
|
||
switch (opcode) {
|
||
/* Replication ops — always allowed (they build majority) */
|
||
case OPCode_heartbeat:
|
||
case OPCode_replConfig:
|
||
case OPCode_replConfigPush:
|
||
case OPCode_upgradeConfigAck:
|
||
case OPCode_hello:
|
||
case OPCode_walAck:
|
||
case OPCode_replEntry:
|
||
case OPCode_replAck:
|
||
case OPCode_replCatchUpDone:
|
||
case OPCode_walCatchupEntry:
|
||
case OPCode_snapshotTransfer:
|
||
/* Read-only queries */
|
||
case OPCode_get:
|
||
case OPCode_probeConfigTag:
|
||
/* Read-only controller status (controller-status 7.1, 7.2):
|
||
* answers the local identity/health view even without quorum —
|
||
* precisely when an operator most wants it. */
|
||
case OPCode_ctrlstatus:
|
||
/* Fact-bearing ops — reports of events that already happened
|
||
* (worker job transitions/rusage, scheduler pend reasons).
|
||
* Rejecting them loses the fact forever: the reporter never
|
||
* retries (bug-396). Admit so the RETAIN path can append and
|
||
* hold them until quorum returns (Req 3.2; umbrella 2.8 —
|
||
* facts are retained, never dropped). */
|
||
case OPCode_notify:
|
||
case OPCode_update:
|
||
case OPCode_setpreason:
|
||
case OPCode_clrpreason:
|
||
return 0;
|
||
default:
|
||
return -XCE_ERR_NOT_READY;
|
||
}
|
||
}
|
||
return 0;
|
||
|
||
case CTRL_INIT:
|
||
/* Allow election RPCs AND primary→standby replication messages.
|
||
* During the startup stagger a primary may already connect and push
|
||
* config — accepting these ensures the node doesn't miss the initial
|
||
* bootstrap and can transition to CTRL_STANDBY immediately. */
|
||
if (opcode == OPCode_requestVote ||
|
||
opcode == OPCode_preVoteRequest ||
|
||
opcode == OPCode_probeConfigTag ||
|
||
opcode == OPCode_heartbeat ||
|
||
opcode == OPCode_hello ||
|
||
opcode == OPCode_nop ||
|
||
opcode == OPCode_replConfigPush ||
|
||
opcode == OPCode_walCatchupEntry ||
|
||
opcode == OPCode_snapshotTransfer ||
|
||
/* Read-only controller status — answerable in every role
|
||
* (controller-status 7.1, 7.2). */
|
||
opcode == OPCode_ctrlstatus)
|
||
return 0;
|
||
return -XCE_ERR_NOT_READY;
|
||
|
||
case CTRL_ELECTING:
|
||
/* Allow election RPCs AND primary→standby replication messages.
|
||
* A node campaigning for leadership must still accept config, WAL,
|
||
* and heartbeats from the current primary — if it wins the election
|
||
* it needs current state; if it loses, it becomes STANDBY and must
|
||
* resume replication without having missed pushes. Only job/client
|
||
* ops are rejected (we're not ready to serve). */
|
||
if (opcode == OPCode_requestVote ||
|
||
opcode == OPCode_preVoteRequest ||
|
||
opcode == OPCode_probeConfigTag ||
|
||
opcode == OPCode_hello ||
|
||
opcode == OPCode_nop ||
|
||
opcode == OPCode_replConfigPush ||
|
||
opcode == OPCode_walCatchupEntry ||
|
||
opcode == OPCode_snapshotTransfer ||
|
||
/* Read-only controller status — answerable in every role
|
||
* (controller-status 7.1, 7.2). */
|
||
opcode == OPCode_ctrlstatus)
|
||
return 0;
|
||
return -XCE_ERR_NOT_READY;
|
||
|
||
case CTRL_STANDBY: {
|
||
/* Allow ops that standbys legitimately handle */
|
||
switch (opcode) {
|
||
case OPCode_nop:
|
||
case OPCode_reply:
|
||
case OPCode_hello:
|
||
case OPCode_join:
|
||
case OPCode_shutdown:
|
||
case OPCode_heartbeat:
|
||
case OPCode_redirect:
|
||
case OPCode_steppingDown:
|
||
case OPCode_replEntry:
|
||
case OPCode_replAck:
|
||
case OPCode_replSnapshot:
|
||
case OPCode_replCatchUpDone:
|
||
case OPCode_replConfig:
|
||
case OPCode_replConfigPush:
|
||
case OPCode_requestVote:
|
||
case OPCode_preVoteRequest:
|
||
case OPCode_probeConfigTag:
|
||
case OPCode_walAck:
|
||
case OPCode_upgradeConfigAck:
|
||
case OPCode_snapshotTransfer:
|
||
case OPCode_walCatchupEntry:
|
||
/* Read-only controller status — a standby answers with its local
|
||
* view (it reports the primary it knows) (controller-status 7.1, 7.2). */
|
||
case OPCode_ctrlstatus:
|
||
return 0;
|
||
default:
|
||
return -XCE_ERR_STANDBY;
|
||
}
|
||
}
|
||
|
||
default:
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/* ha_config_promotion_fenced (ha-term-safety task 4.3, R7.9): a member is
|
||
* non-promotable while config serve-fenced — its latest committed config marker
|
||
* references bytes it does not hold/match, so it must not become authoritative.
|
||
* Pure atomic read via config_serve_fenced (no I/O) — safe on the election path. */
|
||
int ha_config_promotion_fenced(const ha_state_t *ha)
|
||
{
|
||
if (!ha)
|
||
return 0;
|
||
return config_serve_fenced(ha->raft_c) ? 1 : 0;
|
||
}
|
||
|
||
/* ── Redirect helpers used by join.c ───────────────────────── */
|
||
|
||
/*
|
||
* ha_should_redirect: returns 1 when this standby knows the primary's
|
||
* address and should redirect incoming join requests there.
|
||
* Returns 0 for PRIMARY role, or for STANDBY with no known primary yet.
|
||
*/
|
||
int ha_should_redirect(const ha_state_t *ha)
|
||
{
|
||
ctrl_role_t role =
|
||
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
|
||
if (role == CTRL_PRIMARY)
|
||
return 0;
|
||
if (ha->primary_host[0] == '\0' || ha->primary_port == 0)
|
||
return 0;
|
||
return 1;
|
||
}
|
||
|
||
/*
|
||
* ha_no_primary_known: returns 1 when this node is a STANDBY but does not
|
||
* yet know the primary's address. Callers should reply E_XCE_NOMASTER so
|
||
* the client retries rather than treating the error as permanent.
|
||
*/
|
||
int ha_no_primary_known(const ha_state_t *ha)
|
||
{
|
||
ctrl_role_t role =
|
||
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
|
||
if (role == CTRL_PRIMARY)
|
||
return 0;
|
||
return (ha->primary_host[0] == '\0' || ha->primary_port == 0) ? 1 : 0;
|
||
}
|
||
|
||
/*
|
||
* ha_register_persist_hooks: register the three persistence hooks on the
|
||
* cluster's hook system, guarded by the hooks_registered idempotency flag.
|
||
*
|
||
* ha-single-leader-role task 2.2: extracted from ha_promote_to_primary so
|
||
* BOTH roads to PRIMARY (the raft become-leader callback and the non-raft
|
||
* ha_promote_to_primary path) share one hook-registration sequence and
|
||
* cannot diverge. Safe on a zero-init cluster_t (zeroed hooksys →
|
||
* xce_add_hook returns -EINVAL harmlessly; hooks_registered still set to 1).
|
||
* Requires non-NULL c (dereferences &c->hooksys).
|
||
*/
|
||
static void ha_register_persist_hooks(cluster_t *c)
|
||
{
|
||
if (c->ha.hooks_registered)
|
||
return;
|
||
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||
(hook_fn_t)persist_job_hook_fun, &c->persist);
|
||
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
|
||
(hook_fn_t)persist_queue_hook_fun, &c->persist);
|
||
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
(hook_fn_t)persist_worker_hook_fun, &c->persist);
|
||
c->ha.hooks_registered = 1;
|
||
}
|
||
|
||
/*
|
||
* ha_primary_log_append (ha-term-safety task 3.1, R1.1/R1.2/R2.1): the SINGLE
|
||
* primary append entry point. Routes a local primary append (job-state entry or
|
||
* the own-term promotion no-op) through the raft library's raft_log_append, so
|
||
* the LIBRARY stamps the live election term (s->term) and assigns the LSN
|
||
* (s->log_len+1), then drives on_persist_entry — which (primary role) performs
|
||
* the actual WAL write via persist_wal_async. This makes the library index the
|
||
* authority for both the per-entry term (R1) and the LSN, retiring the
|
||
* hand-rolled bypass that assigned the LSN directly through persist_wal_async.
|
||
*
|
||
* Co-tracking invariant (closes the task 2.1 deferral): because raft_log_append
|
||
* is the only primary append path, every primary append advances BOTH the
|
||
* library index (s->log_len) and the persist write cursor (p->current_lsn) by
|
||
* exactly one, in lockstep — so p->current_lsn == s->log_len + 1 always holds and
|
||
* the LSN the library assigns equals the LSN on_persist_entry writes. raft_init
|
||
* (task 2.5) seeds s->log_len from the recovered durable floor, which equals
|
||
* p->current_lsn - 1 at boot, establishing the invariant before the first append.
|
||
*
|
||
* Persist-before-act: raft_log_append calls on_persist_entry BEFORE advancing
|
||
* s->log_len and propagates a non-zero persist rc without mutating the log (no
|
||
* phantom entry) — preserving persist-before-ACK and the bug-850 contiguity
|
||
* guard beneath the WAL write.
|
||
*
|
||
* Legacy fallback (non-raft road): when the raft library is not in RAFT_LEADER
|
||
* state (raft_log_append returns -EPERM) — the init-time non-raft promotion road
|
||
* (ha_promote_to_primary) and engine-less unit-test contexts — the library is
|
||
* not tracking the log, so there is no co-tracking hazard. We append locally
|
||
* through the legacy persist_wal_async path (which still records the live term
|
||
* via the persist_set_term seam set on the prior promotion) so those roads keep
|
||
* working unchanged. Once raft drives leadership in production, the -EPERM
|
||
* branch is never taken and the library is the sole LSN/term source.
|
||
*
|
||
* out_lsn (optional) receives the assigned 1-based LSN on success.
|
||
* Returns 0 on success; propagates a non-zero persist rc on append failure.
|
||
*/
|
||
int ha_primary_log_append(cluster_t *c, uint32_t type,
|
||
const void *payload, uint32_t len, uint64_t *out_lsn)
|
||
{
|
||
if (!c)
|
||
return -EINVAL;
|
||
|
||
/* Pack the in-flight adapter envelope [raft_entry_blob_t{type,len}][payload]
|
||
* — the SAME format on_persist_entry (ha_cb_persist_entry_stub) unpacks, so
|
||
* the encoder and decoder share the single wal_types.h definition. */
|
||
size_t blob_len = sizeof(raft_entry_blob_t) + len;
|
||
uint8_t stackbuf[256];
|
||
uint8_t *blob = (blob_len <= sizeof(stackbuf)) ? stackbuf : malloc(blob_len);
|
||
if (!blob)
|
||
return -ENOMEM;
|
||
|
||
raft_entry_blob_t *bh = (raft_entry_blob_t *)blob;
|
||
bh->type = type;
|
||
bh->payload_len = len;
|
||
if (payload && len > 0)
|
||
memcpy(blob + sizeof(*bh), payload, len);
|
||
|
||
uint64_t idx = 0;
|
||
int rc = raft_log_append(&c->ha.raft, blob, blob_len, &idx);
|
||
|
||
if (rc == -EPERM) {
|
||
/* Non-raft road: the library is not LEADER and is not tracking the log.
|
||
* Append locally through the legacy path (no library index to co-track).
|
||
* Stamp the live election term (ha->term) so the entry still records the
|
||
* leader's term (R1.1) even off the raft road — matching on_persist_entry's
|
||
* persist_set_term on the raft path. */
|
||
persist_set_term(&c->persist,
|
||
atomic_load_explicit(&c->ha.term, memory_order_acquire));
|
||
rc = persist_wal_async(&c->persist, type, payload, len);
|
||
if (rc == 0) {
|
||
pthread_mutex_lock(&c->persist.lock);
|
||
idx = c->persist.current_lsn - 1; /* the LSN just assigned */
|
||
pthread_mutex_unlock(&c->persist.lock);
|
||
}
|
||
}
|
||
|
||
if (blob != stackbuf)
|
||
free(blob);
|
||
|
||
if (rc == 0 && out_lsn)
|
||
*out_lsn = idx;
|
||
return rc;
|
||
}
|
||
|
||
/* ha_repl_ack_abs_peer_idx (ha-term-safety task 3.3 remediation): resolve the
|
||
* ABSOLUTE library peer index from an ACK endpoint's stored peer_idx, branching
|
||
* on endpoint type EXACTLY as the standby_endpoints scan at ha.c:6187-6189.
|
||
*
|
||
* Two endpoint types reach the walAck branch of repl_ack_decode_and_dispatch,
|
||
* and their stored peer_idx has DIFFERENT meaning:
|
||
* - Promoted vote ep (vote_ep_ctx_t, ep_is_vote != 0): peer_idx is a
|
||
* self-EXCLUDED "votes" index (relative). Map it back to the absolute index
|
||
* — the inverse of raft_cb_send_vote_request's votes_idx remap (self_idx ==
|
||
* my_priority): abs = votes_idx < self_idx ? votes_idx : votes_idx + 1.
|
||
* - Heartbeat standby ep (standby_ep_ctx_t, ctx==NULL, ep_is_vote == 0):
|
||
* peer_idx is ALREADY the absolute candidate index — pass it through.
|
||
*
|
||
* The earlier unconditional remap corrupted the heartbeat path: a standby ep's
|
||
* already-absolute index was remapped a second time, crediting the WRONG peer
|
||
* (false quorum / stuck commit) or, near the array bound, an OUT-OF-BOUNDS
|
||
* match_index[] write. This resolver is the single source of truth so the
|
||
* production call site and the unit test exercise identical logic. */
|
||
int ha_repl_ack_abs_peer_idx(ha_state_t *ha, int ep_is_vote, int peer_idx)
|
||
{
|
||
int self_idx = ha->my_priority; /* self_idx == my_priority by convention */
|
||
if (ep_is_vote)
|
||
return (peer_idx < self_idx) ? peer_idx : peer_idx + 1;
|
||
return peer_idx; /* heartbeat standby ep: already absolute */
|
||
}
|
||
|
||
/* ha-term-safety task 3.3 (R4.1/R4.5): drive the raft library on a replication
|
||
* ACK. Route the reply through raft_on_append_entries_reply (updating
|
||
* match_index[]/next_index[]) then raft_advance_commit, so the library
|
||
* recomputes the current-term-gated commit (Figure-8) and MAY fire
|
||
* on_commit_advanced — the task-2.4 callback feeding the synchronous barrier.
|
||
*
|
||
* Peer-index contract: `abs_peer_idx` is the ABSOLUTE library peer index
|
||
* (j != self_idx), the same index space the library uses for
|
||
* match_index[]/next_index[]. The CALLER resolves it per endpoint type via
|
||
* ha_repl_ack_abs_peer_idx (promoted-vote eps carry a relative votes index;
|
||
* heartbeat standby eps carry an already-absolute index) — this seam must NOT
|
||
* remap again or it would corrupt the heartbeat path.
|
||
*
|
||
* Defense-in-depth: a peer index outside [0, raft.n_peers) (and never self) is
|
||
* rejected loudly rather than passed into raft_on_append_entries_reply, whose
|
||
* match_index[] write is unchecked. This fences an OOB / wrong-peer write at the
|
||
* controller boundary (no assert inside src/lib/raft/) and self-heals — the
|
||
* caller simply skips this ACK's library accounting.
|
||
*
|
||
* The library drops a stale-term reply (reply_term != s->term) inside
|
||
* raft_on_append_entries_reply, so a stale ACK never touches match_index[] and
|
||
* never advances commit. raft_advance_commit is leader-only (-EPERM otherwise);
|
||
* on a non-leader the call is a harmless no-op. Neither library call performs
|
||
* I/O — safe on the iomgr ACK path. */
|
||
int ha_raft_on_repl_ack(cluster_t *c, int abs_peer_idx, uint64_t acked_lsn,
|
||
uint64_t reply_term)
|
||
{
|
||
if (!c)
|
||
return -EINVAL;
|
||
|
||
/* Defense-in-depth bounds/self check: the library's match_index[] write is
|
||
* unchecked, so reject a bogus index here (fail loud, never corrupt). */
|
||
int self_idx = c->ha.my_priority;
|
||
int n_peers = c->ha.raft.n_peers;
|
||
if (abs_peer_idx < 0 || abs_peer_idx >= n_peers || abs_peer_idx == self_idx) {
|
||
ERROR("ha_raft_on_repl_ack: rejecting out-of-range peer index %d "
|
||
"(n_peers=%d self_idx=%d) — skipping library ACK accounting",
|
||
abs_peer_idx, n_peers, self_idx);
|
||
return -EINVAL;
|
||
}
|
||
|
||
/* A successful WalAck reports the standby's matched index (acked_lsn). The
|
||
* library applies it only when reply_term == s->term (stale-term drop). */
|
||
(void)raft_on_append_entries_reply(&c->ha.raft, abs_peer_idx, /*success*/1,
|
||
acked_lsn, reply_term);
|
||
|
||
/* Recompute the current-term-gated commit; fires on_commit_advanced exactly
|
||
* once on a strict advance of a current-term entry (else holds). Leader-only:
|
||
* -EPERM off the leader leaves commit_index unchanged with no callback. */
|
||
(void)raft_advance_commit(&c->ha.raft);
|
||
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* ha_arm_own_term_gate: append the own-term WAL_NOOP and arm the commit
|
||
* engine's writable_lsn gate (ha-single-leader-role task 2.2, Req 1.4/2.1/2.2).
|
||
*
|
||
* The shared promotion tail both roads to PRIMARY run. Sequence:
|
||
* 1. persist_wal_async(WAL_NOOP) — appends a header-only entry and
|
||
* broadcasts it to standbys via repl_broadcast (one call does both).
|
||
* 2. derive noop_lsn = current_lsn - 1 under p->lock (the append
|
||
* post-incremented current_lsn; persist.c:1019-1021 pattern).
|
||
* 3. commit_submit(noop_lsn, COMMIT_MODE_RETAIN) — lock-free, non-blocking,
|
||
* registration-free observability bookkeeping (records nothing).
|
||
* 4. commit_set_writable_lsn(noop_lsn) — arms the own-term gate so the
|
||
* engine withholds WAIT completion until committed_lsn >= noop_lsn.
|
||
*
|
||
* Non-blocking by construction: every call here returns without waiting for
|
||
* the no-op to commit (Req 1.4 — status honesty does not wait).
|
||
*
|
||
* Degraded-but-safe (design §Error Strategy): if the no-op append fails,
|
||
* no LSN exists, so the gate stays UNSET; promotion STILL COMPLETES. The
|
||
* universal commit barrier still requires per-entry majority of a new-term
|
||
* entry, so the first state-changing entry of this term becomes the de-facto
|
||
* own-term vehicle. We do NOT block and do NOT abort promotion.
|
||
*/
|
||
static void ha_arm_own_term_gate(cluster_t *c)
|
||
{
|
||
/* ha-term-safety task 3.1: route the own-term promotion no-op through the
|
||
* library (ha_primary_log_append → raft_log_append) so the WAL_NOOP records
|
||
* the newly established term (R1.2) and the library assigns its LSN. On the
|
||
* non-raft promotion road this falls back to the legacy append; either way
|
||
* the returned LSN is the no-op's WAL LSN. */
|
||
uint64_t noop_lsn = 0;
|
||
int rc = ha_primary_log_append(c, WAL_NOOP, NULL, 0, &noop_lsn);
|
||
if (rc < 0) {
|
||
ERROR("HA: own-term WAL_NOOP append failed: %d", rc);
|
||
WARN("HA: own-term gate left UNSET (degraded-but-safe); first "
|
||
"new-term entry becomes the own-term commit vehicle");
|
||
return;
|
||
}
|
||
|
||
/* Observability-only bookkeeping (lock-free, non-blocking, records
|
||
* nothing per entry — commit state derives from registry watermarks). */
|
||
(void)commit_submit(&c->commit, noop_lsn, COMMIT_MODE_RETAIN, 0);
|
||
|
||
/* Arm the own-term gate: completion of any WAIT is withheld until
|
||
* committed_lsn >= noop_lsn. */
|
||
commit_set_writable_lsn(&c->commit, noop_lsn);
|
||
|
||
INFO("HA: own-term noop appended lsn=%" PRIu64 ", writable_lsn gate armed",
|
||
noop_lsn);
|
||
}
|
||
|
||
/*
|
||
* ha_promote_to_primary: the NON-RAFT promotion road (cluster.c init-time,
|
||
* legacy XCE_TESTING evaluation path, tests). This path OWNS its term, so it
|
||
* calls ha_do_promote (which increments the term + sets role=PRIMARY). It
|
||
* then runs the SAME shared sequence the raft road runs: register hooks,
|
||
* start the heartbeat, and arm the own-term gate (noop + writable_lsn).
|
||
*
|
||
* Every road to PRIMARY arms the own-term gate. The noop+gate tail is
|
||
* guarded on c != NULL (callers always pass a real cluster here).
|
||
*/
|
||
int ha_promote_to_primary(cluster_t *c)
|
||
{
|
||
ha_do_promote(&c->ha); /* non-raft path owns its term increment */
|
||
|
||
/* Shared promotion sequence — must match the raft become-leader tail:
|
||
* hooks live BEFORE the no-op append so the persistence seam is active
|
||
* before any racing client write. */
|
||
ha_register_persist_hooks(c);
|
||
|
||
/* Arm periodic heartbeat to standbys — keeps their watchdog alive when
|
||
* the cluster is idle (no WAL entries or config pushes). */
|
||
ha_start_primary_heartbeat(c);
|
||
|
||
/* Own-term no-op + writable_lsn gate (Req 1.4, 2.1, 2.2). */
|
||
ha_arm_own_term_gate(c);
|
||
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* ha_demote_to_standby: disable persistence hooks, then set role=STANDBY
|
||
* and reset the hooks_registered idempotency guard (via ha_do_demote).
|
||
*/
|
||
int ha_demote_to_standby(cluster_t *c)
|
||
{
|
||
/* Disable persistence hooks before changing role */
|
||
if (c->ha.hooks_registered) {
|
||
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
|
||
(hook_fn_t)persist_job_hook_fun);
|
||
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
|
||
(hook_fn_t)persist_queue_hook_fun);
|
||
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
|
||
(hook_fn_t)persist_worker_hook_fun);
|
||
}
|
||
ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */
|
||
return 0;
|
||
}
|
||
|
||
/* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */
|
||
|
||
/*
|
||
* ha_wait_for_heartbeat_or_timeout: called from the startup path after
|
||
* ha_init() when the node needs to discover whether a primary already exists.
|
||
*
|
||
* Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where
|
||
* deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter
|
||
*
|
||
* The heartbeat handler (cluster.c) posts heartbeat_seen when it receives
|
||
* OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits
|
||
* OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler.
|
||
*
|
||
* Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived);
|
||
* ha_do_demote() was called (role = CTRL_STANDBY).
|
||
* Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT);
|
||
* caller is responsible for driving the election.
|
||
*/
|
||
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha)
|
||
{
|
||
/* Seed jitter PRNG on first use */
|
||
if (ha->jitter_seed == 0)
|
||
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
|
||
|
||
/* Compute timeout: election_timeout + priority_step * priority + random jitter.
|
||
* Higher-priority (lower index) nodes time out sooner, giving them a head
|
||
* start in the election before lower-priority nodes begin campaigning. */
|
||
long to_ms = (long)ha->config.election_timeout_ms;
|
||
to_ms += (long)ha->my_priority * (long)ha->config.election_priority_step_ms;
|
||
if (ha->config.election_jitter_max_ms > 0) {
|
||
to_ms += (long)(xorshift64_next(&ha->jitter_seed)
|
||
% ((uint64_t)ha->config.election_jitter_max_ms + 1));
|
||
}
|
||
|
||
/* sem_timedwait requires an absolute CLOCK_REALTIME deadline */
|
||
struct timespec deadline;
|
||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||
deadline.tv_sec += to_ms / 1000;
|
||
deadline.tv_nsec += (to_ms % 1000) * 1000000L;
|
||
if (deadline.tv_nsec >= 1000000000L) {
|
||
deadline.tv_sec++;
|
||
deadline.tv_nsec -= 1000000000L;
|
||
}
|
||
|
||
int rc = sem_timedwait(&ha->heartbeat_seen, &deadline);
|
||
if (rc == 0) {
|
||
/* Heartbeat arrived before the deadline — demote to standby */
|
||
ha_do_demote(ha);
|
||
return CTRL_STANDBY;
|
||
}
|
||
|
||
/* ETIMEDOUT (or any other error): treat as election timeout */
|
||
return CTRL_ELECTING;
|
||
}
|
||
|
||
/* ── Term Durability (Req 9) ────────────────────────────────────────────────
|
||
*
|
||
* ha_term_file_t: on-disk format for ha-term.bin.
|
||
* Fields are in host byte order (single-host durability, no cross-endian migration).
|
||
* CRC32 covers all preceding bytes via zlib crc32().
|
||
*/
|
||
|
||
#define HA_TERM_MAGIC 0xDEADBEEFCAFEBABEULL
|
||
|
||
typedef struct {
|
||
uint64_t magic;
|
||
uint64_t term;
|
||
uint64_t voted_for_term;
|
||
char voted_for_name[XCE_MAXNAME_LEN];
|
||
uint32_t crc32;
|
||
} ha_term_file_t;
|
||
|
||
/*
|
||
* ha_persist_term: atomically write ha_term_file_t to {workdir}/ha-term.bin.
|
||
* Uses write-to-tmp + fsync + rename for durability (existing project pattern).
|
||
* If workdir is NULL, the persist is skipped (test mode or no workdir configured).
|
||
* Req 9.1: must be called before any vote grant or term increment is reported.
|
||
*/
|
||
void ha_persist_term(ha_state_t *ha, const char *workdir)
|
||
{
|
||
if (!workdir)
|
||
return; /* test mode: skip file I/O */
|
||
|
||
ha_term_file_t f;
|
||
memset(&f, 0, sizeof(f));
|
||
f.magic = HA_TERM_MAGIC;
|
||
f.term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
f.voted_for_term = ha->voted_for_term;
|
||
strncpy(f.voted_for_name, ha->voted_for_name, sizeof(f.voted_for_name) - 1);
|
||
f.voted_for_name[sizeof(f.voted_for_name) - 1] = '\0';
|
||
|
||
/* CRC32 covers all bytes before the crc32 field itself */
|
||
size_t crc_len = offsetof(ha_term_file_t, crc32);
|
||
f.crc32 = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len);
|
||
|
||
/* Write to tmp then rename — atomic from filesystem perspective */
|
||
char path[1024], tmp[1040];
|
||
snprintf(path, sizeof(path), "%s/ha-term.bin", workdir);
|
||
snprintf(tmp, sizeof(tmp), "%s/ha-term.bin.tmp", workdir);
|
||
|
||
int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600);
|
||
if (fd < 0)
|
||
return; /* best-effort; caller must not rely on this for correctness */
|
||
|
||
ssize_t written = write(fd, &f, sizeof(f));
|
||
if (written == (ssize_t)sizeof(f))
|
||
fsync(fd);
|
||
close(fd);
|
||
|
||
if (written == (ssize_t)sizeof(f))
|
||
rename(tmp, path);
|
||
}
|
||
|
||
/*
|
||
* ha_restore_term: read {workdir}/ha-term.bin at startup and restore
|
||
* ha->term and ha->voted_for_* into ha_state_t.
|
||
*
|
||
* On missing file (ENOENT): no-op — term stays 0. Req 9.3.
|
||
* On CRC/magic mismatch: log WARN, zero ha->term and ha->voted_for_*. Req 9.2.
|
||
* On valid file: atomically restore ha->term; copy voted_for fields. Req 9.2.
|
||
*
|
||
* Returns 0 on success (including gracefully-handled file-not-found and CRC bad),
|
||
* -errno on unexpected I/O error.
|
||
* If workdir is NULL, function is a no-op (safe for unit tests without a workdir).
|
||
*/
|
||
int ha_restore_term(ha_state_t *ha, const char *workdir)
|
||
{
|
||
if (!workdir)
|
||
return 0;
|
||
|
||
char path[1024];
|
||
snprintf(path, sizeof(path), "%s/ha-term.bin", workdir);
|
||
|
||
int fd = open(path, O_RDONLY);
|
||
if (fd < 0) {
|
||
if (errno == ENOENT)
|
||
return 0; /* first boot — no file yet, term stays 0 (Req 9.3) */
|
||
return -errno;
|
||
}
|
||
|
||
ha_term_file_t f;
|
||
memset(&f, 0, sizeof(f));
|
||
ssize_t n = read(fd, &f, sizeof(f));
|
||
close(fd);
|
||
|
||
if (n != (ssize_t)sizeof(f)) {
|
||
WARN("HA: ha-term.bin short read (%zd/%zu) — ignoring, starting at term 0",
|
||
n, sizeof(f));
|
||
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
|
||
ha->voted_for_term = 0;
|
||
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
|
||
return 0;
|
||
}
|
||
|
||
/* Verify magic */
|
||
if (f.magic != HA_TERM_MAGIC) {
|
||
WARN("HA: ha-term.bin bad magic (0x%016" PRIx64 ") — ignoring, starting at term 0",
|
||
f.magic);
|
||
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
|
||
ha->voted_for_term = 0;
|
||
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
|
||
return 0;
|
||
}
|
||
|
||
/* Verify CRC32 */
|
||
size_t crc_len = offsetof(ha_term_file_t, crc32);
|
||
uint32_t computed = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len);
|
||
if (computed != f.crc32) {
|
||
WARN("HA: ha-term.bin CRC mismatch (expected 0x%08x got 0x%08x) — ignoring, starting at term 0",
|
||
computed, f.crc32);
|
||
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
|
||
ha->voted_for_term = 0;
|
||
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
|
||
return 0;
|
||
}
|
||
|
||
/* Valid file — restore state (Req 9.2) */
|
||
atomic_store_explicit(&ha->term, f.term, memory_order_release);
|
||
ha->voted_for_term = f.voted_for_term;
|
||
strncpy(ha->voted_for_name, f.voted_for_name, sizeof(ha->voted_for_name) - 1);
|
||
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
|
||
|
||
return 0;
|
||
}
|
||
|
||
/* ── ha_heartbeat_is_fresh (Req 6.2) ────────────────────────────────────────
|
||
*
|
||
* Returns 1 if the elapsed time since the last primary message
|
||
* (ha->last_activity, written by ha_on_heartbeat_received) is strictly less
|
||
* than ha->config.heartbeat_timeout_ms (i.e. the primary is still alive).
|
||
* Returns 0 if no primary message has been seen yet (last_activity == 0) or
|
||
* the timeout has expired.
|
||
*
|
||
* Uses CLOCK_MONOTONIC seconds to avoid wall-clock skew; whole-second
|
||
* granularity is sufficient for the 10 s default timeout.
|
||
*/
|
||
int ha_heartbeat_is_fresh(ha_state_t *ha)
|
||
{
|
||
time_t last = ha->last_activity; /* time_t load is atomic on LP64 */
|
||
if (last == 0)
|
||
return 0; /* no primary message seen yet */
|
||
struct timespec now;
|
||
clock_gettime(CLOCK_MONOTONIC, &now);
|
||
long ms = (long)(now.tv_sec - last) * 1000L;
|
||
return ms < (long)ha->config.heartbeat_timeout_ms;
|
||
}
|
||
|
||
/* ── Voter freshness floor (election safety) ───────────────────────────────
|
||
*
|
||
* Compute the voter's effective freshness floor as the per-field max of its
|
||
* local state and its learned-committed cluster state. Outdated candidates
|
||
* (below this floor) are denied so the cluster can never be rolled back below
|
||
* what it has already committed. See ha.h for full contract.
|
||
*/
|
||
void ha_vote_self_freshness(ha_state_t *ha,
|
||
uint64_t local_config_tag,
|
||
uint64_t local_applied_lsn,
|
||
uint64_t *out_tag,
|
||
uint64_t *out_lsn)
|
||
{
|
||
uint64_t learned_tag =
|
||
atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire);
|
||
uint64_t learned_lsn =
|
||
atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire);
|
||
|
||
if (out_tag)
|
||
*out_tag = (local_config_tag > learned_tag) ? local_config_tag : learned_tag;
|
||
if (out_lsn)
|
||
*out_lsn = (local_applied_lsn > learned_lsn) ? local_applied_lsn : learned_lsn;
|
||
}
|
||
|
||
/* ── Candidate advertised LSN (election safety, ha-commit-durability 4.2) ───
|
||
*
|
||
* A candidate advertises EXACTLY its durable WAL tip on the LSN axis — the
|
||
* bare `persist_durable_floor(p)`, NOT the folded
|
||
* max(durable_floor, last_known_committed_lsn) that ha_vote_self_freshness
|
||
* computes for the VOTER floor.
|
||
*
|
||
* Why bare, not folded: ha_vote_self_freshness unconditionally returns
|
||
* max(local_lsn, last_known_committed_lsn). Feeding it durable_floor would let
|
||
* a node restarted with a TRUNCATED log (durable_floor low) but a stale
|
||
* volatile learned_committed (high) advertise the stale-high value and WIN —
|
||
* exactly the truncated-candidate-wins hole this task closes (design §1a/§5a).
|
||
* A candidate may only claim a log position it can durably back.
|
||
*
|
||
* The candidate's TAG axis is unchanged: it still uses
|
||
* max(local_config_tag, learned_tag) (the config epoch is orthogonal to log
|
||
* durability — design §1a). Only the LSN axis bypasses the fold.
|
||
*
|
||
* NULL-safe (persist_durable_floor guards NULL → 0) for the test/seam paths.
|
||
*/
|
||
uint64_t ha_candidate_advertise_lsn(persist_t *p)
|
||
{
|
||
return persist_durable_floor(p);
|
||
}
|
||
|
||
/* bug-848: see ha.h. After local WAL recovery, advertise the recovered durable
|
||
* tip for catch-up so the primary streams only the missing suffix instead of
|
||
* re-streaming + overwriting consistent records. Monotonic: never lowers a
|
||
* higher applied_lsn (e.g. if catch-up already advanced it). */
|
||
void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p)
|
||
{
|
||
if (!ha || !p)
|
||
return;
|
||
|
||
uint64_t recovered = persist_durable_floor(p);
|
||
uint64_t cur = atomic_load_explicit(&ha->applied_lsn,
|
||
memory_order_acquire);
|
||
if (recovered > cur)
|
||
atomic_store_explicit(&ha->applied_lsn, recovered,
|
||
memory_order_release);
|
||
}
|
||
|
||
/* bug-1042: re-sync the raft library's log_len / commit_index / term from the
|
||
* recovered WAL AFTER persist_recover.
|
||
*
|
||
* Init-ordering hazard: ha_init_raft (myapp.c, before the election) sources
|
||
* raft.initial_log_len from persist_durable_floor — but on a STANDBY-REJOIN boot
|
||
* persist_recover has not run yet at that point, so the library comes up with
|
||
* log_len=0. After cluster_init_phase2 runs persist_recover (recovering e.g.
|
||
* durable_floor=7), the library still thinks its log is EMPTY. Every incoming
|
||
* AppendEntries then fails logOk (prevLogIndex>=1 against log_len=0 → reject),
|
||
* so the rejoined member is PERMANENTLY stuck as a non-converging standby and
|
||
* the primary re-streams [tip+1..committed] every heartbeat forever.
|
||
*
|
||
* Fix: after recovery, raise the library's log_len to the recovered WAL tip and
|
||
* its commit_index to the recovered durable floor (clamped to log_len, honoring
|
||
* the commit_index<=log_len invariant), and align raft.term with the persisted
|
||
* term so the per-LSN (term,index) reconcile reads consistent state. Monotonic:
|
||
* never lowers an already-higher library position (e.g. if catch-up advanced it
|
||
* before this runs). The library stores no entry bytes — on_get_entry_term reads
|
||
* the recovered per-LSN term index — so raising log_len is sufficient for logOk. */
|
||
void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p)
|
||
{
|
||
if (!ha || ha->raft_c == NULL || !p)
|
||
return;
|
||
|
||
uint64_t recovered_tip = persist_durable_floor(p);
|
||
if (recovered_tip > ha->raft.log_len)
|
||
ha->raft.log_len = recovered_tip;
|
||
|
||
/* commit_index advances to the recovered durable floor but never past the
|
||
* log tip (library invariant commit_index <= log_len), and never decreases. */
|
||
uint64_t recovered_commit = recovered_tip;
|
||
if (recovered_commit > ha->raft.log_len)
|
||
recovered_commit = ha->raft.log_len;
|
||
if (recovered_commit > ha->raft.commit_index)
|
||
ha->raft.commit_index = recovered_commit;
|
||
|
||
/* Align the library term with the persisted/restored HA term so the
|
||
* (term,lsn) reconcile and own-term gate read a consistent value after a
|
||
* rejoin (ha_restore_term already loaded ha->term before ha_init_raft). */
|
||
uint64_t ha_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
if (ha_term > ha->raft.term)
|
||
ha->raft.term = ha_term;
|
||
}
|
||
|
||
/* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ────────────────────────
|
||
*
|
||
* handle_request_vote: process an incoming OP_requestVote RPC.
|
||
*
|
||
* Decision order (fail-fast):
|
||
* 0. role gate (Req 6.1, 6.2):
|
||
* - CTRL_PRIMARY → deny
|
||
* - CTRL_STANDBY with fresh heartbeat → deny
|
||
* 1. in_catch_up → deny (Req 2.5)
|
||
* 2. req term < my term → deny (Req 2.2)
|
||
* 3. req term > my term → update term, clear voted_for (Req 2.3)
|
||
* 4. already voted for a DIFFERENT candidate in this term → deny (Req 2.6)
|
||
* 5. freshness gate: candidate not at least as fresh → deny (Req 2.4)
|
||
* 6. grant vote, record voted_for, persist (Req 9.1)
|
||
*
|
||
* Returns 1 = granted, 0 = denied, <0 = error.
|
||
*
|
||
* TEST-ONLY (XCE_TESTING): production vote dispatch goes through
|
||
* cluster.c → raft_on_vote_request. This function is the legacy
|
||
* entry point used by test_ha_vote_gate.c and test_ha_election.c
|
||
* to exercise role-gate logic in isolation without a full raft state.
|
||
*/
|
||
#ifdef XCE_TESTING
|
||
int handle_request_vote(ha_state_t *ha,
|
||
uint64_t term,
|
||
const char *candidate_id,
|
||
uint64_t config_tag,
|
||
uint64_t last_lsn,
|
||
uint64_t my_config_tag,
|
||
uint64_t my_last_lsn,
|
||
int in_catch_up_flag)
|
||
{
|
||
/* Req 2.3: higher term → step down BEFORE any role gate.
|
||
*
|
||
* Raft §5.1: "If RPC request or response contains term T > currentTerm:
|
||
* set currentTerm = T, convert to follower."
|
||
*
|
||
* Without this ordering, a CTRL_PRIMARY node at term=N would deny a vote
|
||
* request at term=N+1 without ever seeing the higher term, leaving two
|
||
* nodes both believing they lead → split-brain. */
|
||
uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
if (term > my_term) {
|
||
atomic_store_explicit(&ha->term, term, memory_order_release);
|
||
ha->voted_for_term = 0;
|
||
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
|
||
/* Step down from leader: a higher-term election is in progress and we
|
||
* must not block it with the role gate below. */
|
||
ctrl_role_t cur_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
if (cur_role == CTRL_PRIMARY) {
|
||
ha_do_demote(ha);
|
||
INFO("HA: stepping down (term %" PRIu64 " → %" PRIu64
|
||
") — higher-term election in progress", my_term, term);
|
||
}
|
||
my_term = term;
|
||
}
|
||
|
||
/* Req 2.2: stale term → deny, do not update any state */
|
||
if (term < my_term) {
|
||
return 0;
|
||
}
|
||
|
||
/* Req 6.1, 6.2: role-based deny gate.
|
||
* Evaluated AFTER the term catch-up above: if we just stepped down due to
|
||
* a higher term, role is now CTRL_STANDBY and we fall through to voting.
|
||
* PRIMARY at the current term still denies votes (protecting the live
|
||
* primary from same-term challengers). */
|
||
ctrl_role_t my_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
if (my_role == CTRL_PRIMARY)
|
||
return 0; /* deny — I am primary; protect against same-term challengers */
|
||
if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
|
||
ha_heartbeat_is_fresh(ha))
|
||
return 0; /* deny — my primary is alive (active primary connection) */
|
||
|
||
/* Req 2.5: catching-up standbys deny all votes */
|
||
if (in_catch_up_flag)
|
||
return 0;
|
||
|
||
/* Protect voted_for check+set from concurrent vote requests on different vtasks */
|
||
pthread_mutex_lock(&ha->lock);
|
||
|
||
/* Req 2.6: already voted for a DIFFERENT candidate in this term → deny.
|
||
* Same-candidate re-grant is idempotent (Raft §5.2: grant iff votedFor ∈
|
||
* {Nil, candidateId}) — matches the duplicate-vote check in raft.c. */
|
||
if (ha->voted_for_term == term && ha->voted_for_name[0] != '\0' &&
|
||
strcmp(ha->voted_for_name, candidate_id) != 0) {
|
||
pthread_mutex_unlock(&ha->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* Req 2.4 + election-safety fix #2: freshness gate against the EFFECTIVE
|
||
* floor — the max of the voter's local state and its learned-committed
|
||
* cluster state. A candidate that is below what the cluster has already
|
||
* committed (config_tag or, at equal tag, LSN) would roll the cluster back
|
||
* and must be denied even when this voter's local state is colder.
|
||
* Deny if candidate has a lower config_tag, or equal config_tag with lower
|
||
* LSN; grant if strictly fresher or equally fresh. */
|
||
uint64_t eff_tag = 0, eff_lsn = 0;
|
||
ha_vote_self_freshness(ha, my_config_tag, my_last_lsn, &eff_tag, &eff_lsn);
|
||
if (config_tag < eff_tag) {
|
||
pthread_mutex_unlock(&ha->lock);
|
||
return 0;
|
||
}
|
||
if (config_tag == eff_tag && last_lsn < eff_lsn) {
|
||
pthread_mutex_unlock(&ha->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* Grant the vote: record voted_for and persist before responding (Req 9.1) */
|
||
ha->voted_for_term = term;
|
||
strncpy(ha->voted_for_name, candidate_id, sizeof(ha->voted_for_name) - 1);
|
||
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
|
||
|
||
pthread_mutex_unlock(&ha->lock);
|
||
|
||
/* workdir is not available here — persist with NULL (test mode or caller
|
||
* wraps this function with workdir before sending the network reply).
|
||
* Production dispatch path calls ha_persist_term(ha, workdir) explicitly
|
||
* before this function writes its network reply. */
|
||
ha_persist_term(ha, NULL);
|
||
|
||
return 1;
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
/* ── raft-drive-async Task 2.1: reconnect-offload feature flag (ALL builds) ──
|
||
*
|
||
* drive_async_reconnect_offload gates the peer reconnect path:
|
||
* ON (production default) → primary_heartbeat_fn submits the peer reconnect
|
||
* to the slow lane and returns immediately; the delaymgr thread NEVER
|
||
* performs the blocking connect inline (ha-async-io.md thread-role
|
||
* contract — a blocking connect on the heartbeat/election-timeout thread
|
||
* stalls liveness to healthy peers and triggers spurious elections).
|
||
* OFF → legacy inline blocking connect on the delaymgr thread; retained only
|
||
* so the TDD RED test can demonstrate the pre-change stall.
|
||
*
|
||
* Compiled in ALL builds (production reads it on every heartbeat tick); the
|
||
* setter is the test seam to flip RED↔GREEN. */
|
||
static _Atomic int g_drive_async_reconnect_offload = 1;
|
||
|
||
void ha_set_reconnect_offload_for_test(int on)
|
||
{
|
||
atomic_store_explicit(&g_drive_async_reconnect_offload, on ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
int ha_reconnect_offload_enabled(void)
|
||
{
|
||
return atomic_load_explicit(&g_drive_async_reconnect_offload,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/* raft-drive-async task 3.2: SnapshotSend feature flag.
|
||
*
|
||
* DEFAULT OFF (0): the snapshot transfer runs INLINE on the caller (io_worker/
|
||
* drive) thread — byte-for-byte the legacy behavior. This task lands DORMANT,
|
||
* exactly like the Task 3.1 peer-control dispatch flag, because the offload
|
||
* captures a BORROWED standby endpoint (see snapshot_lane_fn's bug-1197 hazard
|
||
* note): moving the transfer to a lane thread while the io_worker may
|
||
* RDHUP-free that endpoint is a cross-thread UAF. Today (flag OFF) the send
|
||
* and the endpoint teardown are serialized on the single io_worker, so they
|
||
* cannot overlap.
|
||
*
|
||
* Flipping ON is the job of Task 3.4 (receiver-side integration), which
|
||
* establishes the endpoint-lifetime discipline for the pool/lane dispatch
|
||
* model and proves the transfer no longer stalls other peers' heartbeats/votes
|
||
* with the pool near saturation. Until then, bug-1198 (the repl-ack body is
|
||
* still unbounded when run inline) remains open by design.
|
||
*
|
||
* Keyed per standby (Req 1.5) so at most one snapshot per peer is in flight;
|
||
* at most one persist_checkpoint in flight (existing atomic guard). Compiled
|
||
* in ALL builds; the setter is the test seam to flip RED↔GREEN. */
|
||
static _Atomic int g_drive_async_snapshot_offload = 0;
|
||
|
||
void ha_set_snapshot_offload_for_test(int on)
|
||
{
|
||
atomic_store_explicit(&g_drive_async_snapshot_offload, on ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
int ha_snapshot_offload_enabled(void)
|
||
{
|
||
return atomic_load_explicit(&g_drive_async_snapshot_offload,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/* raft-drive-async task 3.3: ConfigPush feature flag.
|
||
*
|
||
* DEFAULT OFF (0): the requester-less config push runs INLINE on the caller
|
||
* (io_worker/drive) thread — byte-for-byte the legacy behavior, i.e.
|
||
* push_config_transport_only's fan-out followed by its blocking
|
||
* wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS). This task lands DORMANT,
|
||
* exactly like the Task 3.1/3.2 flags, because the offload borrows standby
|
||
* endpoints it does not own (bug-1197 family — push_config_transport_only
|
||
* reads c->ha.standby_endpoints): moving the push to a lane thread while the
|
||
* io_worker may RDHUP-free those endpoints is a cross-thread UAF. Today (flag
|
||
* OFF) the push and the endpoint teardown are serialized on the single
|
||
* io_worker, so they cannot overlap.
|
||
*
|
||
* Flipping ON is the job of Task 3.4, which establishes the endpoint-lifetime
|
||
* discipline for the whole pool/lane dispatch model and flips this together
|
||
* with the snapshot + peer-control dispatch flags. Until then this offload is
|
||
* exercised only by unit tests with a stable (or absent) endpoint set.
|
||
*
|
||
* ONLY the requester-less call sites (post-election promotion, heartbeat-driven
|
||
* HA_REPL_PUSH_CONFIG) route through this flag; the admin-upgrade barrier path
|
||
* (push_config_to_standbys) is unaffected. Compiled in ALL builds; the setter
|
||
* is the test seam to flip RED<->GREEN. */
|
||
static _Atomic int g_drive_async_configpush_offload = 0;
|
||
|
||
void ha_set_configpush_offload_for_test(int on)
|
||
{
|
||
atomic_store_explicit(&g_drive_async_configpush_offload, on ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
int ha_configpush_offload_enabled(void)
|
||
{
|
||
return atomic_load_explicit(&g_drive_async_configpush_offload,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/* raft-drive-async task 3.1: PeerCtrlDispatch feature flag.
|
||
*
|
||
* DEFAULT OFF (0): the registered peer-control handler runs its routing body
|
||
* INLINE on the iomgr io_worker thread — byte-for-byte the pre-task behavior,
|
||
* so THIS COMMIT DOES NOT CHANGE PRODUCTION BEHAVIOR. ON (1): the handler
|
||
* hands the decoded packet to the vtask pool and returns promptly. The flag
|
||
* stays OFF until the task 3.4 deadlock gate proves the dispatch path safe.
|
||
*
|
||
* Compiled in ALL builds (production reads it on every peer-control message);
|
||
* the setter is the test seam to flip RED<->GREEN. */
|
||
static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 0;
|
||
|
||
void ha_set_peerctrl_dispatch_for_test(int on)
|
||
{
|
||
atomic_store_explicit(&g_drive_async_peerctrl_dispatch_enabled, on ? 1 : 0,
|
||
memory_order_release);
|
||
}
|
||
|
||
int ha_peerctrl_dispatch_enabled(void)
|
||
{
|
||
return atomic_load_explicit(&g_drive_async_peerctrl_dispatch_enabled,
|
||
memory_order_acquire);
|
||
}
|
||
|
||
/* raft-drive-async task 3.1: test-only observation seam (plain global, NULL in
|
||
* production — zero cost, same idiom as ha_nop_req_hook). Invoked at the top
|
||
* of peer_ctrl_msg_handler_inline before any role routing. */
|
||
void (*ha_peerctrl_body_probe)(endpoint_t *ep) = NULL;
|
||
|
||
/* ── Test-only hook shims (XCE_TESTING only) ──────────────────────────────
|
||
* Type definitions (election_timeout_ctx_t), extern declarations
|
||
* (cluster_init_phase2, push_config_transport_only,
|
||
* ha_send_ctrlhello_to_all_standbys), and static forward declarations
|
||
* (ha_heartbeat_is_fresh) have been moved to the forward-declarations block
|
||
* near the top of this file so that the raft callback implementations
|
||
* (Task 6.2) can reference them.
|
||
*/
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* ha_phase2_hook: test-only seam.
|
||
* When non-NULL, ha_elect_evaluate_and_signal() calls this INSTEAD OF the real
|
||
* cluster_init_phase2() so tests can count phase2 calls without a live cluster_t.
|
||
*/
|
||
int (*ha_phase2_hook)(void *c) = NULL;
|
||
|
||
/*
|
||
* ha_send_ctrlhello_hook: test-only seam for ha_send_ctrlhello_to_all_standbys.
|
||
* When non-NULL, the post-election path calls this INSTEAD OF the real function
|
||
* (cluster.c is not linked in unit-test binary). Req 7.2.
|
||
*/
|
||
int (*ha_send_ctrlhello_hook)(void *c) = NULL;
|
||
|
||
/*
|
||
* ha_push_config_hook: test-only seam for push_config_transport_only.
|
||
* When non-NULL, the post-election path calls this INSTEAD OF the real function
|
||
* (config_repl.c's push_config_transport_only needs a real cluster). Req 7.1.
|
||
*/
|
||
int (*ha_push_config_hook)(void *c) = NULL;
|
||
|
||
/*
|
||
* ha_nop_req_hook: test-only seam for xce_nop_req in primary_heartbeat_fn.
|
||
* When non-NULL, primary_heartbeat_fn calls this INSTEAD OF xce_nop_req so
|
||
* tests can count NOP sends without a live endpoint. Req 3.2, task 2.2.
|
||
*/
|
||
int (*ha_nop_req_hook)(endpoint_t *ep) = NULL;
|
||
|
||
/*
|
||
* ha_heartbeat_req_hook: test-only seam for the HA heartbeat send in
|
||
* primary_heartbeat_fn. When non-NULL, primary_heartbeat_fn calls this INSTEAD
|
||
* OF xce_ha_heartbeat_req with the SAME (term, committed_lsn,
|
||
* committed_config_tag) it stamps onto the wire, so tests can assert the
|
||
* heartbeat carries the primary's own current committed state. Req 1.1, 3.4.
|
||
*/
|
||
int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term,
|
||
uint64_t committed_lsn,
|
||
uint64_t committed_config_tag) = NULL;
|
||
|
||
/*
|
||
* ha_reconnect_connect_hook: test-only seam for the BLOCKING half of the
|
||
* offloaded peer reconnect (raft-drive-async Task 2.1). When non-NULL, the
|
||
* slow-lane reconnect worker calls this INSTEAD OF the real bounded
|
||
* endpoint_connect + iomgr_monitor, so a test can drive the reconnect path
|
||
* deterministically (e.g. park the worker to simulate an unreachable peer).
|
||
* Returns 0 to simulate success, <0 to simulate failure. Req 2.1, 2.2, 2.5.
|
||
*/
|
||
int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx) = NULL;
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* ha_standby_evicted_count_for_test: counts how many stale same-peer outbound
|
||
* channels on_standby_connected evicted while resolving the inbound-vs-outbound
|
||
* connection-ownership race (raft-drive-async Task 2.2, Req 2.6). Test-only
|
||
* observability — never read on the production path.
|
||
*/
|
||
_Atomic int ha_standby_evicted_count_for_test = 0;
|
||
#endif
|
||
|
||
/*
|
||
* ha_repl_push_config_for_standby_hook: test-only seam for the per-standby
|
||
* config push triggered by the heartbeat-reply decision loop (Task 3.3).
|
||
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
|
||
* push_config_transport_only() when the decision is HA_REPL_PUSH_CONFIG.
|
||
* Requirements: 1.4 (ha-repl-hardening)
|
||
*/
|
||
int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx) = NULL;
|
||
|
||
/*
|
||
* ha_repl_stream_wal_hook: test-only seam for ha_stream_wal_to_standby.
|
||
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
|
||
* ha_stream_wal_to_standby() when the decision is HA_REPL_STREAM_WAL.
|
||
* Requirements: 5.1–5.5 (ha-repl-hardening)
|
||
*/
|
||
int (*ha_repl_stream_wal_hook)(void *ep, const char *wal_path,
|
||
uint64_t standby_lsn,
|
||
uint64_t committed_lsn) = NULL;
|
||
|
||
/*
|
||
* ha_repl_send_snapshot_hook: test-only seam for ha_send_snapshot_to_standby.
|
||
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
|
||
* ha_send_snapshot_to_standby() when the decision is HA_REPL_SEND_SNAPSHOT.
|
||
* Requirements: 4.1–4.5 (ha-repl-hardening)
|
||
*/
|
||
int (*ha_repl_send_snapshot_hook)(int peer_idx) = NULL;
|
||
|
||
/*
|
||
* ha_configpush_probe: test-only observation seam for the requester-less
|
||
* config-push offload (raft-drive-async Task 3.3). When non-NULL,
|
||
* ha_config_push_do_send() invokes it right around push_config_transport_only
|
||
* (on BOTH the inline and the lane path), so a test can record the running
|
||
* thread and optionally park the lane worker in flight to prove the
|
||
* cluster-wide single-flight dedup. NULL in production (zero cost).
|
||
*/
|
||
void (*ha_configpush_probe)(void) = NULL;
|
||
|
||
#endif /* XCE_TESTING */
|
||
|
||
/* ── Election Evaluation (Req 2.1, 2.7, 2.8) ───────────────────────────────
|
||
*
|
||
* ha_elect_evaluate_and_signal: evaluate accumulated votes; set ctx->result
|
||
* and post ctx->done when quorum is reached or all peers replied.
|
||
*
|
||
* Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size.
|
||
* ctx->votes_granted starts at 1 (self-vote); ctx->n_peers is n_other_peers.
|
||
* Quorum = ceil((n_peers + 1) / 2) = (n_peers + 1 + 1) / 2 = (n_peers + 2) / 2.
|
||
*
|
||
* Yield gate (Req 2.7): if any responding peer has a higher config_tag, or
|
||
* equal config_tag with a higher LSN than the candidate → yield (CTRL_STANDBY).
|
||
* Candidate's own tag/lsn is stored in ctx->votes[0].my_tag / my_lsn.
|
||
*
|
||
* Fatal gate (Req 2.8): if candidate has config_tag=0 and any peer has tag>0
|
||
* → CTRL_STANDBY (never PRIMARY with no config when peer has config).
|
||
*
|
||
* Returns CTRL_PRIMARY or CTRL_STANDBY; also sets ctx->result.
|
||
*/
|
||
ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx)
|
||
{
|
||
/* Guard: called from handle_request_vote_reply under ctx->lock; called from
|
||
* unit-test path (peers==NULL) without threads — done_posted prevents double
|
||
* execution if two threads both hit quorum simultaneously. */
|
||
if (ctx->done_posted)
|
||
return (ctx->result == 1) ? CTRL_PRIMARY : CTRL_STANDBY;
|
||
|
||
ctrl_role_t role_result;
|
||
|
||
/* Fix 2 (CRITICAL): zero-peer guard — single-node shortcircuit.
|
||
* n_peers==0 means no other nodes; votes[] flexible array has 0 entries,
|
||
* so accessing votes[0] would be out-of-bounds. Self-vote (votes_granted=1)
|
||
* always satisfies quorum=1. No fresher-peer or fatal-gate checks apply. */
|
||
if (ctx->n_peers == 0) {
|
||
ctx->result = 1;
|
||
ctx->done_posted = 1;
|
||
sem_post(&ctx->done);
|
||
role_result = CTRL_PRIMARY;
|
||
goto apply_role;
|
||
}
|
||
|
||
{
|
||
int total_nodes = ctx->n_peers + 1; /* include self */
|
||
int quorum = (total_nodes + 1) / 2; /* ceil(total/2) */
|
||
|
||
/* Req 2.8: fatal gate — candidate has no config but a peer does */
|
||
uint64_t my_tag = ctx->votes[0].my_tag;
|
||
uint64_t my_lsn = ctx->votes[0].my_lsn;
|
||
if (my_tag == 0 && ctx->max_resp_tag > 0) {
|
||
ctx->result = 0;
|
||
ctx->done_posted = 1;
|
||
sem_post(&ctx->done);
|
||
role_result = CTRL_STANDBY;
|
||
goto apply_role;
|
||
}
|
||
|
||
/* Check quorum */
|
||
if (ctx->votes_granted < quorum) {
|
||
ctx->result = 0;
|
||
ctx->done_posted = 1;
|
||
sem_post(&ctx->done);
|
||
role_result = CTRL_STANDBY;
|
||
goto apply_role;
|
||
}
|
||
|
||
/* Req 2.7: yield gate — fresher responding peer */
|
||
if (ctx->max_resp_tag > my_tag) {
|
||
ctx->result = 0;
|
||
ctx->done_posted = 1;
|
||
sem_post(&ctx->done);
|
||
role_result = CTRL_STANDBY;
|
||
goto apply_role;
|
||
}
|
||
if (ctx->max_resp_tag == my_tag && ctx->max_resp_lsn > my_lsn) {
|
||
ctx->result = 0;
|
||
ctx->done_posted = 1;
|
||
sem_post(&ctx->done);
|
||
role_result = CTRL_STANDBY;
|
||
goto apply_role;
|
||
}
|
||
|
||
/* Quorum met and no fresher peer → PRIMARY */
|
||
ctx->result = 1;
|
||
ctx->done_posted = 1;
|
||
sem_post(&ctx->done);
|
||
role_result = CTRL_PRIMARY;
|
||
}
|
||
|
||
apply_role:
|
||
/* ── Task 4.1: apply role and call cluster_init_phase2 ─────────────────
|
||
*
|
||
* After determining the election result, update ha->role and (on first boot)
|
||
* call cluster_init_phase2. This runs once per process lifetime — the
|
||
* phase2_done flag prevents re-entry on subsequent elections (RDHUP failover).
|
||
*
|
||
* The cluster_t pointer is carried in ctx->c (NULL in unit tests).
|
||
*
|
||
* Cancel the election timeout watchdog if we won (it would be a no-op now
|
||
* but the explicit cancel avoids a spurious second evaluate call).
|
||
*/
|
||
{
|
||
cluster_t *c = ctx->c;
|
||
int is_first_boot = !ha->phase2_done;
|
||
|
||
if (role_result == CTRL_PRIMARY) {
|
||
/* Cancel election timeout watchdog (quorum already reached) */
|
||
if (ctx->timeout_ctx) {
|
||
election_timeout_ctx_t *etc =
|
||
(election_timeout_ctx_t *)ctx->timeout_ctx;
|
||
atomic_store(&etc->cancelled, 1);
|
||
}
|
||
|
||
/* First boot: call cluster_init_phase2 to load config+data (Req 9.2) */
|
||
if (is_first_boot) {
|
||
#ifdef XCE_TESTING
|
||
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
|
||
if (ha_phase2_hook != NULL)
|
||
ha_phase2_hook(c);
|
||
#else
|
||
if (c != NULL)
|
||
cluster_init_phase2(c);
|
||
#endif
|
||
ha->phase2_done = 1;
|
||
}
|
||
|
||
/* ha-single-leader-role task 3.1 (Req 1.1, 1.5): enter CTRL_PRIMARY
|
||
* DIRECTLY — symmetric with raft_cb_become_leader's rewrite. There
|
||
* is no transitional COMMITTING state and no committing-timeout
|
||
* watchdog; new-leader safety comes from the own-term WAL_NOOP +
|
||
* writable_lsn gate armed below (ha_arm_own_term_gate), not from a
|
||
* fourth role value. The legacy XCE_TESTING election path OWNS its
|
||
* term (no raft library in test mode); it historically did NOT
|
||
* increment the term in this branch, so we preserve that — no
|
||
* ha_do_promote, no new term store (research.md §4 finding 2). */
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
|
||
memory_order_release);
|
||
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64
|
||
", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire),
|
||
is_first_boot);
|
||
|
||
/* Push config to standbys with lower config_tag (Req 7.1).
|
||
* Task 3.3: requester-less promotion push → offloadable (dormant
|
||
* flag OFF = inline, unchanged; flag ON = slow lane). */
|
||
#ifdef XCE_TESTING
|
||
if (ha_push_config_hook != NULL)
|
||
ha_push_config_hook(c);
|
||
#else
|
||
if (c != NULL)
|
||
ha_config_push_offloadable(c, NULL, 0, NULL, 0);
|
||
#endif
|
||
|
||
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up (Req 7.2) */
|
||
#ifdef XCE_TESTING
|
||
if (ha_send_ctrlhello_hook != NULL)
|
||
ha_send_ctrlhello_hook(c);
|
||
#else
|
||
if (c != NULL)
|
||
ha_send_ctrlhello_to_all_standbys(c);
|
||
#endif
|
||
|
||
/* ha-single-leader-role task 3.1: route promotion side effects
|
||
* through the SAME shared sequence raft_cb_become_leader runs so the
|
||
* test-driven (legacy) and raft-driven promotion paths cannot diverge
|
||
* (design §Legacy Path Symmetry). Guarded on a real cluster_t — the
|
||
* helpers dereference &c->hooksys / &c->persist / &c->commit, and the
|
||
* unit-test evaluation path passes c == NULL. Identical role / hook /
|
||
* heartbeat / no-watchdog / own-term-gate state results either road. */
|
||
if (c) {
|
||
/* Register the persistence hooks BEFORE the own-term no-op append,
|
||
* so the persistence seam is live before any racing client write.
|
||
* Idempotent via the hooks_registered guard. */
|
||
ha_register_persist_hooks(c);
|
||
/* Periodic heartbeat to standbys (idempotency guard prevents
|
||
* double-arming if already started). */
|
||
ha_start_primary_heartbeat(c);
|
||
/* Own-term WAL_NOOP + writable_lsn gate (Req 1.4, 2.1, 2.2) —
|
||
* non-blocking; append failure degrades safe (gate unset,
|
||
* promotion still completes). */
|
||
ha_arm_own_term_gate(c);
|
||
}
|
||
} else {
|
||
/* STANDBY result */
|
||
|
||
/* Cancel election timeout watchdog — it is now a no-op but explicit
|
||
* cancellation matches spec wording and avoids a spurious lock-acquire
|
||
* if the timeout fires after STANDBY is decided. */
|
||
if (ctx->timeout_ctx) {
|
||
election_timeout_ctx_t *etc =
|
||
(election_timeout_ctx_t *)ctx->timeout_ctx;
|
||
atomic_store(&etc->cancelled, 1);
|
||
}
|
||
|
||
/* First boot: call cluster_init_phase2 so structures are ready;
|
||
* config arrives via primary's replication push (Req 9.2, 7.3) */
|
||
if (is_first_boot) {
|
||
#ifdef XCE_TESTING
|
||
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
|
||
if (ha_phase2_hook != NULL)
|
||
ha_phase2_hook(c);
|
||
#else
|
||
if (c != NULL)
|
||
cluster_init_phase2(c);
|
||
#endif
|
||
ha->phase2_done = 1;
|
||
}
|
||
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
|
||
memory_order_release);
|
||
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
|
||
atomic_load_explicit(&ha->term, memory_order_acquire),
|
||
is_first_boot);
|
||
/* Primary connects outbound and pushes config via ctrlhello.
|
||
* Standby is fully passive here — no fetch_config_from_primary. */
|
||
}
|
||
}
|
||
|
||
return role_result;
|
||
}
|
||
|
||
/* ── Vote Reply Handler (Req 2.2–2.6) ──────────────────────────────────────
|
||
*
|
||
* handle_request_vote_reply: process a reply to an OP_requestVote RPC.
|
||
*
|
||
* Called by each per-peer vote-request thread once the peer replies.
|
||
* Updates ctx->votes[peer_idx] with the reply fields, tracks the freshest
|
||
* responding peer in ctx->max_resp_tag/max_resp_lsn, increments counters,
|
||
* and evaluates whether to signal ctx->done.
|
||
*
|
||
* Quorum = ceil((n_peers + 1) / 2) = (n_peers + 2) / 2
|
||
* (n_peers = number of other peers, not counting self; votes_granted starts at 1)
|
||
*
|
||
* Posts ctx->done via ha_elect_evaluate_and_signal when:
|
||
* - votes_granted reaches quorum (fast path — no need to wait for all replies), OR
|
||
* - n_replied reaches n_peers (all peers accounted for, win or lose)
|
||
*/
|
||
/*
|
||
* ha_record_vote_reply: record a decoded vote reply into ctx->votes[peer_idx]
|
||
* WITHOUT signalling the legacy promotion path (bug-828).
|
||
*
|
||
* Records the granting flag, peer config_tag, and peer LSN under ctx->lock with
|
||
* the same idempotency guard handle_request_vote_reply uses, and updates the
|
||
* vote tally + freshest-responder tracking. It does NOT call
|
||
* ha_elect_evaluate_and_signal — so it never posts ctx->done and never runs the
|
||
* legacy apply_role promotion side-effects.
|
||
*
|
||
* This is the seam the PRODUCTION vote_reply_dispatch path calls so the granting
|
||
* voter's advertised LSN reaches ha_compute_safe_committed_lsn (via ctx->votes[])
|
||
* while the raft library remains the SOLE production promotion driver
|
||
* (raft_on_vote_reply → on_become_leader). Routing the record through here (not
|
||
* handle_request_vote_reply) avoids a second, racing promotion signal.
|
||
*
|
||
* Returns non-zero if this reply newly reached the legacy quorum/all-replied
|
||
* threshold (so handle_request_vote_reply knows to signal); 0 otherwise.
|
||
* Must be called WITHOUT ctx->lock held (it takes the lock internally).
|
||
*/
|
||
static int ha_record_vote_reply_locked(ha_elect_ctx_t *ctx,
|
||
int peer_idx,
|
||
int vote_granted,
|
||
uint64_t peer_config_tag,
|
||
uint64_t peer_lsn)
|
||
{
|
||
/* Guard against out-of-range index */
|
||
if (peer_idx < 0 || peer_idx >= ctx->n_peers)
|
||
return 0;
|
||
|
||
pthread_mutex_lock(&ctx->lock);
|
||
|
||
ha_vote_entry_t *entry = &ctx->votes[peer_idx];
|
||
|
||
/* Idempotency guard: ignore duplicate replies */
|
||
if (entry->replied) {
|
||
pthread_mutex_unlock(&ctx->lock);
|
||
return 0;
|
||
}
|
||
|
||
/* Record reply fields */
|
||
entry->replied = 1;
|
||
entry->vote_granted = vote_granted ? 1 : 0;
|
||
entry->peer_tag = peer_config_tag;
|
||
entry->peer_lsn = peer_lsn;
|
||
|
||
/* Update counters */
|
||
if (vote_granted)
|
||
ctx->votes_granted++;
|
||
ctx->n_replied++;
|
||
|
||
/* Track the freshest responding peer for the yield gate (Req 2.7) */
|
||
if (peer_config_tag > ctx->max_resp_tag) {
|
||
ctx->max_resp_tag = peer_config_tag;
|
||
ctx->max_resp_lsn = peer_lsn;
|
||
} else if (peer_config_tag == ctx->max_resp_tag && peer_lsn > ctx->max_resp_lsn) {
|
||
ctx->max_resp_lsn = peer_lsn;
|
||
}
|
||
|
||
/* Compute whether the legacy quorum/all-replied threshold is now reached.
|
||
* Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. */
|
||
int total_nodes = ctx->n_peers + 1;
|
||
int quorum = (total_nodes + 1) / 2;
|
||
int reached = (ctx->votes_granted >= quorum || ctx->n_replied >= ctx->n_peers);
|
||
|
||
pthread_mutex_unlock(&ctx->lock);
|
||
return reached;
|
||
}
|
||
|
||
/* Public record-only seam (bug-828): records the reply but never signals the
|
||
* legacy promotion path. Used by the production vote_reply_dispatch path. */
|
||
void ha_record_vote_reply(ha_state_t *ha,
|
||
ha_elect_ctx_t *ctx,
|
||
int peer_idx,
|
||
uint64_t term,
|
||
int vote_granted,
|
||
uint64_t peer_config_tag,
|
||
uint64_t peer_lsn)
|
||
{
|
||
(void)ha; /* fencing: caller bumps ha->term before calling here */
|
||
(void)term;
|
||
if (!ctx)
|
||
return;
|
||
(void)ha_record_vote_reply_locked(ctx, peer_idx, vote_granted,
|
||
peer_config_tag, peer_lsn);
|
||
}
|
||
|
||
void handle_request_vote_reply(ha_state_t *ha,
|
||
ha_elect_ctx_t *ctx,
|
||
int peer_idx,
|
||
uint64_t term,
|
||
int vote_granted,
|
||
uint64_t peer_config_tag,
|
||
uint64_t peer_lsn)
|
||
{
|
||
(void)term; /* fencing: callers bump ha->term if peer's term > ours, before calling here */
|
||
|
||
if (peer_idx < 0 || peer_idx >= ctx->n_peers)
|
||
return;
|
||
|
||
/* Record the reply (shared with the production record-only seam) and learn
|
||
* whether the legacy quorum/all-replied threshold is now reached. */
|
||
int reached = ha_record_vote_reply_locked(ctx, peer_idx, vote_granted,
|
||
peer_config_tag, peer_lsn);
|
||
|
||
/* Decide whether to signal the waiting thread.
|
||
* ha_elect_evaluate_and_signal takes ctx->lock; its done_posted guard
|
||
* prevents double sem_post if two threads reach quorum simultaneously.
|
||
* Skip if the index was out of range / a duplicate reply (reached==0 and
|
||
* nothing recorded) — the duplicate case is already idempotent in
|
||
* ha_record_vote_reply_locked, which returns 0 without re-counting. */
|
||
if (reached) {
|
||
pthread_mutex_lock(&ctx->lock);
|
||
ha_elect_evaluate_and_signal(ha, ctx);
|
||
pthread_mutex_unlock(&ctx->lock);
|
||
}
|
||
}
|
||
|
||
/* ── Task 2.2: iomgr-native vote send path ──────────────────────────────────
|
||
*
|
||
* vote_ep_ctx_t: per-peer context stored in ep->data for each outgoing vote
|
||
* endpoint. on_vote_connected fires after the TLS handshake completes (when
|
||
* endpoint_t.ssl_handshake_done transitions to 1 in io_handler).
|
||
*/
|
||
|
||
/* vote_ep_ctx_t: typedef moved to Task 6.2 forward-declaration block above */
|
||
|
||
/*
|
||
* peer_ctrl_msg_handler / vote_reply_decode_and_dispatch /
|
||
* repl_ack_decode_and_dispatch: forward declarations.
|
||
* peer_ctrl_msg_handler is the single iomgr handler set in on_vote_connected
|
||
* and never reassigned — it routes by ha->role at dispatch time.
|
||
*/
|
||
static int peer_ctrl_msg_handler(void *ep_raw);
|
||
static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt);
|
||
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx,
|
||
io_packet_t *pkt);
|
||
/* ha-prevote task 5.1: preVoteReply decode/route; defined alongside the
|
||
* real-vote decoder, feeds the later-defined pre_vote_reply_dispatch. */
|
||
static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx,
|
||
io_packet_t *pkt);
|
||
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
|
||
int peer_idx, int granted);
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* xce_vote_request_hook: test-only seam.
|
||
*
|
||
* When non-NULL, on_vote_connected() calls this INSTEAD OF the real
|
||
* xce_request() so tests can capture the call without requiring a live
|
||
* iomgr or SSL endpoint. The hook receives the fully built
|
||
* xce_request_t and may inspect all fields.
|
||
*
|
||
* Pattern mirrors xce_retry_send_hook in request.c.
|
||
*/
|
||
int (*xce_vote_request_hook)(endpoint_t *ep, xce_request_t *req) = NULL;
|
||
#endif /* XCE_TESTING */
|
||
|
||
/*
|
||
* on_vote_connected: iomgr callback fired after TLS handshake completes on an
|
||
* outgoing vote endpoint.
|
||
*
|
||
* Steps:
|
||
* 1. Cast ep->data to vote_ep_ctx_t.
|
||
* 2. Build request_vote_req_t from ctx fields.
|
||
* 3. Build xce_request_t{opcode=OPCode_requestVote, body.requestvotemsg=&vrq}.
|
||
* 4. Call xce_vote_request_hook (test seam, XCE_TESTING only) — then
|
||
* call xce_request() to queue the packet on the iomgr send path.
|
||
* 5. Set ep->msg_handler = vote_reply_msg_handler so the next inbound
|
||
* packet is decoded as a RequestVoteReply.
|
||
*
|
||
* Req 2.3: send vote after TLS handshake.
|
||
* Req 1.2: vote RPC via iomgr path (xce_request enqueues to endpoint_enq).
|
||
*/
|
||
static void on_vote_connected(endpoint_t *ep)
|
||
{
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
|
||
/* Build the RequestVoteRequest payload */
|
||
request_vote_req_t vrq;
|
||
memset(&vrq, 0, sizeof(vrq));
|
||
vrq.term = vctx->my_term;
|
||
vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
|
||
vrq.config_tag = vctx->my_tag;
|
||
vrq.last_log_lsn = vctx->my_lsn;
|
||
/* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */
|
||
vrq.last_log_term = vctx->my_last_log_term;
|
||
vrq.last_log_index = vctx->my_last_log_index;
|
||
|
||
/* Wrap in xce_request_t */
|
||
xce_request_t req;
|
||
memset(&req, 0, sizeof(req));
|
||
req.opcode = OPCode_requestVote;
|
||
req.body_which = Request_body_requestvotemsg;
|
||
req.body.requestvotemsg = &vrq;
|
||
|
||
#ifdef XCE_TESTING
|
||
if (xce_vote_request_hook != NULL)
|
||
xce_vote_request_hook(ep, &req);
|
||
else
|
||
xce_request(ep, &req);
|
||
#else
|
||
xce_request(ep, &req);
|
||
#endif
|
||
|
||
/* Set state-aware dispatcher — routes by ha->role at message-arrival time.
|
||
* Never reassigned: ELECTING → vote reply; PRIMARY → repl ack. */
|
||
ep->msg_handler = peer_ctrl_msg_handler;
|
||
}
|
||
|
||
/*
|
||
* on_pre_vote_connected: iomgr callback fired after the TLS handshake completes
|
||
* on an outgoing PreVote endpoint (ha-prevote task 4.1).
|
||
*
|
||
* Mirrors on_vote_connected but emits OPCode_preVoteRequest (reusing the
|
||
* RequestVoteRequest payload struct + union arm, per the wire schema in task
|
||
* 1.1). prospective_term (current+1) rides in the term field; nothing is
|
||
* persisted. The reply-routing handler for preVoteReply is wired in task 5.1;
|
||
* here we set the same state-aware dispatcher used by the real-vote path.
|
||
*/
|
||
static void on_pre_vote_connected(endpoint_t *ep)
|
||
{
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
|
||
request_vote_req_t vrq;
|
||
memset(&vrq, 0, sizeof(vrq));
|
||
vrq.term = vctx->my_term; /* prospective term = current + 1 */
|
||
vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
|
||
vrq.config_tag = vctx->my_tag;
|
||
vrq.last_log_lsn = vctx->my_lsn;
|
||
/* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */
|
||
vrq.last_log_term = vctx->my_last_log_term;
|
||
vrq.last_log_index = vctx->my_last_log_index;
|
||
|
||
xce_request_t req;
|
||
memset(&req, 0, sizeof(req));
|
||
req.opcode = OPCode_preVoteRequest;
|
||
req.body_which = Request_body_requestvotemsg;
|
||
req.body.requestvotemsg = &vrq;
|
||
|
||
#ifdef XCE_TESTING
|
||
if (xce_vote_request_hook != NULL)
|
||
xce_vote_request_hook(ep, &req);
|
||
else
|
||
xce_request(ep, &req);
|
||
#else
|
||
xce_request(ep, &req);
|
||
#endif
|
||
|
||
ep->msg_handler = peer_ctrl_msg_handler;
|
||
}
|
||
|
||
/*
|
||
* xce_vote_reply_hook: test-only seam for vote_reply_msg_handler.
|
||
*
|
||
* Forward-declared here (before vote_reply_msg_handler) so the handler body
|
||
* can reference it. Defined only under XCE_TESTING; guarded below.
|
||
*
|
||
* When non-NULL, vote_reply_msg_handler() calls this INSTEAD OF the real
|
||
* handle_request_vote_reply() so tests can capture decoded fields without
|
||
* a live iomgr. Receives the decoded scalar fields — no capnp pointers.
|
||
*
|
||
* Pattern mirrors xce_vote_request_hook used in task 2.2.
|
||
*/
|
||
#ifdef XCE_TESTING
|
||
int (*xce_vote_reply_hook)(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx,
|
||
uint64_t term, int vote_granted,
|
||
uint64_t peer_tag, uint64_t peer_lsn) = NULL;
|
||
#endif /* XCE_TESTING */
|
||
|
||
/*
|
||
* vote_reply_dispatch: internal helper — delegates to the raft library or
|
||
* the test hook (when XCE_TESTING and hook is non-NULL).
|
||
*
|
||
* Production path: raft_on_vote_reply owns quorum tracking and fires
|
||
* on_become_leader/on_become_follower via the raft callbacks.
|
||
* Test path: xce_vote_reply_hook captures decoded fields without real iomgr.
|
||
*/
|
||
static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx,
|
||
uint64_t term, int vote_granted,
|
||
uint64_t peer_tag, uint64_t peer_lsn)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
if (xce_vote_reply_hook != NULL) {
|
||
xce_vote_reply_hook(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn);
|
||
return;
|
||
}
|
||
#endif
|
||
/* bug-828 (PRIMARY fix): route the decoded reply into the election context
|
||
* BEFORE driving the raft library. Without this, the granting voter's
|
||
* advertised peer_lsn was discarded — ctx->votes[idx].vote_granted stayed 0,
|
||
* so a clean failover's ha_compute_safe_committed_lsn (read in
|
||
* raft_cb_become_leader / on_become_leader) saw only the self-vote
|
||
* (n_granted=1) and truncated committed entries down to self_lsn.
|
||
*
|
||
* Record-only: ha_record_vote_reply records ctx->votes[idx] under ctx->lock
|
||
* but does NOT post ctx->done / run the legacy apply_role promotion path.
|
||
* raft_on_vote_reply remains the SOLE production promotion driver (it owns
|
||
* the quorum tally and fires on_become_leader) — so there is no double
|
||
* promotion. `idx` is vctx->peer_idx, the local ctx->votes[] index (set in
|
||
* raft_cb_send_vote_request via the absolute→local mapping), which is exactly
|
||
* the index ha_compute_safe_committed_lsn and the test hook use. */
|
||
ha_record_vote_reply(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn);
|
||
raft_on_vote_reply(&ha->raft, idx, vote_granted, term);
|
||
}
|
||
|
||
/*
|
||
* vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch
|
||
* to vote_reply_dispatch. Factored from vote_reply_msg_handler so that
|
||
* peer_ctrl_msg_handler can invoke the same logic without code duplication.
|
||
* Packet ownership (raft-drive-async task 3.1): the caller
|
||
* (peer_ctrl_msg_handler) has ALREADY taken `pkt` off ep->fsm_in.pkt and
|
||
* cleared ep->fsm_in.pkt to NULL — `pkt` is passed in explicitly and this
|
||
* function frees it on every path. Clearing in the caller (not here) is what
|
||
* makes the off-iomgr dispatch path race-free: the body never touches
|
||
* ep->fsm_in.pkt, so a concurrent io_worker read on the same endpoint cannot
|
||
* collide with the in-flight packet. vote_reply_dispatch can still reach
|
||
* raft_cb_become_follower → endpoint_fini(ep) (loser path) safely because
|
||
* ep->fsm_in.pkt is already NULL.
|
||
*/
|
||
static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt)
|
||
{
|
||
ha_state_t *ha = vctx->ha;
|
||
ha_elect_ctx_t *ctx = vctx->ctx;
|
||
int idx = vctx->peer_idx;
|
||
int ok = 0;
|
||
|
||
uint64_t vr_term = 0;
|
||
int vr_granted = 0;
|
||
uint64_t vr_tag = 0;
|
||
uint64_t vr_lsn = 0;
|
||
|
||
if (!pkt || !pkt->data || pkt->sz == 0)
|
||
goto done;
|
||
|
||
{
|
||
struct capn capn_ctx;
|
||
struct Request outer_rq;
|
||
Request_ptr outer_rp;
|
||
struct RequestReply rep_s;
|
||
struct RequestVoteReply vr_raw;
|
||
int capn_ok;
|
||
|
||
capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
|
||
(size_t)pkt->sz, 0);
|
||
if (capn_ok != 0)
|
||
goto done;
|
||
|
||
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
|
||
read_Request(&outer_rq, outer_rp);
|
||
|
||
if (outer_rq.body_which != Request_body_replymsg ||
|
||
!outer_rq.body.replymsg.p.data) {
|
||
capn_free(&capn_ctx);
|
||
goto done;
|
||
}
|
||
|
||
read_RequestReply(&rep_s, outer_rq.body.replymsg);
|
||
|
||
if (rep_s.data_which != RequestReply_data_requestvotereplymsg) {
|
||
capn_free(&capn_ctx);
|
||
goto done;
|
||
}
|
||
|
||
memset(&vr_raw, 0, sizeof(vr_raw));
|
||
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
|
||
|
||
capn_free(&capn_ctx);
|
||
|
||
vr_term = (uint64_t)vr_raw.term;
|
||
vr_granted = vr_raw.voteGranted ? 1 : 0;
|
||
vr_tag = (uint64_t)vr_raw.peerConfigTag;
|
||
vr_lsn = (uint64_t)vr_raw.peerLastLogLsn;
|
||
ok = 1;
|
||
}
|
||
|
||
done:
|
||
if (pkt) { free_packet(pkt); free(pkt); }
|
||
|
||
if (ok) {
|
||
uint64_t cur = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
while (vr_term > cur) {
|
||
if (atomic_compare_exchange_weak_explicit(&ha->term, &cur, vr_term,
|
||
memory_order_acq_rel,
|
||
memory_order_acquire))
|
||
break;
|
||
}
|
||
vote_reply_dispatch(ha, ctx, idx, vr_term, vr_granted, vr_tag, vr_lsn);
|
||
} else {
|
||
vote_reply_dispatch(ha, ctx, idx,
|
||
atomic_load_explicit(&ha->term, memory_order_acquire),
|
||
0, 0, 0);
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* pre_vote_reply_decode_and_dispatch: decode a preVoteReply packet on an
|
||
* outbound PreVote endpoint and feed it to the PreVote tally (ha-prevote task
|
||
* 5.1). Mirrors vote_reply_decode_and_dispatch exactly — the PreVote reply
|
||
* reuses the identical RequestVoteReply payload / union arm — but routes to
|
||
* pre_vote_reply_dispatch (the raft-library PreVote tally) instead of the
|
||
* real-vote tally, and deliberately performs NO term adoption (a PreVote reply
|
||
* never changes our term; Req 1.2). vctx->peer_idx is the ABSOLUTE cluster
|
||
* index that raft_on_pre_vote_reply expects (set in ha_send_pre_vote_request).
|
||
* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly
|
||
* (already taken off ep->fsm_in.pkt by peer_ctrl_msg_handler); this function
|
||
* frees it on every path and never touches ep->fsm_in.pkt.
|
||
*/
|
||
static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx,
|
||
io_packet_t *pkt)
|
||
{
|
||
ha_state_t *ha = vctx->ha;
|
||
cluster_t *c = vctx->c;
|
||
int idx = vctx->peer_idx; /* ABSOLUTE cluster index */
|
||
int ok = 0;
|
||
int vr_granted = 0;
|
||
|
||
if (!pkt || !pkt->data || pkt->sz == 0)
|
||
goto done;
|
||
|
||
{
|
||
struct capn capn_ctx;
|
||
struct Request outer_rq;
|
||
Request_ptr outer_rp;
|
||
struct RequestReply rep_s;
|
||
struct RequestVoteReply vr_raw;
|
||
int capn_ok;
|
||
|
||
capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
|
||
(size_t)pkt->sz, 0);
|
||
if (capn_ok != 0)
|
||
goto done;
|
||
|
||
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
|
||
read_Request(&outer_rq, outer_rp);
|
||
|
||
if (outer_rq.body_which != Request_body_replymsg ||
|
||
!outer_rq.body.replymsg.p.data) {
|
||
capn_free(&capn_ctx);
|
||
goto done;
|
||
}
|
||
|
||
read_RequestReply(&rep_s, outer_rq.body.replymsg);
|
||
|
||
if (rep_s.data_which != RequestReply_data_requestvotereplymsg) {
|
||
capn_free(&capn_ctx);
|
||
goto done;
|
||
}
|
||
|
||
memset(&vr_raw, 0, sizeof(vr_raw));
|
||
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
|
||
|
||
capn_free(&capn_ctx);
|
||
|
||
vr_granted = vr_raw.voteGranted ? 1 : 0;
|
||
ok = 1;
|
||
}
|
||
|
||
done:
|
||
if (pkt) { free_packet(pkt); free(pkt); }
|
||
|
||
/* A malformed / wrong-body reply counts as a non-grant (best-effort
|
||
* delivery, per design Event Contract). No term adoption either way. */
|
||
pre_vote_reply_dispatch(ha, c, idx, ok ? vr_granted : 0);
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* repl_ack_decode_and_dispatch: decode an inbound packet on a promoted vote
|
||
* endpoint while role == CTRL_PRIMARY.
|
||
*
|
||
* - OPCode_replCatchUpDone → INFO no-op (ack-counting removed in task 4.1)
|
||
* - OPCode_reply with stateReportMsg → ha_process_standby_state (Task 3.1)
|
||
* - OPCode_reply with walackreplymsg → repl_on_ack (Task 4.1)
|
||
* - OPCode_reply with replconfigpushreplymsg → repl_on_config_ack
|
||
* (barrier-rollout task 5.2 r1 / ha-config-bootstrap task 8.1)
|
||
* - OPCode_nop or OPCode_reply with any other body → silently ignored
|
||
* - anything else → WARN + drop
|
||
*/
|
||
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx,
|
||
io_packet_t *pkt)
|
||
{
|
||
ha_state_t *ha = vctx->ha;
|
||
cluster_t *c = vctx->c;
|
||
|
||
/* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in
|
||
* explicitly (peer_ctrl_msg_handler already cleared ep->fsm_in.pkt); this
|
||
* function frees it and never touches ep->fsm_in.pkt.
|
||
*
|
||
* raft-drive-async tasks 3.2/3.3 (FUTURE): the long ops reachable from the
|
||
* stateReport branch below — ha_dispatch_repl_action_for_standby's
|
||
* snapshot send / config push — are where the SlowLane offload lands. For
|
||
* task 3.1 the body only relocates onto the pool and still runs bounded
|
||
* decode/route work inline in the vtask entry. */
|
||
if (!pkt || !pkt->data || pkt->sz == 0) {
|
||
if (pkt) { free_packet(pkt); free(pkt); }
|
||
return 0;
|
||
}
|
||
|
||
struct capn capn_ctx;
|
||
struct Request outer_rq;
|
||
Request_ptr outer_rp;
|
||
int capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
|
||
(size_t)pkt->sz, 0);
|
||
free_packet(pkt);
|
||
free(pkt);
|
||
|
||
if (capn_ok != 0)
|
||
return 0;
|
||
|
||
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
|
||
read_Request(&outer_rq, outer_rp);
|
||
int opcode = (int)outer_rq.opcode;
|
||
|
||
if (opcode == OPCode_replCatchUpDone) {
|
||
capn_free(&capn_ctx);
|
||
/* ha-single-leader-role task 4.1: the force-promotion ack-counting
|
||
* machinery has been deleted — promotion is driven solely by the
|
||
* heartbeat-driven decision loop and per-entry majority ack. Standbys
|
||
* still send OPCode_replCatchUpDone and the dispatch gate still admits
|
||
* it; it is now an INFO-logged no-op. */
|
||
(void)ha;
|
||
(void)c;
|
||
INFO("HA: replCatchUpDone received (no-op; ack-counting machinery removed in task 4.1)");
|
||
} else if (opcode == OPCode_reply &&
|
||
outer_rq.body_which == Request_body_replymsg &&
|
||
outer_rq.body.replymsg.p.data) {
|
||
/* Heartbeat NOP reply — decode inner RequestReply for stateReport */
|
||
struct RequestReply rep_s;
|
||
memset(&rep_s, 0, sizeof(rep_s));
|
||
read_RequestReply(&rep_s, outer_rq.body.replymsg);
|
||
|
||
if (rep_s.data_which == RequestReply_data_stateReportMsg) {
|
||
/* Decode stateReport and call convergence tracker (Task 3.1).
|
||
* read_StateReport reads scalar fields directly from the capnp
|
||
* pointer without heap allocation — safe to call before capn_free. */
|
||
struct StateReport sr_raw;
|
||
memset(&sr_raw, 0, sizeof(sr_raw));
|
||
read_StateReport(&sr_raw, rep_s.data.stateReportMsg);
|
||
state_report_t sr;
|
||
sr.applied_lsn = (uint64_t)sr_raw.appliedLsn;
|
||
sr.config_tag = (uint64_t)sr_raw.configTag;
|
||
capn_free(&capn_ctx);
|
||
|
||
/* peer_idx from standby_ep_ctx_t (layout-compatible with vote_ep_ctx_t) */
|
||
int peer_idx = vctx->peer_idx;
|
||
uint64_t committed_lsn = 0;
|
||
uint64_t committed_tag = 0;
|
||
uint64_t last_written = 0; /* latest WAL LSN — may be > committed */
|
||
#ifndef XCE_TESTING
|
||
if (c) {
|
||
committed_lsn = atomic_load_explicit(&c->persist.committed_lsn,
|
||
memory_order_acquire);
|
||
committed_tag = atomic_load_explicit(&ha->committed_config_tag,
|
||
memory_order_acquire);
|
||
/* current_lsn is the next LSN to write; last written = current - 1.
|
||
* Read without lock — monotonic, approximate value is sufficient
|
||
* for the replication decision. */
|
||
last_written = (c->persist.current_lsn > 0)
|
||
? c->persist.current_lsn - 1 : 0;
|
||
}
|
||
#else
|
||
/* In test mode, c may be NULL; caller sets ha fields directly */
|
||
committed_lsn = atomic_load_explicit(&ha->last_known_committed_lsn,
|
||
memory_order_acquire);
|
||
committed_tag = atomic_load_explicit(&ha->committed_config_tag,
|
||
memory_order_acquire);
|
||
(void)c;
|
||
#endif
|
||
ha_process_standby_state(ha, peer_idx,
|
||
sr.config_tag, sr.applied_lsn,
|
||
committed_tag, committed_lsn);
|
||
|
||
/* Derive oldest_wal_lsn from the WAL directory.
|
||
* The oldest retained segment's start LSN is the boundary —
|
||
* standbys below it need a snapshot (Task 3.5b). */
|
||
uint64_t oldest_wal_lsn = 0;
|
||
#ifndef XCE_TESTING
|
||
if (c && c->persist.wal_path[0]) {
|
||
extern uint64_t ha_compute_oldest_wal_lsn(const char *wal_path);
|
||
oldest_wal_lsn = ha_compute_oldest_wal_lsn(c->persist.wal_path);
|
||
}
|
||
#endif
|
||
/* Replication target: use max(committed_lsn, last_written) so
|
||
* standbys can catch up even when committed_lsn is stuck at 0
|
||
* (no replicator-registered standbys to advance it). */
|
||
uint64_t target_lsn = committed_lsn;
|
||
if (last_written > target_lsn)
|
||
target_lsn = last_written;
|
||
|
||
ha_dispatch_repl_action_for_standby(ha, c, ep, peer_idx,
|
||
committed_tag, target_lsn,
|
||
oldest_wal_lsn);
|
||
} else if (rep_s.data_which == RequestReply_data_walackreplymsg) {
|
||
/* Task 4.1: WAL ACK reply from standby — extract lsn + success,
|
||
* call repl_on_ack to update acked_lsn and signal majority waiter. */
|
||
struct WalAckReply war_raw;
|
||
memset(&war_raw, 0, sizeof(war_raw));
|
||
read_WalAckReply(&war_raw, rep_s.data.walackreplymsg);
|
||
uint64_t acked_lsn = (uint64_t)war_raw.lsn;
|
||
uint64_t reply_term = (uint64_t)war_raw.term;
|
||
int success = war_raw.success ? 1 : 0;
|
||
capn_free(&capn_ctx);
|
||
|
||
if (success && c) {
|
||
repl_on_ack(&c->persist.repl, ep, acked_lsn);
|
||
TRACE("repl_ack_decode_and_dispatch: WAL ACK from standby"
|
||
" lsn=%lu success=1", (unsigned long)acked_lsn);
|
||
/* ha-term-safety task 3.3 (R4.1/R4.5): route the ACK through the
|
||
* raft library so it updates match_index[]/next_index[] and
|
||
* recomputes the current-term-gated commit (raft_advance_commit).
|
||
* This fires on_commit_advanced (task 2.4) ONLY when a quorum
|
||
* agrees on a CURRENT-TERM entry — making the library the live
|
||
* commit source alongside the durable barrier; a stale-term reply
|
||
* is dropped inside the library. Driven on the same iomgr ACK
|
||
* path; neither library call performs I/O.
|
||
*
|
||
* Resolve the ABSOLUTE library peer index per endpoint type FIRST
|
||
* (remediation): this branch fires for BOTH promoted-vote eps
|
||
* (vctx->ctx != NULL, peer_idx is a relative votes index) AND
|
||
* heartbeat standby eps (vctx->ctx == NULL, peer_idx is already
|
||
* absolute) — see the authoritative scan at ha.c:6187-6189. A
|
||
* blind remap would double-map the heartbeat path onto the wrong
|
||
* (or out-of-range) match_index[] slot. */
|
||
int ack_abs_idx = ha_repl_ack_abs_peer_idx(
|
||
&c->ha, /*ep_is_vote*/ vctx->ctx != NULL, vctx->peer_idx);
|
||
ha_raft_on_repl_ack(c, ack_abs_idx, acked_lsn, reply_term);
|
||
/* ha-convergence-recovery task 4.2 (D6, Req 5.1/5.2/7.1):
|
||
* ACK-driven catch-up — continue streaming to a standby still
|
||
* behind target, or re-include it at parity. Shares the exact
|
||
* decision the unit test drives via ha_walack_continuation_for_test.
|
||
* Scheduling is O(1) on iomgr; all WAL file I/O runs in the
|
||
* deferred continuation task. */
|
||
ha_walack_on_ack(c, ep, vctx->peer_idx, acked_lsn);
|
||
} else if (!success) {
|
||
WARN("repl_ack_decode_and_dispatch: WAL NACK from standby"
|
||
" lsn=%lu success=0 — ignoring", (unsigned long)acked_lsn);
|
||
}
|
||
} else if (rep_s.data_which == RequestReply_data_replconfigpushreplymsg) {
|
||
/* Config-push ACK from standby (ha-commit-barrier-rollout task
|
||
* 5.2 remediation r1; completes the unlanded half of
|
||
* ha-config-bootstrap task 8.1): the standby replies to
|
||
* OP_replConfigPush with replconfigpushreplymsg carrying
|
||
* {accepted, versionTag} (config_repl.c:744-752). Decode the
|
||
* acked tag and apply it to the replicator so the configAck
|
||
* barrier (repl_wait_majority_ack) can observe the covering
|
||
* ACK — without this branch every multi-member config upgrade
|
||
* would time out. read_ReplConfigPushReply reads scalars only
|
||
* — safe to call before capn_free. */
|
||
struct ReplConfigPushReply rcp_raw;
|
||
memset(&rcp_raw, 0, sizeof(rcp_raw));
|
||
read_ReplConfigPushReply(&rcp_raw, rep_s.data.replconfigpushreplymsg);
|
||
uint64_t acked_tag = (uint64_t)rcp_raw.versionTag;
|
||
int accepted = rcp_raw.accepted ? 1 : 0;
|
||
capn_free(&capn_ctx);
|
||
|
||
if (accepted && c) {
|
||
repl_on_config_ack(&c->persist.repl, ep, acked_tag);
|
||
TRACE("repl_ack_decode_and_dispatch: config ACK from standby"
|
||
" tag=%016" PRIx64 " accepted=1", acked_tag);
|
||
} else if (!accepted) {
|
||
WARN("repl_ack_decode_and_dispatch: config push rejected by"
|
||
" standby (tag=%016" PRIx64 ") — ignoring", acked_tag);
|
||
}
|
||
} else {
|
||
capn_free(&capn_ctx);
|
||
}
|
||
} else if (opcode == OPCode_nop) {
|
||
/* Plain NOP with no body — silently ignore */
|
||
capn_free(&capn_ctx);
|
||
} else {
|
||
capn_free(&capn_ctx);
|
||
WARN("repl_ack_decode_and_dispatch: unexpected opcode %d on promoted ep",
|
||
opcode);
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* peer_ctrl_msg_handler_inline: the state-aware peer-control routing body.
|
||
* Routes by ha->role at dispatch time — no explicit handler reassignment needed.
|
||
* CTRL_ELECTING → vote_reply_decode_and_dispatch
|
||
* CTRL_PRIMARY → repl_ack_decode_and_dispatch
|
||
* CTRL_STANDBY + PreVote → pre_vote_reply_decode_and_dispatch (task 5.1)
|
||
* any other role → drop packet (WARN)
|
||
*
|
||
* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly —
|
||
* the caller (peer_ctrl_msg_handler) has already taken it off ep->fsm_in.pkt and
|
||
* cleared that slot to NULL. This body and the routers it calls free `pkt` on
|
||
* every path and NEVER touch ep->fsm_in.pkt, which is what lets this body run
|
||
* off the iomgr thread (on the vtask pool) without colliding with a concurrent
|
||
* io_worker read on the same endpoint.
|
||
*
|
||
* This body is run either INLINE on the io_worker thread (dispatch flag OFF —
|
||
* the production default; identical to the pre-task behavior) or on a vtask
|
||
* pool worker (flag ON; via peer_ctrl_vtask_entry).
|
||
*
|
||
* PreVote note (ha-prevote task 5.1): a preVoteReply arrives on the candidate's
|
||
* OUTBOUND PreVote endpoint while the candidate's role is still CTRL_STANDBY
|
||
* (a PreVote round performs no role change — Req 1.1). Without this branch the
|
||
* reply would hit the default case and be dropped, stranding the round until it
|
||
* times out. We gate on raft.pre_vote_in_progress so a STANDBY node with no
|
||
* active round still drops unexpected traffic.
|
||
*/
|
||
static int peer_ctrl_msg_handler_inline(endpoint_t *ep, io_packet_t *pkt)
|
||
{
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
ha_state_t *ha = vctx->ha;
|
||
|
||
/* raft-drive-async task 3.1: test-only observation seam — fires before any
|
||
* routing so a test can count/timestamp/park the body. NULL in production. */
|
||
if (ha_peerctrl_body_probe)
|
||
ha_peerctrl_body_probe(ep);
|
||
|
||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
|
||
/* PreVote replies land while CTRL_STANDBY with a round in progress. */
|
||
if (role == CTRL_STANDBY && ha->raft.pre_vote_in_progress)
|
||
return pre_vote_reply_decode_and_dispatch(vctx, pkt);
|
||
|
||
switch (role) {
|
||
case CTRL_ELECTING:
|
||
return vote_reply_decode_and_dispatch(vctx, pkt);
|
||
|
||
case CTRL_PRIMARY:
|
||
return repl_ack_decode_and_dispatch(ep, vctx, pkt);
|
||
|
||
default:
|
||
if (pkt) { free_packet(pkt); free(pkt); }
|
||
WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role);
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
/*
|
||
* peer_ctrl_vtask_entry (raft-drive-async task 3.1): vtask pool entry that runs
|
||
* the peer-control routing body off the iomgr thread. Extracts (ep, pkt) from
|
||
* the VTASK_IO task data — the SAME io_task_data_t shape the lib vtask path
|
||
* uses — takes ownership of pkt out of the task data (so vtask_fini, which only
|
||
* frees the data struct, cannot reach it) and hands it to the body, which frees
|
||
* it. No double-free: pkt is freed exactly once inside the body; run_task then
|
||
* frees the task + data struct. endpoint_unblock_tasks (per-endpoint
|
||
* serialization, Req 4.1/4.2) is driven by run_task after this returns.
|
||
*/
|
||
static int peer_ctrl_vtask_entry(vtask_t *task)
|
||
{
|
||
io_task_data_t *iod = &task->data->u.iodata;
|
||
endpoint_t *ep = iod->ep;
|
||
io_packet_t *pkt = iod->pkt;
|
||
|
||
iod->pkt = NULL; /* body owns pkt now; nothing else may reference it */
|
||
|
||
if (ep == NULL) {
|
||
if (pkt) { free_packet(pkt); free(pkt); }
|
||
return -EINVAL;
|
||
}
|
||
return peer_ctrl_msg_handler_inline(ep, pkt);
|
||
}
|
||
|
||
/*
|
||
* peer_ctrl_classify (raft-drive-async task 3.1): assign the LivenessAdmission
|
||
* task class for an inbound peer-control message. This handler decodes vote
|
||
* replies, PreVote replies and WAL/config ACKs — all RECEIPT decode + commit
|
||
* eval, which is bounded liveness work. Per the design's deadlock-freedom
|
||
* argument (R4.5), ack/commit-advance decode MUST be VTASK_LIVENESS so it is
|
||
* NEVER admission-gated: a quorum of acks can always find a reserved worker to
|
||
* advance committed_lsn even when the pool is saturated by *_BLOCK tasks.
|
||
*
|
||
* (The vote-GRANT term fsync — VTASK_LIVE_BLOCK, R4.6 — is handled on the
|
||
* INBOUND RequestVote handler, not on these reply/ack endpoints; and the long
|
||
* config-push/snapshot ops are offloaded to the SlowLane in tasks 3.2/3.3. So
|
||
* everything this handler dispatches is liveness work today.)
|
||
*
|
||
* ┌─ HIGH FLAG-ON HAZARD (raft-drive-async finding-2; bug-1198) ─────────────┐
|
||
* │ LIVENESS is correct ONLY once the snapshot send runs OFF the pool. The │
|
||
* │ CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT → │
|
||
* │ persist_checkpoint (a BLOCKING fdatasync) in │
|
||
* │ repl_ack_decode_and_dispatch → ha_dispatch_repl_action_for_standby. │
|
||
* │ Running that blocking op while tagged VTASK_LIVENESS (never gated) breaks │
|
||
* │ the deadlock-freedom reservation — a burst of snapshot sends could occupy │
|
||
* │ every liveness worker and starve the acks that advance committed_lsn. │
|
||
* │ │
|
||
* │ Task 3.2 has LANDED the slow-lane offload (ha_snapshot_send_dispatch), │
|
||
* │ but it is DORMANT: g_drive_async_snapshot_offload defaults OFF (the send │
|
||
* │ still runs INLINE) because the offload borrows a standby endpoint whose │
|
||
* │ lifetime is not yet pool-safe (bug-1197). So bug-1198 is NOT yet closed │
|
||
* │ in the default build. Task 3.4 is the single gate that flips BOTH │
|
||
* │ g_drive_async_snapshot_offload AND g_drive_async_peerctrl_dispatch ON │
|
||
* │ together (offload bounds the body → LIVENESS correct) and proves the │
|
||
* │ deadlock-freedom + endpoint-lifetime properties under load. │
|
||
* │ DO NOT flip g_drive_async_peerctrl_dispatch_enabled ON until task 3.4's │
|
||
* │ gate is green WITH the snapshot offload also ON. │
|
||
* └──────────────────────────────────────────────────────────────────────────┘
|
||
*/
|
||
static vtask_class_t peer_ctrl_classify(ha_state_t *ha, io_packet_t *pkt)
|
||
{
|
||
(void)ha;
|
||
(void)pkt;
|
||
return VTASK_LIVENESS;
|
||
}
|
||
|
||
/*
|
||
* peer_ctrl_dispatch_to_pool (raft-drive-async task 3.1): hand the decoded
|
||
* peer-control packet to the vtask pool and return promptly. Per-endpoint
|
||
* arrival-order serialization is automatic — bind_vtask calls
|
||
* endpoint_block_tasks(ep) and the dispatcher won't bind a second task for the
|
||
* same endpoint until this one completes (Req 4.1/4.2).
|
||
*
|
||
* Controllers NEVER self-terminate (CLAUDE.md): on any failure to enqueue
|
||
* (pool unwired, OOM, or admission reject) we fail loud and run the body INLINE
|
||
* rather than drop a consensus message — staying alive and correct.
|
||
*
|
||
* ┌─ CRITICAL FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197) ──────────┐
|
||
* │ Enabling this dispatch (g_drive_async_peerctrl_dispatch_enabled == 1) │
|
||
* │ introduces an endpoint use-after-free that does NOT exist while the body │
|
||
* │ runs inline. raft_cb_become_follower (ha.c, see its endpoint_fini loop) │
|
||
* │ runs on a POOL worker and calls endpoint_fini() on ALL peer vote-eps — │
|
||
* │ freeing each ep->data (vctx) via FREEUP. Meanwhile the single io_worker │
|
||
* │ thread can be concurrently inside peer_ctrl_msg_handler_inline(ep_j, ...) │
|
||
* │ for a same-batch queued event on another ep, reading ep_j->data → races a │
|
||
* │ freed vctx (null-deref / UAF). Safe TODAY only because the flag is OFF and │
|
||
* │ the io_worker processes peer-control sequentially. │
|
||
* │ FIX REQUIRED BEFORE task 3.4 FLIPS THE FLAG ON: every pool-reachable │
|
||
* │ teardown that frees a peer ep (raft_cb_become_follower's endpoint_fini │
|
||
* │ loop) MUST iomgr_unmonitor(ep) BEFORE endpoint_fini(ep) so the io_worker │
|
||
* │ can never deliver a new event on a freed ep->data. Do NOT enable dispatch │
|
||
* │ until that ordering is in place. │
|
||
* └──────────────────────────────────────────────────────────────────────────┘
|
||
*/
|
||
static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt)
|
||
{
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
ha_state_t *ha = vctx->ha;
|
||
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
|
||
|
||
if (pool == NULL) {
|
||
WARN("peer_ctrl_dispatch_to_pool: vtask pool not wired — running "
|
||
"inline (stay alive, never drop a peer-control message)");
|
||
return peer_ctrl_msg_handler_inline(ep, pkt);
|
||
}
|
||
|
||
vtask_data_t *data = (vtask_data_t *)calloc(1, sizeof(*data));
|
||
vtask_t *task = (vtask_t *)calloc(1, sizeof(*task));
|
||
if (data == NULL || task == NULL) {
|
||
WARN("peer_ctrl_dispatch_to_pool: OOM allocating vtask — running "
|
||
"inline (stay alive)");
|
||
free(data);
|
||
free(task);
|
||
return peer_ctrl_msg_handler_inline(ep, pkt);
|
||
}
|
||
|
||
data->vtype = VTASK_IO;
|
||
data->u.iodata.ep = ep; /* drives per-endpoint serialization in bind_vtask */
|
||
data->u.iodata.pkt = pkt; /* ownership transferred into the task */
|
||
vtask_init(task, (vtask_func_t)peer_ctrl_vtask_entry, data);
|
||
/* raft-drive-async task 3.1: the task is the SOLE owner of pkt (the shim
|
||
* already cleared ep->fsm_in.pkt). Mark it so taskq_item_killer frees pkt
|
||
* if this task is destroyed while still queued (never executed) at
|
||
* vtaskmgr_fini — without this the pkt would leak on shutdown. The
|
||
* executed path frees pkt inside peer_ctrl_vtask_entry (and NULLs
|
||
* iod->pkt), so there is no double-free. */
|
||
task->owns_io_pkt = 1;
|
||
|
||
vtask_class_t cls = peer_ctrl_classify(ha, pkt);
|
||
int rc = vtask_enq_classified(pool, task, cls);
|
||
if (rc < 0) {
|
||
/* Task NOT enqueued — we still own task/data/pkt. LIVENESS is never
|
||
* gated so this should not happen; if it does, fail loud and run inline
|
||
* (never drop). Free the wrapper structs only — pkt is freed by the
|
||
* inline body below. */
|
||
WARN("peer_ctrl_dispatch_to_pool: vtask_enq_classified failed (rc=%d) "
|
||
"— running inline (stay alive)", rc);
|
||
free(data);
|
||
free(task);
|
||
return peer_ctrl_msg_handler_inline(ep, pkt);
|
||
}
|
||
|
||
return 0; /* returns promptly; the body runs on a pool worker */
|
||
}
|
||
|
||
/*
|
||
* peer_ctrl_msg_handler: the registered iomgr msg_handler for peer-control
|
||
* endpoints (set in on_vote_connected / on_pre_vote_connected / on_standby_
|
||
* connected). Runs on the io_worker thread.
|
||
*
|
||
* raft-drive-async task 3.1: takes ownership of the inbound packet off
|
||
* ep->fsm_in.pkt and clears that slot IMMEDIATELY (so a subsequent io_worker
|
||
* read on this endpoint allocates a fresh packet and cannot collide with the
|
||
* one we are about to process), then either runs the routing body inline
|
||
* (dispatch flag OFF — production default, byte-for-byte the prior behavior) or
|
||
* hands it to the vtask pool and returns promptly (flag ON).
|
||
*/
|
||
static int peer_ctrl_msg_handler(void *ep_raw)
|
||
{
|
||
endpoint_t *ep = (endpoint_t *)ep_raw;
|
||
io_packet_t *pkt = ep->fsm_in.pkt;
|
||
|
||
ep->fsm_in.pkt = NULL; /* take ownership before any routing/dispatch */
|
||
|
||
if (!ha_peerctrl_dispatch_enabled())
|
||
return peer_ctrl_msg_handler_inline(ep, pkt);
|
||
|
||
return peer_ctrl_dispatch_to_pool(ep, pkt);
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
|
||
/*
|
||
* vote_reply_build_pkt_for_test: test helper that encodes a RequestVoteReply
|
||
* into a newly allocated io_packet_t that vote_reply_msg_handler_for_test can decode.
|
||
*
|
||
* Wire format mirrors the real peer reply:
|
||
* Request{opcode=reply, body.replymsg=RequestReply{
|
||
* data_which=requestvotereplymsg,
|
||
* data.requestvotereplymsg=RequestVoteReply{term, voteGranted, ...}
|
||
* }}
|
||
*
|
||
* Returns a freshly allocated io_packet_t* on success, NULL on failure.
|
||
* Handler takes ownership: it calls free_packet()+free() when done.
|
||
* Tests set ep->fsm_in.pkt to the returned pointer before calling the handler.
|
||
*/
|
||
io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
|
||
uint64_t peer_tag, uint64_t peer_lsn)
|
||
{
|
||
struct capn cs;
|
||
struct capn_segment *seg;
|
||
struct RequestVoteReply vr_capn;
|
||
RequestVoteReply_ptr vr_ptr;
|
||
struct RequestReply rr_capn;
|
||
RequestReply_ptr rr_ptr;
|
||
struct Request rq_capn;
|
||
Request_ptr rq_ptr;
|
||
io_packet_t *pkt;
|
||
int rc;
|
||
|
||
capn_init_malloc(&cs);
|
||
seg = capn_root(&cs).seg;
|
||
|
||
/* RequestVoteReply — all scalar fields */
|
||
memset(&vr_capn, 0, sizeof(vr_capn));
|
||
vr_capn.term = term;
|
||
vr_capn.voteGranted = vote_granted ? 1 : 0;
|
||
vr_capn.peerConfigTag = peer_tag;
|
||
vr_capn.peerLastLogLsn = peer_lsn;
|
||
vr_ptr = new_RequestVoteReply(seg);
|
||
write_RequestVoteReply(&vr_capn, vr_ptr);
|
||
|
||
/* RequestReply envelope */
|
||
memset(&rr_capn, 0, sizeof(rr_capn));
|
||
rr_capn.seq = 1;
|
||
rr_capn.opcode = OPCode_requestVote;
|
||
rr_capn.status = 0;
|
||
rr_capn.data_which = RequestReply_data_requestvotereplymsg;
|
||
rr_capn.data.requestvotereplymsg = vr_ptr;
|
||
rr_ptr = new_RequestReply(seg);
|
||
write_RequestReply(&rr_capn, rr_ptr);
|
||
|
||
/* Outer Request{opcode=reply} */
|
||
memset(&rq_capn, 0, sizeof(rq_capn));
|
||
rq_capn.opcode = OPCode_reply;
|
||
rq_capn.seq = 1;
|
||
rq_capn.body_which = Request_body_replymsg;
|
||
rq_capn.body.replymsg = rr_ptr;
|
||
rq_ptr = new_Request(seg);
|
||
write_Request(&rq_capn, rq_ptr);
|
||
capn_setp(capn_root(&cs), 0, rq_ptr.p);
|
||
|
||
pkt = calloc(1, sizeof(*pkt));
|
||
if (!pkt) { capn_free(&cs); return NULL; }
|
||
|
||
pkt->sz = capn_size(&cs);
|
||
rc = alloc_packet(pkt);
|
||
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
|
||
|
||
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
|
||
capn_free(&cs);
|
||
|
||
return pkt;
|
||
}
|
||
|
||
/*
|
||
* vote_reply_msg_handler_for_test: test entry point for vote-reply decode path.
|
||
* Temporarily forces role to CTRL_ELECTING so peer_ctrl_msg_handler routes to
|
||
* vote_reply_decode_and_dispatch, then restores the original role.
|
||
* Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
|
||
*/
|
||
int vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
|
||
{
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
ha_state_t *ha = vctx ? vctx->ha : NULL;
|
||
int32_t saved_role = ha ? atomic_load_explicit(&ha->role, memory_order_acquire) : 0;
|
||
if (ha)
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||
ep->fsm_in.pkt = pkt;
|
||
int rc = peer_ctrl_msg_handler(ep);
|
||
if (ha)
|
||
atomic_store_explicit(&ha->role, saved_role, memory_order_release);
|
||
return rc;
|
||
}
|
||
|
||
/*
|
||
* vote_ep_ctx_build_for_test: test helper that allocates a vote_ep_ctx_t,
|
||
* fills it from the supplied parameters, and wires up ep->data and
|
||
* ep->on_connected so the test can drive on_vote_connected indirectly.
|
||
*
|
||
* Returns ep (same pointer passed in) for call-chain convenience.
|
||
* Caller is responsible for free(ep->data) after use.
|
||
*/
|
||
endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
|
||
ha_state_t *ha,
|
||
ha_elect_ctx_t *ctx,
|
||
int peer_idx,
|
||
uint64_t my_term,
|
||
uint64_t my_tag,
|
||
uint64_t my_lsn,
|
||
const char *my_name)
|
||
{
|
||
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
|
||
if (!vctx) return ep; /* test will fail when on_connected is NULL */
|
||
|
||
vctx->ha = ha;
|
||
vctx->c = NULL;
|
||
vctx->ctx = ctx;
|
||
vctx->peer_idx = peer_idx;
|
||
vctx->my_term = my_term;
|
||
vctx->my_tag = my_tag;
|
||
vctx->my_lsn = my_lsn;
|
||
vctx->my_name = my_name;
|
||
|
||
ep->data = vctx;
|
||
ep->on_connected = on_vote_connected;
|
||
return ep;
|
||
}
|
||
|
||
/*
|
||
* election_timeout_fn_for_test: test helper that exercises the election timeout
|
||
* callback logic with an explicit cancelled_val instead of an atomic field.
|
||
* Allows testing the cancelled-gate without setting up a real delaymgr.
|
||
*
|
||
* Returns 0 in all cases (mirrors the real callback's return value).
|
||
*/
|
||
int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
|
||
int cancelled_val)
|
||
{
|
||
if (cancelled_val)
|
||
return 0;
|
||
pthread_mutex_lock(&ctx->lock);
|
||
ha_elect_evaluate_and_signal(ha, ctx);
|
||
pthread_mutex_unlock(&ctx->lock);
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* repl_catchup_done_pkt_for_test: build a minimal io_packet_t encoding
|
||
* Request{opcode=replCatchUpDone, body_which=nodata}.
|
||
* Used by peer_ctrl_msg_handler tests to verify the repl-ack dispatch path.
|
||
*/
|
||
io_packet_t *repl_catchup_done_pkt_for_test(void)
|
||
{
|
||
struct capn cs;
|
||
struct capn_segment *seg;
|
||
struct Request rq_capn;
|
||
Request_ptr rq_ptr;
|
||
io_packet_t *pkt;
|
||
int rc;
|
||
|
||
capn_init_malloc(&cs);
|
||
seg = capn_root(&cs).seg;
|
||
|
||
memset(&rq_capn, 0, sizeof(rq_capn));
|
||
rq_capn.opcode = OPCode_replCatchUpDone;
|
||
rq_capn.seq = 1;
|
||
rq_capn.body_which = Request_body_nomsg;
|
||
rq_ptr = new_Request(seg);
|
||
write_Request(&rq_capn, rq_ptr);
|
||
capn_setp(capn_root(&cs), 0, rq_ptr.p);
|
||
|
||
pkt = calloc(1, sizeof(*pkt));
|
||
if (!pkt) { capn_free(&cs); return NULL; }
|
||
pkt->sz = capn_size(&cs);
|
||
rc = alloc_packet(pkt);
|
||
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
|
||
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
|
||
capn_free(&cs);
|
||
return pkt;
|
||
}
|
||
|
||
/*
|
||
* vote_ep_err_handler_for_test: call the static vote_ep_err_handler on ep.
|
||
* Allows tests to invoke the err_handler without going through iomgr/endpoint machinery.
|
||
*/
|
||
int vote_ep_err_handler_for_test(endpoint_t *ep)
|
||
{
|
||
return vote_ep_err_handler((void *)ep);
|
||
}
|
||
|
||
/*
|
||
* peer_ctrl_msg_handler_for_test: expose peer_ctrl_msg_handler for tests.
|
||
* Sets ep->fsm_in.pkt then invokes the handler. Packet ownership transferred.
|
||
*/
|
||
int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
|
||
{
|
||
ep->fsm_in.pkt = pkt;
|
||
return peer_ctrl_msg_handler(ep);
|
||
}
|
||
|
||
/*
|
||
* pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL
|
||
* decode/route path (peer_ctrl_msg_handler) while the candidate is CTRL_STANDBY
|
||
* with a PreVote round in progress — the exact production condition under which
|
||
* a PreVote reply arrives. Unlike vote_reply_msg_handler_for_test it does NOT
|
||
* force the role: it proves the STANDBY+pre_vote_in_progress branch added in
|
||
* task 5.1 actually routes the reply to the PreVote tally (instead of dropping
|
||
* it). Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
|
||
*/
|
||
int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
|
||
{
|
||
ep->fsm_in.pkt = pkt;
|
||
return peer_ctrl_msg_handler(ep);
|
||
}
|
||
|
||
/*
|
||
* vote_ep_ctx_set_cluster_for_test: set the cluster_t pointer inside the
|
||
* vote_ep_ctx_t stored at ep->data. Tests that need the promoted-endpoint
|
||
* dispatch path to reach ha_promote_to_primary(c) with a live cluster pointer
|
||
* must call this after vote_ep_ctx_build_for_test (which sets vctx->c = NULL
|
||
* by default).
|
||
* Requirements: task 7.3 (ha-peer-connection-reuse).
|
||
*/
|
||
void vote_ep_ctx_set_cluster_for_test(endpoint_t *ep, cluster_t *c)
|
||
{
|
||
if (!ep || !ep->data) return;
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
vctx->c = c;
|
||
}
|
||
|
||
/*
|
||
* raft_cb_become_leader_for_test: drive the leader promotion path without
|
||
* a live cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress
|
||
* production-only calls (phase2, push_config, ctrlhello).
|
||
*/
|
||
void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term)
|
||
{
|
||
ha->raft_c = NULL;
|
||
raft_cb_become_leader((void *)ha, term);
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/* ha_arm_promotion_fence_timer_for_test (ha-commit-durability task 4.4):
|
||
* expose the file-static bounded-window arm path so the durability suite can
|
||
* drive the exact deferral wired into raft_cb_become_leader without staging a
|
||
* full election. Production never references this symbol (XCE_TESTING only). */
|
||
void ha_arm_promotion_fence_timer_for_test(cluster_t *c, bool truncated,
|
||
uint64_t durable_floor,
|
||
uint64_t target_hwm)
|
||
{
|
||
ha_arm_promotion_fence_timer(&c->ha, c, truncated, durable_floor,
|
||
target_hwm);
|
||
}
|
||
#endif
|
||
|
||
/*
|
||
* raft_cb_become_follower_for_test: drive the follower path without a live
|
||
* cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production
|
||
* calls (phase2, fetch_config); ha_arm_heartbeat_timeout_vtask is a no-op
|
||
* when c=NULL so no watchdog thread is spawned.
|
||
*/
|
||
void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term)
|
||
{
|
||
ha->raft_c = NULL;
|
||
raft_cb_become_follower((void *)ha, term, -1);
|
||
}
|
||
|
||
/*
|
||
* raft_cb_become_follower_with_cluster_for_test (ha-convergence-recovery
|
||
* task 2.1): drive the follower path with a LIVE cluster pointer so the
|
||
* cluster-dependent demotion-convergence branch (task 2.2) is exercised
|
||
* against a real persist/hooksys/delaymgr fixture. Sets ha->raft_c = c so
|
||
* the become-follower callback sees a real cluster (unlike the NULL-cluster
|
||
* shim above). raft_cb_become_follower is file-static, so this seam must
|
||
* live here in ha.c.
|
||
*/
|
||
void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term)
|
||
{
|
||
c->ha.raft_c = c;
|
||
raft_cb_become_follower((void *)&c->ha, term, -1);
|
||
}
|
||
|
||
/*
|
||
* raft_cb_become_leader_with_cluster_for_test (fix-promotion-standby-registration):
|
||
* drive the leader promotion path with a LIVE cluster pointer so the
|
||
* cluster-dependent standby-registration branch (repl_add_standby for each
|
||
* surviving vote endpoint whose votes[i].peer is set) is exercised against a
|
||
* real persist/repl/commit/delaymgr fixture. Sets ha->raft_c = c so the
|
||
* become-leader callback sees a real cluster (unlike the NULL-cluster shim
|
||
* raft_cb_become_leader_for_test above, which suppresses the c-dependent
|
||
* registration block entirely). raft_cb_become_leader is file-static, so this
|
||
* seam must live here in ha.c. Symmetric with
|
||
* raft_cb_become_follower_with_cluster_for_test.
|
||
*/
|
||
void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, uint64_t term)
|
||
{
|
||
c->ha.raft_c = c;
|
||
raft_cb_become_leader((void *)&c->ha, term);
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
/* ── Task 2.4: vote_ep_err_handler ─────────────────────────────────────────
|
||
*
|
||
* Called by iomgr when the outgoing vote endpoint encounters an error
|
||
* (connect failure, SSL error, remote close before reply).
|
||
* Counts the peer as denied: grant=0, tag=0, lsn=0.
|
||
*/
|
||
static int vote_ep_err_handler(void *ep_raw)
|
||
{
|
||
endpoint_t *ep = (endpoint_t *)ep_raw;
|
||
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
|
||
if (!vctx) return 0;
|
||
|
||
ha_state_t *ha = vctx->ha;
|
||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
|
||
if (role == CTRL_PRIMARY) {
|
||
/* Promoted connection dropped — remove from standby_endpoints */
|
||
if (ha->standby_endpoints) {
|
||
pthread_mutex_lock(&ha->lock);
|
||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||
if (zarray_get(ha->standby_endpoints, i) == ep) {
|
||
zarray_del(ha->standby_endpoints, i);
|
||
break;
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&ha->lock);
|
||
}
|
||
/* Task 2.2: deregister from WAL replicator.
|
||
* Req 1.2: when a standby disconnects, the HA layer shall deregister
|
||
* the endpoint from the replicator via repl_remove_standby.
|
||
* vctx->c may be NULL in test mode — guard accordingly. */
|
||
if (vctx->c)
|
||
repl_remove_standby(&vctx->c->persist.repl, ep);
|
||
|
||
/* Quorum-transition observability (Req 8.1): a standby just dropped — the
|
||
* write-quorum count changed. Record the loss edge once (silent if
|
||
* quorum is still held after the removal). */
|
||
ha_log_quorum_transition(ha);
|
||
} else {
|
||
/* Election phase: count as denied vote */
|
||
handle_request_vote_reply(ha, vctx->ctx, vctx->peer_idx,
|
||
vctx->my_term,
|
||
0, /* vote_granted = false */
|
||
0, 0);
|
||
}
|
||
|
||
/* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data.
|
||
* vctx is freed in ha_election_cleanup() when all per-peer contexts release. */
|
||
return 0;
|
||
}
|
||
|
||
/* ── ha_init_raft: one-time raft state machine initialization ────────────────
|
||
*
|
||
* Called once at startup after ha_restore_term() to wire the raft state
|
||
* machine with the correct cluster size and callbacks. Subsequent elections
|
||
* use raft_start_election() / raft_on_leader_lost() on this same state.
|
||
*
|
||
* n_peers: total candidates_count from cluster config (includes self).
|
||
*/
|
||
void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers)
|
||
{
|
||
/* Map persisted voted_for_name back to a peer index for initial_voted_for.
|
||
* This lets raft_check_and_start_election (RDHUP path) see the prior vote
|
||
* without re-reading the file. */
|
||
int restored_voted_for = -1;
|
||
if (ha->voted_for_name[0] != '\0' && c && c->myconf) {
|
||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||
if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) {
|
||
restored_voted_for = i;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
|
||
/* ha-term-safety task 2.5 (R1.5, R2.1): restore the durable log/commit floor
|
||
* from the recovered WAL so the library boots at the recovered tip — not an
|
||
* empty log at LSN 0. The library never reads storage; the caller restores
|
||
* initial_log_len / initial_commit_index here (same discipline as
|
||
* initial_term / initial_voted_for above).
|
||
*
|
||
* initial_log_len = recovered durable last-log index
|
||
* (persist_durable_floor: re-derived from the WAL tip
|
||
* by persist_recover; 0 on empty first boot).
|
||
* initial_commit_index = recovered committed LSN (committed_lsn). Recovery
|
||
* re-derives the durable tip but the post-election
|
||
* safe-commit point governs the committed position;
|
||
* we pass whatever committed prefix is established at
|
||
* init (0 on a fresh boot). Clamp to log_len to honor
|
||
* the library invariant commit_index <= log_len.
|
||
*
|
||
* The recovered last-log TERM is surfaced on demand via on_get_entry_term(
|
||
* log_len) (the library's logOk freshness gate reads it) — there is no
|
||
* separate initial last-log-term field. */
|
||
uint64_t initial_log_len = c ? persist_durable_floor(&c->persist) : 0;
|
||
uint64_t initial_commit_index = c
|
||
? atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire)
|
||
: 0;
|
||
if (initial_commit_index > initial_log_len)
|
||
initial_commit_index = initial_log_len;
|
||
|
||
raft_config_t rcfg;
|
||
memset(&rcfg, 0, sizeof(rcfg));
|
||
rcfg.n_peers = n_peers;
|
||
rcfg.self_idx = ha->my_priority;
|
||
rcfg.election_timeout_ms = ha->config.election_timeout_ms;
|
||
rcfg.my_priority = ha->my_priority;
|
||
rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
rcfg.initial_voted_for = restored_voted_for;
|
||
rcfg.initial_log_len = initial_log_len;
|
||
rcfg.initial_commit_index = initial_commit_index;
|
||
rcfg.cbs = &s_real_cbs;
|
||
rcfg.ctx = ha;
|
||
raft_init(&ha->raft, &rcfg);
|
||
|
||
ha->n_peers = n_peers;
|
||
ha->raft_c = c;
|
||
INFO("HA: raft initialized (n_peers=%d, self=%d, term=%" PRIu64
|
||
", log_len=%" PRIu64 ", commit_index=%" PRIu64 ")",
|
||
n_peers, ha->my_priority,
|
||
atomic_load_explicit(&ha->term, memory_order_acquire),
|
||
initial_log_len, initial_commit_index);
|
||
}
|
||
|
||
/* ── ha-prevote task 4.1: candidate-side PreVote orchestration ─────────────
|
||
*
|
||
* The PreVote round is a side-effect-free majority probe interposed BEFORE the
|
||
* destructive truncate + CTRL_ELECTING + real-election sequence. During the
|
||
* round the node stays CTRL_STANDBY and mutates no persistent state (term /
|
||
* voted_for / role). Only on a WIN does it enter the unchanged real-election
|
||
* path; on a LOSS / round-timeout it stays STANDBY, re-arms the liveness
|
||
* watchdog, and logs campaign abandonment.
|
||
*
|
||
* This task implements the round machinery + the WON/LOST branches. The
|
||
* failover triggers (ha_on_primary_ctrl_fd_rdhup / liveness_poll_fn /
|
||
* ha_on_heartbeat_received) are rewired to call this in task 4.2 — until then
|
||
* the round is INERT in production.
|
||
*/
|
||
|
||
/*
|
||
* prevote_round_ctx_t: delaymgr context for the PreVote round timeout.
|
||
* cancelled is set when the round is decided (WON or LOST) before the timeout
|
||
* fires, so a late timeout task is a no-op.
|
||
*/
|
||
typedef struct {
|
||
_Atomic int cancelled;
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
} prevote_round_ctx_t;
|
||
|
||
/* Active round-timeout context. _Atomic so arm (vtask) vs timeout-fire
|
||
* (delaymgr) vs cancel-on-decision is well-defined. */
|
||
static prevote_round_ctx_t * _Atomic s_prevote_round_ctx = NULL;
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* ha_prevote_rearm_liveness_hook: test-only seam. When non-NULL, the LOST /
|
||
* timeout branch calls this INSTEAD OF ha_arm_heartbeat_timeout_vtask so a unit
|
||
* test can count re-arms without a live delaymgr (Req 5.4).
|
||
*/
|
||
int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c) = NULL;
|
||
#endif
|
||
|
||
/* ha_prevote_rearm_liveness: re-arm the standby liveness watchdog after a lost
|
||
* round so a later genuine failure still triggers a new PreVote round (the
|
||
* watchdog does not re-arm itself once it has fired). */
|
||
static void ha_prevote_rearm_liveness(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
#ifdef XCE_TESTING
|
||
if (ha_prevote_rearm_liveness_hook != NULL) {
|
||
ha_prevote_rearm_liveness_hook(ha, c);
|
||
return;
|
||
}
|
||
#endif
|
||
ha_arm_heartbeat_timeout_vtask(ha, c);
|
||
}
|
||
|
||
/* ha_prevote_cancel_round_timeout: mark the active round-timeout context
|
||
* cancelled (the delaymgr task frees it on fire) and clear the static pointer
|
||
* so a later decision/timeout sees no active round. */
|
||
static void ha_prevote_cancel_round_timeout(void)
|
||
{
|
||
prevote_round_ctx_t *old = atomic_load_explicit(&s_prevote_round_ctx,
|
||
memory_order_acquire);
|
||
if (old) {
|
||
atomic_store_explicit(&old->cancelled, 1, memory_order_release);
|
||
atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release);
|
||
}
|
||
}
|
||
|
||
/*
|
||
* ha_enter_real_election: the WON-branch sequence — truncate uncommitted WAL
|
||
* above the committed boundary, then enter the UNCHANGED real-election path
|
||
* (ha_start_election_async transitions to CTRL_ELECTING and delegates the term
|
||
* bump + self-vote to raft_start_election). Factored here so both the PreVote
|
||
* WON branch (task 4.1) and the rewired trigger (task 4.2) can call it; this
|
||
* task does NOT change ha_on_primary_ctrl_fd_rdhup's own behaviour.
|
||
*
|
||
* c == NULL (unit-test mode): skip truncate + the live election; just transition
|
||
* the role to CTRL_ELECTING so the WON transition is observable.
|
||
*/
|
||
static void ha_enter_real_election(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
if (!c) {
|
||
/* Test mode: the real election needs iomgr/delaymgr; just mark the
|
||
* destructive transition so the WON branch is observable. */
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
|
||
/* bug-828 final slice (Leader Completeness): do NOT pre-truncate the WAL
|
||
* here. The earlier code truncated above ha->last_known_committed_lsn
|
||
* BEFORE the vote — but that watermark is the LOCALLY-LEARNED commit point,
|
||
* which LAGS the true cluster commit point (the primary commits an entry on
|
||
* a durable majority, then dies before a heartbeat carries that committed
|
||
* LSN to this standby). persist_truncate_wal_above clamps durable_lsn down
|
||
* to the cutoff, so a standby that had DURABLY persisted committed entries
|
||
* [committed+1 .. tip] would discard them and then advertise the stale-low
|
||
* cutoff as its durable floor. Every voter doing the same made
|
||
* safe-committed-from-votes collapse to the stale watermark, and the
|
||
* AUTHORITATIVE post-election truncate (raft_cb_become_leader,
|
||
* persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale
|
||
* prefix — losing committed jobs on a clean failover (bug-828/795).
|
||
*
|
||
* Raft model: a candidate does NOT truncate its log before campaigning. It
|
||
* keeps its full durable log and advertises its real durable tip
|
||
* (ha_candidate_advertise_lsn == persist_durable_floor), so voters holding
|
||
* more committed entries deny a colder candidate and the winner has every
|
||
* committed entry (Leader Completeness). Divergent UNCOMMITTED tails are
|
||
* removed ONLY by the authoritative post-election safe-commit truncate
|
||
* (above the majority-held point) and by the library's per-entry (term,lsn)
|
||
* reconcile (on_truncate_from) during subsequent AppendEntries — never by a
|
||
* stale-watermark pre-truncate that can erase committed state. */
|
||
|
||
/* Candidate freshness for the RequestVote payload (ha-commit-durability
|
||
* task 4.2). The TAG axis uses the same config-epoch fold the voter side
|
||
* computes (ha_vote_self_freshness), but the LSN axis advertises the BARE
|
||
* durable WAL tip — a candidate may only claim a log position it can
|
||
* durably back, so a truncated-restart node cannot over-advertise a stale
|
||
* volatile learned-committed value and win (design §1a/§5a). */
|
||
uint64_t cand_tag = 0, cand_lsn = 0;
|
||
if (c->myconf) {
|
||
uint64_t local_tag = c->myconf->config_version_tag;
|
||
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
|
||
memory_order_acquire);
|
||
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL);
|
||
cand_lsn = ha_candidate_advertise_lsn(&c->persist);
|
||
}
|
||
|
||
int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
|
||
const char *my_name = (c->myconf) ? c->myconf->name : "";
|
||
|
||
ha->raft_c = c;
|
||
|
||
#ifdef XCE_TESTING
|
||
/* Test seam: the WON-branch real election is the same step the pre-rewire
|
||
* rdhup body fired, so the existing rdhup-integration tests intercept it
|
||
* here (capturing my_tag/my_lsn / counting election starts) — they now
|
||
* drive it through the PreVote→WON path (ha-prevote task 4.2).
|
||
*
|
||
* Mark the role CTRL_ELECTING before invoking the hook: the real
|
||
* ha_start_election_async transitions to CTRL_ELECTING as its first step,
|
||
* so a test that intercepts the election must still observe the destructive
|
||
* role transition the WON branch performs (the spy hook does not start a
|
||
* real election that would set it). */
|
||
if (ha_rdhup_election_hook != NULL) {
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
|
||
memory_order_release);
|
||
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
|
||
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
|
||
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
|
||
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
|
||
int np = 0;
|
||
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
|
||
if (strcmp(cfg->candidates[i].name,
|
||
c->myhost ? c->myhost->name : "") != 0)
|
||
peer_ptrs[np++] = &cfg->candidates[i];
|
||
}
|
||
ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn,
|
||
my_name);
|
||
}
|
||
return;
|
||
}
|
||
#endif
|
||
|
||
/* ha_start_election_async transitions to CTRL_ELECTING and drives the
|
||
* UNCHANGED raft_start_election (term++ + self-vote). Peer connections are
|
||
* driven by raft_cb_send_vote_request; the peers[] arg is unused. */
|
||
ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name);
|
||
}
|
||
|
||
/*
|
||
* ha_start_prevote_async: start a PreVote round (ha-prevote task 4.1).
|
||
*
|
||
* Snapshots self tag/lsn into the round metadata carrier (ha->elect_ctx, read
|
||
* by raft_cb_send_pre_vote_request), calls raft_begin_pre_vote (self-grant +
|
||
* per-peer send via the callback), and arms a round-timeout bounded by
|
||
* election_timeout_ms (Req 5.4 — no unbounded delay). The node stays
|
||
* CTRL_STANDBY; NOTHING persistent is mutated (no term/voted_for/role change).
|
||
*
|
||
* If raft_begin_pre_vote returns RAFT_PRE_VOTE_WON immediately (single-node
|
||
* cluster: self-grant is already a majority), the WON branch runs at once.
|
||
*
|
||
* c == NULL (unit-test mode): the send callback returns early and the round
|
||
* timeout is a no-op; the round state lives entirely in raft_state_t so the
|
||
* reply-driven branches are still fully exercisable via the test seam.
|
||
*/
|
||
void ha_start_prevote_async(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
if (!ha) return;
|
||
|
||
ha->raft_c = c;
|
||
|
||
/* Snapshot self freshness for the PreVote request payload (same floor as a
|
||
* real vote). Stored in the metadata carrier ha->elect_ctx so
|
||
* raft_cb_send_pre_vote_request can read my_tag/my_lsn. */
|
||
uint64_t cand_tag = 0, cand_lsn = 0;
|
||
int n_peers = ha->n_peers > 0 ? ha->n_peers : ha->raft.n_peers;
|
||
if (c && c->myconf) {
|
||
uint64_t local_tag = c->myconf->config_version_tag;
|
||
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
|
||
memory_order_acquire);
|
||
/* Task 4.2: TAG via the config-epoch fold, LSN as the bare durable WAL
|
||
* tip (PreVote must not over-advertise either — design §1a). */
|
||
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL);
|
||
cand_lsn = ha_candidate_advertise_lsn(&c->persist);
|
||
n_peers = c->myconf->peers_cfg.candidates_count;
|
||
|
||
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
|
||
ha_elect_ctx_t *mc = calloc(1, sizeof(*mc) +
|
||
(size_t)n_other * sizeof(mc->votes[0]));
|
||
if (mc) {
|
||
mc->n_peers = n_other;
|
||
mc->result = -1;
|
||
mc->c = c;
|
||
mc->self_tag = cand_tag;
|
||
mc->self_lsn = cand_lsn;
|
||
pthread_mutex_init(&mc->lock, NULL);
|
||
sem_init(&mc->done, 0, 0);
|
||
if (n_other > 0) {
|
||
mc->votes[0].my_tag = cand_tag;
|
||
mc->votes[0].my_lsn = cand_lsn;
|
||
}
|
||
/* Do NOT free a prior ctx (live vote endpoints may hold a bare
|
||
* pointer to it — same UAF caution as the rdhup path). */
|
||
atomic_store_explicit(&ha->elect_ctx, mc, memory_order_release);
|
||
}
|
||
ha->n_peers = n_peers;
|
||
}
|
||
|
||
INFO("HA: starting PreVote round (prospective term=%" PRIu64 ", n_peers=%d) "
|
||
"— staying CTRL_STANDBY, no persist",
|
||
ha->raft.term + 1, n_peers);
|
||
|
||
/* raft_begin_pre_vote: self-grant + per-peer send; mutates only the
|
||
* pre_vote_* tally in raft_state_t (no term/voted_for/role change). */
|
||
int rc = raft_begin_pre_vote(&ha->raft);
|
||
|
||
/* Arm the round timeout (bounded by election_timeout_ms). Production only;
|
||
* the round state is in raft_state_t so tests exercise the branches via the
|
||
* reply seam without the timer. */
|
||
ha_prevote_cancel_round_timeout();
|
||
if (c && c->delaymgr.running) {
|
||
prevote_round_ctx_t *tctx = calloc(1, sizeof(*tctx));
|
||
if (tctx) {
|
||
atomic_init(&tctx->cancelled, 0);
|
||
tctx->ha = ha;
|
||
tctx->c = c;
|
||
atomic_store_explicit(&s_prevote_round_ctx, tctx,
|
||
memory_order_release);
|
||
extern int ha_prevote_round_timeout_fn(void *arg);
|
||
int trc = delay_task_ms(&c->delaymgr,
|
||
(int)ha->config.election_timeout_ms, 1,
|
||
(delay_task_entry_fn_t)ha_prevote_round_timeout_fn,
|
||
tctx);
|
||
if (trc < 0) {
|
||
WARN("HA: ha_start_prevote_async: round-timeout schedule failed: %d",
|
||
trc);
|
||
atomic_store_explicit(&s_prevote_round_ctx, NULL,
|
||
memory_order_release);
|
||
free(tctx);
|
||
}
|
||
}
|
||
}
|
||
|
||
/* Single-node: the self-grant is already a majority — win immediately. */
|
||
if (rc == RAFT_PRE_VOTE_WON) {
|
||
ha_prevote_cancel_round_timeout();
|
||
INFO("HA: PreVote won immediately (single-node) — entering real election");
|
||
ha_enter_real_election(ha, c);
|
||
}
|
||
}
|
||
|
||
/*
|
||
* pre_vote_reply_dispatch: feed a PreVote reply into the raft tally and branch
|
||
* on its outcome (ha-prevote task 4.1). peer_idx is the ABSOLUTE cluster index
|
||
* of the replying peer (the same index raft_begin_pre_vote sent to).
|
||
*
|
||
* RAFT_PRE_VOTE_WON → cancel round timeout, run the real-election sequence
|
||
* (truncate + CTRL_ELECTING + ha_start_election_async).
|
||
* RAFT_PRE_VOTE_LOST → stay CTRL_STANDBY, re-arm liveness, log abandonment.
|
||
* RAFT_PRE_VOTE_PENDING → no-op (await more replies).
|
||
*
|
||
* Mirrors vote_reply_dispatch → raft_on_vote_reply, but the raft library makes
|
||
* NO role/term change here — the controller acts on the returned status.
|
||
*
|
||
* Production caller (task 5.1): pre_vote_reply_decode_and_dispatch, reached from
|
||
* peer_ctrl_msg_handler when a preVoteReply arrives on the outbound PreVote
|
||
* endpoint. Also driven by the XCE_TESTING reply seam.
|
||
*/
|
||
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
|
||
int peer_idx, int granted)
|
||
{
|
||
int outcome = raft_on_pre_vote_reply(&ha->raft, peer_idx, granted);
|
||
|
||
if (outcome == RAFT_PRE_VOTE_WON) {
|
||
ha_prevote_cancel_round_timeout();
|
||
INFO("HA: PreVote round WON — entering real election (term will be %"
|
||
PRIu64 ")", ha->raft.term + 1);
|
||
ha_enter_real_election(ha, c);
|
||
return;
|
||
}
|
||
|
||
if (outcome == RAFT_PRE_VOTE_LOST) {
|
||
ha_prevote_cancel_round_timeout();
|
||
/* Req 7.3: log campaign abandonment distinctly from a genuine failover
|
||
* so operators can tell a suppressed disruptive election apart. */
|
||
INFO("HA: PreVote round LOST — abandoning campaign, staying CTRL_STANDBY "
|
||
"(no term increment); re-arming liveness watchdog");
|
||
/* Stay CTRL_STANDBY — no persist, no role change (Req 1.4). */
|
||
ha_prevote_rearm_liveness(ha, c);
|
||
return;
|
||
}
|
||
|
||
/* RAFT_PRE_VOTE_PENDING — await more replies, do nothing. */
|
||
}
|
||
|
||
/*
|
||
* ha_prevote_round_timeout_fn: delaymgr task body for the PreVote round timeout.
|
||
* A timeout that fires before a decision is treated as a LOSS (Req 5.4).
|
||
*/
|
||
int ha_prevote_round_timeout_fn(void *arg)
|
||
{
|
||
prevote_round_ctx_t *ctx = (prevote_round_ctx_t *)arg;
|
||
|
||
if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) {
|
||
#ifndef XCE_TESTING
|
||
free(ctx);
|
||
#endif
|
||
return 0;
|
||
}
|
||
|
||
ha_state_t *ha = ctx->ha;
|
||
cluster_t *c = ctx->c;
|
||
|
||
/* Only act if the round is still in progress (not already decided). */
|
||
if (ha->raft.pre_vote_in_progress) {
|
||
ha->raft.pre_vote_in_progress = 0;
|
||
atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release);
|
||
WARN("HA: PreVote round timed out before a decision — treating as LOSS, "
|
||
"staying CTRL_STANDBY; re-arming liveness watchdog (Req 5.4)");
|
||
ha_prevote_rearm_liveness(ha, c);
|
||
}
|
||
|
||
#ifndef XCE_TESTING
|
||
free(ctx);
|
||
#endif
|
||
return 0;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* pre_vote_reply_dispatch_for_test: drive the real pre_vote_reply_dispatch with
|
||
* decoded (peer_idx, granted) fields, bypassing iomgr (ha-prevote task 4.1).
|
||
*/
|
||
void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
|
||
int peer_idx, int granted)
|
||
{
|
||
pre_vote_reply_dispatch(ha, c, peer_idx, granted);
|
||
}
|
||
|
||
/*
|
||
* ha_prevote_round_timeout_fire_for_test: simulate the round timeout firing
|
||
* before a decision, driving the LOST branch (Req 5.4).
|
||
*/
|
||
void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
prevote_round_ctx_t ctx;
|
||
memset(&ctx, 0, sizeof(ctx));
|
||
ctx.ha = ha;
|
||
ctx.c = c;
|
||
atomic_init(&ctx.cancelled, 0);
|
||
ha_prevote_round_timeout_fn(&ctx);
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
/* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ────
|
||
*
|
||
* Non-blocking election initiator. Sets up the raft state machine and
|
||
* delegates all election mechanics to raft_start_election().
|
||
*
|
||
* raft_start_election() handles:
|
||
* - term increment and self-vote (via on_persist callback)
|
||
* - single-node shortcircuit (fires on_become_leader immediately)
|
||
* - multi-node RPC fan-out (via on_send_vote_request callback per peer)
|
||
* - election timeout scheduling (via on_schedule_timeout callback)
|
||
*
|
||
* ha_elect_ctx_t is still allocated here because raft_cb_send_vote_request
|
||
* reads ha->elect_ctx to retrieve my_tag/my_lsn for the capnp VoteRequest
|
||
* payload. Vote counting and quorum decisions are now owned by the raft
|
||
* library (raft_on_vote_reply fires on_become_leader/follower callbacks).
|
||
*
|
||
* Requirements: 1.3, 2.1 (shortcircuit), 2.4 (async vote send), 9.2
|
||
*/
|
||
void ha_start_election_async(ha_state_t *ha, cluster_t *c,
|
||
int n_peers, xce_candidate_t **peers,
|
||
uint64_t my_config_tag, uint64_t my_lsn,
|
||
const char *my_name)
|
||
{
|
||
(void)peers; /* peer connections driven by raft_cb_send_vote_request */
|
||
(void)my_name; /* self-vote name stored in on_persist via raft_start_election */
|
||
|
||
/* Step 0: raft is initialized once at startup by ha_init_raft(); nothing
|
||
* to do here. ha->raft_c and ha->n_peers are already set. */
|
||
|
||
/* Step 1: transition to CTRL_ELECTING before firing RPCs.
|
||
* raft_start_election fires on_become_leader / on_send_vote_request below;
|
||
* the role must already be ELECTING so ha_dispatch_gate() admits vote RPCs. */
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||
INFO("HA FSM: → CTRL_ELECTING (async, n_peers=%d)", n_peers);
|
||
|
||
/* Step 2: pre-allocate ha_elect_ctx_t so raft_cb_send_vote_request can
|
||
* read my_tag/my_lsn from it when building the capnp VoteRequest payload.
|
||
* Vote counting is now owned by the raft library; this ctx is only a
|
||
* metadata carrier for tag/lsn. For multi-node elections it is kept alive
|
||
* until all per-peer vote_ep_ctx_t endpoints finish (iomgr lifetime).
|
||
* For single-node, raft_start_election fires on_become_leader immediately
|
||
* so no ctx reads occur after return.
|
||
*
|
||
* n_peers = total cluster size (including self), matching raft convention.
|
||
* standby_converged[] is indexed by votes_idx (0 .. n_peers-2), so
|
||
* convergence loops iterate i < n_peers - 1 (skip self). */
|
||
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
|
||
|
||
/* Store total cluster size (including self) — matches raft_config_t.n_peers */
|
||
ha->n_peers = n_peers;
|
||
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
|
||
(size_t)n_other * sizeof(ctx->votes[0]));
|
||
if (!ctx) {
|
||
WARN("ha_start_election_async: calloc ctx failed");
|
||
return;
|
||
}
|
||
ctx->n_peers = n_other;
|
||
ctx->votes_granted = 1; /* self-vote placeholder (raft tracks actual count) */
|
||
ctx->n_replied = 0;
|
||
ctx->max_resp_tag = 0;
|
||
ctx->max_resp_lsn = 0;
|
||
ctx->result = -1;
|
||
ctx->done_posted = 0;
|
||
ctx->c = c;
|
||
pthread_mutex_init(&ctx->lock, NULL);
|
||
sem_init(&ctx->done, 0, 0);
|
||
|
||
/* Store candidate's own tag/lsn unconditionally in self_lsn/self_tag (task 6.3):
|
||
* ha_compute_safe_committed_lsn uses these for the majority computation,
|
||
* including the single-node case where votes[] is empty. */
|
||
ctx->self_lsn = my_lsn;
|
||
ctx->self_tag = my_config_tag;
|
||
|
||
/* Also store in votes[0] for raft_cb_send_vote_request (multi-node only) */
|
||
if (n_other > 0) {
|
||
ctx->votes[0].my_tag = my_config_tag;
|
||
ctx->votes[0].my_lsn = my_lsn;
|
||
}
|
||
|
||
/* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx */
|
||
atomic_store_explicit(&ha->elect_ctx, ctx, memory_order_release);
|
||
|
||
/* Step 4: delegate election to raft library.
|
||
*
|
||
* A node with no valid local config (my_config_tag == 0) must not start an
|
||
* election — any peer with a real config tag will beat it in the freshness
|
||
* gate and the node would waste a term increment. It stays in CTRL_ELECTING,
|
||
* accepts incoming vote requests (granting to fresher peers), and will
|
||
* become STANDBY after the winner sends it a heartbeat/ctrlhello.
|
||
* n_peers and raft state are already initialised above so RDHUP retries work.
|
||
*
|
||
* raft_start_election when allowed:
|
||
* - increments ha->raft.term and records self-vote (fires on_persist)
|
||
* - single-node: fires on_become_leader → raft_cb_become_leader
|
||
* - multi-node: fires on_send_vote_request per peer → raft_cb_send_vote_request
|
||
* then fires on_schedule_timeout → raft_cb_schedule_timeout
|
||
*
|
||
* No direct ha->term++ or ha->voted_for_* writes happen in this function
|
||
* after this call; the raft library owns those via on_persist. */
|
||
/* Req 4.3 / ha-config-bootstrap Req 2.8:
|
||
* A node with no local config (my_config_tag == 0) must not start an
|
||
* election if any peer has a valid config — it cannot win (freshness gate
|
||
* denies it) and it would waste a term increment. It stays in CTRL_ELECTING,
|
||
* accepts incoming vote requests (granting to fresher peers), and becomes
|
||
* STANDBY after the winner sends a heartbeat.
|
||
*
|
||
* Exception: if ALL peers also have config_tag=0 (cluster cold-start with no
|
||
* configured node), someone must win — allow election normally.
|
||
*
|
||
* my_config_tag comes from myconf->config_version_tag set by load_primary_config;
|
||
* n_peers and the peer list from --peers are always available via the parameters.
|
||
* We check if any peer returned a non-zero tag via the xce_probe_config_tag path
|
||
* already called in myapp_run before ha_start_election_async. */
|
||
if (my_config_tag == 0 && n_peers > 1) {
|
||
INFO("ha_start_election_async: no local config (tag=0, n_peers=%d) — "
|
||
"participating as voter only, not starting own election", n_peers);
|
||
/* Transition to CTRL_STANDBY so myapp_run schedules fetch_config_from_primary.
|
||
* The primary will push config via ctrlhello once it becomes CTRL_PRIMARY. */
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||
/* Free ctx — no vote sends will happen from this node this round. */
|
||
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
|
||
pthread_mutex_destroy(&ctx->lock);
|
||
sem_destroy(&ctx->done);
|
||
free(ctx);
|
||
return;
|
||
}
|
||
raft_start_election(&ha->raft);
|
||
|
||
/* For single-node, raft_start_election fires on_become_leader synchronously.
|
||
* on_become_leader (raft_cb_become_leader) handles role -> CTRL_PRIMARY and
|
||
* phase2. Free the ctx here since no raft callbacks
|
||
* will reference it after raft_start_election returns for single-node. */
|
||
if (n_peers == 1) {
|
||
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
|
||
pthread_mutex_destroy(&ctx->lock);
|
||
sem_destroy(&ctx->done);
|
||
free(ctx);
|
||
return;
|
||
}
|
||
|
||
INFO("ha_start_election_async: returning (n_peers=%d, raft election started)",
|
||
n_peers);
|
||
/* Returns immediately — election result delivered via raft on_become_leader/follower */
|
||
}
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* ha_rdhup_election_hook: test-only seam for ha_start_election_async in the
|
||
* RDHUP failover path. When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls
|
||
* this instead of the real ha_start_election_async(). Req 6.3, 6.5, 9.3.
|
||
*/
|
||
void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
|
||
int n_peers, xce_candidate_t **peers,
|
||
uint64_t my_config_tag, uint64_t my_lsn,
|
||
const char *my_name) = NULL;
|
||
#endif
|
||
|
||
/* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ───────────────────
|
||
*
|
||
* Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback),
|
||
* 6.4 (no heartbeat within timeout → elect), 6.8 (role flip)
|
||
* 10.5 (INFO logs on FSM transitions)
|
||
*/
|
||
|
||
/*
|
||
* ha_begin_failover: the single failover entry point (ha-prevote task 4.2).
|
||
*
|
||
* Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and
|
||
* the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route
|
||
* here AFTER their connection-teardown bookkeeping and the config-less
|
||
* voter-only guard. It starts a side-effect-free PreVote round and returns;
|
||
* it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those
|
||
* destructive steps are deferred to PreVote success (ha_enter_real_election,
|
||
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
|
||
* and term untouched (Req 1.1, 4.3, 4.4).
|
||
*/
|
||
/* ha_election_stagger_ms: see ha.h. Same formula as the startup stagger
|
||
* (myapp.c) — priority baseline plus fresh random jitter, re-randomized each
|
||
* call so repeated split-vote collisions converge (Raft §5.2; bug-606). */
|
||
uint32_t ha_election_stagger_ms(ha_state_t *ha)
|
||
{
|
||
if (!ha) return 0;
|
||
if (ha->jitter_seed == 0)
|
||
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
|
||
|
||
uint32_t step = ha->config.election_priority_step_ms;
|
||
uint32_t jmax = ha->config.election_jitter_max_ms;
|
||
uint32_t delay = (uint32_t)ha->my_priority * step;
|
||
if (jmax > 0)
|
||
delay += (uint32_t)(xorshift64_next(&ha->jitter_seed)
|
||
% ((uint64_t)jmax + 1));
|
||
return delay;
|
||
}
|
||
|
||
/* failover_stagger_ctx_t: carries ha/c into the delaymgr-scheduled PreVote start.
|
||
* Freed by ha_failover_stagger_fn (one-shot). */
|
||
typedef struct {
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
} failover_stagger_ctx_t;
|
||
|
||
/* ha_failover_stagger_fn: delaymgr body that starts the PreVote round after the
|
||
* per-node stagger (bug-606). Guard: if a primary connected during the stagger
|
||
* (we are a STANDBY with a fresh heartbeat), the cluster already has a leader —
|
||
* skip the PreVote entirely instead of needlessly probing a live primary. */
|
||
static int ha_failover_stagger_fn(void *arg)
|
||
{
|
||
failover_stagger_ctx_t *ctx = (failover_stagger_ctx_t *)arg;
|
||
ha_state_t *ha = ctx->ha;
|
||
cluster_t *c = ctx->c;
|
||
|
||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
|
||
INFO("HA: failover stagger elapsed but a primary is alive — skipping "
|
||
"PreVote (bug-606)");
|
||
free(ctx);
|
||
return 0;
|
||
}
|
||
|
||
ha_start_prevote_async(ha, c);
|
||
free(ctx);
|
||
return 0;
|
||
}
|
||
|
||
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
if (!ha) return;
|
||
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
|
||
"PreVote wins) — staying CTRL_STANDBY");
|
||
|
||
/* Stagger the PreVote start per node so two survivors that lost the primary
|
||
* simultaneously do not campaign in lock-step and split the vote forever
|
||
* (Raft §5.2; bug-606). Production only: in unit-test mode (c == NULL) or
|
||
* with no running delaymgr, start synchronously so the existing failover /
|
||
* PreVote tests observe the unchanged immediate behaviour. */
|
||
uint32_t stagger = ha_election_stagger_ms(ha);
|
||
if (c && c->delaymgr.running && stagger > 0) {
|
||
failover_stagger_ctx_t *sctx = calloc(1, sizeof(*sctx));
|
||
if (sctx) {
|
||
sctx->ha = ha;
|
||
sctx->c = c;
|
||
INFO("HA: staggering PreVote start by %ums (priority=%d) to avoid "
|
||
"split vote (bug-606)", stagger, ha->my_priority);
|
||
int rc = delay_task_ms(&c->delaymgr, (int)stagger, 1,
|
||
ha_failover_stagger_fn, sctx);
|
||
if (rc >= 0)
|
||
return;
|
||
/* Scheduling failed — fall through to a synchronous start. */
|
||
WARN("HA: ha_begin_failover: stagger schedule failed (%d) — "
|
||
"starting PreVote immediately", rc);
|
||
free(sctx);
|
||
}
|
||
}
|
||
|
||
ha_start_prevote_async(ha, c);
|
||
}
|
||
|
||
void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
if (!ha) return;
|
||
|
||
/* Cancel the liveness poll task first (Req 5.4): prevents a concurrent
|
||
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
|
||
ha_disarm_heartbeat_timeout_vtask(ha);
|
||
|
||
/* Close and null the primary control fd. Connection-teardown bookkeeping
|
||
* stays at the trigger regardless of the PreVote outcome (ha-prevote task
|
||
* 4.2). */
|
||
if (ha->primary_ctrl_fd >= 0) {
|
||
close(ha->primary_ctrl_fd);
|
||
ha->primary_ctrl_fd = -1;
|
||
}
|
||
|
||
/* ha-prevote task 4.2: the failover trigger NO LONGER transitions to
|
||
* CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and
|
||
* STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is
|
||
* deferred to PreVote success (ha_enter_real_election). The node only ever
|
||
* inflates its term / unseats a healthy primary if a PreVote majority is
|
||
* first obtained (Req 1.1, 4.3, 4.4). */
|
||
INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying "
|
||
"CTRL_STANDBY until PreVote wins) (Req 1.1)");
|
||
|
||
if (!c) {
|
||
/* Unit-test mode: no persist / no live transport. Still begin the
|
||
* round so the deferred-destructive behaviour is observable (the round
|
||
* state lives in raft_state_t; ha_enter_real_election with c==NULL only
|
||
* flips the role on a WIN). */
|
||
ha_begin_failover(ha, c);
|
||
return;
|
||
}
|
||
|
||
/* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening
|
||
* Req 10.5): a config-less node (config_version_tag == 0) must NEVER start
|
||
* an RDHUP re-election while a configured peer exists. Two such nodes would
|
||
* grant each other (the freshness gate only denies a LOWER tag; tag0==tag0
|
||
* with equal lsn is granted), forming a majority and electing a config-less
|
||
* primary over the node that actually holds the config — which then pushes
|
||
* an empty config and breaks replication.
|
||
*
|
||
* This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a
|
||
* config-less node can never win a PreVote (the freshness gate denies it),
|
||
* so there is no point starting a round — and starting one would needlessly
|
||
* probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote
|
||
* still GRANTS incoming vote requests to the fresher configured peer
|
||
* regardless of role, and this node transitions/fetches config when the
|
||
* configured winner connects and pushes config via ctrlhello.
|
||
*
|
||
* Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false,
|
||
* so a lone node still self-elects (its PreVote trivially wins). */
|
||
uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0;
|
||
int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
|
||
if (my_config_tag == 0 && n_peers_guard > 1) {
|
||
INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — "
|
||
"voter only, not starting PreVote/RDHUP re-election; awaiting "
|
||
"configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
|
||
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
|
||
/* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async
|
||
* (it snapshots self freshness into ha->elect_ctx so the PreVote request
|
||
* payload — and, on a WIN, the deferred real election — advertise the
|
||
* effective freshness floor, NOT the stale persist.committed_lsn: see the
|
||
* bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */
|
||
ha_begin_failover(ha, c);
|
||
}
|
||
|
||
/* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */
|
||
|
||
/*
|
||
* liveness_poll_ctx_t: context for the delaymgr-based standby liveness task.
|
||
*
|
||
* Requirements: 1.2, 1.4, 1.5, 1.6, 5.4
|
||
*
|
||
* Replaces hb_vtask_t + hb_timeout_thread. No pthread, no semaphore.
|
||
* The poll fn reads ha->last_activity (written by ha_on_heartbeat_received)
|
||
* and fires ha_on_primary_ctrl_fd_rdhup if elapsed >= heartbeat_timeout_ms.
|
||
*
|
||
* cancelled is _Atomic so that writes from ha_disarm_heartbeat_timeout_vtask
|
||
* (main/vtask context) are visible to liveness_poll_fn (delaymgr worker thread)
|
||
* without a data race.
|
||
*/
|
||
typedef struct {
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
_Atomic int cancelled;
|
||
} liveness_poll_ctx_t;
|
||
|
||
/* Static pointer to the active context. _Atomic so concurrent arm/disarm
|
||
* vs poll-fn execution is well-defined. Access pattern:
|
||
* arm — allocates and stores (single writer, vtask context)
|
||
* disarm — sets cancelled, frees (single writer, vtask context)
|
||
* poll_fn — reads via passed arg only; never dereferences s_liveness_ctx
|
||
*/
|
||
static liveness_poll_ctx_t * _Atomic s_liveness_ctx = NULL;
|
||
|
||
/*
|
||
* liveness_poll_fn: delaymgr task replacing hb_timeout_thread.
|
||
*
|
||
* Checks whether the primary has been silent for >= heartbeat_timeout_ms.
|
||
* If so, calls ha_on_primary_ctrl_fd_rdhup and does not re-arm.
|
||
* If not, re-arms itself for the next poll interval.
|
||
* If cancelled, returns immediately without any action.
|
||
*
|
||
* Also skips if role != CTRL_STANDBY (Req 5.4): prevents duplicate failover
|
||
* if EPOLLRDHUP already transitioned the role to CTRL_ELECTING.
|
||
*/
|
||
static int liveness_poll_fn(void *arg)
|
||
{
|
||
liveness_poll_ctx_t *ctx = (liveness_poll_ctx_t *)arg;
|
||
|
||
/* Check cancelled flag first — set by ha_arm_heartbeat_timeout_vtask when
|
||
* re-arming replaces this ctx. When cancelled, free the ctx here (the
|
||
* re-armer cannot safely free it because this task may already be running
|
||
* on a delaymgr thread). Do NOT re-arm.
|
||
*
|
||
* In XCE_TESTING builds callers pass stack-allocated ctxs, so we must
|
||
* not free in that path. Production builds always heap-allocate via
|
||
* ha_arm_heartbeat_timeout_vtask. */
|
||
if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) {
|
||
#ifndef XCE_TESTING
|
||
free(ctx);
|
||
#endif
|
||
return 0;
|
||
}
|
||
|
||
ha_state_t *ha = ctx->ha;
|
||
cluster_t *c = ctx->c;
|
||
|
||
/* Skip if not standby — EPOLLRDHUP may have already changed the role */
|
||
if (atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY)
|
||
return 0;
|
||
|
||
/* Compute elapsed time since last primary message */
|
||
struct timespec now_ts;
|
||
clock_gettime(CLOCK_MONOTONIC, &now_ts);
|
||
time_t now = now_ts.tv_sec;
|
||
time_t last = ha->last_activity; /* time_t write is atomic on LP64 */
|
||
time_t elapsed = (last > 0) ? (now - last) : 0;
|
||
uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000;
|
||
|
||
if (last > 0 && (uint32_t)elapsed >= timeout_s) {
|
||
/* Silence exceeded timeout — trigger failover */
|
||
WARN("HA: liveness poll: no primary activity for %lus (timeout=%us) → failover",
|
||
(unsigned long)elapsed, timeout_s);
|
||
ha_on_primary_ctrl_fd_rdhup(ha, c);
|
||
/* Do NOT re-arm — failover handles re-election */
|
||
return 0;
|
||
}
|
||
|
||
/* Still fresh — re-arm for next poll interval */
|
||
if (c && c->delaymgr.running) {
|
||
uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2;
|
||
if (poll_ms < 100) poll_ms = 100;
|
||
delay_task_ms(&c->delaymgr, (int)poll_ms, 1,
|
||
(delay_task_entry_fn_t)liveness_poll_fn, ctx);
|
||
}
|
||
return 0;
|
||
}
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* liveness_poll_fn_test_entry: test-only entry point that calls liveness_poll_fn
|
||
* directly with a pre-constructed ctx, bypassing delaymgr.
|
||
* Only compiled when XCE_TESTING is defined.
|
||
*/
|
||
void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx)
|
||
{
|
||
liveness_poll_fn(ctx);
|
||
}
|
||
|
||
/*
|
||
* primary_heartbeat_fn_for_test: call primary_heartbeat_fn directly with c.
|
||
* Bypasses delaymgr scheduling so tests can drive the idle-NOP logic without
|
||
* a live timer. Only compiled when XCE_TESTING is defined. (Task 2.2)
|
||
*/
|
||
int primary_heartbeat_fn_for_test(cluster_t *c)
|
||
{
|
||
return primary_heartbeat_fn(c);
|
||
}
|
||
#endif
|
||
|
||
void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
if (!ha) return;
|
||
|
||
/* Disarm any existing poll ctx by setting cancelled=1.
|
||
* Do NOT free it here — the delaymgr task may already be running on
|
||
* another thread and will free the ctx itself when it sees cancelled=1.
|
||
* Clearing s_liveness_ctx prevents a second re-arm attempt from operating
|
||
* on the stale pointer. */
|
||
liveness_poll_ctx_t *old = atomic_load_explicit(&s_liveness_ctx,
|
||
memory_order_acquire);
|
||
if (old) {
|
||
atomic_store_explicit(&old->cancelled, 1, memory_order_release);
|
||
atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release);
|
||
}
|
||
|
||
liveness_poll_ctx_t *ctx = calloc(1, sizeof(*ctx));
|
||
if (!ctx) return;
|
||
|
||
ctx->ha = ha;
|
||
ctx->c = c;
|
||
atomic_init(&ctx->cancelled, 0);
|
||
|
||
/* Initialise last_activity to now so the first poll computes elapsed from
|
||
* the arm time rather than from zero. */
|
||
struct timespec ts;
|
||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||
ha->last_activity = ts.tv_sec;
|
||
|
||
atomic_store_explicit(&s_liveness_ctx, ctx, memory_order_release);
|
||
|
||
uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2;
|
||
if (poll_ms < 100) poll_ms = 100;
|
||
|
||
INFO("HA: standby liveness poll armed (timeout=%ums, poll=%ums)",
|
||
ha->config.heartbeat_timeout_ms, poll_ms);
|
||
|
||
if (c && c->delaymgr.running) {
|
||
int rc = delay_task_ms(&c->delaymgr, (int)poll_ms, 1,
|
||
(delay_task_entry_fn_t)liveness_poll_fn, ctx);
|
||
if (rc < 0)
|
||
WARN("HA: ha_arm_heartbeat_timeout_vtask: delay_task_ms failed: %d", rc);
|
||
}
|
||
}
|
||
|
||
/* ha_discover_primary: if CTRL_ELECTING, step down to STANDBY on receiving
|
||
* a valid primary message. A campaigning node that sees a live primary must
|
||
* defer — continuing the election risks split-brain (bug-353). */
|
||
void ha_discover_primary(ha_state_t *ha, cluster_t *c)
|
||
{
|
||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
if (role != CTRL_ELECTING)
|
||
return;
|
||
|
||
uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
INFO("HA: discovered live primary while ELECTING (term=%" PRIu64 ") — stepping down",
|
||
term);
|
||
raft_cb_become_follower(ha, term, -1);
|
||
}
|
||
|
||
void ha_on_heartbeat_received(ha_state_t *ha)
|
||
{
|
||
if (!ha) return;
|
||
struct timespec ts;
|
||
clock_gettime(CLOCK_MONOTONIC, &ts);
|
||
ha->last_activity = ts.tv_sec;
|
||
tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec);
|
||
|
||
/* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader
|
||
* is alive — a PreVote round started by a transient blip must be ABANDONED.
|
||
* Cancel the in-progress round (clear the raft round state + the controller
|
||
* round-timeout handle) and re-attach as a standby. No persistent term /
|
||
* voted_for / role change happens here: the node never left CTRL_STANDBY
|
||
* during the round, so re-attaching is a no-op on role/term — exactly the
|
||
* non-disruptive rejoin the spec requires. */
|
||
if (ha->raft.pre_vote_in_progress) {
|
||
ha->raft.pre_vote_in_progress = 0;
|
||
ha->raft.pre_votes_granted = 0;
|
||
ha_prevote_cancel_round_timeout();
|
||
INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round "
|
||
"and re-attaching as CTRL_STANDBY (leader alive, no term change) "
|
||
"(Req 4.2)");
|
||
}
|
||
}
|
||
|
||
/*
|
||
* ha_observe_heartbeat_committed: reconcile-free monotonic learn of the
|
||
* primary's committed position from a heartbeat (ha-heartbeat-committed-
|
||
* propagation, Req 2.1, 2.2, 2.3, 3.1, 3.3).
|
||
*
|
||
* Single-receiver-thread path (controller request handling), so a relaxed
|
||
* compare-then-store is sufficient. The config tag is written BEFORE the LSN
|
||
* (both release-ordered) so a concurrent acquire-load reader that observes the
|
||
* advanced LSN never pairs it with a stale tag.
|
||
*
|
||
* Advisory only: NO ha_rejoin_reconcile / NO WAL truncation / NO change to any
|
||
* produced/persisted committed, role, term, or applied state. This is the
|
||
* explicit distinction from cluster_apply_ctrlhello_reply (which truncates).
|
||
*/
|
||
void ha_observe_heartbeat_committed(cluster_t *c, uint64_t committed_lsn,
|
||
uint64_t committed_config_tag)
|
||
{
|
||
if (!c) return;
|
||
ha_state_t *ha = &c->ha;
|
||
|
||
uint64_t cur = atomic_load_explicit(&ha->last_known_committed_lsn,
|
||
memory_order_acquire);
|
||
/* Monotonic no-op: a zero (absent/liveness-only) or non-advancing LSN must
|
||
* leave the learned position — and its config tag — untouched (Req 2.2). */
|
||
if (committed_lsn == 0 || committed_lsn <= cur)
|
||
return;
|
||
|
||
/* Tag moves with the LSN, never independently regressed (Req 2.3): publish
|
||
* the tag first so the LSN's release store carries the matching tag. */
|
||
atomic_store_explicit(&ha->last_known_committed_config_tag,
|
||
committed_config_tag, memory_order_release);
|
||
atomic_store_explicit(&ha->last_known_committed_lsn, committed_lsn,
|
||
memory_order_release);
|
||
/* Reconcile-free: nothing else is touched (Req 3.1, 3.3). */
|
||
tlog(TLOG_DEBUG3,
|
||
"HA: learned committed_lsn=%" PRIu64 " config_tag=%" PRIu64
|
||
" from heartbeat (was %" PRIu64 ")",
|
||
committed_lsn, committed_config_tag, cur);
|
||
}
|
||
|
||
/*
|
||
* ha_consume_heartbeat_committed: standby OP_nop consume branch — the
|
||
* non-primary gate (Req 3.4) + the stale-term guard (Req 3.2), then the
|
||
* monotonic observe (Req 1.2). Reconcile-free; updates nothing on a NULL body,
|
||
* a failed gate, or a zero committed value (Req 4.2). Pure additive helper —
|
||
* the caller runs the unchanged liveness/reply path first (Req 4.1).
|
||
*/
|
||
void ha_consume_heartbeat_committed(cluster_t *c, const heartbeat_msg_t *hb)
|
||
{
|
||
if (!c || !hb)
|
||
return;
|
||
|
||
/* (3.4) A primary uses its own authoritative committed — never an inbound
|
||
* heartbeat value. Only non-primary nodes consume. */
|
||
ctrl_role_t role =
|
||
(ctrl_role_t)atomic_load_explicit(&c->ha.role, memory_order_acquire);
|
||
if (role == CTRL_PRIMARY)
|
||
return;
|
||
|
||
/* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat through the raft
|
||
* library as a 0-entry AppendEntries so the LIBRARY performs UpdateTerm — a
|
||
* heartbeat at a strictly-higher term makes this node adopt the term and step
|
||
* down. This REPLACES the prior hand-rolled term-guard that only DROPPED a
|
||
* stale heartbeat (hb->term < myterm) and never adopted a higher term.
|
||
*
|
||
* The heartbeat carries no entries and no prev tuple, so we present the
|
||
* node's own log tip as (prevLogIndex,prevLogTerm) — logOk holds (already-
|
||
* have at the tip) for an equal/higher-term heartbeat, and UpdateTerm runs
|
||
* first regardless. raft_init must have wired the raft state (production
|
||
* boot); guard for the pre-init placeholder where raft_c is unset. */
|
||
if (c->ha.raft_c == c) {
|
||
uint64_t my_log_len = c->ha.raft.log_len;
|
||
uint64_t my_tip_term = (my_log_len > 0)
|
||
? persist_entry_term(&c->persist, my_log_len) : 0;
|
||
|
||
wal_catchup_entry_t hbae;
|
||
memset(&hbae, 0, sizeof(hbae));
|
||
hbae.n_entries = 0; /* heartbeat: zero entries */
|
||
hbae.term = hb->term; /* leader's live term (UpdateTerm) */
|
||
hbae.prev_log_index = my_log_len; /* already-have at our tip */
|
||
hbae.prev_log_term = my_tip_term;
|
||
hbae.leader_commit = hb->committed_lsn;
|
||
(void)ha_consume_append_entries(c, &hbae, /*leader_idx*/-1);
|
||
}
|
||
|
||
/* (3.2) Stale-term guard for the committed LEARN below: after UpdateTerm a
|
||
* heartbeat from a primary whose term is behind ours is stale/partitioned —
|
||
* do not let its committed inflate the vote-freshness floor. Re-read our term
|
||
* (it may have advanced via the library UpdateTerm above). */
|
||
uint64_t myterm = atomic_load_explicit(&c->ha.term, memory_order_acquire);
|
||
if (hb->term < myterm)
|
||
return;
|
||
|
||
/* (1.2) Monotonic, reconcile-free learn of the committed position. Zero/
|
||
* non-advancing LSN no-ops inside the observe (Req 4.2). */
|
||
ha_observe_heartbeat_committed(c, hb->committed_lsn,
|
||
hb->committed_config_tag);
|
||
}
|
||
|
||
void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)
|
||
{
|
||
(void)ha;
|
||
liveness_poll_ctx_t *ctx = atomic_load_explicit(&s_liveness_ctx,
|
||
memory_order_acquire);
|
||
if (!ctx) return;
|
||
atomic_store_explicit(&ctx->cancelled, 1, memory_order_release);
|
||
atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release);
|
||
}
|
||
|
||
/* ── Primary heartbeat sender + late-joiner outbound connect ─────────────── */
|
||
|
||
/*
|
||
* standby_ep_ctx_t: per-peer context for outbound standby connections opened
|
||
* by the primary during the heartbeat task.
|
||
*
|
||
* Layout deliberately matches vote_ep_ctx_t for the first 4 fields so that
|
||
* peer_ctrl_msg_handler (which casts ep->data to vote_ep_ctx_t*) can safely
|
||
* read ha, c, and peer_idx. ctx is always NULL here (no active election).
|
||
*/
|
||
typedef struct {
|
||
ha_state_t *ha;
|
||
cluster_t *c;
|
||
ha_elect_ctx_t *ctx; /* always NULL — not a vote endpoint */
|
||
int peer_idx; /* index into c->myconf->peers_cfg.candidates[] */
|
||
/* Task 1.1 (ha-heartbeat-liveness): last time a push/nop was sent to this
|
||
* standby (CLOCK_MONOTONIC seconds). Initialized to 0 at allocation;
|
||
* written by on_standby_connected and primary_heartbeat_fn (task 2.2). */
|
||
time_t last_push_time;
|
||
} standby_ep_ctx_t;
|
||
|
||
#ifdef XCE_TESTING
|
||
/*
|
||
* standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data,
|
||
* and return ep. last_push_time is initialised to 0 (calloc).
|
||
* Caller must free(ep->data) after the test. (Task 2.2)
|
||
*/
|
||
endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep,
|
||
ha_state_t *ha,
|
||
cluster_t *c,
|
||
int peer_idx)
|
||
{
|
||
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
|
||
if (!sctx) return NULL;
|
||
sctx->ha = ha;
|
||
sctx->c = c;
|
||
sctx->ctx = NULL;
|
||
sctx->peer_idx = peer_idx;
|
||
/* last_push_time = 0 from calloc — tests set it explicitly */
|
||
ep->data = sctx;
|
||
return ep;
|
||
}
|
||
|
||
/*
|
||
* on_standby_connected_for_test: call on_standby_connected directly.
|
||
* ep->data must be a valid standby_ep_ctx_t (use standby_ep_ctx_build_for_test).
|
||
* xce_request will fail on a null fd — tests only check last_push_time. (Task 2.2)
|
||
*/
|
||
int on_standby_connected_for_test(endpoint_t *ep)
|
||
{
|
||
on_standby_connected(ep);
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* standby_ep_err_handler_primary_for_test: expose the outbound-standby error
|
||
* handler so a test can tear down a heap endpoint the production way (remove
|
||
* from standby_endpoints + replicator, free ep->data) before reclaiming the
|
||
* endpoint itself. (raft-drive-async Task 2.2)
|
||
*/
|
||
int standby_ep_err_handler_primary_for_test(endpoint_t *ep)
|
||
{
|
||
return standby_ep_err_handler_primary(ep);
|
||
}
|
||
#endif /* XCE_TESTING */
|
||
|
||
/*
|
||
* on_standby_connected: iomgr on_connected callback for outbound standby
|
||
* endpoints created by the heartbeat task.
|
||
*
|
||
* Sends OPCode_hello (CtrlHelloRequest) so the receiving standby runs
|
||
* handle_ctrlhello and triggers WAL catch-up + config push. Registers the
|
||
* endpoint in ha->standby_endpoints so future heartbeats reach it.
|
||
*/
|
||
static void on_standby_connected(endpoint_t *ep)
|
||
{
|
||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
|
||
ha_state_t *ha = sctx->ha;
|
||
cluster_t *c = sctx->c;
|
||
|
||
/*
|
||
* Register in standby_endpoints, resolving the inbound-vs-outbound
|
||
* connection-ownership race (raft-drive-async Task 2.2, Req 2.6/4.4;
|
||
* ha-async-io.md connection ownership).
|
||
*
|
||
* The hazard: BOTH raft_cb_become_leader (promotes surviving vote eps) and
|
||
* this path (late-joiner / reconnect) zarray_store into standby_endpoints
|
||
* keyed only by ep-POINTER identity. So while an outbound reconnect to
|
||
* peer X is in flight, a DIFFERENT ep for the same peer X (a just-promoted
|
||
* vote-ep, or a stale prior channel whose RDHUP has not yet been processed)
|
||
* may already be registered. Leaving both would create two live outbound
|
||
* state-push channels to one peer — heartbeats/WAL fan out to a dead/stale
|
||
* duplicate (lost/reordered messages, Req 2.6) and the older endpoint is
|
||
* orphaned in the replicator.
|
||
*
|
||
* Resolution (deterministic, per-peer): the freshly-connected channel wins.
|
||
* Under ha->lock collect any DIFFERENT ep with the same peer_idx and drop
|
||
* it from the array; then register `ep`. The actual teardown of the stale
|
||
* ep runs AFTER releasing ha->lock because standby_ep_err_handler_primary
|
||
* re-takes ha->lock (and touches the replicator) — avoiding self-deadlock
|
||
* and holding the lock across no I/O (R4.4: iomgr/registration state mutated
|
||
* only via the lock-guarded interface, never racing the event loop).
|
||
*/
|
||
endpoint_t *stale_same_peer = NULL;
|
||
if (ha->standby_endpoints) {
|
||
pthread_mutex_lock(&ha->lock);
|
||
int already = 0;
|
||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||
endpoint_t *cur = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
|
||
if (cur == ep) { already = 1; continue; }
|
||
/* Different ep, same peer → stale duplicate to evict. */
|
||
standby_ep_ctx_t *cur_sctx =
|
||
cur ? (standby_ep_ctx_t *)cur->data : NULL;
|
||
if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx) {
|
||
stale_same_peer = cur;
|
||
zarray_del(ha->standby_endpoints, i);
|
||
/* count shrank by one; the surviving scan continues but we
|
||
* only ever expect a single same-peer duplicate. */
|
||
break;
|
||
}
|
||
}
|
||
if (!already)
|
||
zarray_store(ha->standby_endpoints, ep);
|
||
pthread_mutex_unlock(&ha->lock);
|
||
}
|
||
|
||
/* Tear down the evicted stale channel OUTSIDE ha->lock. Mirror the library
|
||
* disconnect path (handle_error → iomgr_unmonitor → err_handler) so we
|
||
* never free an endpoint whose fd is still in the epoll set (R4.4: mutate
|
||
* iomgr/registration state ONLY via the lock-guarded interface, never
|
||
* racing the event loop). err-handler removes it from standby_endpoints +
|
||
* the replicator and frees ep->data; we then reclaim the endpoint itself.
|
||
* Never self-terminate — this is routine ownership reconciliation. */
|
||
if (stale_same_peer) {
|
||
WARN("HA: peer[%d] reconnected — evicting stale duplicate outbound "
|
||
"channel to keep exactly one live connection (Req 2.6)",
|
||
sctx->peer_idx);
|
||
if (stale_same_peer->iomgr)
|
||
iomgr_unmonitor((iomgr_t *)stale_same_peer->iomgr, stale_same_peer);
|
||
(void)standby_ep_err_handler_primary(stale_same_peer);
|
||
/* err-handler set ep->data = NULL and freed sctx; endpoint_fini then
|
||
* reclaims outq/ssl/packets, and we free the calloc'd endpoint itself
|
||
* (endpoint_get uses calloc; there is no endpoint_put). */
|
||
endpoint_fini(stale_same_peer);
|
||
free(stale_same_peer);
|
||
#ifdef XCE_TESTING
|
||
atomic_fetch_add(&ha_standby_evicted_count_for_test, 1);
|
||
#endif
|
||
}
|
||
|
||
/* Reset failure counter so the next disconnect restarts logging from attempt 1. */
|
||
ha->peer_connect_fail_count[sctx->peer_idx] = 0;
|
||
|
||
/* Record last_push_time before any send so the first heartbeat tick after
|
||
* connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */
|
||
{
|
||
struct timespec _ts;
|
||
clock_gettime(CLOCK_MONOTONIC, &_ts);
|
||
sctx->last_push_time = _ts.tv_sec;
|
||
}
|
||
|
||
/* Send ctrlhello — triggers handle_ctrlhello on the standby side */
|
||
uint64_t lsn = atomic_load_explicit(&c->persist.committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t tag = c->myconf ? c->myconf->config_version_tag : 0;
|
||
char *name = c->myconf ? c->myconf->name : "";
|
||
|
||
ctrl_hello_req_t hello = { .name = name, .key = NULL,
|
||
.applied_lsn = lsn, .config_tag = tag };
|
||
xce_request_t req;
|
||
memset(&req, 0, sizeof(req));
|
||
req.opcode = OPCode_hello;
|
||
req.body_which = Request_body_ctrlhellomsg;
|
||
req.body.ctrlhellomsg = &hello;
|
||
|
||
int rc = xce_request(ep, &req);
|
||
if (rc < 0)
|
||
WARN("HA: ctrlhello to peer[%d] failed: %d", sctx->peer_idx, rc);
|
||
else
|
||
INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent",
|
||
sctx->peer_idx);
|
||
|
||
/* Task 2.1: register late-joiner with WAL replicator.
|
||
* Req 1.1: primary must register standby with repl_add_standby when
|
||
* establishing the standby control channel (late-joiner outbound connect).
|
||
* Host/port from candidates[peer_idx] — the peer we connected to. */
|
||
if (c && c->myconf &&
|
||
sctx->peer_idx >= 0 &&
|
||
sctx->peer_idx < c->myconf->peers_cfg.candidates_count) {
|
||
const xce_candidate_t *peer =
|
||
&c->myconf->peers_cfg.candidates[sctx->peer_idx];
|
||
int rc = repl_add_standby(&c->persist.repl, ep,
|
||
peer->hostname, (uint32_t)peer->port);
|
||
if (rc >= 0)
|
||
INFO("HA: repl registered late-joiner standby peer[%d] (slot %d, %s:%u)",
|
||
sctx->peer_idx, rc, peer->hostname, (uint32_t)peer->port);
|
||
else
|
||
WARN("HA: repl_add_standby late-joiner peer[%d] failed: %d",
|
||
sctx->peer_idx, rc);
|
||
}
|
||
|
||
/* Quorum-transition observability (Req 8.2): a late-joiner standby connected —
|
||
* the write-quorum count changed. Record the restore edge once (silent if
|
||
* quorum was already held). */
|
||
ha_log_quorum_transition(ha);
|
||
|
||
/* Set state-aware dispatcher for subsequent messages on this endpoint */
|
||
ep->msg_handler = peer_ctrl_msg_handler;
|
||
}
|
||
|
||
/*
|
||
* standby_ep_err_handler_primary: error handler for outbound standby
|
||
* endpoints created by the heartbeat task. Removes from standby_endpoints
|
||
* and frees the context — the next heartbeat tick will attempt reconnect.
|
||
*/
|
||
static int standby_ep_err_handler_primary(void *ep_raw)
|
||
{
|
||
endpoint_t *ep = (endpoint_t *)ep_raw;
|
||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
|
||
if (!sctx) return 0;
|
||
|
||
ha_state_t *ha = sctx->ha;
|
||
cluster_t *c = sctx->c;
|
||
|
||
if (ha->standby_endpoints) {
|
||
pthread_mutex_lock(&ha->lock);
|
||
for (int i = 0; i < ha->standby_endpoints->count; i++) {
|
||
if (zarray_get(ha->standby_endpoints, i) == ep) {
|
||
zarray_del(ha->standby_endpoints, i);
|
||
break;
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&ha->lock);
|
||
}
|
||
|
||
/* Task 2.2: deregister late-joiner standby from WAL replicator before
|
||
* freeing the context. Req 1.2: RDHUP or connection error triggers
|
||
* repl_remove_standby so the replicator stops sending WAL entries to it. */
|
||
if (c)
|
||
repl_remove_standby(&c->persist.repl, ep);
|
||
|
||
free(sctx);
|
||
ep->data = NULL;
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* reconnect_ctx_t: owns the prepared endpoint across the in-flight reconnect
|
||
* window (raft-drive-async Task 2.1). Built on the delaymgr tick by the
|
||
* non-blocking prep (ha_connect_prep_peer), consumed once on a slow-lane
|
||
* worker by reconnect_lane_fn, then freed. The ctx keeps `ep`/`sctx` valid
|
||
* for the whole window so the lane worker never touches freed memory while the
|
||
* delaymgr thread races ahead to the next tick (Req 2.3 lifetime).
|
||
*/
|
||
typedef struct {
|
||
cluster_t *c;
|
||
int peer_idx; /* ABSOLUTE candidate index */
|
||
endpoint_t *ep; /* prepared, not yet connected */
|
||
standby_ep_ctx_t *sctx; /* == ep->data */
|
||
int should_log;
|
||
} reconnect_ctx_t;
|
||
|
||
/*
|
||
* ha_connect_prep_peer: the NON-BLOCKING half of the outbound peer connect.
|
||
* Allocates the endpoint context, initialises the endpoint, sets the SSL
|
||
* client state and the on-connected / err handlers — all of which are
|
||
* non-blocking and safe to run on the delaymgr tick. Does NOT perform the
|
||
* blocking connect (that is ha_connect_finish_peer, run on the slow lane).
|
||
*
|
||
* Returns a heap reconnect_ctx_t* (caller owns) on success, or NULL on an
|
||
* allocation failure (nothing leaked).
|
||
*/
|
||
static reconnect_ctx_t *ha_connect_prep_peer(cluster_t *c, int peer_idx)
|
||
{
|
||
ha_state_t *ha = &c->ha;
|
||
|
||
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
|
||
if (!sctx) return NULL;
|
||
sctx->ha = ha;
|
||
sctx->c = c;
|
||
sctx->peer_idx = peer_idx;
|
||
|
||
endpoint_t *ep = endpoint_get();
|
||
if (!ep) { free(sctx); return NULL; }
|
||
|
||
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
|
||
endpoint_set_peer(ep, c->myhost);
|
||
ep->data = sctx;
|
||
ep->iomgr = &c->iomgr;
|
||
|
||
if (ha->client_ssl_ctx) {
|
||
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
|
||
ep->ssl = SSL_new(client_ctx);
|
||
if (ep->ssl)
|
||
SSL_set_connect_state(ep->ssl);
|
||
}
|
||
|
||
endpoint_set_on_connected(ep, on_standby_connected);
|
||
endpoint_set_err_handler(ep, standby_ep_err_handler_primary);
|
||
|
||
int n = ++ha->peer_connect_fail_count[peer_idx];
|
||
int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0);
|
||
|
||
reconnect_ctx_t *rc_ctx = calloc(1, sizeof(*rc_ctx));
|
||
if (!rc_ctx) {
|
||
ep->data = NULL;
|
||
endpoint_fini(ep);
|
||
free(ep); /* endpoint_get() calloc'd it; fini frees inner only */
|
||
free(sctx);
|
||
return NULL;
|
||
}
|
||
rc_ctx->c = c;
|
||
rc_ctx->peer_idx = peer_idx;
|
||
rc_ctx->ep = ep;
|
||
rc_ctx->sctx = sctx;
|
||
rc_ctx->should_log = should_log;
|
||
return rc_ctx;
|
||
}
|
||
|
||
/*
|
||
* ha_connect_finish_peer: the BLOCKING half of the outbound peer connect —
|
||
* the bounded connect + lock-guarded iomgr registration. MUST run on a
|
||
* slow-lane worker, NEVER on the delaymgr tick (Req 2.1, 2.2; ha-async-io.md
|
||
* thread-role contract).
|
||
*
|
||
* On success the endpoint is connecting and registered with iomgr; on_connected
|
||
* (on_standby_connected) fires later from the iomgr loop. On failure the
|
||
* endpoint is torn down here and the controller simply retries on the next
|
||
* heartbeat tick (controllers never self-terminate).
|
||
*
|
||
* Returns 0 on a successful connect+register, <0 otherwise.
|
||
*
|
||
* NOTE (Task 2.2 seam): this implements the happy-path connect+register +
|
||
* teardown-on-failure. The inbound-vs-outbound connection-ownership race
|
||
* resolution, the no-lost/no-reordered-message guarantee on success, and the
|
||
* richer failure/retry hardening are owned by Task 2.2 — they layer on top of
|
||
* this happy path without changing this function's contract (return 0/<0,
|
||
* caller clears the per-peer in-flight flag).
|
||
*/
|
||
static int ha_connect_finish_peer(reconnect_ctx_t *rc_ctx)
|
||
{
|
||
cluster_t *c = rc_ctx->c;
|
||
endpoint_t *ep = rc_ctx->ep;
|
||
standby_ep_ctx_t *sctx = rc_ctx->sctx;
|
||
int peer_idx = rc_ctx->peer_idx;
|
||
const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[peer_idx];
|
||
|
||
#ifdef XCE_TESTING
|
||
if (ha_reconnect_connect_hook) {
|
||
/* Deterministic test path: the hook stands in for the real bounded
|
||
* connect + register so a test can park the worker (unreachable peer)
|
||
* or return a synthetic success/failure without a live socket. On
|
||
* the test path we do NOT touch the real endpoint fd. */
|
||
int hrc = ha_reconnect_connect_hook(c, peer_idx);
|
||
/* The hook stands in for the real connect+iomgr_monitor: it NEVER
|
||
* registers the synthetic endpoint with iomgr, so the prepped ep/sctx
|
||
* are unowned on BOTH success and failure here — reclaim them in either
|
||
* case (test-only path; production reclaims via iomgr/err-handler).
|
||
* Reverting this to free only on failure re-introduces the 2.1
|
||
* GREEN-test endpoint leak masked by detect_leaks=0. */
|
||
ep->data = NULL;
|
||
endpoint_fini(ep);
|
||
free(ep); /* endpoint_get() calloc'd ep; fini frees its inner
|
||
* members only, not the struct itself */
|
||
free(sctx);
|
||
return hrc;
|
||
}
|
||
#endif
|
||
|
||
if (rc_ctx->should_log)
|
||
INFO("HA: peer[%d] (%s) not connected — initiating outbound connect "
|
||
"(attempt %d)", peer_idx, peer->hostname,
|
||
c->ha.peer_connect_fail_count[peer_idx]);
|
||
|
||
int rc = endpoint_connect(ep, peer->hostname, peer->port,
|
||
DEFAULT_CONNECT_TIMEOUT_MS);
|
||
if (rc < 0) {
|
||
if (rc_ctx->should_log)
|
||
WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
|
||
peer_idx, peer->hostname, peer->port, rc);
|
||
/* endpoint_fini calls FREEUP(ep->data) which frees sctx — do not
|
||
* double-free sctx here. endpoint_get() calloc'd ep itself, so free
|
||
* the struct after fini (fini reclaims only its inner members). */
|
||
ep->data = NULL;
|
||
endpoint_fini(ep);
|
||
free(ep);
|
||
free(sctx);
|
||
return rc;
|
||
}
|
||
rc = iomgr_monitor(ep); /* lock-guarded epoll registration (Req 4.4) */
|
||
if (rc < 0) {
|
||
ep->data = NULL;
|
||
endpoint_fini(ep);
|
||
free(ep);
|
||
free(sctx);
|
||
return rc;
|
||
}
|
||
INFO("HA: connecting outbound to peer[%d] (%s:%u)",
|
||
peer_idx, peer->hostname, peer->port);
|
||
return 0;
|
||
}
|
||
|
||
/*
|
||
* reconnect_lane_fn: the slow-lane worker entry (raft-drive-async Task 2.1).
|
||
* Runs the bounded blocking connect off the delaymgr thread, then ALWAYS
|
||
* clears the per-peer in-flight flag (success or failure) so the dedup never
|
||
* wedges and the next heartbeat tick can retry an unreachable peer (Req 2.5,
|
||
* 2.4 — never self-terminates). Frees the reconnect ctx.
|
||
*/
|
||
static void reconnect_lane_fn(void *arg)
|
||
{
|
||
reconnect_ctx_t *rc_ctx = (reconnect_ctx_t *)arg;
|
||
if (!rc_ctx) return;
|
||
|
||
cluster_t *c = rc_ctx->c;
|
||
int peer_idx = rc_ctx->peer_idx;
|
||
|
||
(void)ha_connect_finish_peer(rc_ctx);
|
||
|
||
/* Clear in-flight LAST, after the endpoint is fully connected/registered
|
||
* or torn down — so a concurrent heartbeat tick cannot enqueue a second
|
||
* reconnect that races this one's endpoint setup (Req 2.5). */
|
||
atomic_store_explicit(&c->ha.reconnect_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
free(rc_ctx);
|
||
}
|
||
|
||
/*
|
||
* ha_connect_to_peer: open a new outbound TLS connection to a peer candidate
|
||
* and register it as a standby channel. Called from primary_heartbeat_fn for
|
||
* peers that are absent from standby_endpoints.
|
||
*
|
||
* Offload flag ON (default, Task 2.1): prepares the endpoint (non-blocking)
|
||
* and submits the bounded connect to the slow lane with a per-peer dedup key,
|
||
* then returns immediately — the delaymgr thread NEVER blocks on connect.
|
||
*
|
||
* Offload flag OFF (legacy / TDD RED only): runs the blocking connect inline
|
||
* on the calling (delaymgr) thread. Retained solely so the RED test can show
|
||
* the pre-change stall; production runs with the flag ON.
|
||
*/
|
||
static void ha_connect_to_peer(cluster_t *c, int peer_idx)
|
||
{
|
||
ha_state_t *ha = &c->ha;
|
||
|
||
if (!ha_reconnect_offload_enabled()) {
|
||
/* ── Legacy inline path (flag OFF) — blocks the caller. ── */
|
||
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
|
||
if (!rc_ctx) return;
|
||
(void)ha_connect_finish_peer(rc_ctx);
|
||
free(rc_ctx);
|
||
return;
|
||
}
|
||
|
||
/* ── Offload path (flag ON) — non-blocking on the delaymgr thread. ──
|
||
*
|
||
* Per-peer single-flight: CAS the in-flight flag 0->1. If it was already
|
||
* 1, a reconnect to this peer is already in flight on the lane — enqueue
|
||
* nothing (Req 2.5). This guard is the authoritative dedup; the lane's
|
||
* own per-key dedup is a redundant backstop. */
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong_explicit(
|
||
&ha->reconnect_inflight[peer_idx], &expected, 1,
|
||
memory_order_acq_rel, memory_order_acquire)) {
|
||
return; /* already reconnecting this peer */
|
||
}
|
||
|
||
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
|
||
if (!rc_ctx) {
|
||
/* prep failed (alloc) — release the in-flight guard so the next tick
|
||
* retries; controller stays alive (Req 2.4). */
|
||
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
|
||
/* Submit the blocking connect to the slow lane. Dedup key is the
|
||
* per-peer absolute index (+1 so it is non-zero; key 0 disables dedup). */
|
||
uint64_t key = (uint64_t)(peer_idx + 1);
|
||
int srv = slowlane_submit(&c->slowlane, key, reconnect_lane_fn, rc_ctx);
|
||
if (srv == -EEXIST) {
|
||
/* Lane still holds this peer's dedup key from a PRIOR reconnect task.
|
||
* This is the narrow window where that task already cleared our
|
||
* reconnect_inflight flag (its last action) but the lane has not yet
|
||
* removed its key. Our CAS therefore succeeded (flag was 0) even
|
||
* though the lane refuses the submit. We MUST release the flag we just
|
||
* set — otherwise no task owns its clear and dedup wedges permanently.
|
||
* Tear down our duplicate prep; the next tick retries cleanly. */
|
||
rc_ctx->ep->data = NULL;
|
||
endpoint_fini(rc_ctx->ep);
|
||
free(rc_ctx->ep); /* endpoint_get() calloc'd it; fini frees inner only */
|
||
free(rc_ctx->sctx);
|
||
free(rc_ctx);
|
||
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
if (srv < 0 && srv != -EAGAIN) {
|
||
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
|
||
* failure: the task is still queued and will run, so we keep the flag
|
||
* set and let reconnect_lane_fn clear it. Only on a true failure do
|
||
* we tear down and release the guard so the next tick retries. */
|
||
WARN("HA: slowlane_submit reconnect peer[%d] failed: %d — retry next tick",
|
||
peer_idx, srv);
|
||
rc_ctx->ep->data = NULL;
|
||
endpoint_fini(rc_ctx->ep);
|
||
free(rc_ctx->ep); /* endpoint_get() calloc'd it; fini frees inner only */
|
||
free(rc_ctx->sctx);
|
||
free(rc_ctx);
|
||
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
|
||
memory_order_release);
|
||
return;
|
||
}
|
||
/* 0 or -EAGAIN: the task is enqueued and will run exactly once on a lane
|
||
* worker; reconnect_lane_fn clears the in-flight flag on completion. */
|
||
}
|
||
|
||
/*
|
||
* primary_heartbeat_fn: self-perpetuating one-shot delaymgr task.
|
||
*
|
||
* On each tick:
|
||
* 1. Sends OPCode_nop to all connected standbys (keeps watchdog alive).
|
||
* 2. Scans the peers list; for any peer NOT in standby_endpoints, opens a
|
||
* fresh outbound TLS connection so late joiners are bootstrapped without
|
||
* the standby ever initiating an outbound connect.
|
||
*
|
||
* Stops naturally on demotion: the role check prevents re-arming.
|
||
*/
|
||
static int primary_heartbeat_fn(void *arg)
|
||
{
|
||
cluster_t *c = (cluster_t *)arg;
|
||
ha_state_t *ha = &c->ha;
|
||
|
||
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||
memory_order_acquire);
|
||
if (role != CTRL_PRIMARY)
|
||
return 0; /* demoted — stop sending NOPs */
|
||
|
||
/* 1. Heartbeat all connected standbys — idle-triggered NOP (Req 3.1-3.4).
|
||
*
|
||
* For each standby ep, compute elapsed time since the last outbound push.
|
||
* Send NOP only when elapsed >= heartbeat_interval_ms so that WAL entries,
|
||
* config pushes, and ctrlhello suppress redundant NOPs (Req 3.4).
|
||
*/
|
||
{
|
||
struct timespec _hb_ts;
|
||
clock_gettime(CLOCK_MONOTONIC, &_hb_ts);
|
||
time_t now = _hb_ts.tv_sec;
|
||
|
||
int interval_s = (int)(ha->config.heartbeat_interval_ms / 1000);
|
||
if (interval_s <= 0)
|
||
interval_s = HA_DEFAULT_HEARTBEAT_INTERVAL_MS / 1000;
|
||
|
||
if (ha->standby_endpoints) {
|
||
/* Liveness eviction threshold (wall clock, matches ep->ts_last_in). */
|
||
time_t now_wall = time(NULL);
|
||
uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000;
|
||
if (timeout_s == 0)
|
||
timeout_s = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS / 1000;
|
||
|
||
/* Stamp the primary's OWN current committed position + term onto
|
||
* every heartbeat this tick (Req 1.1, 3.4). Read atomically here,
|
||
* once per tick, so every standby sees a consistent snapshot. Use
|
||
* the primary's authoritative state — NOT the standby-side
|
||
* last_known_* fields. */
|
||
uint64_t hb_term =
|
||
atomic_load_explicit(&ha->term, memory_order_acquire);
|
||
uint64_t hb_committed_lsn =
|
||
atomic_load_explicit(&c->persist.committed_lsn,
|
||
memory_order_acquire);
|
||
uint64_t hb_committed_config_tag =
|
||
atomic_load_explicit(&ha->committed_config_tag,
|
||
memory_order_acquire);
|
||
|
||
/* Endpoints evicted this tick; their repl deregistration runs AFTER
|
||
* ha->lock is released (repl lock-ordering — mirrors the err handlers). */
|
||
endpoint_t *evicted[MAX_HA_CANDIDATES];
|
||
int n_evicted = 0;
|
||
|
||
pthread_mutex_lock(&ha->lock);
|
||
int n = ha->standby_endpoints->count;
|
||
for (int i = 0; i < n; i++) {
|
||
endpoint_t *ep = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
|
||
if (!ep) continue;
|
||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
|
||
if (!sctx) continue;
|
||
|
||
/* Liveness eviction (symmetric with the standby's liveness poll):
|
||
* a standby that was responsive at least once (ts_last_in > 0) but
|
||
* has sent nothing for longer than heartbeat_timeout is dead or was
|
||
* restarted. A wedged/dead endpoint left in the array makes the
|
||
* reconnect scan below treat the peer as still-connected, so the
|
||
* restarted node never receives a fresh ctrlhello and loops
|
||
* elections forever (terms climbing unbounded). Evict it here; the
|
||
* reconnect scan re-establishes a fresh channel on this/next tick.
|
||
* Removal from standby_endpoints is the load-bearing action — the
|
||
* endpoint object itself is reclaimed by its err handler when the
|
||
* dead connection finally surfaces an epoll event (bounded). */
|
||
if (ep->ts_last_in > 0 &&
|
||
(now_wall - ep->ts_last_in) > (time_t)timeout_s) {
|
||
WARN("HA: standby[%d] silent for %lds (timeout=%us) — evicting "
|
||
"dead endpoint, will reconnect",
|
||
i, (long)(now_wall - ep->ts_last_in), timeout_s);
|
||
zarray_del(ha->standby_endpoints, i);
|
||
if (n_evicted < MAX_HA_CANDIDATES)
|
||
evicted[n_evicted++] = ep;
|
||
i--; n--;
|
||
continue;
|
||
}
|
||
|
||
time_t elapsed = now - sctx->last_push_time;
|
||
if (elapsed >= (time_t)interval_s) {
|
||
/* Heartbeat carries the primary's own committed + term
|
||
* (Req 1.1, 3.4). */
|
||
#ifdef XCE_TESTING
|
||
int rc;
|
||
if (ha_heartbeat_req_hook)
|
||
rc = ha_heartbeat_req_hook(ep, hb_term,
|
||
hb_committed_lsn,
|
||
hb_committed_config_tag);
|
||
else if (ha_nop_req_hook)
|
||
rc = ha_nop_req_hook(ep);
|
||
else
|
||
rc = xce_ha_heartbeat_req(ep, hb_term,
|
||
hb_committed_lsn,
|
||
hb_committed_config_tag);
|
||
#else
|
||
int rc = xce_ha_heartbeat_req(ep, hb_term,
|
||
hb_committed_lsn,
|
||
hb_committed_config_tag);
|
||
#endif
|
||
if (rc < 0)
|
||
WARN("HA: heartbeat nop to standby[%d] failed: %d", i, rc);
|
||
else
|
||
sctx->last_push_time = now;
|
||
}
|
||
}
|
||
pthread_mutex_unlock(&ha->lock);
|
||
|
||
for (int k = 0; k < n_evicted; k++)
|
||
repl_remove_standby(&c->persist.repl, evicted[k]);
|
||
}
|
||
}
|
||
|
||
/* 2. Reconnect to any peer missing from standby_endpoints */
|
||
if (c->myconf) {
|
||
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
|
||
for (int i = 0; i < pcfg->candidates_count; i++) {
|
||
if (i == ha->my_priority) continue; /* skip self */
|
||
|
||
/* Check if this peer already has a standby endpoint.
|
||
*
|
||
* Two endpoint types can be in standby_endpoints:
|
||
* - Promoted vote eps: ep->data = vote_ep_ctx_t, peer_idx is
|
||
* votes_idx (relative, self excluded). Convert absolute i to
|
||
* votes_idx using the same formula as raft_cb_send_vote_request.
|
||
* - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL),
|
||
* peer_idx is absolute candidate index. Compare directly with i.
|
||
*/
|
||
int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1;
|
||
int found = 0;
|
||
if (ha->standby_endpoints) {
|
||
pthread_mutex_lock(&ha->lock);
|
||
for (int j = 0; j < ha->standby_endpoints->count; j++) {
|
||
endpoint_t *sep = (endpoint_t *)zarray_get(
|
||
ha->standby_endpoints, j);
|
||
if (!sep || !sep->data) continue;
|
||
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)sep->data;
|
||
int match_idx = (sctx->ctx != NULL)
|
||
? votes_idx_for_i /* promoted vote ep: relative */
|
||
: i; /* standby ep: absolute */
|
||
if (sctx->peer_idx == match_idx) { found = 1; break; }
|
||
}
|
||
pthread_mutex_unlock(&ha->lock);
|
||
}
|
||
|
||
if (!found)
|
||
ha_connect_to_peer(c, i);
|
||
}
|
||
}
|
||
|
||
/* runcount=-1 (infinite repeat): the delaymgr re-enqueues this task
|
||
* automatically. The role check at entry stops work on demotion. */
|
||
return 0;
|
||
}
|
||
|
||
void ha_start_primary_heartbeat(cluster_t *c)
|
||
{
|
||
if (!c) return;
|
||
ha_state_t *ha = &c->ha;
|
||
int interval_ms = (int)ha->config.heartbeat_interval_ms;
|
||
if (interval_ms <= 0)
|
||
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
|
||
|
||
/* Idempotency: only arm once per process lifetime. With runcount=-1 the
|
||
* delaymgr re-enqueues the task automatically; duplicate tasks would stack
|
||
* on every re-election. */
|
||
{
|
||
static _Atomic int s_armed = 0;
|
||
int expected = 0;
|
||
if (!atomic_compare_exchange_strong(&s_armed, &expected, 1))
|
||
return; /* already armed — task is still running */
|
||
}
|
||
|
||
/* Guard: skip if delaymgr is not running (e.g. unit tests with zeroed cluster_t).
|
||
* In production the delaymgr is always started before any promotion. */
|
||
if (!c->delaymgr.running) {
|
||
INFO("HA: heartbeat skipped (delaymgr not running)");
|
||
return;
|
||
}
|
||
|
||
/* runcount=-1 (infinite): delaymgr re-enqueues automatically.
|
||
* The role check in primary_heartbeat_fn no-ops on demotion. */
|
||
int rc = delay_task_ms(&c->delaymgr, interval_ms, -1,
|
||
(delay_task_entry_fn_t)primary_heartbeat_fn, c);
|
||
if (rc < 0)
|
||
WARN("ha_start_primary_heartbeat: delay_task_ms failed: %d", rc);
|
||
else
|
||
INFO("HA: primary heartbeat armed (interval=%dms, rc=%d)", interval_ms, rc);
|
||
}
|
||
|