wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity
myxce/src/controller/ha.c
Rongsong Shen 6d798df35d docs(raft-adapter-conformance): document dormant AppendEntries sender + non-regression guard 
ha_cb_send_append_entries_stub's doc-comment now plainly states it is
reachable only via raft_send_append_entries (zero production callers,
grep-confirmed), and names repl_broadcast/ha_stream_wal_to_standby as
the actual production send/retry/backoff mechanism (Req 3.3, amended
to permit this documentation-only resolution over retire-or-wire-in).

Adds an XCE_TESTING-only ha_send_append_entries_stub_call_count
counter, mirroring the ha_wal_ack_send_hook idiom, as the guard against
this ever silently regaining a production caller. Task 4.2 will drive
the real production path and assert it stays zero.

1063 -> 1064 tests, 0 failures. Zero behavior change to any production
send path.
2026-07-29 15:04:07 +00:00

9852 lines
460 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#define _GNU_SOURCE
#include "ha.h"
#include <arpa/inet.h>
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <netdb.h>
#include <poll.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
#include <time.h>
#include <unistd.h>
#include <zlib.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/sha.h>
/* Circular-include safety: cluster_def.h includes ha.h (include-guarded),
* and ha.h only forward-declares cluster_t — no cycle. */
#include "xce/cluster_def.h"
#include "xce/errnox.h"
#include "controller.capnp.h"
#include "persist.h"
#include "request.h"
#include "utils.h"
/* ── HA dispatch gate error codes (defined here; declared extern in errnox.h) ── */
int XCE_ERR_NOT_READY = E_XCE_NOT_READY;
int XCE_ERR_STANDBY = E_XCE_STANDBY;
/* No-op stubs removed: s_real_cbs (below) is used exclusively since task 6.2. */
/* ── Forward declarations needed by real raft callbacks (Task 6.2) ────────
* These static functions and types are defined later in this file; declared
* here so the callback implementations can reference them. */
/* vote_ep_ctx_t: per-peer context for outgoing vote endpoints (full def below) */
typedef struct {
ha_state_t *ha;
cluster_t *c;
ha_elect_ctx_t *ctx;
int peer_idx;
uint64_t my_term;
uint64_t my_tag;
uint64_t my_lsn;
/* ha-term-safety task 1.2: candidate's own Raft (lastLogTerm, lastLogIndex)
* advertised in its RequestVote/PreVote so a voter's library logOk gate can
* compare. my_last_log_index == candidate WAL log_len; my_last_log_term ==
* term of that last entry (recovered/live WAL term). */
uint64_t my_last_log_term;
uint64_t my_last_log_index;
const char *my_name;
} vote_ep_ctx_t;
/* election_timeout_ctx_t: delaymgr callback context for raft election timeout */
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
ha_elect_ctx_t *ctx;
} election_timeout_ctx_t;
/* cluster_init_phase2 / push_config_transport_only /
* ha_send_ctrlhello_to_all_standbys: defined in cluster.c / config_repl.c;
* only referenced in production builds.
* (XCE_TESTING uses hook shims and does not link cluster.c.)
* ha.c uses push_config_transport_only ONLY (task 5.2 remediation r1):
* every ha.c push site is reachable from peer_ctrl_msg_handler / io_worker
* (vote replies, heartbeat replies), where the push_config_to_standbys
* barrier wait would block the single iomgr epoll thread — and the config
* ACK it waits for would arrive on that same blocked thread (structural
* deadlock). Convergence is owned by the OP_replConfig anti-entropy pull
* and the heartbeat decision loop, not by these best-effort pushes. */
#ifndef XCE_TESTING
extern int cluster_init_phase2(cluster_t *c);
extern void ha_send_ctrlhello_to_all_standbys(cluster_t *c);
extern int push_config_transport_only(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag, int *n_out);
#endif
/* raft-drive-async Task 3.3: requester-less config-push offload router (defined
* below, near the snapshot helpers). Forward-declared because
* raft_cb_become_leader (earlier) is one of its two call sites. */
static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag);
/* config_repl.c is linked in BOTH production and XCE_TESTING builds.
* ha-term-safety task 4.3 (remediation r1): the commit-advance hook activates a
* newly-committed config marker (Finding B); ha_dispatch_gate / the pre-promotion
* gate read the serve-fence (Finding C). */
extern int config_activate_committed(cluster_t *c);
extern int config_serve_fenced(const cluster_t *c);
/* task 4.3 (remediation r2, Finding D): the commit-advance path schedules an
* off-iomgr retry that re-activates a newly-committed config marker and clears a
* stale serve-fence (non-blocking ordq_put only). */
extern int config_schedule_activate_retry(cluster_t *c);
/* ── config-upgrade-snapshot task 6.1: Activation Fence ───────────────────
* upgrade_materialization_verify (below) is a SEPARATE fence from the
* config-byte serve-fence above — see ha_state_t's upgrade_materialization_
* fence field doc comment (ha.h) for the full "why a second fence" rationale.
* It reads the SAME committed-marker authority config_activate_committed
* itself reads. */
extern int config_committed_marker(const cluster_t *c, uint64_t *version_tag,
uint8_t sha256_out[32]);
/* upgrade_materialization_read_live_config_sha256 (config-upgrade-snapshot
* task 6.1, Req 3.2/3.4): resolve and hash THIS MEMBER'S CURRENTLY-ACTIVE
* config bytes fresh, right now — never a cached/frozen copy (in particular,
* never c->persist.cfgref_sha, which is only a snapshot-time REFERENCE
* captured at whatever moment the last checkpoint ran, not proof the bytes
* are still current this instant).
*
* DELIBERATE DESIGN CHOICE: this calls the same two already-PUBLIC
* config_repl.c entry points (conf_read_current_filename, conf_read_raw)
* that config_activate_committed's own local_config_sha256 helper uses
* internally, rather than re-deriving the <workdir>/<name>/conf/current ->
* cluster.bin.<hex> resolution a third time (persist.c's
* read_active_config_bytes already duplicates that resolution inline once,
* with its own documented rationale for why persist.c specifically avoids a
* persist.c -> config_repl.c call edge). ha.c carries no such constraint —
* it already depends on config_repl.c directly for config_activate_
* committed/config_serve_fenced/config_schedule_activate_retry/conf_read_raw
* above, so calling these two existing, already-public accessors here is
* in-boundary, ordinary integration along an established edge, not a new
* layering violation or a third duplicate implementation of path validation.
*
* Returns 0 with out_sha populated on success; -ENOENT when no local config
* bytes exist yet (benign — e.g. c->myconf unset, or no conf/current pointer
* written yet); another negative errno surfaces a genuine read/validation
* failure from the underlying calls. */
static int upgrade_materialization_read_live_config_sha256(cluster_t *c,
uint8_t out_sha[32])
{
extern int conf_read_current_filename(const char *workdir, char *out,
size_t outsz);
extern int conf_read_raw(const char *conffile, uint8_t **bytes,
size_t *len);
myconf_t *mc = c->myconf;
if (!mc || !mc->workdir || !mc->name)
return -ENOENT;
/* effective_dir = workdir/name — must match conf_save_config's own path
* construction (config_repl.c) exactly, mirroring local_config_sha256. */
char conf_dir[4096];
snprintf(conf_dir, sizeof(conf_dir), "%s/%s", mc->workdir, mc->name);
char fname[256] = {0};
int rc = conf_read_current_filename(conf_dir, fname, sizeof(fname));
if (rc != 0)
return -ENOENT; /* no local bytes yet — caller fences + retries */
char path[4352];
snprintf(path, sizeof(path), "%s/conf/%s", conf_dir, fname);
uint8_t *bytes = NULL;
size_t blen = 0;
rc = conf_read_raw(path, &bytes, &blen);
if (rc != 0)
return -ENOENT;
SHA256(bytes, blen, out_sha);
free(bytes);
return 0;
}
/* upgrade_materialization_verify (config-upgrade-snapshot task 6.1, Req 1.8,
* 3.2, 3.4): the Activation Fence's own verify function — mirrors
* config_activate_committed's exact shape (config_repl.c:304-377) for a
* DIFFERENT condition. config_serve_fence (config_repl.c) asks "do the
* locally-held config BYTES hash-match the committed marker?"; THIS fence
* asks "has THIS MEMBER'S OWN materialization (the upgrade-snapshot
* checkpoint, config-upgrade-snapshot task 5) actually completed for the
* CURRENT committed marker, with matching config bytes?" — a distinct,
* independently-trackable condition: bytes can already hash-match
* (config_serve_fence clear) while this member's own checkpoint for that
* exact marker has not run yet, e.g. immediately after a marker commits but
* before apply_committed_entry's WAL_CONFIG_MARKER case has actually fired
* and written a checkpoint.
*
* "Materialization complete" is determined via c->persist.cfgref_present /
* cfgref_tag / cfgref_sha — persist.c's scan_latest_committed_config_ref,
* invoked from EVERY checkpoint (persist_checkpoint_write_locked), stamps
* these fields with the latest committed WAL_CONFIG_MARKER as of that
* checkpoint. cfgref_present && cfgref_tag == marker_tag && cfgref_sha ==
* marker_sha therefore means "some checkpoint on THIS member has already
* captured a reference to this exact committed marker" — materialization for
* it has happened. This is the SAME field config_committed_marker's own
* compaction fallback already reads (config_repl.c), so this is a read of an
* existing, already-correct signal, not a new one.
*
* That alone is not sufficient: the checkpoint's cfgref_sha is a REFERENCE
* captured AT CHECKPOINT TIME, not proof the bytes are STILL current right
* now (the WAL marker's raft-commit and the R7 config-push byte delivery are
* two independent channels with no strict ordering guarantee). So this
* function ALSO re-reads the live on-disk config bytes fresh on every call
* (never c->persist.cfgref_sha, never any other cached/embedded copy) via
* upgrade_materialization_read_live_config_sha256 (above), and hash-verifies
* that fresh read against the committed marker too. Both must hold for the
* fence to clear.
*
* Never exits/aborts on this condition (controller/CLAUDE.md's never-self-
* terminate invariant) — always fail loud + fence + stay alive + retriable,
* mirroring config_activate_committed exactly: logs loud only on the 0->1
* fence transition (never spams on repeated failed retries), returns
* -EAGAIN on failure so the caller retries, clears + returns 0 on success. */
int upgrade_materialization_verify(cluster_t *c)
{
if (!c)
return -EINVAL;
/* 1. The committed marker is the authority for the target upgrade. An
* absent marker (no configuration change has ever committed on this
* cluster, or the marker has not committed yet) is NOT a divergence —
* clear any stale fence and succeed, mirroring config_activate_
* committed's own -ENOENT handling exactly. */
uint64_t marker_tag = 0;
uint8_t marker_sha[32];
int rc = config_committed_marker(c, &marker_tag, marker_sha);
if (rc == -ENOENT) {
atomic_store_explicit(&c->ha.upgrade_materialization_fence, 0,
memory_order_release);
return 0;
}
if (rc != 0)
return rc; /* -EINVAL / -EIO surfaced to the caller */
/* 2. Materialization-complete signal: has ANY checkpoint on THIS member
* already captured a reference to this EXACT committed marker? */
bool materialized = c->persist.cfgref_present &&
c->persist.cfgref_tag == marker_tag &&
memcmp(c->persist.cfgref_sha, marker_sha, 32) == 0;
/* 3. Hash-verify: re-read the LIVE, currently-active config bytes fresh
* on every attempt and compare against the committed marker. */
uint8_t live_sha[32];
int live_rc = upgrade_materialization_read_live_config_sha256(c, live_sha);
bool bytes_verified = (live_rc == 0) &&
(memcmp(live_sha, marker_sha, 32) == 0);
if (!materialized || !bytes_verified) {
int was = atomic_exchange_explicit(&c->ha.upgrade_materialization_fence,
1, memory_order_acq_rel);
if (!was)
ERROR("upgrade materialization fence: committed config marker "
"tag=%016" PRIx64 " -- materialization %s "
"(cfgref_present=%d cfgref_tag=%016" PRIx64 "), live config "
"bytes %s (rc=%d) -- member's OWN upgrade-snapshot "
"materialization for this marker is INCOMPLETE or "
"MISMATCHED; member NON-SERVING and NON-PROMOTABLE under the "
"new configuration, staying alive and retrying "
"(config-upgrade-snapshot R1.8/3.2/3.4)",
marker_tag, materialized ? "captured" : "NOT YET captured",
c->persist.cfgref_present, c->persist.cfgref_tag,
bytes_verified ? "hash-match" : "do NOT hash-match", live_rc);
return -EAGAIN;
}
atomic_store_explicit(&c->ha.upgrade_materialization_fence, 0,
memory_order_release);
INFO("upgrade materialization VERIFIED for committed config marker "
"tag=%016" PRIx64 " -- materialization complete, live config bytes "
"hash-verified", marker_tag);
return 0;
}
/* upgrade_materialization_fenced: trivial atomic-read accessor — mirrors
* config_serve_fenced (config_repl.c:262-268) exactly, for the new
* independent fence. Pure atomic read (no I/O), safe on the election/
* dispatch path. */
int upgrade_materialization_fenced(const cluster_t *c)
{
if (!c)
return 0;
return atomic_load_explicit(&c->ha.upgrade_materialization_fence,
memory_order_acquire) ? 1 : 0;
}
/* ── config-upgrade-snapshot task 6.2: Activation Fence self-triggering retry ──
* Mirrors config_schedule_activate_retry's exact four-function shape
* (config_repl.c:409-507) for the upgrade_materialization_fence instead of
* config_serve_fence: a one-shot NON-BLOCKING delaymgr task offloads the
* actual blocking re-verify work (upgrade_materialization_verify's disk
* reads) onto a VTASK_LIVE_BLOCK vtask, so the delaymgr thread itself never
* blocks (ha-async-io.md thread-role contract). */
/* upgrade_materialization_retry_fn: re-runs upgrade_materialization_verify so
* a member whose materialization was incomplete or hash-mismatched re-checks
* and clears the fence once conditions allow. Does the actual blocking disk
* I/O; callers must invoke it off the delaymgr thread (see
* upgrade_materialization_retry_vtask_entry /
* schedule_upgrade_materialization_retry below). Public so tests can drive it
* directly without a vtask pool — mirrors config_activate_retry_fn. */
int upgrade_materialization_retry_fn(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
if (!upgrade_materialization_fenced(c))
return 0; /* already cleared by another path — nothing to retry */
int rc = upgrade_materialization_verify(c);
if (rc == 0)
INFO("upgrade materialization retry: fence CLEARED — this member's "
"own materialization for the committed config marker now "
"verified");
else if (rc == -EAGAIN)
TRACE("upgrade materialization retry: still fenced (materialization "
"incomplete or hash-mismatched) — will retry on the next "
"commit advance");
else
WARN("upgrade materialization retry: upgrade_materialization_verify "
"rc=%d", rc);
return rc;
}
/* upgrade_materialization_retry_vtask_entry: vtask entry wrapping
* upgrade_materialization_retry_fn so its blocking disk I/O runs on a vtask
* worker, never inline on the single delaymgr thread. Mirrors
* config_activate_retry_vtask_entry. */
static int upgrade_materialization_retry_vtask_entry(vtask_t *task)
{
cluster_t *c = (cluster_t *)task->data->u.generic;
upgrade_materialization_retry_fn(c);
return 0;
}
/* upgrade_materialization_retry_delaymgr_trampoline: the delaymgr task body.
* Delaymgr must only trigger + enqueue, never block — dispatch the actual
* retry work to a vtask (VTASK_LIVE_BLOCK, the same class
* config_activate_retry_delaymgr_trampoline uses) and return immediately.
* Fails loud and runs inline (stay alive, never silently drop a scheduled
* retry) if the pool is unavailable. Mirrors
* config_activate_retry_delaymgr_trampoline. */
static int upgrade_materialization_retry_delaymgr_trampoline(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
vtaskmgr_t *pool = &c->vtaskmgr;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = vtask_alloc();
if (data == NULL || task == NULL) {
WARN("upgrade materialization retry: vtask alloc failed — running "
"inline (stay alive)");
free(data);
free(task);
return upgrade_materialization_retry_fn(c);
}
data->vtype = VTASK_GENERIC;
data->u.generic = c;
vtask_init(task, (vtask_func_t)upgrade_materialization_retry_vtask_entry,
data);
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
WARN("upgrade materialization retry: vtask enqueue failed — running "
"inline (stay alive)");
free(data);
free(task);
return upgrade_materialization_retry_fn(c);
}
return 0;
}
/* schedule_upgrade_materialization_retry: enqueue the retry trampoline on the
* delaymgr (NON-BLOCKING — delay_task_ms only does ordq_put; the trampoline
* itself offloads the blocking work to a vtask, see above). Called from the
* commit-advance path (iomgr ACK thread) so the retry's disk I/O runs OFF
* that thread (ha-async-io invariant). No-op when the member is NOT
* upgrade-materialization fenced (steady state schedules nothing) or when
* the delaymgr is not running (the unit-test fixture path, which never inits
* delaymgr — avoids leaking a task into an un-drained queue). Mirrors
* config_schedule_activate_retry exactly. Returns 0 on enqueue/no-op,
* negative on enqueue failure. */
int schedule_upgrade_materialization_retry(cluster_t *c)
{
if (!c)
return -EINVAL;
if (!upgrade_materialization_fenced(c))
return 0; /* steady state: nothing to retry, schedule nothing */
if (!c->delaymgr.running)
return 0; /* delaymgr not running (test fixture) — caller drives directly */
/* One-shot, minimal delay: run promptly but off the iomgr ACK thread. */
int rc = delay_task_ms(&c->delaymgr, 1, 1,
(delay_task_entry_fn_t)upgrade_materialization_retry_delaymgr_trampoline,
c);
if (rc < 0)
WARN("upgrade materialization retry: delay_task_ms failed: %d — "
"fence will clear on a later commit advance", rc);
return rc;
}
/* ha_phase2_retry_fn / ha_phase2_retry_vtask_entry /
* ha_phase2_retry_delaymgr_trampoline / ha_schedule_phase2_retry (bug-1477):
* see the doc comments in ha.h. Structurally identical to
* upgrade_materialization_retry_fn's own 3-layer shape (blocking retry_fn ->
* vtask entry -> delaymgr trampoline) just above, except the trampoline
* re-arms itself on failure since nothing else recurring would otherwise
* ever retry cluster_init_phase2 for a node stuck on myapp_run's passive
* skip_election path. */
int ha_phase2_retry_fn(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
if (c->ha.phase2_done)
return 0; /* already done by another path -- nothing to retry */
int rc;
#ifdef XCE_TESTING
rc = (ha_phase2_hook != NULL) ? ha_phase2_hook(c) : 0;
#else
rc = cluster_init_phase2(c);
#endif
if (rc == 0) {
c->ha.phase2_done = 1;
INFO("HA: cluster_init_phase2 retry succeeded — commit engine now "
"wired (bug-1477)");
} else {
WARN("HA: cluster_init_phase2 retry rc=%d — member stays alive and "
"will retry again (bug-1477, never self-terminate)", rc);
ha_schedule_phase2_retry(c);
}
return rc;
}
static int ha_phase2_retry_vtask_entry(vtask_t *task)
{
cluster_t *c = (cluster_t *)task->data->u.generic;
ha_phase2_retry_fn(c);
return 0;
}
static int ha_phase2_retry_delaymgr_trampoline(void *arg)
{
cluster_t *c = (cluster_t *)arg;
if (!c)
return -EINVAL;
vtaskmgr_t *pool = &c->vtaskmgr;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = vtask_alloc();
if (data == NULL || task == NULL) {
WARN("HA: cluster_init_phase2 retry: vtask alloc failed — running "
"inline (stay alive)");
free(data);
free(task);
return ha_phase2_retry_fn(c);
}
data->vtype = VTASK_GENERIC;
data->u.generic = c;
vtask_init(task, (vtask_func_t)ha_phase2_retry_vtask_entry, data);
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
WARN("HA: cluster_init_phase2 retry: vtask enqueue failed — running "
"inline (stay alive)");
free(data);
free(task);
return ha_phase2_retry_fn(c);
}
return 0;
}
int ha_schedule_phase2_retry(cluster_t *c)
{
if (!c)
return -EINVAL;
if (c->ha.phase2_done)
return 0; /* steady state: nothing to retry, schedule nothing */
if (!c->delaymgr.running)
return 0; /* delaymgr not running (test fixture) — caller drives directly */
uint32_t interval_ms = c->ha.config.heartbeat_interval_ms;
if (interval_ms == 0)
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
int rc = delay_task_ms(&c->delaymgr, (int)interval_ms, 1,
(delay_task_entry_fn_t)ha_phase2_retry_delaymgr_trampoline,
c);
if (rc < 0)
WARN("HA: cluster_init_phase2 retry: delay_task_ms failed: %d — "
"will retry on the next natural re-trigger", rc);
return rc;
}
static void on_vote_connected(endpoint_t *ep);
static void on_pre_vote_connected(endpoint_t *ep);
static int vote_ep_err_handler(void *ep_raw);
/* raft-adapter-conformance task 3.1: PreVote's own on_err handler, registered
* at raft_cb_send_pre_vote_request's endpoint-setup call site (mirrors the
* real-vote vote_ep_err_handler forward-declared above). */
static int pre_vote_ep_err_handler(void *ep_raw);
/* bug-1423: forward-declared so raft_cb_send_vote_request's connect/monitor
* failure branches can route a synthetic "no vote" reply through the SAME
* production path a real reply uses (see vote_reply_dispatch's own comment,
* below, for why this must never be the legacy handle_request_vote_reply). */
static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx,
uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn);
/* bug-1423 (PreVote twin): same rationale as vote_reply_dispatch above, for
* raft_cb_send_pre_vote_request's connect/monitor failure branches. */
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted);
/* bug-1197: defer a vote-ep teardown onto the per-ep FIFO of the vtask pool so
* it runs strictly after the last live use of the ep (flag-ON loser path). */
static void vote_ep_teardown_async(ha_state_t *ha, endpoint_t *ep);
/* 2026-07-10 controller-invariant audit: forward-declared so the catch-up
* ingest path (ha_apply_wal_catchup_entry, earlier in this file) can kick the
* standby apply pump against the already-known commit position without
* waiting for the next heartbeat. Defined near ha_observe_heartbeat_committed. */
static void ha_standby_advance_committed_and_apply(cluster_t *c,
uint64_t learned_committed_lsn);
/* ha-single-leader-role task 2.2: shared promotion seam — every road to
* PRIMARY registers the persist hooks and arms the own-term gate. */
static void ha_register_persist_hooks(cluster_t *c);
static void ha_arm_own_term_gate(cluster_t *c);
int ha_heartbeat_is_fresh(ha_state_t *ha); /* exposed for PreVote denial-reason logging (cluster.c) */
static int primary_heartbeat_fn(void *arg);
static void on_standby_connected(endpoint_t *ep);
static int standby_ep_err_handler_primary(void *ep_raw);
/* ── ha-elect-ctx-lifecycle task 2.1: election carrier reference-counted
* lifetime ──────────────────────────────────────────────────────────────
*
* ha_elect_ctx_release: decrement ctx->refcount; on reaching zero, destroy
* the carrier's mutex/semaphore and free it. This is the ONLY place
* ha_elect_ctx_t is destroyed/freed — every release path (vote-endpoint
* teardown completion via vote_ep_on_fini, and round supersession in
* ha_start_election_async / ha_start_prevote_async) routes through here, so
* the carrier is freed exactly once, from whichever release is last.
*/
static void ha_elect_ctx_release(ha_elect_ctx_t *ctx)
{
if (ctx == NULL)
return;
if (atomic_fetch_sub_explicit(&ctx->refcount, 1, memory_order_acq_rel) != 1)
return; /* not the last reference */
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
}
#ifdef XCE_TESTING
/* ha_elect_ctx_release_for_test: drive ha_elect_ctx_release directly so unit
* tests can verify the release/free mechanism without staging a full
* election round. */
void ha_elect_ctx_release_for_test(ha_elect_ctx_t *ctx)
{
ha_elect_ctx_release(ctx);
}
#endif
/*
* vote_ep_on_fini: endpoint_t.on_fini completion hook wired onto every vote
* endpoint at capture time (raft_cb_send_vote_request /
* raft_cb_send_pre_vote_request). Fires from inside endpoint_fini()
* immediately before ep->data (the vote_ep_ctx_t) is freed — releases the
* carrier reference this endpoint held.
*/
static void vote_ep_on_fini(endpoint_t *ep)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
if (vctx == NULL)
return;
ha_elect_ctx_release(vctx->ctx);
}
/* ── Real raft callback implementations (Task 6.2) ─────────────
*
* These are registered as s_real_cbs in ha_init, replacing the no-op stubs.
* They are wired into the active election path in task 6.3.
*
* All callbacks receive ctx = ha_state_t*.
* ha->raft_c is set by ha_start_election_async before any raft_* calls.
* When ha->raft_c is NULL (unit test mode), callbacks that require cluster_t
* are no-ops.
*/
/*
* raft_cb_send_vote_request: encode and send a capnp VoteRequest to peer.
*
* peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[].
* Uses the existing iomgr vote-send path (on_vote_connected / vote_ep_ctx_t).
* Reads the current election context from ha->elect_ctx (set before raft_start_election).
* Returns 0 on success; non-zero if no cluster context or alloc failure.
*/
static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c || !c->myconf) return 1; /* test mode */
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
if (peer_idx < 0 || peer_idx >= pcfg->candidates_count)
return 1;
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (!ectx) return 1; /* no election in progress */
xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx];
/* Compute the local index of peer_idx within ectx->votes[]:
* votes[] has n_peers - 1 = n_other entries for peers excluding self.
* Map: votes_idx = peer_idx if peer_idx < self_idx, else peer_idx - 1. */
int self_idx = ha->my_priority; /* self_idx == my_priority by convention */
int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1;
if (votes_idx < 0 || votes_idx >= ectx->n_peers)
return 1;
/* Allocate per-peer context */
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
if (!vctx) {
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
return 1;
}
vctx->ha = ha;
vctx->c = c;
vctx->ctx = ectx;
vctx->peer_idx = votes_idx;
vctx->my_term = term;
/* tag and lsn come from votes[0] which holds our own values */
vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0;
vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0;
/* ha-term-safety task 1.2: advertise our Raft (lastLogTerm, lastLogIndex).
* lastLogIndex == our WAL log length (the library's log_len == WAL tip LSN);
* lastLogTerm == the term STAMPED ON THAT LAST ENTRY.
*
* bug-851 / task 5.5: lastLogTerm MUST be the term of the entry at log_len —
* sourced via persist_entry_term, the SAME accessor voters use to compute
* their own last-log term (on_get_entry_term → persist_entry_term, ha.c:1556).
* The old stopgap advertised c->persist.term (the persist layer's RUNNING
* term), which is stale/0 after a restart and does NOT equal the last entry's
* stored term. That made a freshly-restarted but freshest survivor advertise
* a too-low lastLogTerm, so every voter's logOk rejected it (asymmetric with
* how the voter computes its own term) and the election stalled. */
vctx->my_last_log_index = ha->raft.log_len;
vctx->my_last_log_term = (c && ha->raft.log_len > 0)
? persist_entry_term(&c->persist, ha->raft.log_len)
: 0;
vctx->my_name = c->myconf ? c->myconf->name : "";
/* Get a fresh endpoint */
endpoint_t *ep = endpoint_get();
if (!ep) {
free(vctx);
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
return 1;
}
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
if (c) endpoint_set_peer(ep, c->myhost);
ep->data = vctx;
ep->iomgr = &c->iomgr;
/* ha-elect-ctx-lifecycle task 2.2: this endpoint now references ectx via
* vctx->ctx — take a reference before any path that could reach
* endpoint_fini (connect failure below, and beyond). Released exactly
* once by vote_ep_on_fini (wired below) when this endpoint's teardown
* completes, regardless of which teardown path gets there. */
atomic_fetch_add_explicit(&ectx->refcount, 1, memory_order_relaxed);
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
if (ep->ssl)
SSL_set_connect_state(ep->ssl);
}
endpoint_set_on_connected(ep, on_vote_connected);
endpoint_set_err_handler(ep, vote_ep_err_handler);
endpoint_set_on_fini(ep, vote_ep_on_fini);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) {
endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (not monitored) */
/* bug-1423: a connect failure (e.g. peer dead/unreachable) must still
* register as a "no vote" reply through the SAME path a real reply
* uses. Without this, raft_state_t.replies_received never reaches
* n_peers-1 whenever any peer is unreachable, so the library's
* "everyone's replied, decide now" fast path (raft.c "Step 6:
* all-replied-no-quorum") is structurally unreachable — recovery then
* depends solely on the raw election timeout, observed to leave a
* candidate stuck in CTRL_ELECTING far longer than expected in a
* 3-node cluster with one node down. Test mode (c==NULL) keeps the
* legacy handle_request_vote_reply call that existing unit tests
* depend on; production routes through vote_reply_dispatch, the same
* call a genuine reply makes (ha.c vote_reply_dispatch). */
if (!c)
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
else
vote_reply_dispatch(ha, ectx, votes_idx, term, /*vote_granted=*/0, 0, 0);
return 1;
}
rc = iomgr_monitor(ep);
if (rc < 0) {
endpoint_unref(ep); /* bug-1298: fini frees ep->data (vctx), then frees ep */
/* bug-1423: same fix as the connect-failure branch above — this was
* unconditionally calling the legacy handle_request_vote_reply even in
* production, bypassing raft_on_vote_reply (the sole production
* promotion driver per vote_reply_dispatch's own comment). */
if (!c)
handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0);
else
vote_reply_dispatch(ha, ectx, votes_idx, term, /*vote_granted=*/0, 0, 0);
return 1;
}
/* Track endpoint for promotion/close after election result.
* Also record the candidate pointer: raft_cb_become_leader uses
* votes[i].peer->hostname / ->port to register the surviving vote
* endpoint with the WAL replicator (repl_add_standby). Without this,
* the promotion registration block (guarded by votes[i].peer != NULL)
* is skipped — the standbys never enter c->persist.repl.standbys[],
* commit/quorum see zero standbys, and every write fails NOT_READY.
* `peer` points into c->myconf->peers_cfg.candidates[] (persistent;
* outlives the election — repl_add_standby copies host/port).
* (fix-promotion-standby-registration, Req 1.1) */
ectx->votes[votes_idx].ep = ep;
ectx->votes[votes_idx].peer = peer;
return 0;
}
/*
* raft_cb_send_pre_vote_request: encode and send a capnp PreVote request to a
* peer (ha-prevote task 4.1).
*
* Mirrors raft_cb_send_vote_request exactly — same iomgr connect path, same
* vote_ep_ctx_t carrier, same per-peer endpoint tracking — but the connected
* handler (on_pre_vote_connected) emits OPCode_preVoteRequest instead of
* OPCode_requestVote. prospective_term is the candidate's CURRENT+1 term; it
* is NEVER persisted (PreVote is a side-effect-free straw poll).
*
* peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[].
* Returns 0 on success; non-zero in test mode (no cluster) or on alloc failure.
*
* Transport-side reply routing for preVoteReply is wired separately in task 5.1;
* this callback only emits the request.
*/
static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx,
uint64_t prospective_term)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c || !c->myconf) return 1; /* test mode — caller owns transport */
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
if (peer_idx < 0 || peer_idx >= pcfg->candidates_count)
return 1;
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (!ectx) return 1; /* no round metadata carrier */
xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx];
/* Map absolute peer_idx → votes[] local index (same convention as the
* real-vote send callback). */
int self_idx = ha->my_priority;
int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1;
if (votes_idx < 0 || votes_idx >= ectx->n_peers)
return 1;
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
if (!vctx) return 1;
vctx->ha = ha;
vctx->c = c;
vctx->ctx = ectx;
/* Carry the ABSOLUTE peer index: the PreVote tally in raft_state_t is keyed
* by absolute cluster index (raft_begin_pre_vote loops over i in
* [0, n_peers) and raft_on_pre_vote_reply dedups by that same index). */
vctx->peer_idx = peer_idx;
vctx->my_term = prospective_term;
vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0;
vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0;
/* ha-term-safety task 1.2 + bug-851/task 5.5: advertise lastLogTerm as the
* term of the entry at log_len (persist_entry_term — the same source voters
* use), NOT the stale c->persist.term. Identical sourcing to the real-vote
* send path above; see the full rationale there. */
vctx->my_last_log_index = ha->raft.log_len;
vctx->my_last_log_term = (c && ha->raft.log_len > 0)
? persist_entry_term(&c->persist, ha->raft.log_len)
: 0;
vctx->my_name = c->myconf ? c->myconf->name : "";
endpoint_t *ep = endpoint_get();
if (!ep) { free(vctx); return 1; }
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
if (c) endpoint_set_peer(ep, c->myhost);
ep->data = vctx;
ep->iomgr = &c->iomgr;
/* ha-elect-ctx-lifecycle task 2.3: same mechanism as the real-vote send
* path above — take a reference before any path that could reach
* endpoint_fini. */
atomic_fetch_add_explicit(&ectx->refcount, 1, memory_order_relaxed);
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
if (ep->ssl)
SSL_set_connect_state(ep->ssl);
}
endpoint_set_on_connected(ep, on_pre_vote_connected);
endpoint_set_err_handler(ep, pre_vote_ep_err_handler);
endpoint_set_on_fini(ep, vote_ep_on_fini);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) {
endpoint_unref(ep); /* bug-1298: fini+free */
/* bug-1423 (PreVote twin): a connect failure (dead/unreachable peer)
* must still register as a "not granted" reply with the raft library's
* PreVote tally, same as the real-vote send path. PreVote's own round
* timeout (ha_prevote_round_timeout_fn) is an unconditional fallback
* that always resolves the round regardless, so this gap is not a
* permanent-deadlock class bug like the real-vote one was — but
* leaving it means every round against a down peer wastes the full
* round timeout instead of concluding as soon as the live peer
* replies. */
if (c)
pre_vote_reply_dispatch(ha, c, peer_idx, /*granted=*/0);
return 1;
}
rc = iomgr_monitor(ep);
if (rc < 0) {
endpoint_unref(ep); /* bug-1298: fini+free */
/* bug-1423 (PreVote twin): same fix as the connect-failure branch above. */
if (c)
pre_vote_reply_dispatch(ha, c, peer_idx, /*granted=*/0);
return 1;
}
return 0;
}
/*
* raft_cb_vote_eligible: freshness gate delegated from raft library.
*
* from_idx is the absolute candidate index of the requester.
* Applies the same checks as handle_request_vote:
* - role gate: PRIMARY → deny
* - fresh heartbeat on STANDBY → deny
* - in_catch_up → deny
* - freshness: candidate config_tag/lsn must be ≥ ours
* Returns 1 to grant, 0 to deny.
*/
static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term)
{
(void)from_idx; (void)term;
ha_state_t *ha = (ha_state_t *)ctx;
/* Role gate */
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_PRIMARY)
return 0;
/* Fresh heartbeat on STANDBY WITH a live primary connection → primary is
* alive, deny. BOTH conditions are required (mirrors the handle_request_vote
* gate at ~ha.c:2205). Without the primary_ctrl_fd>=0 guard a node whose
* primary has DIED keeps a falsely-"fresh" status: ha_heartbeat_is_fresh
* derives from last_activity, which ha_arm_heartbeat_timeout_vtask re-arms to
* now on every failover retry — so the node would deny every legitimate
* higher-term PreVote for minutes, preventing convergence (bug-660).
* primary_ctrl_fd is -1 after the primary RDHUPs / the liveness watchdog
* fires / on a fresh boot, so a dead-or-absent primary correctly does NOT
* deny. */
if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(ha))
return 0;
/* In catch-up WHILE a live primary is feeding us → deny.
*
* raft-ha-core-safety task 5.3 (bug-1272): this deny used to be
* UNCONDITIONAL, which deadlocks the election under overlapping churn — after
* several rapid kills the only fully-up-to-date node is killed and the two
* survivors are both restarted-and-lagging (in_catch_up=1) with NO live
* primary left to catch up FROM (so the flag never clears). Each holds the
* full committed prefix and is a SAFE candidate, yet they mutually deny every
* PreVote → permanent stall (round-N election stall, 7.2 soak).
*
* Gate the deny on a LIVE primary, mirroring the heartbeat-fresh deny above:
* the purpose of the in_catch_up deny is to avoid disrupting a primary that is
* ACTIVELY replicating to us — not to fence a survivor whose primary is gone.
* Once the primary is unreachable, raft_log_ok (already applied in Step 2 of
* raft_on_pre_vote_request / raft_on_vote_request, BEFORE this callback)
* guarantees the candidate's (lastLogTerm,lastLogIndex) >= ours, so granting a
* fresher candidate is safe (Leader Completeness preserved). */
if (atomic_load_explicit(&ha->in_catch_up, memory_order_acquire) &&
ha->primary_ctrl_fd >= 0 && ha_heartbeat_is_fresh(ha))
return 0;
/* Freshness gate requires knowing the candidate's config_tag/lsn.
* The raft library does not pass these; they are carried in the capnp
* RequestVoteRequest fields. For the adapter, freshness is checked in
* handle_request_vote (called from the iomgr decode path in task 6.3).
* Here we return 1 (eligible) so the raft library proceeds; the full
* freshness gate is enforced by the caller before invoking raft_on_vote_request.
* This is safe because the caller (task 6.3 dispatcher) already applies
* the freshness check from handle_request_vote before delegating here. */
return 1;
}
/* raft_election_timeout_fn: delaymgr callback for raft election timeout.
* Fires raft_on_election_timeout after delay_ms if not cancelled.
* Owns the election_timeout_ctx_t — frees it on exit. */
static int raft_election_timeout_fn(void *arg)
{
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
ha_state_t *ha = t->ha;
/* Clear the stored pointer before checking cancelled so on_cancel_timeout
* is a safe no-op if it races with this callback. */
ha->raft_election_timeout_ctx = NULL;
if (atomic_load(&t->cancelled)) {
free(t);
return 0;
}
/* bug-1435: serialize against concurrent vote-reply/vote-request
* processing on the pool (see ha.h raft_lock doc). */
pthread_mutex_lock(&ha->raft_lock);
raft_on_election_timeout(&ha->raft);
pthread_mutex_unlock(&ha->raft_lock);
free(t);
return 0;
}
/*
* raft_cb_schedule_timeout: schedule election timeout via delaymgr.
*
* Creates an election_timeout_ctx_t and calls delay_task_ms.
* The pointer is stored in ha->raft_election_timeout_ctx for cancellation.
*/
static void raft_cb_schedule_timeout(void *ctx, uint32_t delay_ms)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c) return; /* test mode */
election_timeout_ctx_t *t = calloc(1, sizeof(*t));
if (!t) return;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->ctx = NULL; /* raft path does not use ha_elect_ctx_t */
/* task 5.7: randomize the per-campaign watchdog to [delay_ms, 2*delay_ms]
* (Ongaro §3.4). The library passes a FIXED election_timeout_ms; arming the
* watchdog at exactly that value lets a constant cold-connection grant RTT
* land just past every deadline forever (resonance → unbounded term climb).
* Re-randomizing here breaks the resonance without touching the pure
* (sim-tested) library. */
delay_ms = ha_jittered_election_delay(ha, delay_ms);
int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1,
raft_election_timeout_fn, t);
if (rc < 0) {
WARN("raft_cb_schedule_timeout: delay_task_ms failed: %d", rc);
free(t);
return;
}
ha->raft_election_timeout_ctx = t;
}
/*
* raft_cb_cancel_timeout: cancel the pending election timeout.
* Sets cancelled=1 on the stored election_timeout_ctx_t.
*/
static void raft_cb_cancel_timeout(void *ctx)
{
ha_state_t *ha = (ha_state_t *)ctx;
election_timeout_ctx_t *t =
(election_timeout_ctx_t *)ha->raft_election_timeout_ctx;
if (t)
atomic_store(&t->cancelled, 1);
ha->raft_election_timeout_ctx = NULL;
}
/* raft_election_start_vtask_entry: vtask entry wrapping raft_check_and_start_
* election. raft_start_election's on_persist callback (raft_cb_persist ->
* ha_persist_term) does a synchronous fsync -- must never run inline on the
* delaymgr thread (ha-async-io.md thread-role contract: delaymgr only fires
* timers and triggers work, then returns; it never blocks, since the same
* thread also drives heartbeats and every other election timeout). */
static int raft_election_start_vtask_entry(vtask_t *task)
{
ha_state_t *ha = (ha_state_t *)task->data->u.generic;
/* bug-1435: serialize against concurrent vote-reply/vote-request
* processing on the pool (see ha.h raft_lock doc). */
pthread_mutex_lock(&ha->raft_lock);
raft_check_and_start_election(&ha->raft);
pthread_mutex_unlock(&ha->raft_lock);
return 0;
}
/* ha_dispatch_election_start: offload raft_check_and_start_election onto a
* vtask (VTASK_LIVE_BLOCK -- the same class already used for the vote-GRANT
* term fsync, R4.6) so its persist-before-campaign fsync never blocks the
* delaymgr thread. On pool unavailability/enqueue failure, fail loud and run
* inline anyway (stay alive, never silently drop a scheduled election --
* mirrors persist_apply_pump_kick's and vote_ep_teardown_enqueue's fallback). */
static void ha_dispatch_election_start(ha_state_t *ha)
{
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = vtask_alloc();
if (pool == NULL || data == NULL || task == NULL) {
WARN("ha_dispatch_election_start: vtask unavailable (pool=%p data=%p "
"task=%p) — starting election inline (stay alive)",
(void *)pool, (void *)data, (void *)task);
free(data);
free(task);
pthread_mutex_lock(&ha->raft_lock);
raft_check_and_start_election(&ha->raft);
pthread_mutex_unlock(&ha->raft_lock);
return;
}
data->vtype = VTASK_GENERIC;
data->u.generic = ha;
vtask_init(task, (vtask_func_t)raft_election_start_vtask_entry, data);
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
WARN("ha_dispatch_election_start: vtask enqueue failed — starting "
"election inline (stay alive)");
free(data);
free(task);
pthread_mutex_lock(&ha->raft_lock);
raft_check_and_start_election(&ha->raft);
pthread_mutex_unlock(&ha->raft_lock);
}
}
/* raft_rdhup_delay_fn: delaymgr callback for RDHUP-triggered delayed election.
* Owns its ctx (election_timeout_ctx_t) — frees it on exit.
* Dispatches the actual election start off the delaymgr thread (see
* ha_dispatch_election_start) only if this node hasn't already voted. */
static int raft_rdhup_delay_fn(void *arg)
{
election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg;
ha_state_t *ha = t->ha;
if (!atomic_load(&t->cancelled))
ha_dispatch_election_start(ha);
free(t);
return 0;
}
/*
* raft_cb_start_election_after_delay: schedule RDHUP retry timer.
* After delay_ms, calls raft_check_and_start_election to start election
* only if this node hasn't already voted for another candidate.
*/
static void raft_cb_start_election_after_delay(void *ctx, uint32_t delay_ms)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
if (!c) return; /* test mode */
/* Priority 0 → delay_ms == 0: highest-priority node starts immediately.
* Dispatched off the calling thread (see ha_dispatch_election_start) since
* this callback's caller is not guaranteed to be vtask-safe. */
if (delay_ms == 0) {
ha_dispatch_election_start(ha);
return;
}
election_timeout_ctx_t *t = calloc(1, sizeof(*t));
if (!t) return;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->ctx = NULL;
int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1,
raft_rdhup_delay_fn, t);
if (rc < 0) {
WARN("raft_cb_start_election_after_delay: delay_task_ms failed: %d", rc);
free(t);
}
}
/* ── Task 4.1 (ha-repl-hardening): safe commit point computation ─────────────
*
* ha_compute_safe_committed_lsn: determine the safe committed LSN and config
* tag after winning an election, using the vote replies collected in ctx.
*
* Algorithm (design §Flow 0: Post-Election — Determine Safe Commit Point):
* 1. Collect LSNs from all granted voters:
* - Self always granted: self_lsn = ctx->self_lsn (stored unconditionally)
* - Each peer with vote_granted=1: lsn = votes[i].peer_lsn
* 2. Sort granted LSNs descending.
* 3. safe_committed_lsn = sorted[floor(n_granted / 2)]
* (the majority-threshold element — the highest LSN held by a majority)
* 4. Same logic for config tags (ctx->self_tag / votes[i].peer_tag).
*
* Single-node (n_peers == 0): votes[] is empty; safe_committed_lsn = ctx->self_lsn.
* This prevents WAL truncation to 0 on re-election in a single-node cluster.
*
* Requirements: 2.3 (ha-repl-hardening spec)
* Boundary: Controller FSM (ha.c)
*/
static int _cmp_u64_desc(const void *a, const void *b)
{
uint64_t ua = *(const uint64_t *)a;
uint64_t ub = *(const uint64_t *)b;
if (ua > ub) return -1;
if (ua < ub) return 1;
return 0;
}
void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx,
uint64_t *safe_lsn_out,
uint64_t *safe_tag_out)
{
if (!ctx || !safe_lsn_out || !safe_tag_out) return;
/* Single-node: no votes[] entries — self is the only majority.
* Return self_lsn so WAL is not truncated to 0 on re-election. */
if (ctx->n_peers == 0) {
*safe_lsn_out = ctx->self_lsn;
*safe_tag_out = ctx->self_tag;
return;
}
/* Collect granted voter LSNs and tags.
* Maximum granted voters = n_peers + 1 (self + all other peers). */
int max_voters = ctx->n_peers + 1;
uint64_t *lsns = calloc((size_t)max_voters, sizeof(uint64_t));
uint64_t *tags = calloc((size_t)max_voters, sizeof(uint64_t));
if (!lsns || !tags) {
free(lsns);
free(tags);
*safe_lsn_out = 0;
*safe_tag_out = 0;
return;
}
int n_granted = 0;
/* Self always granted: self's lsn/tag from the dedicated self_lsn/self_tag fields */
lsns[n_granted] = ctx->self_lsn;
tags[n_granted] = ctx->self_tag;
n_granted++;
/* Each peer with vote_granted=1 */
for (int i = 0; i < ctx->n_peers; i++) {
if (ctx->votes[i].vote_granted) {
lsns[n_granted] = ctx->votes[i].peer_lsn;
tags[n_granted] = ctx->votes[i].peer_tag;
n_granted++;
}
}
/* Sort descending */
qsort(lsns, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc);
qsort(tags, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc);
/* Majority-threshold element: index = floor(n_granted / 2)
* For n_granted=3: index=1 (2nd element, 0-based) — majority=2 out of 3.
* For n_granted=1: index=0 (self only). */
int idx = n_granted / 2;
*safe_lsn_out = lsns[idx];
*safe_tag_out = tags[idx];
free(lsns);
free(tags);
INFO("HA: safe_committed_lsn=%" PRIu64 " safe_config_tag=%" PRIu64
" (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted);
}
/* ── bug-1060: never truncate the new leader's own durable log ───────────────
*
* The post-election WAL truncation cutoff must never fall below the leader's own
* durable tip, or it destroys entries committed by a prior leader (which this
* node had acked as a standby, and so are committed) — a Raft §5.4 Leader
* Completeness violation that diverges WALs and deadlocks the next election.
* See ha.h for the full rationale. */
uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip)
{
return safe_lsn > durable_tip ? safe_lsn : durable_tip;
}
/* ── bug-1462: never floor committed_config_tag below the leader's own
* loaded config — see ha.h for full rationale. */
uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag)
{
return safe_tag > self_tag ? safe_tag : self_tag;
}
/* ── raft-drive-async task 1.1: peer-scaled liveness reservation ─────────────
*
* Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is
* applied LAST so deadlock-freedom (>=1 guaranteed-free liveness worker) holds
* even when the n_vcpus ceiling would otherwise drive the value to 0 (e.g.
* n_vcpus == 1). See ha.h for the full rationale; the caller WARNs on the
* under-provisioned (n_vcpus < n_peers) clamp — this helper never logs. */
int drive_async_liveness_reserved(int n_peers, int n_vcpus)
{
int desired = n_peers - 1; /* worst-case concurrent liveness work */
int ceiling = n_vcpus - 1; /* keep >=1 commit (REQ_BLOCK) slot */
int reserved = desired < ceiling ? desired : ceiling; /* min(desired, ceiling) */
if (reserved < 1) /* floor applied LAST — always wins */
reserved = 1;
return reserved;
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────────
*
* ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot
* Policy Decision Table inputs from the promoting member's recovery/durability
* state and delegate to persist_recover_decision so the 3.2/3.3 table is the
* single source of truth (DRY). "Behind the HWM" is durable_floor < safe_hwm;
* a truncated log is treated as behind regardless of its volatile position
* (design.md note). A hollow node (truncated OR behind) can never be ELIGIBLE
* until it catches up — independent of quorum size (Req 2.5).
*/
recover_decision_t ha_promotion_gate_decision(bool recovery_truncated,
uint64_t durable_floor,
uint64_t safe_hwm,
bool peer_reachable,
bool catch_up_succeeded)
{
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < safe_hwm),
.healthy_peer_reachable = peer_reachable,
.catch_up_succeeded = catch_up_succeeded,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
return persist_recover_decision(&in);
}
/* ha_evaluate_promotion_fence: live gate over the existing in_catch_up fence
* (Req 2.2, 2.6). Reuses ha->in_catch_up — does NOT invent a parallel fence.
* - ELIGIBLE → clear the fence, return ELIGIBLE (a member that caught up past
* the committed boundary regains eligibility, Req 2.6).
* - FENCE → set the fence, return FENCE (refuse promotion; catch up from a
* healthy peer).
* - FENCE_UNRECOVERABLE → run the 3.3 loud-fence action (logs LOUDLY + sets
* the sticky non-promotable flag) and ALSO set the in_catch_up
* fence. The member STAYS ALIVE, non-promotable, retrying
* catch-up — it NEVER self-exits (Req 2.4).
*/
recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha,
bool recovery_truncated,
uint64_t durable_floor,
uint64_t safe_hwm,
bool peer_reachable,
bool catch_up_succeeded)
{
recover_decision_t d = ha_promotion_gate_decision(
recovery_truncated, durable_floor, safe_hwm, peer_reachable,
catch_up_succeeded);
switch (d) {
case RECOVER_DECISION_ELIGIBLE:
/* Release the fence: catch-up brought the member to a
* committed-consistent state at/above the cluster boundary (Req 2.6). */
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
break;
case RECOVER_DECISION_FENCE:
/* Refuse promotion: extend the catch-up fence so the member catches up
* from a healthy peer before it can lead (Req 2.2). */
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
INFO("HA: promotion FENCED — recovery_truncated=%d durable_floor=%"
PRIu64 " < cluster HWM=%" PRIu64 " (catch up before leading; "
"ha-commit-durability Req 2.2)",
recovery_truncated, durable_floor, safe_hwm);
break;
case RECOVER_DECISION_FENCE_UNRECOVERABLE: {
/* Unrecoverable: route through the 3.3 loud-fence action — logs LOUDLY
* and sets the sticky non-promotable flag. Also hold the in_catch_up
* fence so the member refuses promotion. The process STAYS ALIVE and
* keeps retrying catch-up; it NEVER self-exits (Req 2.4). */
recover_policy_inputs_t in = {
.verdict = recovery_truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < safe_hwm),
.healthy_peer_reachable = peer_reachable,
.catch_up_succeeded = catch_up_succeeded,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = safe_hwm,
};
persist_t *p = (ha != NULL && ha->raft_c != NULL) ? &ha->raft_c->persist
: NULL;
persist_recover_loud_fence_if_unrecoverable(p, &in);
if (ha != NULL)
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
break;
}
}
return d;
}
/* ── ha-commit-durability task 4.4: bounded-window catch-up re-alert ─────────
*
* (never-self-exit rework 2026-06-18.) A member that wins an election while
* recovery-truncated / behind the HWM with no peer reachable AT THE PROMOTION
* INSTANT is FENCED (in_catch_up set, non-promotable) and STAYS ALIVE, retrying
* catch-up indefinitely. This one-shot timer exists ONLY for operator
* VISIBILITY: when it fires, if the member is STILL unrecoverable (still
* truncated/behind AND no peer AND no progress) it RE-EMITS the loud,
* operator-visible alert via persist_recover_loud_fence_if_unrecoverable (which
* also keeps the sticky non-promotable flag set) — it NEVER terminates the
* process. If the member has caught up (not truncated AND durable_floor >=
* HWM) the timer RELEASES the fence (Req 2.6). If progress is possible (peer
* back, or durable floor advanced) it leaves the member fenced under the
* existing catch-up machinery.
*
* Earlier this timer DEFERRED-then-ABORTED. The abort is GONE: a controller
* MUST NEVER self-terminate as a designed response to an operational condition
* (user invariant, 2026-06-18). The task-5.2 soak quorum collapse was caused
* by that self-abort; staying alive + retrying self-heals when a peer returns.
*
* The window is a settable seam so the timer test runs fast. Default =
* DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */
static _Atomic uint32_t s_promotion_fence_window_ms =
DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
uint32_t ha_promotion_fence_window_ms(void)
{
uint32_t w = atomic_load_explicit(&s_promotion_fence_window_ms,
memory_order_acquire);
return w ? w : DEFAULT_RECOVERY_CATCHUP_WINDOW_MS;
}
#ifdef XCE_TESTING
void ha_set_promotion_fence_window_ms(uint32_t ms)
{
atomic_store_explicit(&s_promotion_fence_window_ms, ms,
memory_order_release);
}
#endif
/* promotion_fence_ctx_t: delaymgr context for the bounded catch-up window.
* Captures the cluster committed high-water mark (target_hwm) and the durable
* floor AT SCHEDULE TIME (baseline_durable_floor) so the timer can detect
* catch-up progress (durable_floor advanced toward the HWM). `cancelled` is set
* by ha_cancel_promotion_fence_timer (recovery / shutdown) so a fired-but-
* cancelled callback is a safe no-op. */
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
bool was_truncated; /* recovery-truncated at schedule time */
uint64_t target_hwm; /* cluster committed high-water mark */
uint64_t baseline_durable_floor; /* durable floor at schedule time */
} promotion_fence_ctx_t;
/* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer.
* Idempotent; safe to call when no timer is armed. Used when the member catches
* up (fence released) or when the controller is shutting down — so neither a
* caught-up nor a stopped member ever reaches the deferred FATAL. */
void ha_cancel_promotion_fence_timer(ha_state_t *ha)
{
if (!ha) return;
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)atomic_exchange_explicit(
&ha->promotion_fence_ctx, NULL, memory_order_acq_rel);
if (t)
atomic_store_explicit(&t->cancelled, 1, memory_order_release);
}
/* Forward decl: the timeout callback re-arms itself for periodic re-alert. */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm);
/* promotion_fence_timeout_fn: the bounded-window re-alert timer callback. Owns
* its ctx — frees it on exit. Re-evaluates the LIVE recovery/durability state.
* It NEVER terminates the process (never-self-exit invariant, 2026-06-18):
* - caught up (NOT truncated AND durable_floor >= target_hwm) → release the
* in_catch_up fence + clear the sticky unrecoverable flag; eligible again
* (Req 2.6).
* - a healthy peer is reachable now, OR catch-up is making progress → leave
* the member fenced under the existing catch-up machinery (it will release
* via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not
* re-arm the re-alert while progress is happening.
* - STILL unrecoverable (truncated/behind AND no peer AND no progress) →
* RE-EMIT the loud operator alert via persist_recover_loud_fence_if_-
* unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the
* timer so the alert keeps repeating until a peer returns. The process
* STAYS ALIVE and keeps retrying — it NEVER aborts. */
static int promotion_fence_timeout_fn(void *arg)
{
promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg;
ha_state_t *ha = t->ha;
cluster_t *c = t->c;
/* Clear the stored handle before checking cancelled so a concurrent
* ha_cancel_promotion_fence_timer is a safe no-op (matches the
* raft_election_timeout_fn convention). */
promotion_fence_ctx_t *cur = (promotion_fence_ctx_t *)atomic_load_explicit(
&ha->promotion_fence_ctx, memory_order_acquire);
if (cur == t)
atomic_store_explicit(&ha->promotion_fence_ctx, NULL,
memory_order_release);
if (atomic_load_explicit(&t->cancelled, memory_order_acquire)) {
free(t);
return 0;
}
if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */
/* Re-read the LIVE state after the window. */
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool below_hwm = durable_floor < t->target_hwm;
bool made_progress = durable_floor > t->baseline_durable_floor;
bool peer_reachable = (ha->repl != NULL)
? (repl_active_standby_count(ha->repl) > 0)
: false;
/* raft-apply-decoupling task 5.3 (round-2 remediation, Req 8.4): a fence
* set for an APPLY_DIVERGENT reason must NEVER be live-cleared by this
* purely-structural re-check. apply_verdict is only ever recomputed by
* persist_recover() (a boot-time-once replay, cluster.c) — more WAL
* bytes arriving or a healthy peer appearing cannot prove a local
* apply-dispatch defect is fixed, so this periodic timer must treat an
* apply-divergent fence as untouchable until the next full restart
* replays cleanly (persist_recover_decision fences apply_divergent
* unconditionally for the same reason — see persist.c). */
bool apply_divergent = persist_apply_verdict(&c->persist) == APPLY_DIVERGENT;
/* self_restart_pending: same unconditional, never-live-cleared refusal as
* apply_divergent — see the field's doc comment in ha.h. This purely
* structural re-check (durable floor vs HWM) must not release a fence set
* because a demoted ex-primary's execv() failed; that node's WAL state can
* look fully caught-up while its in-memory state is still stale. */
bool self_restart_pending =
atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
/* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
* 9.4/9.5): same unconditional, never-live-cleared-by-structure refusal
* as apply_divergent/self_restart_pending above — a node stuck on a
* resource-exhaustion apply failure (task 4.1/4.2's Live Apply Resource
* Fence) can have a perfectly healthy WAL (not truncated, durable_floor
* >= HWM) by construction, since apply success and WAL durability are
* orthogonal. This purely structural re-check must not release the fence
* on WAL health alone — only the apply pump's own successful retry of the
* stuck LSN clears it (persist_clear_apply_resource_fence, called from
* inside persist_apply_pump_run itself). */
bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
/* Caught up: not truncated AND durable position reached the boundary AND
* the current apply verdict is not DIVERGENT AND no self-restart is
* pending AND no live apply resource fence is engaged → release the
* fence + clear the sticky flag; the member is eligible on its next
* election (Req 2.6, Req 9.5). */
if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending
&& !apply_resource_fenced) {
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%"
PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)",
durable_floor, t->target_hwm);
free(t);
return 0;
}
/* Still hollow, but a peer is reachable or catch-up advanced the durable
* floor: progress is possible. Leave the member fenced (in_catch_up stays
* set) under the existing catch-up machinery; do not re-arm the re-alert
* while progress is happening. */
if (peer_reachable || made_progress) {
INFO("HA: promotion fence catch-up window elapsed but progress is "
"possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%"
PRIu64 " HWM=%" PRIu64 ") — remaining fenced, catching up "
"(ha-commit-durability Req 2.2)",
peer_reachable, durable_floor, t->baseline_durable_floor,
t->target_hwm);
free(t);
return 0;
}
/* STILL unrecoverable: truncated/behind AND no healthy peer AND no catch-up
* progress. RE-EMIT the loud operator alert and keep the member fenced +
* non-promotable + ALIVE (Req 2.4) — NEVER terminate. Re-arm the re-alert
* timer so the operator keeps hearing about it until a peer returns. */
uint64_t target_hwm = t->target_hwm; /* capture before free (UAF guard) */
{
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = below_hwm,
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
/* Loud, operator-visible alert + sticky non-promotable flag. STAYS
* ALIVE: no abort/exit anywhere on this path. */
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
}
free(t);
/* Re-arm so the loud alert repeats periodically while the member remains
* unrecoverable. The member keeps retrying catch-up; the next firing
* re-evaluates and releases the fence once a peer returns and it catches up. */
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, target_hwm);
return 0;
}
/* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule
* the one-shot bounded-window RE-ALERT timer. Cancels any previously-armed
* timer first so a re-evaluation never leaks or double-fires. On a failure to
* allocate/schedule the timer it emits the loud unrecoverable fence ONCE (stays
* alive, non-promotable) — it NEVER aborts (never-self-exit invariant). */
static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c,
bool truncated, uint64_t durable_floor,
uint64_t target_hwm)
{
/* Fence first (Req 2.2): refuse promotion while hollow. */
atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release);
/* Cancel a stale timer (e.g. a prior promotion attempt). */
ha_cancel_promotion_fence_timer(ha);
promotion_fence_ctx_t *t = calloc(1, sizeof(*t));
if (!t)
goto loud_fence_no_timer;
atomic_init(&t->cancelled, 0);
t->ha = ha;
t->c = c;
t->was_truncated = truncated;
t->target_hwm = target_hwm;
t->baseline_durable_floor = durable_floor;
uint32_t window_ms = ha_promotion_fence_window_ms();
int rc = delay_task_ms(&c->delaymgr, (int)window_ms, 1,
promotion_fence_timeout_fn, t);
if (rc < 0) {
WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc);
free(t);
goto loud_fence_no_timer;
}
atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release);
INFO("HA: promotion FENCED + catch-up re-alert window armed (%u ms) — "
"recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64
"; staying ALIVE + non-promotable, retrying catch-up "
"(ha-commit-durability Req 2.2/2.4)",
window_ms, truncated, durable_floor, target_hwm);
return;
loud_fence_no_timer:
{
/* Could not arm the re-alert timer — still emit the loud unrecoverable
* fence once so the operator sees it; the member stays fenced
* (in_catch_up set above) + non-promotable + ALIVE. NO abort. */
recover_policy_inputs_t in = {
.verdict = truncated ? RECOVER_TRUNCATED
: RECOVER_CLEAN,
.durable_below_hwm = (durable_floor < target_hwm),
.healthy_peer_reachable = false,
.catch_up_succeeded = false,
.durable_floor_lsn = durable_floor,
.cluster_hwm_lsn = target_hwm,
};
persist_recover_loud_fence_if_unrecoverable(&c->persist, &in);
}
}
/*
* raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly.
*
* Equivalent of ha_elect_evaluate_and_signal CTRL_PRIMARY path:
* - enter CTRL_PRIMARY (no transitional state)
* - call cluster_init_phase2 on first boot
* - push config, send ctrlhello
*
* ha->raft_c must be non-NULL (production path only; test mode uses noop stubs).
*/
static void raft_cb_become_leader(void *ctx, uint64_t term)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
/* Cancel any pending election timeout */
raft_cb_cancel_timeout(ctx);
/* Sync ha->term from raft term (raft owns term increments) */
atomic_store_explicit(&ha->term, term, memory_order_release);
/* The cluster committed high-water mark for this promotion — derived from
* the voter replies (NOT a stale ctrlhello). The task-4.3 pre-promotion
* eligibility gate (below) compares the local CLAMPED durable position
* against it. Lifted to function scope so the gate can read it after the
* truncate-clamp. */
uint64_t promotion_safe_lsn = 0;
/* Task 4.1 (ha-repl-hardening): compute safe_committed_lsn from vote replies.
* Uses voter LSNs collected during election (ha->elect_ctx->votes[]) to find
* the highest LSN held by a majority without an extra round-trip.
* Design §Flow 0: Post-Election — Determine Safe Commit Point. */
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
uint64_t safe_lsn = 0;
uint64_t safe_tag = 0;
ha_compute_safe_committed_lsn(ectx, &safe_lsn, &safe_tag);
promotion_safe_lsn = safe_lsn;
/* Store results for use in WAL truncation and post-election commit.
* safe_lsn is the highest LSN on a majority; entries above it
* may not have been committed on all nodes and must be truncated.
*
* bug-1060 (Leader Completeness, Raft §5.4): the truncation cutoff is
* clamped to the leader's OWN durable tip so it never destroys entries
* this node holds durably — those were acked by this node as a standby
* and so were committed by the prior leader. Truncating to the bare
* majority element (which can sit BELOW the leader's durable tip when
* a slower peer is in the granting majority) diverged WALs and dead-
* locked the next election. With the clamp, persist_truncate_wal_above
* is a no-op on the leader's durable log; safe_lsn only governs removal
* of a non-durable divergent tail strictly above the durable tip. */
if (c) {
uint64_t durable_tip = persist_durable_floor(&c->persist);
uint64_t trunc_floor =
ha_promotion_truncate_floor(safe_lsn, durable_tip);
persist_truncate_wal_above(&c->persist, trunc_floor);
/* 2026-07-10 (chaotic-soak finding): persist_truncate_wal_above
* only advances persist->committed_lsn on the branches that
* perform a REAL truncation; its no-op path (this leader's own
* log already covers safe_lsn -- the common case) leaves
* committed_lsn untouched at its pre-election value, stalling
* commit-gated apply until the normal ack-driven commit path
* organically catches up seconds later. See
* persist_promotion_advance_committed's doc comment. */
persist_promotion_advance_committed(&c->persist, safe_lsn);
}
/* Task 4.2 (ha-repl-hardening): set committed_lsn = safe_lsn.
* persist_promotion_advance_committed (above) advances
* c->persist.committed_lsn in the production path (c != NULL),
* durable-capped and monotonic. Also update
* ha->last_known_committed_lsn so the test-mode decision loop
* (XCE_TESTING path in standby heartbeat reply handler) and any
* code reading ha->last_known_committed_lsn see the safe commit
* point immediately after promotion.
* Design §Flow 0 step 5: committed_lsn = safe_committed_lsn.
* Req 2.3 (ha-repl-hardening). */
atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn,
memory_order_release);
/* Update committed config tag baseline for the post-election decision
* loop. bug-1462: clamp to the leader's OWN loaded config tag —
* unlike WAL LSN, a node's actively-loaded config is never
* speculative, so the bare majority floor (correct for LSN) must
* not be allowed to strand a real config behind an all-zero floor
* reported by voters that never received a live config push.
* See ha_promotion_config_tag_floor's doc comment (ha.h). */
uint64_t committed_tag_floor =
ha_promotion_config_tag_floor(safe_tag, ectx->self_tag);
atomic_store_explicit(&ha->committed_config_tag, committed_tag_floor,
memory_order_release);
}
}
/* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ───────
* Refuse promotion BEFORE the node becomes authoritative CTRL_PRIMARY if it
* carries the recovery-truncated flag (task 3.1 — a structurally corrupt log
* must never lead) OR its CLAMPED durable position is below the cluster
* committed high-water mark (promotion_safe_lsn, from the voter replies).
* The durable position is read AFTER persist_truncate_wal_above above, so it
* reflects the 2.2 truncate-clamp (a node whose durable tip was clamped down
* is correctly seen as behind). A hollow node (truncated OR behind) is
* refused regardless of quorum size — combined with the 4.2 freshness floor
* it can neither win the vote nor self-promote (Req 2.5). On refusal we set
* the in_catch_up fence and return WITHOUT becoming primary; the member
* catches up from a healthy peer and a later promotion attempt passes the
* gate (Req 2.2/2.6). Only the production path (real persist) is gated; the
* c==NULL unit shim has no persist and skips it.
*
* Belt-and-suspenders (never-self-exit rework 2026-06-18): a member that has
* been marked sticky-unrecoverable (persist_recovery_unrecoverable) must
* NEVER become primary while that flag is set — it stays alive, fenced, and
* keeps retrying catch-up. The flag clears automatically on catch-up. */
if (c != NULL) {
bool truncated = persist_recovery_truncated(&c->persist);
uint64_t durable_floor = persist_durable_floor(&c->persist);
bool unrecoverable = persist_recovery_unrecoverable(&c->persist);
/* raft-apply-decoupling task 5.3 (round-2 remediation, Req 8.4):
* same guard as promotion_fence_timeout_fn above — a fence set for
* an APPLY_DIVERGENT reason must never be live-cleared, and this
* gate must never promote while the current apply verdict is
* divergent, REGARDLESS of what `unrecoverable` currently reads
* (defense-in-depth: it must not rely on the other call site alone
* having kept `unrecoverable` accurate). apply_verdict is only ever
* recomputed by a fresh boot-time persist_recover() replay. */
bool apply_divergent = persist_apply_verdict(&c->persist) == APPLY_DIVERGENT;
/* self_restart_pending (2026-07-10, ha-convergence-recovery Req 1.4
* supersession): a demoted ex-primary whose execv() re-exec failed
* must never become primary again while carrying its stale pre-
* demotion in-memory state — same unconditional-refuse treatment as
* apply_divergent, and for the same reason: this node's WAL/durability
* state can look perfectly ELIGIBLE while the in-memory state the
* restart exists to discard is still live. No live-clear path exists;
* only a successful restart (a fresh process) clears it. */
bool self_restart_pending =
atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire);
/* apply_resource_fenced (job-array-single-wal-record task 4.3, Req
* 9.4/9.5): same unconditional-refuse treatment as apply_divergent/
* self_restart_pending above — a node stuck on a resource-exhaustion
* apply failure (task 4.1/4.2's Live Apply Resource Fence) must never
* become primary while the fence is engaged, regardless of how
* structurally ELIGIBLE its WAL looks (truncated/durable_floor are
* orthogonal to apply success). Only the apply pump's own successful
* retry of the stuck LSN clears it — no election/HWM condition can. */
bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist);
/* A healthy peer is reachable for catch-up iff some voter granted with a
* live endpoint survives this election (these become the standby
* channels promoted below). */
bool peer_reachable = false;
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
for (int i = 0; i < ectx->n_peers; i++) {
if (ectx->votes[i].vote_granted && ectx->votes[i].ep != NULL) {
peer_reachable = true;
break;
}
}
}
}
/* catch_up_succeeded is false at the promotion-eligibility evaluation:
* a hollow member never silently proceeds to leadership. We evaluate
* the PURE 3.2/3.3 decision here. A FENCE or an UNRECOVERABLE
* (truncated/behind + no peer right now) both fence the member
* (in_catch_up) and arm the catch-up re-alert timer; the member STAYS
* ALIVE and keeps retrying — it NEVER self-exits (never-self-exit
* invariant, 2026-06-18; this is what fixes the task-5.2 soak collapse
* where a self-aborting recovering winner repeatedly dropped quorum). */
recover_decision_t gate = ha_promotion_gate_decision(
truncated, durable_floor, promotion_safe_lsn,
peer_reachable, /*catch_up_succeeded=*/false);
if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent
&& !self_restart_pending && !apply_resource_fenced) {
/* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, no
* pending self-restart, AND no live apply resource fence engaged:
* release the fence (a caught-up member regains eligibility, Req
* 2.6), clear the sticky flag if set, and cancel any pending
* re-alert timer. Proceed to authoritative promotion below. */
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release);
persist_clear_recovery_unrecoverable(&c->persist);
ha_cancel_promotion_fence_timer(ha);
} else {
/* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent,
* self-restart-pending, or apply-resource-fenced: refuse
* promotion — do NOT become CTRL_PRIMARY. Fence the member and
* arm the catch-up re-alert timer. The member STAYS ALIVE,
* non-promotable, retrying catch-up (Req 2.2/2.4). NEVER
* self-exit, never fence silently. An apply-divergent,
* self-restart-pending, or apply-resource-fenced member is
* refused unconditionally (Req 8.4; ha-convergence-recovery Req
* 1.4; job-array-single-wal-record Req 9.4/9.5) — no peer/HWM
* condition can clear any of these triggers; apply-resource-
* fenced clears only from inside the apply pump itself on a
* successful retry (task 4.2), the others only on a fresh boot
* (clean restart replay, or a successful re-exec). */
WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d "
"apply_divergent=%d self_restart_pending=%d "
"apply_resource_fenced=%d) — "
"recovery_truncated=%d durable_floor=%" PRIu64
" cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + "
"arming catch-up re-alert window, staying ALIVE "
"(ha-commit-durability Req 2.2/2.4; raft-apply-decoupling "
"Req 8.4; ha-convergence-recovery Req 1.4; "
"job-array-single-wal-record Req 9.4/9.5)",
(int)gate, unrecoverable, apply_divergent, self_restart_pending,
apply_resource_fenced, truncated, durable_floor,
promotion_safe_lsn, peer_reachable);
ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor,
promotion_safe_lsn);
return;
}
}
/* ── ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): config
* serve-fence promotion gate ──────────────────────────────────────────────
* A member whose latest committed config marker references bytes it does not
* hold (config_serve_fenced) MUST NOT become authoritative CTRL_PRIMARY — it
* cannot serve under the committed configuration. Refuse promotion and STAY
* ALIVE (controller-never-self-terminate): the member remains non-promotable
* and keeps retrying; once the correct bytes arrive out-of-band and re-verify
* clears the fence, a later promotion attempt passes this gate. The check is a
* pure atomic read (no I/O) — safe on the election path. */
if (config_serve_fenced(c)) {
WARN("HA: promotion REFUSED — config serve-FENCED (committed config "
"marker references bytes not present/matching locally); member "
"stays ALIVE, non-promotable, awaiting out-of-band config bytes "
"(ha-term-safety R7.9)");
return; /* do NOT become CTRL_PRIMARY; never self-exit */
}
/* ── config-upgrade-snapshot task 6.2 (Req 1.8, 3.2, 3.3, 4.3, 5.5):
* Activation Fence promotion gate ──────────────────────────────────────
* A member whose OWN upgrade-snapshot materialization for the current
* committed config marker has not completed and hash-verified must not
* become authoritative CTRL_PRIMARY — doing so would let it win an
* election (Raft's log-freshness gate only proves it has the committed
* MARKER, not that it has finished MATERIALIZING under it) and then
* immediately violate the serving gate above the moment it starts
* accepting requests. This is a SEPARATE, SEQUENTIAL check from the
* config serve-fence directly above (same rationale as the dispatch-gate
* composition: byte-mismatch and materialization-incomplete are distinct
* conditions) — never a new promotion-eligibility mechanism for WAL/log
* completeness (Req 3.1 is explicit that Raft's own freshness gate
* already owns that; this fence is layered on top, purely about
* materialization completeness). Refuse promotion and STAY ALIVE
* (controller-never-self-terminate): the member remains non-promotable
* and keeps retrying (schedule_upgrade_materialization_retry, driven off
* the commit-advance path); once its own materialization completes and
* re-verify clears the fence, a later promotion attempt passes this
* gate. Pure atomic read (no I/O) — safe on the election path. */
if (upgrade_materialization_fenced(c)) {
WARN("HA: promotion REFUSED — upgrade materialization FENCED (this "
"member's own upgrade-snapshot materialization for the "
"committed config marker is incomplete or hash-mismatched); "
"member stays ALIVE, non-promotable, retrying materialization "
"(config-upgrade-snapshot Req 1.8/3.2/3.3/4.3/5.5)");
return; /* do NOT become CTRL_PRIMARY; never self-exit */
}
int is_first_boot = !ha->phase2_done;
/* First boot: call cluster_init_phase2 */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
if (ha_phase2_hook != NULL)
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
p2rc = cluster_init_phase2(c);
#endif
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure, and do NOT
* proceed to CTRL_PRIMARY with an unwired commit engine — mirror
* the upgrade_materialization_fenced refusal just above (stay
* ALIVE, non-promotable, retry; a later promotion attempt
* succeeds once ha_schedule_phase2_retry completes). Before this
* fix, phase2_done was latched unconditionally, so this failure
* path never existed for the code below to worry about. */
WARN("HA: promotion REFUSED — cluster_init_phase2 failed "
"(rc=%d); member stays ALIVE, non-promotable, retrying "
"(bug-1477, never self-terminate)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
return; /* do NOT become CTRL_PRIMARY with an unwired commit engine */
}
/* bug-1445 fix: cluster_init_phase2() above just ran this process's
* very FIRST persist_recover()/replay_wal() -- the ONLY thing that
* can engage the Live Apply Resource Fence (job-array-single-wal-
* record tasks 4.1/4.2, extended to boot replay by task 7.2) for
* THIS node. The eligibility gate above (~line 1337) necessarily
* read persist_apply_resource_fenced() as false: a fresh persist_t
* defaults to unfenced (persist_init) and this is the first replay
* that could ever have set it -- so that gate's ELIGIBLE decision
* was made blind to whatever this first replay just did. Re-check
* here, once, right after the only call that could have changed the
* answer: if this node's own first-ever replay just fenced it,
* refuse promotion now instead of falling through to CTRL_PRIMARY
* below on a decision that is now stale. Every later election in
* this process's lifetime is unaffected -- is_first_boot is false
* by then, so the gate above already covers it correctly (this is
* exactly what the sibling test,
* test_replay_wal_engages_fence_on_resource_exhaustion_failure,
* proves). Mirrors the existing refusal branch above verbatim (same
* WARN shape, same ha_arm_promotion_fence_timer() call, same
* unconditional-refuse treatment as apply_divergent/
* self_restart_pending) rather than inventing a second mechanism. */
if (c != NULL && persist_apply_resource_fenced(&c->persist)) {
WARN("HA: promotion REFUSED — this node's own first-ever boot "
"replay (cluster_init_phase2, just completed) engaged the "
"Live Apply Resource Fence; the earlier eligibility check "
"in this same election evaluation necessarily missed it "
"(bug-1445) — member stays ALIVE, non-promotable, "
"retrying apply (job-array-single-wal-record Req 9.4/9.5)");
ha_arm_promotion_fence_timer(ha, c,
persist_recovery_truncated(&c->persist),
persist_durable_floor(&c->persist),
promotion_safe_lsn);
return;
}
}
/* ha-single-leader-role (Req 1.1): enter CTRL_PRIMARY directly — there is
* no transitional COMMITTING state. New-leader safety comes from the
* own-term commit rule (the WAL_NOOP + writable_lsn gate armed below),
* not from a fourth role value. The raft-owned term was already synced
* at entry (~line 463), so we do NOT call ha_do_promote here (it would
* double-increment the term, research.md §4 finding 2). This store sits
* OUTSIDE the c != NULL guard so the c==NULL test shim
* (raft_cb_become_leader_for_test) reaches it. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
memory_order_release);
INFO("HA FSM (raft): → CTRL_PRIMARY (term=%" PRIu64
", first_boot=%d)", term, is_first_boot);
/* 2026-07-10 (chaotic-soak regression finding, bug-1043 family recurrence):
* a node promoted from standby can carry an apply-lag backlog -- entries
* already committed per c->persist.committed_lsn (just finalized above by
* the truncate-clamp / safe_lsn assignment) but never materialized into
* the job/queue table, because standby-side materialization is now
* commit-gated and only kicked by heartbeat consumption or catch-up
* ingest (ha_standby_advance_committed_and_apply) -- BOTH of which stop
* firing the instant this node becomes CTRL_PRIMARY (ha_consume_heartbeat_
* committed explicitly skips primaries). Without draining here, that
* backlog sits unapplied: expect-job/joblist on the new primary would
* report a genuinely committed (WAL-consistent, walcompare-clean) job as
* missing.
*
* SYNCHRONOUS on purpose (persist_apply_pump_run_sync, not the async
* persist_apply_pump_kick): the async kick only ENQUEUES the drain onto a
* vtask and returns immediately, so this function (and thus promotion)
* could complete -- and the node could start answering expect-job/joblist
* queries as primary -- before the vtask worker actually finished
* applying the backlog. A live chaotic HA soak caught exactly that race
* (2026-07-10): the async version still lost committed jobs on a freshly
* promoted primary.
*
* THREAD-CONTEXT CORRECTION (2026-07-10, controller-invariant-audit): this
* comment previously claimed promotion "already runs on a VTASK_LIVE_BLOCK
* vtask" -- that is TRUE only for the single-node self-grant and RDHUP-
* failover election-start paths (ha_dispatch_election_start). It is FALSE
* for the common real multi-node case: a winning RequestVoteReply is
* dispatched via peer_ctrl_classify, which hardcodes VTASK_LIVENESS (never
* admission-gated, documented as never expected to block -- see the
* bug-1198 hazard box above peer_ctrl_classify's definition, which closed
* this EXACT class of bug for HA_REPL_SEND_SNAPSHOT's persist_checkpoint).
* persist_apply_pump_run_sync's pread()s of recently-written WAL segments
* are lower practical risk than a bare fdatasync (almost always page-cache
* hot, since the entries were just appended) but are still, strictly,
* blocking I/O on a task class the design says never blocks. This residual
* gap is tracked (see buglog.json) rather than fixed inline here: closing
* it properly means either reclassifying the winning-vote-reply case in
* peer_ctrl_classify or dispatching promotion's tail onto a fresh
* VTASK_LIVE_BLOCK vtask, both larger changes needing their own dedicated
* soak-validation pass. Do NOT remove this call to "fix" the classification
* gap -- it is what closes the bug-1043-family apply-lag/job-loss bug;
* removing it reopens that regression. The backlog itself stays inherently
* small (only entries committed since the last heartbeat-driven kick). */
if (c)
persist_apply_pump_run_sync(&c->persist);
/* Req 2.1 (ha-repl-hardening): re-initialize per-standby convergence
* tracking for THIS primary term. Stale converged flags from a
* previous term (demote → re-elect cycle) must not satisfy the
* majority-convergence gate before standbys re-report their state. */
memset(ha->standby_reported_tag, 0, sizeof(ha->standby_reported_tag));
memset(ha->standby_reported_lsn, 0, sizeof(ha->standby_reported_lsn));
memset(ha->standby_converged, 0, sizeof(ha->standby_converged));
/* Promote surviving vote endpoints to standby replication channels.
* Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to
* ha->standby_endpoints so push_config and ctrlhello reach them. */
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx && ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ectx->n_peers; i++) {
if (ectx->votes[i].ep != NULL) {
zarray_store(ha->standby_endpoints, ectx->votes[i].ep);
INFO("HA: promoted vote ep[%d] → standby channel", i);
/* Task 2.1: register promoted ep with WAL replicator.
* Req 1.1: primary must register standby with repl_add_standby
* when establishing the standby control channel.
* peer->hostname / peer->port from the vote entry's peer pointer. */
if (c && ectx->votes[i].peer != NULL) {
int rc = repl_add_standby(&c->persist.repl,
ectx->votes[i].ep,
ectx->votes[i].peer->hostname,
(uint32_t)ectx->votes[i].peer->port);
if (rc >= 0)
INFO("HA: repl registered standby ep[%d] (slot %d, %s:%u)",
i, rc,
ectx->votes[i].peer->hostname,
ectx->votes[i].peer->port);
else
WARN("HA: repl_add_standby ep[%d] failed: %d", i, rc);
}
}
}
pthread_mutex_unlock(&ha->lock);
}
}
/* Quorum-transition observability (Req 8.1, 8.2): promotion registered the
* surviving vote endpoints as standbys — the write-quorum count just changed.
* Record any loss/restore edge once (no-op when quorum is held at promotion). */
ha_log_quorum_transition(ha);
if (c) {
/* Push config to standbys — read raw bytes from disk so standbys
* can save the file. NULL bytes are passed only in test mode. */
#ifdef XCE_TESTING
if (ha_push_config_hook != NULL)
ha_push_config_hook(c);
#else
{
uint8_t *cfg_bytes = NULL;
size_t cfg_len = 0;
extern int conf_read_raw(const char *, uint8_t **, size_t *);
if (c->myconf && c->myconf->conffile &&
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
/* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON).
* The offload deep-copies cfg_bytes, so this free is still
* correct regardless of routing. */
ha_config_push_offloadable(c, cfg_bytes, cfg_len,
c->myconf->config_sha256,
c->myconf->config_version_tag);
free(cfg_bytes);
} else {
ha_config_push_offloadable(c, NULL, 0, NULL,
c->myconf ? c->myconf->config_version_tag : 0);
}
}
#endif
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up */
#ifdef XCE_TESTING
if (ha_send_ctrlhello_hook != NULL)
ha_send_ctrlhello_hook(c);
#else
ha_send_ctrlhello_to_all_standbys(c);
#endif
/* ha-single-leader-role task 2.2: register the persistence hooks FIRST,
* before the own-term no-op append, so the persistence seam is live
* before any racing client write can reach the cluster. Idempotent
* via the hooks_registered guard. */
ha_register_persist_hooks(c);
/* Start primary heartbeat immediately so standbys receive NOP and their
* liveness poll does not fire. ha_start_primary_heartbeat is also
* called in ha_promote_to_primary; the idempotency guard (role check
* at fn top) prevents double-arming. */
ha_start_primary_heartbeat(c);
/* ha-single-leader-role task 2.2 (Req 1.4, 2.1, 2.2): append the
* own-term WAL_NOOP and arm the commit engine's writable_lsn gate.
* This is the shared promotion tail both roads to PRIMARY run.
* Non-blocking: promotion never waits for the no-op to commit. */
ha_arm_own_term_gate(c);
}
}
/*
* raft_cb_become_follower: node lost election or saw higher term — enter STANDBY.
*
* Syncs ha->term from the raft-reported term, sets role to CTRL_STANDBY,
* calls cluster_init_phase2 on first boot (standbys also need it to
* receive config replication push from the new primary).
*/
static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx)
{
(void)leader_idx;
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
/* Cancel any pending election timeout */
raft_cb_cancel_timeout(ctx);
/* Sync ha->term */
atomic_store_explicit(&ha->term, term, memory_order_release);
int is_first_boot = !ha->phase2_done;
/* First boot: call cluster_init_phase2 so structures are ready */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
if (ha_phase2_hook != NULL)
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
p2rc = cluster_init_phase2(c);
#endif
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure — a standby
* (unlike a promoted primary) can safely stay passive while
* ha_schedule_phase2_retry retries; nothing below depends on
* phase2 having succeeded. */
WARN("HA: cluster_init_phase2 on become-follower failed (rc=%d) "
"— member stays alive, will retry (bug-1477)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
}
}
/* Close all outgoing vote endpoints (loser path).
* The primary will push config and ctrlhello via promoted connections;
* the standby waits passively on its listener for the inbound connection.
*
* FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197): when
* g_drive_async_peerctrl_dispatch_enabled is ON this loop runs on a POOL
* worker and frees ep->data while the io_worker may still deliver a queued
* peer-control event on another ep → UAF. Before task 3.4 flips the flag
* ON, each endpoint_fini below MUST be preceded by iomgr_unmonitor(ep). See
* the full hazard note at peer_ctrl_dispatch_to_pool. Safe today (flag OFF). */
{
ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit(
&ha->elect_ctx, memory_order_acquire);
if (ectx) {
for (int i = 0; i < ectx->n_peers; i++) {
endpoint_t *ep = ectx->votes[i].ep;
if (ep == NULL)
continue;
INFO("HA: closing vote ep[%d] (loser path)", i);
if (ha_peerctrl_dispatch_enabled() && ha->pool != NULL) {
/* bug-1197: this callback runs on a pool worker; another
* worker may be mid-vtask (or have a queued vtask) on this
* same ep reading ep->data. Defer teardown onto the ep's
* per-ep FIFO so endpoint_fini+free runs strictly after the
* last live use, and gate late dispatch (tearing_down). */
vote_ep_teardown_async(ha, ep);
} else {
/* bug-1298: drop the owner ref (deferred fini+free if a pool
* vtask still references this vote-ep). Also fixes the old
* flag-OFF struct leak (was endpoint_fini-only). */
endpoint_unref(ep);
}
ectx->votes[i].ep = NULL;
}
}
}
/* ha-convergence-recovery Req 1.1-1.3, 1.5: demotion convergence.
*
* Read the PRIOR role BEFORE the CTRL_STANDBY store below. If this node was
* the PRIMARY, it may hold an uncommitted, potentially divergent WAL suffix
* and provisional in-memory state. Synchronously disable the three
* persistence hooks HERE (before any further event can be persisted — Req
* 1.1, reusing ha_demote_to_standby's disable sequence), then schedule a
* one-shot delaymgr task that truncates above committed_lsn and self-restarts
* (Req 1.2/1.3/1.5) off this callback thread (D1: no file I/O or exec on the
* iomgr/raft callback thread).
*
* Non-PRIMARY demotions (STANDBY/INIT/ELECTING) take the unchanged path —
* they have no registered hooks and no provisional suffix to discard, so
* their behavior is byte-equivalent to before this spec. */
ctrl_role_t prior_role = (ctrl_role_t)atomic_load_explicit(
&ha->role, memory_order_acquire);
if (prior_role == CTRL_PRIMARY && c != NULL) {
/* Req 1.1: disable persistence hooks SYNCHRONOUSLY, before the role
* store and before scheduling, so no provisional entry can be produced
* after the higher term was observed. Mirrors ha_demote_to_standby
* (ha.c) — disable each hook, then clear the idempotency guard. */
if (ha->hooks_registered) {
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun);
ha->hooks_registered = 0;
}
/* raft-apply-decoupling task 1.4 (Req 1.3, 1.4, 9.3, 9.4): a demoted
* PRIMARY can never advance committed_lsn again for anything it had
* pending -- resolve every currently registered commit continuation
* as FAILED now, synchronously with the rest of this "stop being
* primary" sequence, rather than leaving a requester parked until
* the commit engine's own timeout-sweep thread (task 1.3) eventually
* times it out. lsn_floor=0 resolves ALL pending continuations (lsn
* 0 is never a real registration -- the free-slot sentinel). Uses
* the same -XCE_ERR_STANDBY vocabulary the HA dispatch gate already
* uses to reject writes to a non-primary. Reachable only when this
* node was already PRIMARY, which means cluster_init_phase2 (and
* therefore commit_engine_init) already ran, so c->commit is always
* initialized here. */
commit_resolve_pending_above(&c->commit, 0, -XCE_ERR_STANDBY);
}
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
INFO("HA FSM (raft): → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
term, is_first_boot);
/* Req 1.2/1.3/1.5: schedule the deferred truncate-above-committed-LSN +
* self-restart now that hooks are disabled and the role is STANDBY. Only
* the was-PRIMARY edge converges; all other edges skip this entirely. */
if (prior_role == CTRL_PRIMARY && c != NULL) {
WARN("HA: PRIMARY demoted (term=%" PRIu64 ") — scheduling demotion "
"convergence (truncate above committed_lsn + self-restart)", term);
(void)ha_schedule_demotion_convergence(c);
}
/* Arm heartbeat watchdog — fires ha_on_primary_ctrl_fd_rdhup if primary
* sends no ctrlhello within election_timeout_ms. (This also resets
* last_activity = now, granting a full grace window before the first poll.) */
ha_arm_heartbeat_timeout_vtask(ha, c);
}
/*
* raft_cb_persist: persist term + voted_for to disk before granting vote.
*
* Maps raft integer voted_for index to a candidate name using cluster config.
* Updates ha->term, ha->voted_for_term, ha->voted_for_name then calls
* ha_persist_term(ha, workdir).
*
* voted_for == -1 means "no vote"; voted_for >= 0 is the candidate index.
* Returns 0 on success, non-zero on error (causes raft library to deny vote).
*/
static int raft_cb_persist(void *ctx, uint64_t term, int voted_for)
{
ha_state_t *ha = (ha_state_t *)ctx;
cluster_t *c = ha->raft_c;
/* Store ha-term.bin under {workdir}/{myname}/data/ alongside WAL/snapshot. */
char data_dir[1024] = {0};
if (c && c->myconf && c->myconf->workdir && c->myconf->name)
snprintf(data_dir, sizeof(data_dir), "%s/%s/data",
c->myconf->workdir, c->myconf->name);
const char *workdir = data_dir[0] ? data_dir : NULL;
/* Update ha->term */
atomic_store_explicit(&ha->term, term, memory_order_release);
/* Map voted_for index to name */
pthread_mutex_lock(&ha->lock);
if (voted_for == -1) {
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
} else {
ha->voted_for_term = term;
/* Look up candidate name by index */
if (c && c->myconf &&
voted_for < c->myconf->peers_cfg.candidates_count) {
const char *name = c->myconf->peers_cfg.candidates[voted_for].name;
strncpy(ha->voted_for_name, name, sizeof(ha->voted_for_name) - 1);
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
} else {
/* Test mode or out-of-range: use "self" placeholder */
if (voted_for == ha->my_priority) {
strncpy(ha->voted_for_name, "self",
sizeof(ha->voted_for_name) - 1);
} else {
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
}
}
}
pthread_mutex_unlock(&ha->lock);
ha_persist_term(ha, workdir);
return 0;
}
/* raft_cb_trace: emit raft state-machine events at DEBUG3 level.
* Active when controller is started with -vvv; suppressed otherwise. */
static void raft_cb_trace(void *ctx, const char *event, const char *detail)
{
ha_state_t *ha = (ha_state_t *)ctx;
tlog(TLOG_DEBUG3, "RAFT[%s] term=%" PRIu64 " event=%s%s%s",
ha->my_priority == 0 ? "p0" :
ha->my_priority == 1 ? "p1" : "p2",
atomic_load_explicit(&ha->term, memory_order_relaxed),
event,
detail ? " " : "", detail ? detail : "");
}
/* ── ha-term-safety: MINIMAL log/commit callback placeholders ────────────────
*
* TODO(ha-term-safety task 2.x): REPLACE these stubs with the real bodies.
* The raft library's raft_init() now REQUIRES all five non-NULL, and
* raft_on_vote_request() internally calls on_get_entry_term for its logOk gate.
* These placeholders exist ONLY to restore the controller build (tasks 1.1/1.2);
* they perform NO persistence, NO replication, and NO truncation. See CONCERNS.
*
* on_persist_entry → task 2.1 (durable WAL append/overwrite) — IMPLEMENTED
* on_get_entry_term → task 2.2 (WAL entry-term lookup; persist_entry_term, 1.3) — IMPLEMENTED
* on_send_append_entries→ task 2.3 (AppendEntries transmit) — IMPLEMENTED
* on_truncate_from → task 2.2 (conflict truncation) — IMPLEMENTED
* on_commit_advanced → task 2.4 (commit-barrier trigger / commit source) — IMPLEMENTED
*
* on_get_entry_term returns 0 (== "absent") for now: with the log-replication
* path unwired, the library's local last-log term reads as 0, so the logOk gate
* grants on lastLogTerm >= 0 — i.e. it does NOT yet tighten the existing
* config_tag/last_log_lsn freshness gate already enforced in cluster.c. The real
* accessor (task 2.2/1.3) makes the gate effective. */
/* on_persist_entry (ha-term-safety task 2.1, R1.1–R1.4): durably append/overwrite
* the log entry at 1-based `index` (== WAL LSN) stamping the LEADER-supplied
* `term` into hdr.term, then return the persist rc.
*
* Persist-before-act: this is called BEFORE the library reports the append/accept
* (raft_log_append / raft_on_append_entries_request). On a non-zero (failure) rc
* the library does NOT advance its log and a standby does NOT ACK (the existing
* persist-before-ACK + bug-850 contiguity guard sit beneath the WAL write and are
* preserved — we do not bypass them). We never false-ACK on failure.
*
* `entry` is the adapter envelope (raft_entry_blob_t prefix + payload) packed by
* the append/replication driver (tasks 3.1/3.2); the library holds no entry bytes
* and treats it as opaque. We decode (type, payload, payload_len) from it for the
* existing persist write path.
*
* Path selection by role:
* - PRIMARY own-append: persist_wal_async — assigns the contiguous next LSN
* (== index here, since the library's index == log_len+1 == current_lsn) and
* defers fsync to the group-commit flusher (the steady-state append path).
* - STANDBY replicated apply: persist_wal_write_replicated — writes at the GIVEN
* lsn=index with the contiguity guard + inline persist-before-ACK fsync.
*
* NOTE (deferred to 2.3/3.2): the leader's per-entry term is carried on the repl
* envelope by task 2.3/3.2; until then the standby path stamps the term we set
* here via persist_set_term — which IS the supplied `term` argument — so a test
* that drives this callback already records the leader term. The envelope wiring
* makes the live replication path source the same value. */
static int ha_cb_persist_entry_stub(void *ctx, uint64_t index, uint64_t term,
const void *entry, size_t len)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return -EINVAL; /* test mode without a cluster: cannot persist */
persist_t *p = &ha->raft_c->persist;
/* Decode the adapter entry envelope: a fixed raft_entry_blob_t prefix
* carrying the WAL type + payload_len, followed by the payload bytes. */
uint32_t type = WAL_NOOP;
const void *payload = NULL;
uint32_t plen = 0;
if (entry && len >= sizeof(raft_entry_blob_t)) {
const raft_entry_blob_t *blob = (const raft_entry_blob_t *)entry;
type = blob->type;
plen = blob->payload_len;
/* Reject a malformed envelope whose declared payload exceeds the buffer
* — fail loud rather than read past the blob. */
if ((size_t)plen + sizeof(*blob) > len)
return -EINVAL;
if (plen > 0)
payload = (const uint8_t *)entry + sizeof(*blob);
} else if (entry && len > 0) {
/* Defensive: an un-enveloped blob is treated as malformed. */
return -EINVAL;
}
/* Stamp the live election term so the WAL header records the LEADER's term
* for this entry (R1.1/R1.2/R1.4). */
persist_set_term(p, term);
/* Persist at LSN=index through the existing write path (persist-before-ACK
* and the bug-850 contiguity guard are preserved beneath these). */
if (ha_is_primary(ha)) {
/* Own-append: persist_wal_async assigns the contiguous next LSN, which
* equals `index` for the leader (library index == current_lsn). */
return persist_wal_async(p, type, payload, plen);
}
return persist_wal_write_replicated(p, index, type, payload, plen);
}
/* on_get_entry_term (ha-term-safety task 2.2, R1.5/R3.1): return the recorded
* Raft term for the entry at 1-based LSN `index`, sourced from the WAL term
* index via persist_entry_term. Returns 0 when the LSN is absent (no entry).
* The library calls this for its logOk / (term,lsn) reconcile gate.
*
* Legacy-0 caveat (from the task 1.3 review MEDIUM): persist_entry_term maps a
* present legacy-0 entry (hdr.term == 0, written before this feature) to
* WAL_LEGACY_TERM (== 1). In a mixed pre/post-feature cluster where a
* post-feature leader is genuinely in term 1, a (term,lsn) EQUALITY check
* cannot tell a legacy-0 entry (mapped to 1) apart from a real term-1 entry by
* term alone. This is safe because the Raft log-matching invariant (same term
* + same index ⇒ same entry, enforced across the prefix) plus the bounded
* one-time upgrade window (every entry written after upgrade carries a real
* term) mean the only collision is at the legacy/term-1 boundary, where the
* prefixes already agree. We note it explicitly rather than special-casing it. */
static uint64_t ha_cb_get_entry_term_stub(void *ctx, uint64_t index)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return 0; /* test mode without a cluster: no log → absent */
return persist_entry_term(&ha->raft_c->persist, index);
}
/* ha_read_wal_entry_at: source ONE local WAL entry at LSN `lsn` for the
* AppendEntries send path (ha-term-safety task 2.3). The library holds no entry
* bytes, so the leader reads its own WAL segment files (seg-%016llx.wal under
* wal_path) and returns the entry's type + payload. The per-LSN term is NOT read
* from the on-disk header here — the caller takes it from persist_entry_term
* (the authoritative in-memory term index), per the library contract that the
* wire entry's term == the term recorded for that LSN. `payload` is malloc'd and
* owned by the caller (freed after the send). Returns 0 on success, negative on
* absence/read error. */
static int ha_read_wal_entry_at(const char *wal_path, uint64_t lsn,
uint8_t *type_out, uint8_t **payload_out,
uint32_t *plen_out)
{
*type_out = 0;
*payload_out = NULL;
*plen_out = 0;
DIR *d = opendir(wal_path);
if (!d)
return -errno ? -errno : -EIO;
/* Enumerate segment start-LSNs, descending, so we open the latest segment
* whose start LSN is <= the target first. */
uint64_t seg_lsns[4096];
int nseg = 0;
struct dirent *de;
while ((de = readdir(d)) != NULL &&
nseg < (int)(sizeof(seg_lsns) / sizeof(*seg_lsns))) {
uint64_t sl = 0;
if (sscanf(de->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1)
seg_lsns[nseg++] = sl;
}
closedir(d);
if (nseg == 0)
return -ENOENT;
/* insertion sort ascending */
for (int i = 1; i < nseg; i++) {
uint64_t key = seg_lsns[i];
int j = i - 1;
while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; }
seg_lsns[j + 1] = key;
}
int rc = -ENOENT;
for (int si = 0; si < nseg; si++) {
/* Skip a segment whose successor already starts past the target. */
if (si + 1 < nseg && seg_lsns[si + 1] <= lsn)
continue;
if (seg_lsns[si] > lsn)
break;
char seg_path[4096];
snprintf(seg_path, sizeof(seg_path),
"%s/seg-%016" PRIx64 ".wal", wal_path, seg_lsns[si]);
int fd = open(seg_path, O_RDONLY);
if (fd < 0)
continue;
uint32_t off = 0;
for (;;) {
wal_entry_hdr_t hdr;
ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off);
if (n < (ssize_t)sizeof(hdr)) break;
if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break;
if (hdr.lsn == lsn) {
*type_out = (uint8_t)hdr.type;
if (hdr.payload_len > 0) {
uint8_t *pbuf = malloc(hdr.payload_len);
if (!pbuf) { rc = -ENOMEM; }
else {
ssize_t pr = pread(fd, pbuf, hdr.payload_len,
(off_t)(off + sizeof(hdr)));
if (pr == (ssize_t)hdr.payload_len) {
*payload_out = pbuf;
*plen_out = hdr.payload_len;
rc = 0;
} else {
free(pbuf);
rc = -EIO;
}
}
} else {
rc = 0; /* zero-payload entry (e.g. WAL_NOOP) */
}
close(fd);
return rc;
}
off += (uint32_t)(sizeof(hdr) + hdr.payload_len);
}
close(fd);
}
return rc;
}
/* on_send_append_entries (ha-term-safety task 2.3, R1.3/R3.1): transmit ONE
* AppendEntries to the standby at slot `peer_idx`, carrying the prev tuple
* (prev_log_index, prev_log_term) + leader_commit, and — for n_entries==1 — the
* single entry at LSN prev_log_index+1 sourced from this leader's OWN WAL (the
* library holds no entry bytes). The entry's term is taken from
* persist_entry_term (the authoritative per-LSN term index), so the standby
* persists the LEADER's recorded term, not a substitute. n_entries==0 emits a
* heartbeat (zero entries) carrying only the prev tuple + leader_commit.
*
* Non-blocking (ha-async-io): delegates to repl_send_append_entries which
* enqueues via the async repl transport — never blocks in the callback.
* peer_idx is the library's raft peer index, mapped 1:1 to the repl standby
* registry slot (the same index match_index/next_index use).
*
* DORMANT IN PRODUCTION (raft-adapter-conformance task 4.1, Requirement 3.3):
* this callback is wired as on_send_append_entries (see s_real_cbs below) and
* is fully protocol-conformant, but it is reachable ONLY via the raft
* library's raft_send_append_entries (src/lib/raft/raft.c:595), which has
* ZERO production call sites — every caller is either
* src/lib/raft/raft_sim.c (the pure library simulation harness) or the
* XCE_TESTING-only test helper ha_cb_send_append_entries_for_test (below).
* Nothing in the running controller ever DRIVES raft_send_append_entries, so
* this callback is registered but never invoked in production.
*
* The mechanism that ACTUALLY drives AppendEntries-equivalent
* send/retry/backoff in production today is a different, coarser
* substitute: repl_broadcast (src/controller/repl.c) fans out each
* freshly-appended WAL entry to every standby as it is written (retrying via
* ha_raft_on_repl_nack on a NACK), and ha_stream_wal_to_standby
* (src/controller/config_repl.c) streams a standby's catch-up range once it
* falls behind. Together these two are the sole production
* AppendEntries-equivalent send/retry/backoff mechanism for Requirement 3.3
* — NOT this per-peer, nextIndex-driven sender.
*
* This is a deliberate documentation-only resolution, not an oversight: see
* research.md's "Documentation-only resolution for the dead nextIndex-driven
* AppendEntries sender (Requirement 3)" decision and
* src/controller/CLAUDE.md's TLA+ Action-to-Code Mapping for the recorded
* disposition. The dead path is kept as an intentionally-dormant,
* protocol-exact reference implementation (retiring or wiring it in were
* both rejected — see research.md) rather than silently left undocumented.
* Its dormancy is guarded against silent regression by the XCE_TESTING-only
* ha_send_append_entries_stub_call_count counter incremented immediately
* below: a future change that accidentally gives raft_send_append_entries a
* production caller moves this counter off zero, which task 4.2's
* non-regression test (test_ha_core_safety.c) asserts against while
* confirming repl_broadcast/ha_stream_wal_to_standby fire for the same
* scenario.
*
* See also the independently-derived reachability trace at
* test_ha_core_safety.c:2436-2464 and the ha_apply_single_entry comment
* above (raft-ha-core-safety task 6.2), which confirm the same
* zero-production-caller finding for this function's only caller
* (repl_send_append_entries). */
#ifdef XCE_TESTING
/* ha_send_append_entries_stub_call_count (raft-adapter-conformance task 4.1,
* Requirement 3.3): non-regression guard counting invocations of
* ha_cb_send_append_entries_stub (the on_send_append_entries callback).
* Expected to stay at 0 across any test that drives only the production
* replication path (repl_broadcast / ha_stream_wal_to_standby) — see the
* DORMANT IN PRODUCTION comment above. External linkage under XCE_TESTING
* only, so a test can read/reset it directly (declared extern in ha.h,
* mirroring the ha_wal_ack_send_hook seam). NULL/0 semantics: always 0 in a
* production build (the whole symbol does not exist outside XCE_TESTING). */
uint64_t ha_send_append_entries_stub_call_count = 0;
#endif
static int ha_cb_send_append_entries_stub(void *ctx, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries)
{
#ifdef XCE_TESTING
ha_send_append_entries_stub_call_count++;
#endif
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return -EINVAL; /* test mode without a cluster: cannot send */
persist_t *p = &ha->raft_c->persist;
replicator_t *r = ha->repl ? ha->repl : &p->repl;
if (n_entries == 0) {
/* Heartbeat: prev tuple + leader_commit, no entry. */
return repl_send_append_entries(r, peer_idx, prev_log_index,
prev_log_term, leader_commit,
/*n_entries*/0,
/*entry_lsn*/0, /*type*/0, /*term*/0,
NULL, 0);
}
/* n_entries == 1: source the entry at prev_log_index+1 from the local WAL. */
uint64_t entry_lsn = prev_log_index + 1;
uint64_t entry_term = persist_entry_term(p, entry_lsn);
uint8_t type = 0;
uint8_t *payload = NULL;
uint32_t plen = 0;
int rc = ha_read_wal_entry_at(p->wal_path, entry_lsn, &type, &payload, &plen);
if (rc != 0) {
WARN("on_send_append_entries: cannot source local WAL entry at lsn=%lu "
"(rc=%d) — not sending to peer %d",
(unsigned long)entry_lsn, rc, peer_idx);
return rc;
}
rc = repl_send_append_entries(r, peer_idx, prev_log_index, prev_log_term,
leader_commit, /*n_entries*/1,
entry_lsn, type, entry_term, payload, plen);
free(payload);
return rc;
}
/* on_truncate_from (ha-term-safety task 2.2, R1.8/R3.2): delete the local log
* entries at LSN `index..end` because they conflict with the leader's log at a
* (term,lsn) the library detected. persist_truncate_wal_above(p, cutoff) removes
* every entry with LSN > cutoff, so to drop LSN >= index we pass cutoff =
* index - 1. This REPLACES the committed-boundary truncate heuristic on the
* library-driven path: the library, not a watermark, decides the cut point.
*
* Returns the truncate rc. A non-zero rc propagates so the library/leader
* retries; the member stays non-promotable per R3.5 (that fence is existing
* behavior and is not touched here). persist_truncate_wal_above also shrinks
* the per-LSN term index for the removed LSNs, so a later on_get_entry_term on
* a truncated LSN reads back 0 (absent). */
static int ha_cb_truncate_from_stub(void *ctx, uint64_t index)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return -EINVAL; /* test mode without a cluster: cannot truncate */
if (index == 0)
return -EINVAL; /* LSN 0 is reserved — never an entry to truncate */
return persist_truncate_wal_above(&ha->raft_c->persist, index - 1);
}
/* on_commit_advanced (ha-term-safety task 2.4, R4.1/R4.5/R5.4): feed the raft
* library's current-term-gated `commit_index` into the existing synchronous
* commit barrier as the commit SOURCE. The library fired this ONLY on a strict
* advance of its Figure-8 current-term commit index (raft.c), so the barrier
* now advances committed_lsn only when BOTH a durable majority agrees AND the
* library has admitted the entry — the term-blind safe-commit
* (ha_compute_safe_committed_lsn) is no longer the steady-state commit source.
*
* This callback only TRIGGERS the barrier (records the ceiling + re-evaluates);
* it performs NO durability itself. The barrier mechanism — majority self-cap at
* the durable floor, persist-before-ACK, monotonic committed_lsn — is unchanged
* (ha-commit-durability). The own-term writable_lsn gate is RETAINED as a
* complementary WAIT-completion predicate (it never caps committed_lsn, so it
* cannot deadlock against this source; see ha_arm_own_term_gate).
*
* Non-blocking (ha-async-io): commit_set_raft_commit_index takes only ce->lock
* and never performs I/O — safe on the iomgr ACK path that drove
* raft_advance_commit. */
static void ha_cb_commit_advanced_stub(void *ctx, uint64_t commit_index)
{
ha_state_t *ha = (ha_state_t *)ctx;
if (!ha || !ha->raft_c)
return; /* test mode without a cluster: no barrier to trigger */
commit_set_raft_commit_index(&ha->raft_c->commit, commit_index);
/* ha-term-safety task 4.3 (remediation r1, Finding B): a config marker may
* have just entered the committed prefix. Activation is NOT performed HERE:
* this callback runs on the iomgr ACK path, where the async-I/O invariant
* forbids the disk reads config_activate_committed performs
* (scan_latest_committed_marker pread + load_primary_config). The committed
* marker is instead consumed off the iomgr thread by the call sites that make
* §7.10 true in production:
* - boot/recovery (myapp.c step 7): activates the latest committed marker,
* hash-verified, after persist recovery.
* - handle_repl_config_push: after the out-of-band bytes are saved, gates
* activation on the committed marker (verify-before-activate).
*
* ha-term-safety task 4.3 (remediation r2, Finding D / R7.9): the two sites
* above are NOT sufficient when the BYTES arrive (and are saved) BEFORE the
* NEW marker commits — the config MARKER (WAL) and the config BYTES (out-of-
* band push) travel on INDEPENDENT transports. In that ordering the push path
* hash-checks the freshly-saved NEW bytes against the still-OLDER committed
* marker, MISMATCHES, and sets the serve-fence; when the NEW marker later
* commits HERE, nothing re-runs activation (this callback can't do disk I/O,
* and the primary's config-behind re-push net does not fire because the push
* already advanced this member's advertised config_version_tag) — so the
* standby would stay PERMANENTLY fenced. Close the gap: while serve-fenced,
* schedule an OFF-IOMGR delaymgr task (config_schedule_activate_retry, NON-
* BLOCKING ordq_put only) that re-runs config_activate_committed on a thread
* allowed to block on disk; once the NEW marker is in the committed prefix and
* the saved bytes verify, the retry clears the fence and activates. No-op once
* unfenced (no busy-spin); a still-genuine mismatch reschedules on the next
* commit advance. */
if (config_serve_fenced(ha->raft_c))
config_schedule_activate_retry(ha->raft_c);
/* config-upgrade-snapshot task 6.2 (Req 3.4, 5.5): symmetric commit-
* advance trigger for the Activation Fence. Materialization technically
* completes on APPLY (persist.c's WAL_CONFIG_MARKER case), which lags
* commit under raft-apply-decoupling — this hook does not need to fire
* at the PRECISE instant materialization becomes possible, only to
* eventually re-check and clear the fence once conditions allow (Req
* 3.4/5.5's "keep retrying" framing, not a precise event-driven
* guarantee). Commit advancing is a reasonable, precedented "something
* changed, worth re-checking" signal, and schedule_upgrade_materialization_
* retry itself is a no-op when unfenced, so this adds no busy-spin in the
* steady state. */
if (upgrade_materialization_fenced(ha->raft_c))
schedule_upgrade_materialization_retry(ha->raft_c);
}
static const raft_callbacks_t s_real_cbs = {
.on_send_vote_request = raft_cb_send_vote_request,
.on_send_pre_vote_request = raft_cb_send_pre_vote_request,
.on_vote_eligible = raft_cb_vote_eligible,
.on_become_leader = raft_cb_become_leader,
.on_become_follower = raft_cb_become_follower,
.on_persist = raft_cb_persist,
.on_schedule_timeout = raft_cb_schedule_timeout,
.on_cancel_timeout = raft_cb_cancel_timeout,
.on_start_election_after_delay = raft_cb_start_election_after_delay,
/* ha-term-safety: MINIMAL placeholders to satisfy raft_init's non-NULL
* requirement — REPLACE in task 2.x (see stubs + CONCERNS). */
.on_persist_entry = ha_cb_persist_entry_stub,
.on_get_entry_term = ha_cb_get_entry_term_stub,
.on_send_append_entries = ha_cb_send_append_entries_stub,
.on_truncate_from = ha_cb_truncate_from_stub,
.on_commit_advanced = ha_cb_commit_advanced_stub,
.on_trace = raft_cb_trace,
};
/* ── PRNG seeding ───────────────────────────────────────────── */
void ha_seed_jitter(uint64_t *seed, int my_priority)
{
uint8_t buf[8] = {0};
int fd = open("/dev/urandom", O_RDONLY);
if (fd >= 0) {
ssize_t n = 0;
while (n < 8) {
ssize_t r = read(fd, buf + n, (size_t)(8 - n));
if (r <= 0) break;
n += r;
}
close(fd);
if (n == 8) {
memcpy(seed, buf, 8);
if (*seed != 0) return;
}
}
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
*seed = (uint64_t)ts.tv_nsec
^ ((uint64_t)my_priority << 56)
^ ((uint64_t)getpid() << 16);
}
int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority)
{
if (!ha || !cfg)
return -EINVAL;
/* Zero first so ha_fini() is safe even if validation fails below */
memset(ha, 0, sizeof(*ha));
if (cfg->heartbeat_interval_ms < 2000)
return -EINVAL;
if (cfg->heartbeat_timeout_ms < cfg->heartbeat_interval_ms)
return -EINVAL;
if (cfg->election_timeout_ms < 500)
return -EINVAL;
ha->config = *cfg;
ha->my_priority = my_priority;
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
atomic_store_explicit(&ha->applied_lsn, 0ULL, memory_order_release);
atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed);
atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed);
atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed);
/* failover_requested / watchdog_stop deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */
pthread_mutex_init(&ha->lock, NULL);
/* bug-1435: dedicated lock serializing raft_state_t's election/vote entry
* points against concurrent pool-worker dispatch (see ha.h field doc).
* RECURSIVE: a WON/quorum outcome legitimately re-enters the state machine
* synchronously on the SAME thread (e.g. raft_on_pre_vote_reply's WON
* branch -> ha_enter_real_election -> ha_start_election_async ->
* raft_start_election, all before the outer call returns) — that is a
* same-thread call chain, not a race, and must not self-deadlock. A plain
* mutex would deadlock on that legitimate nesting; PTHREAD_MUTEX_RECURSIVE
* still fully serializes the DIFFERENT-thread case this lock exists to fix. */
{
pthread_mutexattr_t raft_lock_attr;
pthread_mutexattr_init(&raft_lock_attr);
pthread_mutexattr_settype(&raft_lock_attr, PTHREAD_MUTEX_RECURSIVE);
pthread_mutex_init(&ha->raft_lock, &raft_lock_attr);
pthread_mutexattr_destroy(&raft_lock_attr);
}
ha->ssl_ctx = NULL;
ha->jitter_seed = 0;
ha->standby_endpoints = NULL;
ha->primary_ctrl_fd = -1;
/* election-iomgr-refactor Task 2.1: explicit zero-init for documentation clarity
* (memset above already zeroes these, but being explicit is better documentation) */
ha->client_ssl_ctx = NULL;
ha->phase2_done = 0;
ha->n_peers = 0;
/* ha-heartbeat-liveness Task 1.1: zero-init liveness timestamp.
* Written by ha_on_heartbeat_received (task 2.1); read by liveness_poll_fn. */
ha->last_activity = 0;
/* ha-repl-hardening Task 1.4: per-standby convergence tracking arrays.
* memset(ha, 0, sizeof(*ha)) above already zeroes these arrays; the explicit
* note documents the invariant: on promotion to CTRL_PRIMARY every
* standby starts as not-converged (tag=0, lsn=0, converged=false). */
/* heartbeat_seen starts at 0: no heartbeat seen yet.
* ha_wait_for_heartbeat_or_timeout() calls sem_timedwait on this. */
sem_init(&ha->heartbeat_seen, 0, 0);
/* ── Task 6.1/6.2: Initialize embedded raft_state_t ─────────────────────
*
* Wire real callbacks (s_real_cbs) so the state machine is ready when
* ha_start_election_async re-initialises raft with the correct n_peers /
* self_idx / initial_term after ha_restore_term().
*
* n_peers = 1 and self_idx = 0 are safe placeholders here; they are
* overwritten by ha_raft_reinit() (called from ha_start_election_async)
* which runs after ha_restore_term() and knows the real cluster size.
*
* initial_term = 0 / initial_voted_for = -1 are first-boot defaults;
* ha_raft_reinit() restores the persisted values before any election fires.
*
* ha->raft_c is NULL until ha_start_election_async sets it.
*/
ha->raft_c = NULL;
ha->raft_election_timeout_ctx = NULL;
raft_config_t raft_cfg;
memset(&raft_cfg, 0, sizeof(raft_cfg));
raft_cfg.n_peers = 1; /* placeholder; overwritten by ha_raft_reinit */
raft_cfg.self_idx = my_priority; /* self_idx == priority by convention */
raft_cfg.election_timeout_ms = cfg->election_timeout_ms;
raft_cfg.my_priority = my_priority;
raft_cfg.initial_term = 0; /* ha_restore_term() updates ha->term after ha_init */
raft_cfg.initial_voted_for = -1; /* no vote on fresh init */
raft_cfg.cbs = &s_real_cbs;
raft_cfg.ctx = ha;
raft_init(&ha->raft, &raft_cfg);
return 0;
}
/* SIGUSR1 handler / ha_install_sigusr1() deleted — failover_requested replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
void ha_do_promote(ha_state_t *ha)
{
/* Monotonically increasing fencing token — standbys reject entries from lower terms */
atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel);
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release);
/* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */
ha->last_received_crc = WAL_CRC_NONE;
/* Req 10.5: log FSM transition to PRIMARY */
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")",
atomic_load_explicit(&ha->term, memory_order_acquire));
}
void ha_do_demote(ha_state_t *ha)
{
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
/* Reset so the next ha_promote_to_primary() re-registers hooks idempotently */
ha->hooks_registered = 0;
/* Hook disable requires hooksys_t — done by ha_demote_to_standby(cluster_t*) */
/* Req 10.5: log FSM transition to STANDBY */
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ")",
atomic_load_explicit(&ha->term, memory_order_acquire));
}
/* probe_thread() / wait_any_probe() deleted — only called by am_i_primary() (election-iomgr-refactor) */
/* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */
/*
* ha_is_primary: lock-free role check.
*/
int ha_is_primary(const ha_state_t *ha)
{
return (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire)
== CTRL_PRIMARY;
}
/* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */
/* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */
/* ── P3-6: Test-only failover shim ────────────────────────────── */
#ifdef XCE_TESTING
uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha)
{
ha_do_demote(ha);
ha_do_promote(ha);
return atomic_load_explicit(&ha->term, memory_order_acquire);
}
/* ha-term-safety task 2.1: drive the WIRED on_persist_entry callback so a unit
* test exercises the real adapter body (term stamp + WAL write) without
* raft_log_append (task 3.1). Exposed only under XCE_TESTING. */
int ha_cb_persist_entry_for_test(ha_state_t *ha, uint64_t index, uint64_t term,
const void *entry, size_t len)
{
return s_real_cbs.on_persist_entry(ha, index, term, entry, len);
}
/* ha-term-safety task 2.2: drive the WIRED on_get_entry_term callback so a unit
* test exercises the real entry-term accessor. Exposed only under XCE_TESTING. */
uint64_t ha_cb_get_entry_term_for_test(ha_state_t *ha, uint64_t index)
{
return s_real_cbs.on_get_entry_term(ha, index);
}
/* ha-term-safety task 2.2: drive the WIRED on_truncate_from callback so a unit
* test exercises the real conflict truncation. Exposed only under XCE_TESTING. */
int ha_cb_truncate_from_for_test(ha_state_t *ha, uint64_t index)
{
return s_real_cbs.on_truncate_from(ha, index);
}
/* ha-term-safety task 2.3: drive the WIRED on_send_append_entries callback so a
* unit test exercises the real AppendEntries send body (peer→ep map, prev tuple
* + leader_commit carry, local-WAL entry sourcing). Pair with
* repl_send_ae_capture_hook to assert envelope content. Exposed only under
* XCE_TESTING. */
int ha_cb_send_append_entries_for_test(ha_state_t *ha, int peer_idx,
uint64_t prev_log_index,
uint64_t prev_log_term,
uint64_t leader_commit, int n_entries)
{
return s_real_cbs.on_send_append_entries(ha, peer_idx, prev_log_index,
prev_log_term, leader_commit,
n_entries);
}
/* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback so a unit
* test exercises the real commit-source trigger (feed the library commit_index
* into the cluster's commit barrier). Exposed only under XCE_TESTING. */
void ha_cb_commit_advanced_for_test(ha_state_t *ha, uint64_t commit_index)
{
s_real_cbs.on_commit_advanced(ha, commit_index);
}
/* ha-term-safety task 3.1: drive the (static) own-term gate arm — the WAL_NOOP
* append now routed through ha_primary_log_append — so a unit test can assert
* the promotion no-op records the live term via the library. Exposed only under
* XCE_TESTING. */
void ha_arm_own_term_gate_for_test(cluster_t *c)
{
ha_arm_own_term_gate(c);
}
#endif
/* ha_check_heartbeat_nocluster() / ha_watchdog_thread() / ha_start_watchdog() /
* ha_stop_watchdog() deleted — watchdog pthread replaced by
* ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */
void ha_fini(ha_state_t *ha)
{
if (!ha)
return;
/* ha-commit-durability task 4.4: cancel any pending bounded-window FATAL
* timer so a stopped member never reaches the deferred FATAL. The delaymgr
* task ctx is freed by promotion_fence_timeout_fn when it fires; cancelling
* makes that firing a no-op (it observes cancelled=1 and frees). */
ha_cancel_promotion_fence_timer(ha);
pthread_mutex_destroy(&ha->lock);
pthread_mutex_destroy(&ha->raft_lock);
sem_destroy(&ha->heartbeat_seen);
/* Fix ASAN leak: free the SSL context if one was allocated */
if (ha->ssl_ctx) {
SSL_CTX_free((SSL_CTX *)ha->ssl_ctx);
ha->ssl_ctx = NULL;
}
}
/* ── Task 2.1/2.2: WAL catch-up entry application (standby side) ─────────────
*
* ha_apply_wal_catchup_entry: apply WAL catch-up entries received from the
* primary to local persistent storage and advance applied_lsn on success.
*
* applied_lsn semantic (bug-991): the LAST CONTIGUOUSLY-HELD LSN — the same
* meaning recovery (= durable_floor), checkpoint (= checkpoint_lsn), and init
* (= 0) assign it, and the watermark the standby advertises to the leader
* (sr->applied_lsn) so the leader's catch-up streamer streams the half-open
* range (applied_lsn, committed]. NEVER lsn+1 (next-to-apply): a +1 makes the
* standby advertise one LSN too high, so the leader skips the real next-needed
* entry and the standby's library logOk wedges forever (no WalAck → commit stall).
*
* Two modes (Req 5.3, 5.5):
*
* Batch mode (entry->n_entries > 0):
* Iterate entry->entries[0..n_entries-1] in order. Persist each
* sub-entry individually (entity materialization happens later, once
* committed — see ha_standby_advance_committed_and_apply). NULL
* sub-entries are treated as corrupt and skipped (Req 3.3). After ALL
* sub-entries are processed,
* advance applied_lsn to the last-successfully-applied lsn so the next
* heartbeat reply reflects post-batch state (Req 5.3).
*
* Single-entry mode (entry->n_entries == 0):
* Backward-compatible path from task 2.1. Apply entry->lsn/entry_type
* directly and advance applied_lsn to lsn on success.
*
* In both modes:
* - applied_lsn is advanced atomically and monotonically.
* - On error for any individual entry: log ERROR, skip, continue (Req 3.3).
* - applied_lsn is in-memory only; not persisted to disk (design §Durability).
* - Standby stays in CTRL_STANDBY throughout (Req 3.4).
*
* Returns 0 on success (even if some batch entries were skipped due to errors).
* Returns -EINVAL if ha, p, or entry is NULL.
*/
/* ha_advance_applied_lsn: atomically advance ha->applied_lsn to new_lsn
* if new_lsn > current value (monotonic). */
static void ha_advance_applied_lsn(ha_state_t *ha, uint64_t new_lsn)
{
uint64_t old_lsn = (uint64_t)atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
while (new_lsn > old_lsn) {
if (atomic_compare_exchange_weak_explicit(
&ha->applied_lsn, &old_lsn, new_lsn,
memory_order_acq_rel, memory_order_acquire))
break;
}
}
/* ha_apply_single_entry: durably persist one decoded wal_catchup_entry_t
* sub-entry (name retained for history; despite the name it no longer
* materializes entity state here — see ha_standby_advance_committed_and_apply,
* 2026-07-10). Returns 0 on success, non-zero on error (caller logs and skips).
*
* raft-ha-core-safety task 6.2 — TERM-BLIND DIRECT-PERSIST BYPASS, SAFE-BY-
* GUARD (documented, not structurally retired): this function does NOT go
* through raft_on_append_entries_request's logOk/(term,lsn) reconcile — it
* calls persist_wal_write_replicated directly. Task 2.4 deferred deciding
* this bypass's fate to 6.2; 6.2's reachability trace (see the two call
* sites below and ha_apply_wal_catchup_entry) found it is UNREACHABLE with
* real production data today: every real sender (repl_broadcast for live
* replication; ha_stream_wal_to_standby for catch-up) always supplies
* n_entries>=1 with a non-zero term (persist_entry_term's legacy-zero rule
* maps a stored term of 0 to WAL_LEGACY_TERM for any present LSN — it never
* returns 0 for a real entry). The ONLY function that could ever drive a
* term==0 / n_entries==0 message into this path (repl_send_append_entries)
* is called solely from the raft library's raft_send_append_entries, which
* has zero production callers (only lib/raft/raft_sim.c and an XCE_TESTING
* helper call it) — so that trigger is dormant, not live, code. Even if this
* path were reached, the task-2.2 (term,lsn,payload-crc)-aware guard inside
* persist_wal_write_replicated already refuses to degrade a stored non-empty
* entry to empty and raises the sticky non-promotable fence on an
* unrepairable below-tip conflict — pinned by
* test_2_3_termblind_route_payload_guarded and
* test_2_3_unrepairable_catchup_raises_nonpromotable_fence
* (test_ha_core_safety.c). See ha_apply_wal_catchup_entry below and
* test_6_2_real_senders_never_produce_termblind_shape for the reachability
* evidence this comment summarizes. */
static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e)
{
/* Persist-before-apply-before-ACK: durably write the replicated entry to
* this standby's own WAL BEFORE applying it in memory (and before the
* caller ACKs). The ACK feeds the primary's commit majority, so it must
* imply durability — a standby must not ACK an entry it could lose on
* restart (Raft log[i] includes standbys; ha-async-io.md).
*
* persist_wal_write_replicated does NOT assign a new lsn (uses e->lsn) and
* does NOT re-broadcast; it fsyncs before returning and is idempotent on
* re-stream. WAL_CHECKPOINT is never streamed, but it would be rejected
* by both this primitive and apply_wal_entry. */
int prc = persist_wal_write_replicated(p, e->lsn, (uint32_t)e->entry_type,
e->raw_payload,
(uint32_t)e->raw_payload_len);
if (prc == -EAGAIN) {
/* Contiguity guard (bug-850): this entry is ahead of our log tail
* (a forward gap) — e.g. a live broadcast that arrived before the
* in-order catch-up of the prefix. NOT an error: drop it quietly and do
* NOT ack, so the catch-up continuation re-streams from current_lsn and
* the log fills in order. */
TRACE("ha_apply_single_entry: lsn=%" PRIu64 " ahead of log tail — "
"dropped (will arrive via in-order catch-up)", e->lsn);
return prc;
}
if (prc != 0) {
ERROR("ha_apply_single_entry: persist_wal_write_replicated failed for "
"lsn=%" PRIu64 " type=%u rc=%d — not applying/acking",
e->lsn, (unsigned)e->entry_type, prc);
/* No apply, no ACK on persist failure (mirrors no-ACK-on-apply-fail). */
return prc;
}
/* commit-then-apply on the standby (2026-07-10): entity materialization
* (restore_job/restore_job_state) does NOT happen here on receipt. This
* function's job ends at durable persist + ACK — the entry becomes
* in-memory state only once this node's OWN p->committed_lsn reaches it,
* driven by ha_standby_advance_committed_and_apply (heartbeat-learned
* commit position, clamped to this node's local durable tip) and applied
* through apply_committed_entry() by the same pump the primary's own
* online-apply and restart replay use. Applying eagerly here — before
* this node (or the sender) knows the entry reached cluster majority —
* could materialize a job the sender never actually committed: if a
* later (term,lsn) conflict truncate removes the WAL bytes (the sender
* lost leadership before reaching majority), nothing rolls back the
* already-applied entity mutation, leaving a permanent phantom job with
* no backing WAL entry (2026-07-10 controller-invariant audit finding).
* This mirrors bug-1043's own precedent of decoupling durability/ACK from
* entity materialization, extended one step further: apply now waits for
* COMMIT, not just local durability. */
return 0;
}
#ifdef XCE_TESTING
void (*ha_wal_ack_send_hook)(uint64_t term, uint64_t lsn, int success) = NULL;
#endif
/* ha_send_wal_ack_reply_ex: send WalAckReply {term, lsn, success} back to
* the primary over ep. Skipped when ep or c is NULL (test/offline path),
* unless ha_wal_ack_send_hook is set (XCE_TESTING), in which case the hook is
* called with the built {term, lsn, success} INSTEAD of sending over ep --
* lets a unit test observe a NACK (success=0) without a live endpoint
* (bug-1479). Task 3.1 (ha-wal-replication): ACK/NACK each applied/rejected
* WAL entry. */
static void ha_send_wal_ack_reply_ex(ha_state_t *ha, struct endpoint_t *ep,
cluster_t *c, uint64_t lsn, int success)
{
uint64_t term = (uint64_t)atomic_load_explicit(&ha->term, memory_order_acquire);
#ifdef XCE_TESTING
if (ha_wal_ack_send_hook) {
ha_wal_ack_send_hook(term, lsn, success);
return;
}
#endif
if (!ep || !c)
return;
wal_ack_reply_t ack;
ack.term = term;
ack.lsn = lsn;
ack.success = success ? 1 : 0;
xce_reply_t reply;
memset(&reply, 0, sizeof(reply));
reply.opcode = OPCode_walAck;
reply.status = 0;
reply.data_which = RequestReply_data_walackreplymsg;
reply.data.walackreplymsg = &ack;
xce_reply(ep, c->myhost, &reply);
}
/* ha_send_wal_ack_reply: send a successful WalAckReply {term, lsn,
* success=1} -- the common case, every existing call site. */
static void ha_send_wal_ack_reply(ha_state_t *ha, struct endpoint_t *ep,
cluster_t *c, uint64_t lsn)
{
ha_send_wal_ack_reply_ex(ha, ep, c, lsn, /*success*/1);
}
/* ha_apply_term_entry_via_library: route ONE term-carrying replicated entry
* (a live broadcast or a catch-up sub-entry) through the raft library
* (raft_on_append_entries_request via ha_consume_append_entries) so the LIBRARY
* does logOk + (term,lsn) reconcile and on_persist_entry stamps the LEADER's
* term — never the local term-0 direct-apply path (ha-term-safety task 3.4).
*
* `e` is the single entry (its term/prev_log_index/prev_log_term carry the AE
* metadata, set by the leader's broadcast/stream path). On accept (the entry's
* term is now recorded at its LSN) the caller ACKs and advances applied. Returns
* 1 if the entry was accepted (recorded), 0 otherwise (reject / not yet
* applicable — e.g. a logOk-failing forward gap that catch-up will backfill). */
static int ha_apply_term_entry_via_library(ha_state_t *ha, persist_t *p,
cluster_t *c, struct endpoint_t *ep,
wal_catchup_entry_t *e)
{
/* bug-1475 / bug-1466: resync the raft library's log_len from this node's
* OWN recovered/durable WAL tip before consuming the AE, exactly mirroring
* ha_sync_raft_log_from_recovery's boot-time computation (persist_durable_
* floor), but applied live, JUST-IN-TIME, on this same single AE-receive
* call path — never touching persist.c or adding any new cross-thread
* mutation of ha->raft (ha_apply_wal_catchup_entry / this function already
* own that field on this thread, same as every other s->log_len write in
* the AE-receive path below).
*
* Root cause: persist_write_checkpoint_marker (a LOCAL-only, never-
* replicated WAL_CHECKPOINT entry written by ANY role — primary or
* standby — whenever a committed WAL_CONFIG_MARKER triggers an immediate
* checkpoint) advances persist_t's current_lsn directly, WITHOUT going
* through raft_log_append — so s->log_len silently falls one behind the
* true physical WAL tip for every checkpoint written since the last
* resync (documented gap, bug-1466). Once behind, the very next AE for
* the entry immediately following the checkpoint carries a prev_log_index
* equal to the checkpoint's own LSN (config_repl.c's streamer correctly
* advances its tracker across the skipped checkpoint) — but a stale
* s->log_len makes logOk's `prev_log_index <= s->log_len` bound fail
* forever, permanently wedging this standby's catch-up (every subsequent
* heartbeat re-streams the identical doomed range with zero progress).
*
* persist_durable_floor is a lock-free atomic read of durable_lsn (the
* flusher's own fsync-confirmed watermark, which DOES count a checkpoint's
* consumed LSN slot, same as at boot) — safe to call from any thread. The
* write below (raising ha->raft.log_len) happens on the same thread that
* already owns every other s->log_len mutation in this call chain
* (raft_on_append_entries_request), so no new lock is introduced.
* Monotonic: only ever raises log_len, mirroring
* ha_sync_raft_log_from_recovery exactly. Deliberately does NOT touch the
* term index (on_get_entry_term(checkpoint_lsn) stays 0/absent on every
* member, by design — see persist_write_checkpoint_marker's own comment)
* — recording a synthetic term here would risk a cross-member mismatch
* (each member's checkpoint stamps ITS OWN current term at write time,
* which need not agree with a peer's), whereas leaving it absent keeps
* the logOk equality check (0==0) trivially and SAFELY consistent on
* every member.
*
* bug-1475 follow-up: the "stays absent on every member" claim above only
* holds because persist.c's term index now uses a TERM_INDEX_UNSET
* sentinel to keep a never-explicitly-recorded LSN (like a checkpoint's)
* reading back absent PERMANENTLY. Before that fix, persist.c's array-
* growth zero-fill silently reclassified the checkpoint's gap slot as
* present-with-WAL_LEGACY_TERM the instant any LATER lsn was recorded —
* which happens immediately on the sender (it streams after its own
* later entries exist) but not yet on a receiver that hasn't appended
* past its own checkpoint, so the two sides read DIFFERENT terms for the
* SAME checkpoint LSN and logOk permanently rejected. See
* TERM_INDEX_UNSET's comment (persist.c) and
* test_bug1475_catchup_survives_midstream_checkpoint (test_config_repl.c)
* for the full mechanism and live-instrumentation evidence. */
if (p) {
uint64_t recovered_tip = persist_durable_floor(p);
if (recovered_tip > ha->raft.log_len)
ha->raft.log_len = recovered_tip;
}
/* Wrap the single entry in an AE container whose TOP-LEVEL fields carry the
* prev tuple + term that ha_consume_append_entries reads, with entries[0]
* pointing at the entry itself (the same shape repl_send_append_entries /
* repl_broadcast produce on the wire). */
wal_catchup_entry_t *sub_arr[1] = { e };
wal_catchup_entry_t ae;
memset(&ae, 0, sizeof(ae));
ae.n_entries = 1;
ae.entries = sub_arr;
ae.term = e->term;
ae.prev_log_index = e->prev_log_index;
ae.prev_log_term = e->prev_log_term;
ae.leader_commit = e->leader_commit;
uint64_t before_term = persist_entry_term(p, e->lsn);
int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/ -1);
if (rc != 0)
return 0;
uint64_t after_term = persist_entry_term(p, e->lsn);
if (after_term == e->term) {
/* Accepted (appended / conflict-truncate-then-persist) or already-have
* (idempotent re-stream): ACK + advance applied on that signal.
*
* bug-991: applied_lsn is the "last contiguously-held LSN" (the same
* semantic recovery/checkpoint/init use, and the watermark the leader's
* catch-up streamer reads via sr->applied_lsn to stream (standby_lsn,
* committed]). Advance to e->lsn (the LSN we just held), NOT e->lsn + 1:
* a +1 over-advertises one LSN, so the leader's next stream begins at
* e->lsn + 2 and SKIPS the real next-needed entry e->lsn + 1, wedging
* logOk and stalling commit. */
(void)before_term;
/* commit-then-apply on the standby (2026-07-10, supersedes the prior
* bug-1043 re-apply-here fix): entity materialization does NOT happen
* on receipt/persist. bug-1043's original problem — a caught-up
* standby held entries on disk but not in the queryable job table —
* is now solved by ha_standby_advance_committed_and_apply, driven by
* this node's heartbeat-learned commit position, applying through the
* SAME apply_committed_entry() seam the primary's online-apply and
* restart replay use. Applying here instead (right on receipt, before
* this entry is known to have reached cluster majority) risks a
* permanent phantom job: a later (term,lsn) conflict truncate removes
* the WAL bytes with no entity rollback (2026-07-10 controller-
* invariant audit finding). */
ha_advance_applied_lsn(ha, e->lsn);
ha_send_wal_ack_reply(ha, ep, c, e->lsn);
return 1;
}
/* logOk reject (e.g. forward gap before the prefix arrives, or a genuine
* (term,lsn) conflict): bug-1479 -- send a NACK (success=0) so the
* primary's ha_raft_on_repl_nack can back off nextIndex and immediately
* mark this standby for catch-up, matching raft.tla's AppendEntries
* action (the receiver ALWAYS replies, success = logOk). Before this fix
* NOTHING was sent on reject: the primary's NACK-handling branch
* (repl_ack_decode_and_dispatch) was correct but unreachable, since no
* production path ever produced the success=0 reply it was waiting for
* -- the standby would loop rejecting the identical re-broadcast entry
* forever, discovered live via the config-upgrade soak's force-elect
* rotation. */
ha_send_wal_ack_reply_ex(ha, ep, c, e->lsn, /*success*/0);
return 0;
}
int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p,
wal_catchup_entry_t *entry,
struct endpoint_t *ep, cluster_t *c)
{
if (!ha || !p || !entry)
return -EINVAL;
/* ── ha-term-safety task 3.2: term-carrying AppendEntries → library route ──
* A term-safe AppendEntries (one sent by repl_send_append_entries / the
* library send path) carries a non-zero entry term stamp. Route it through
* raft_on_append_entries_request so the LIBRARY performs UpdateTerm (G5),
* logOk (G2), and the (term,lsn) conflict-truncate (G3) — its on_persist_entry
* callback does the WAL write (persist-before-ACK + bug-850 guard beneath).
* This REPLACES the direct persist_wal_write_replicated apply for term-safe
* entries; the legacy term-blind broadcast (entry term 0, no prev tuple) keeps
* the direct apply below until the live send path is retired (task 3.3).
*
* Single-entry term-safe AE only (n_entries==1 with a term). The library
* computes the target index from the prev tuple, then ACK on accept. */
if (c && c->ha.raft_c == c && entry->n_entries == 1 &&
entry->entries && entry->entries[0] && entry->entries[0]->term != 0) {
ha_apply_term_entry_via_library(ha, p, c, ep, entry->entries[0]);
/* 2026-07-10: a fresh entry just landed on disk -- kick the apply
* pump against the already-known commit position so a standby mid
* catch-up doesn't wait for the next heartbeat tick to materialize
* entries it's already known to have committed. */
ha_standby_advance_committed_and_apply(
c, atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire));
return 0;
}
/* ── Batch mode: entry->n_entries > 0 ─────────────────────────────────── */
if (entry->n_entries > 0) {
/* Apply each sub-entry in LSN order. Track the last successfully
* applied LSN so we can advance applied_lsn after the full batch.
* Req 5.5: applied_lsn advances only after the full batch is written.
* Req 3.3: NULL or failed entries are logged and skipped; no crash. */
uint64_t last_good_lsn = 0;
int had_good = 0;
for (int i = 0; i < entry->n_entries; i++) {
wal_catchup_entry_t *sub = entry->entries[i];
if (!sub) {
ERROR("ha_apply_wal_catchup_entry: NULL sub-entry at "
"batch index %d — skipping", i);
continue;
}
/* ha-term-safety task 3.4: a term-carrying catch-up sub-entry routes
* through the LIBRARY (logOk + (term,lsn) reconcile + on_persist_entry
* stamps the LEADER's term). Each sub-entry carries its own prev tuple
* (wal_entry_to_wire encodes prevLog{Index,Term} per element). This
* retires the term-0 direct-apply path for live/catch-up replication.
* Entries stream in ascending LSN order, so the library's contiguity
* (prev tuple) is satisfied in sequence; a not-yet-applicable entry
* (logOk reject) is left for the catch-up continuation.
*
* raft-ha-core-safety task 6.2: the `sub->term != 0` fallthrough
* below is the term-blind direct-persist bypass (ha_apply_single_
* entry) — documented SAFE-BY-GUARD, not structurally retired.
* ha_stream_wal_to_standby (the only real batch-catch-up sender)
* always stamps sub->term via persist_entry_term, which maps a
* legacy on-disk term of 0 to WAL_LEGACY_TERM for any present LSN,
* so a real streamed sub-entry is never term==0 — this fallthrough
* is unreachable with production data (see the ha_apply_single_
* entry comment for the full reachability trace + the pin test
* test_6_2_real_senders_never_produce_termblind_shape). */
if (c && c->ha.raft_c == c && sub->term != 0) {
if (ha_apply_term_entry_via_library(ha, p, c, ep, sub)) {
last_good_lsn = sub->lsn;
had_good = 1;
}
continue;
}
int rc = ha_apply_single_entry(p, sub);
if (rc != 0) {
/* -EAGAIN = contiguity gap (bug-850): entry ahead of our tail,
* dropped quietly — not an error (catch-up will backfill). */
if (rc != -EAGAIN)
ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed "
"for batch index %d lsn=%" PRIu64 " type=%u rc=%d "
"— skipping entry",
i, sub->lsn, (unsigned)sub->entry_type, rc);
/* Task 3.2: no ACK for failed/dropped entries */
continue;
}
/* Task 3.1: ACK each successfully applied sub-entry */
ha_send_wal_ack_reply(ha, ep, c, sub->lsn);
last_good_lsn = sub->lsn;
had_good = 1;
}
/* Advance applied_lsn to last_good_lsn after the full batch (bug-991:
* last contiguously-held LSN, NOT last_good_lsn + 1 — see the
* ha_apply_term_entry_via_library comment for why the +1 wedged the
* leader's catch-up streamer). Only advance if at least one entry was
* applied successfully. */
if (had_good)
ha_advance_applied_lsn(ha, last_good_lsn);
/* 2026-07-10: kick the apply pump against the already-known commit
* position so a batch catch-up materializes promptly instead of
* waiting for the next heartbeat tick (see the single-entry-term-safe
* branch above for the full rationale). */
if (had_good)
ha_standby_advance_committed_and_apply(
c, atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire));
return 0;
}
/* ── Single-entry mode (task 2.1 backward-compatible path) ──────────────
* raft-ha-core-safety task 6.2: this branch calls ha_apply_single_entry
* UNCONDITIONALLY — no term check at all — so it is the other half of the
* term-blind bypass documented at ha_apply_single_entry above. It is only
* ever reached with entry->n_entries==0, the 0-entry AppendEntries
* heartbeat shape built by repl_send_append_entries. That function is
* called solely from the raft library's raft_send_append_entries, which
* has ZERO production call sites (only lib/raft/raft_sim.c and an
* XCE_TESTING test helper call it) — so in the CURRENT codebase this
* branch is dormant/unreachable from any live sender, not a legacy-
* rollout shape carrying real data. Documented SAFE-BY-GUARD (not
* structurally retired): see the ha_apply_single_entry comment for the
* full reachability trace and the task-2.2 guard that would still bound
* the damage if this ever became reachable. */
int rc = ha_apply_single_entry(p, entry);
if (rc != 0) {
/* -EAGAIN = contiguity gap (bug-850): dropped quietly, backfilled by
* the in-order catch-up; any other rc is a real failure. */
if (rc != -EAGAIN)
ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed for "
"lsn=%" PRIu64 " type=%u rc=%d — skipping entry",
entry->lsn, (unsigned)entry->entry_type, rc);
/* Task 3.2: no ACK on apply failure */
return rc;
}
/* Advance applied_lsn atomically to the last contiguously-held LSN
* (bug-991: entry->lsn, NOT entry->lsn + 1 — see the
* ha_apply_term_entry_via_library comment for the streamer-skip wedge a +1
* caused). This is the same last-held semantic recovery/checkpoint/init use,
* and matches the (standby_lsn, committed] range the leader streams. */
ha_advance_applied_lsn(ha, entry->lsn);
/* Task 3.1: ACK the successfully applied single entry */
ha_send_wal_ack_reply(ha, ep, c, entry->lsn);
/* 2026-07-10: kick the apply pump against the already-known commit
* position (see the batch-mode branch above for the full rationale). */
ha_standby_advance_committed_and_apply(
c, atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire));
return 0;
}
/* ── ha-term-safety task 3.2: library-routed AppendEntries intake ───────────
*
* ha_consume_append_entries: route ONE incoming heartbeat (0-entry) or
* replication (1-entry) AppendEntries through the raft library
* (raft_on_append_entries_request) so the LIBRARY — not hand-rolled term-blind
* code — performs:
* - UpdateTerm first on EVERY RPC incl. heartbeat (G5, R2.6): a higher-term
* AE makes the node adopt the term + step down (via on_become_follower).
* - logOk (G2, R3.1): (prevLogIndex,prevLogTerm) match against the local log
* via on_get_entry_term — a mismatch is REJECTED (no false convergence).
* - (term,lsn) conflict-truncate (G3, R3.2/R3.3): a divergent same-LSN tail is
* truncated (on_truncate_from) then the leader's entry re-persisted
* (on_persist_entry), never touching the agreed prefix below the divergence.
*
* This REPLACES the term-blind term handling in ha_consume_heartbeat_committed
* and the committed-boundary truncate in ha_rejoin_reconcile: the library's
* (term,lsn) decision, not a watermark, drives truncation.
*
* `entry` is a decoded wal_catchup_entry_t whose TOP-LEVEL fields carry the AE
* metadata (prev_log_index, prev_log_term, leader_commit) and, for a single
* entry, the entry itself (lsn, entry_type, term, raw_payload). `leader_idx` is
* the raft peer index of the sending leader.
*
* The library callbacks own all persistence/truncation; on a REJECT (stale term
* or logOk fail) the library mutates nothing and the member keeps its current
* state to retry on a later catch-up (R3.5/R3.6 — never report false
* convergence). The loud non-promotable fence for an unreconcilable member is
* the EXISTING ha-commit-durability promotion-time fence; this intake never
* advertises convergence it did not achieve.
*
* Single-receiver-thread model (controller request handling), matching the
* raft_on_vote_request intake: no ha->lock is taken (the raft state is serial
* per the receive thread). Returns 0 (no crash) on every path; the AE result is
* reflected in the library/WAL state, and a future ACK reply (task 3.3) carries
* the success/match_index to the leader.
*/
int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry,
int leader_idx)
{
if (!c || !entry)
return -EINVAL;
int n_entries = (entry->n_entries > 0) ? 1 : 0; /* 0 = heartbeat, 1 = entry */
/* AppendEntries metadata from the top-level container. */
uint64_t prev_log_index = entry->prev_log_index;
uint64_t prev_log_term = entry->prev_log_term;
uint64_t leader_commit = entry->leader_commit;
/* The incoming AE's term: for a single entry, the entry's term stamp; for a
* heartbeat, the top-level container's term (set by the sender to its live
* term). raft_on_append_entries_request uses this for UpdateTerm (G5). */
uint64_t ae_term;
const void *blob = NULL;
size_t blob_len = 0;
uint64_t entry_term = 0;
/* For a single entry, repack the [raft_entry_blob_t{type,len}][payload]
* envelope that on_persist_entry unpacks (the shared wal_types.h format), so
* the library-driven persist writes the same bytes the leader sent. */
uint8_t stackbuf[512];
uint8_t *heapbuf = NULL;
if (n_entries == 1) {
/* Single-entry mode: the entry lives in entries[0] (batch) or on the
* top-level container (single). Prefer the sub-entry when present. */
wal_catchup_entry_t *e =
(entry->entries && entry->entries[0]) ? entry->entries[0] : entry;
uint32_t type = e->entry_type;
const void *payload = e->raw_payload;
uint32_t plen = (uint32_t)e->raw_payload_len;
entry_term = e->term;
/* bug-1067: the AppendEntries HEADER term (UpdateTerm/stale-check) is the
* leader's CURRENT term, NOT the entry's stored term. For a LIVE append
* the two are equal (the leader appends at its current term), but for a
* CATCH-UP entry written by a PRIOR leader the stored term is LOWER than
* the current leader's term — using it as the AE term made the library
* stale-reject (term < s->term) every historical entry, wedging cross-term
* catch-up forever. The streamer stamps leader_current_term = ha->term;
* fall back to the stored term only when it is unset (legacy sender). */
ae_term = (e->leader_current_term != 0) ? e->leader_current_term
: entry_term;
blob_len = sizeof(raft_entry_blob_t) + plen;
uint8_t *buf = (blob_len <= sizeof(stackbuf)) ? stackbuf
: (heapbuf = malloc(blob_len));
if (!buf)
return -ENOMEM;
raft_entry_blob_t *bh = (raft_entry_blob_t *)buf;
bh->type = type;
bh->payload_len = plen;
if (payload && plen > 0)
memcpy(buf + sizeof(*bh), payload, plen);
blob = buf;
} else {
/* Heartbeat: the top-level container's term carries the leader's live
* term (the sender stamps it). No entry to persist. */
ae_term = entry->term;
}
raft_append_result_t res;
memset(&res, 0, sizeof(res));
int rc = raft_on_append_entries_request(&c->ha.raft, leader_idx, ae_term,
prev_log_index, prev_log_term,
leader_commit, blob, blob_len,
entry_term, n_entries, &res);
if (heapbuf)
free(heapbuf);
if (rc != 0) {
/* Library-internal error (not a protocol reject — those return RAFT_OK
* with res.success==0). Log loud; the member stays available to retry. */
WARN("ha_consume_append_entries: raft_on_append_entries_request rc=%d "
"(leader_idx=%d term=%" PRIu64 " prevIdx=%" PRIu64 ")",
rc, leader_idx, ae_term, prev_log_index);
return 0; /* never crash the receive thread */
}
TRACE("ha_consume_append_entries: leader_idx=%d term=%" PRIu64
" prevIdx=%" PRIu64 " n=%d -> success=%d match=%" PRIu64,
leader_idx, ae_term, prev_log_index, n_entries,
res.success, res.match_index);
/* res.success / res.match_index are consumed by the ACK reply path (task
* 3.3). On a reject (res.success==0) the library left the log untouched —
* R3.5/R3.6: no false convergence, the member reconciles on a later AE. */
return 0;
}
/* ha_handle_snapshot_chunk — standby snapshot receive handler (Task 3.5c).
*
* Each call appends payload_bytes to snap_dir/snap.part.
* When chunk->is_last is non-zero:
* 1. Verify total bytes written matches chunk->total_size.
* 2. Atomically rename snap.part → snap-{snapshot_lsn:016x}.bin.
* 3. Load the snapshot via persist_load_snapshot().
* 4. Set ha->applied_lsn = p->checkpoint_lsn (in-memory).
*
* Mid-transfer reconnect (Req 4.4): the primary restarts every transfer at
* chunk 0, so chunk_index == 0 discards any stale snap.part left by an
* interrupted transfer before appending — no cross-chunk state needed.
* Returns 0 on success, negative errno on error.
* Requirements: 4.2, 4.3, 4.4 (ha-repl-hardening spec)
*/
int ha_handle_snapshot_chunk(ha_state_t *ha, persist_t *p,
const char *snap_dir,
snapshot_transfer_t *chunk,
const uint8_t *payload_bytes,
size_t payload_len)
{
if (!ha || !p || !snap_dir || !chunk)
return -EINVAL;
char part_path[4096 + 16];
snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir);
/* Req 4.4: a transfer always restarts from chunk 0 — discard any stale
* partial file from an interrupted previous transfer, and reset the
* completion bookkeeping below for the fresh attempt. */
pthread_mutex_lock(&ha->lock);
if (chunk->chunk_index == 0) {
if (unlink(part_path) != 0 && errno != ENOENT)
WARN("ha_handle_snapshot_chunk: unlink stale snap.part failed: %s",
strerror(errno));
ha->snapshot_recv_count = 0;
ha->snapshot_recv_total = 0;
}
pthread_mutex_unlock(&ha->lock);
/* bug-1389 triage: crc32 of the exact bytes this chunk arrived with,
* taken right after decode and before any local write — correlate
* against ha_send_snapshot_to_standby's send-side crc32 log
* (config_repl.c) by (snapshot_lsn, chunk_index) to localize whether a
* mismatch is introduced before send, in the wire transport, or during
* receive-side assembly. */
if (payload_bytes && payload_len > 0)
TRACE("ha_handle_snapshot_chunk: chunk crc32 "
"checkpoint_lsn=%" PRIu64 " chunk_index=%u len=%zu crc32=%08x",
chunk->snapshot_lsn, chunk->chunk_index, payload_len,
(unsigned)crc32(0L, (const Bytef *)payload_bytes, (uInt)payload_len));
/* bug-1389: chunks of one transfer are dispatched across the vtask pool
* and are NOT guaranteed to arrive/execute in chunk_index order (a live
* soak observed e.g. chunk_index 2 processed AFTER 3 and 4) — a live
* crc32 correlation confirmed every chunk's CONTENT survives the wire
* intact, but the OLD append-in-arrival-order write put reordered
* chunks' bytes at the wrong file position. pwrite() at the chunk's own
* chunk_offset makes assembly order-independent and safe for concurrent
* writers (positioned writes to disjoint byte ranges of the same fd
* don't race each other). */
if (payload_bytes && payload_len > 0) {
int fd = open(part_path, O_CREAT | O_WRONLY, 0600);
if (fd < 0) {
int err = errno;
ERROR("ha_handle_snapshot_chunk: cannot open snap.part: %s",
strerror(err));
return -err;
}
ssize_t written = pwrite(fd, payload_bytes, payload_len,
(off_t)chunk->chunk_offset);
close(fd);
if (written != (ssize_t)payload_len) {
ERROR("ha_handle_snapshot_chunk: short write to snap.part "
"(%zd of %zu bytes) at offset %" PRIu64,
written, payload_len, chunk->chunk_offset);
return -EIO;
}
}
/* bug-1389: finalize once every chunk index 0..N-1 has actually been
* written, not merely once THIS call happens to carry is_last — under
* out-of-order arrival, is_last can be processed before an earlier
* chunk. snapshot_recv_total is claimed (reset to 0) by whichever
* thread wins the race so a late straggler chunk can never trigger a
* second finalize. */
pthread_mutex_lock(&ha->lock);
ha->snapshot_recv_count++;
if (chunk->is_last)
ha->snapshot_recv_total = chunk->chunk_index + 1;
int should_finalize = (ha->snapshot_recv_total != 0 &&
ha->snapshot_recv_count >= ha->snapshot_recv_total);
if (should_finalize)
ha->snapshot_recv_total = 0;
pthread_mutex_unlock(&ha->lock);
if (!should_finalize)
return 0;
/* All chunks accounted for: verify size, rename, load, set applied_lsn */
/* Verify total file size equals chunk->total_size */
struct stat st;
if (stat(part_path, &st) != 0) {
int err = errno;
ERROR("ha_handle_snapshot_chunk: stat(snap.part) failed: %s",
strerror(err));
return -err;
}
if ((uint64_t)st.st_size != chunk->total_size) {
ERROR("ha_handle_snapshot_chunk: size mismatch — "
"written=%" PRIu64 " expected=%" PRIu64,
(uint64_t)st.st_size, chunk->total_size);
return -EIO;
}
/* Atomic rename snap.part → snap-{lsn:016x}.bin */
char final_path[4096 + 32];
snprintf(final_path, sizeof(final_path),
"%s/snap-%016" PRIx64 ".bin", snap_dir, chunk->snapshot_lsn);
if (rename(part_path, final_path) != 0) {
int err = errno;
ERROR("ha_handle_snapshot_chunk: rename snap.part → %s failed: %s",
final_path, strerror(err));
return -err;
}
/* Load the snapshot — sets p->checkpoint_lsn and p->current_lsn.
* ha-entity-snapshots task 6.1: entities_restored is consumed below,
* purely for observability — it does not change the rc-gates-applied_lsn
* ordering here (persist_load_snapshot/load_snapshot already gate their
* own position advance on entity-restore success, task 5.2). */
bool entities_restored = false;
int rc = persist_load_snapshot(p, final_path, chunk->snapshot_lsn,
&entities_restored);
if (rc != 0) {
ERROR("ha_handle_snapshot_chunk: persist_load_snapshot failed rc=%d", rc);
return rc;
}
/* raft-ha-core-safety task 3.2 (Req 3.1/5.1) — Snapshot Boundary
* Reconcile: authoritatively overwrite the per-LSN term index at the
* checkpoint LSN with the checkpoint's own term (as carried on the
* wire). Establishes a verifiable (term, lsn) boundary so a divergent
* pre-snapshot entry's STALE local term at this exact LSN is reconciled
* rather than silently surviving the install. Applies identically to
* both a legacy (position-only) and an entity-bearing checkpoint — the
* term reconcile is orthogonal to entity content. chunk->checkpoint_term
* == 0 is unambiguous and means the checkpoint LSN is genuinely absent
* from the sender's own term index, never "legacy term 0, present"
* (bug-1475 Finding 2 — see persist_reconcile_checkpoint_term's own doc
* comment, persist.h). persist_reconcile_checkpoint_term itself
* special-cases a wire 0 to TERM_INDEX_UNSET before recording, so the
* receiver reads this LSN back as absent too, matching the sender. */
persist_reconcile_checkpoint_term(p, chunk->snapshot_lsn,
chunk->checkpoint_term);
/* Advance applied_lsn to checkpoint_lsn (in-memory only — Req 4.2) */
atomic_store_explicit(&ha->applied_lsn, p->checkpoint_lsn,
memory_order_release);
/* Observability only (Requirements 2.4, 4.1, 4.2, 5.1, 5.2) — a legacy
* (position-only) checkpoint is an expected rollout state, not an error:
* log it as informational and let the member continue through the
* existing incremental catch-up path for the remaining history. */
if (entities_restored) {
INFO("HA: snapshot loaded lsn=%" PRIu64 " with full job/queue entity "
"restoration — applied_lsn set", p->checkpoint_lsn);
} else {
INFO("HA: snapshot lsn=%" PRIu64 " is a legacy position-only "
"checkpoint (no entity content) — applied_lsn set, continuing "
"via normal incremental catch-up", p->checkpoint_lsn);
}
return 0;
}
/* ha_nop_reply_state_report — populate *sr from ha state and wire into reply.
* Reads ha->applied_lsn (atomic) and myconf->config_version_tag, stores them
* in *sr, then sets reply_out->data_which = stateReportMsg and
* reply_out->data.state_report = sr.
* Caller keeps *sr alive until xce_reply() serialises the message.
* Requirements: 1.1, 1.3, 1.4 (ha-repl-hardening spec) */
void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf,
state_report_t *sr, xce_reply_t *reply_out)
{
sr->applied_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
sr->config_tag = myconf ? myconf->config_version_tag : 0;
sr->term = atomic_load_explicit(&ha->term, memory_order_acquire);
reply_out->data_which = RequestReply_data_stateReportMsg;
reply_out->data.state_report = sr;
}
/*
* ha_evaluate_repl_action — pure decision function for the unified heartbeat-
* driven replication protocol.
*
* Implements the decision tree from the ha-repl-hardening design:
*
* if standby_tag < committed_tag → HA_REPL_PUSH_CONFIG
* elif standby_lsn + 1 < oldest_wal_lsn → HA_REPL_SEND_SNAPSHOT
* elif standby_lsn < committed_lsn → HA_REPL_STREAM_WAL
* else (tag >= committed AND lsn >= committed) → HA_REPL_CONVERGED
*
* No side effects. All callers must use the returned action to drive the
* replication send; they must not cache the result beyond the current cycle.
*
* Requirements: 2.2, 2.3, 2.4 (ha-repl-hardening spec)
* Boundary: Primary Heartbeat Handler (ha.c)
*/
ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag,
uint64_t standby_lsn,
uint64_t committed_tag,
uint64_t committed_lsn,
uint64_t oldest_wal_lsn)
{
if (standby_tag < committed_tag)
return HA_REPL_PUSH_CONFIG;
/* Snapshot only when the standby's NEXT-needed entry (standby_lsn + 1) has
* been garbage-collected below the oldest retained WAL entry — i.e. it can
* no longer be served by streaming. An empty standby (standby_lsn=0) on a
* fresh cluster whose oldest retained entry IS lsn 1 (post reserve-lsn-0)
* needs lsn 1, which is retained: 0 + 1 < 1 is false → STREAM, no snapshot.
* Using standby_lsn + 1 (not standby_lsn - ... ) avoids unsigned underflow
* when oldest_wal_lsn == 0 (empty WAL): 0 + 1 < 0 is false → not snapshot. */
if (standby_lsn + 1 < oldest_wal_lsn)
return HA_REPL_SEND_SNAPSHOT;
if (standby_lsn < committed_lsn)
return HA_REPL_STREAM_WAL;
return HA_REPL_CONVERGED;
}
/*
* ha_process_standby_state — update per-standby convergence tracking.
*
* Called on each heartbeat reply from a standby (CTRL_PRIMARY).
* Stores (reported_tag, reported_lsn) into the per-standby
* arrays, evaluates convergence, and logs INFO on the first
* not-converged → converged transition (Req 7.2). Regression resets the flag
* (Req 2.5).
*
* Convergence: reported_tag >= committed_tag AND reported_lsn >= committed_lsn.
*
* Requirements: 1.2, 2.2, 2.5 (ha-repl-hardening spec)
*/
void ha_process_standby_state(ha_state_t *ha, int peer_idx,
uint64_t reported_tag, uint64_t reported_lsn,
uint64_t committed_tag, uint64_t committed_lsn)
{
if (peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES)
return;
/* Store reported values */
ha->standby_reported_tag[peer_idx] = reported_tag;
ha->standby_reported_lsn[peer_idx] = reported_lsn;
/* Evaluate convergence */
bool was_converged = ha->standby_converged[peer_idx];
bool now_converged = (reported_tag >= committed_tag) &&
(reported_lsn >= committed_lsn);
ha->standby_converged[peer_idx] = now_converged;
/* Log INFO on first transition to converged (Req 7.2) */
if (now_converged && !was_converged) {
INFO("HA: standby[%d] converged: config_tag=%" PRIu64
" applied_lsn=%" PRIu64 " (committed_tag=%" PRIu64
" committed_lsn=%" PRIu64 ")",
peer_idx, reported_tag, reported_lsn,
committed_tag, committed_lsn);
}
}
/*
* ha_compute_repl_target_lsn: the replication/convergence target that
* ha_dispatch_repl_action_for_standby's heartbeat-driven decision compares
* every standby's reported applied_lsn against.
*
* The correct target is simply the real, raft/commit-engine-authoritative
* committed_lsn — the only position a standby can safely (and eventually)
* be told it must catch up to. The ONE documented exception is cluster
* BOOTSTRAP: when committed_lsn is genuinely stuck at 0 (no standby has ever
* acked anything yet, so the commit engine's majority computation has never
* had anything to advance against), last_written (the raw locally-appended
* WAL tip) is used instead, so a first-ever connecting standby is told to
* catch up to real data instead of being handed "nothing to stream".
*
* bug-1478: this fallback must apply ONLY in that committed_lsn==0 bootstrap
* case — NOT unconditionally whenever last_written happens to exceed
* committed_lsn. last_written (persist.current_lsn - 1) counts EVERY
* locally-appended WAL entry, including a local-only, NEVER-replicated
* WAL_CHECKPOINT (persist_write_checkpoint_marker bumps current_lsn
* directly, bypassing raft_log_append/the commit engine entirely — the
* same architectural gap flagged in bug-1466). Once such a checkpoint
* trails an already-advancing (non-zero) committed_lsn, last_written
* permanently sits one LSN past it — and since a checkpoint's own consumed
* LSN slot is, by design, never streamed or applied by any standby
* (ha_stream_wal_to_standby / persist_wal_write_replicated /
* persist_wal_async all explicitly refuse WAL_CHECKPOINT), no standby's
* applied_lsn can EVER legitimately reach that value. Taking the
* max(committed_lsn, last_written) unconditionally turned the bootstrap
* convenience into a permanent, unsatisfiable catch-up target for every
* standby the instant a marker-triggered checkpoint (config-upgrade-
* snapshot task 5) or a periodic WAL-threshold checkpoint interleaves ahead
* of the next real committed advance — ha_evaluate_repl_action's
* standby_lsn < committed_lsn check then never reaches HA_REPL_CONVERGED,
* and ha_stream_wal_to_standby is re-invoked every heartbeat forever,
* finding nothing new to send each time (bug-1478).
*
* Pure, side-effect-free — safe to unit test directly.
*/
uint64_t ha_compute_repl_target_lsn(uint64_t committed_lsn,
uint64_t last_written)
{
if (committed_lsn == 0 && last_written > 0)
return last_written;
return committed_lsn;
}
/* bug-1479 (gap 3): see full doc comment in ha.h. Distinct from
* ha_compute_repl_target_lsn above (the CONVERGENCE target) -- this is the
* STREAM ceiling: how far a leader may actually push WAL to a standby, which
* must extend past committed_lsn or a standby whose own ack is required to
* advance committed_lsn can never receive the entries it needs to ack. */
uint64_t ha_compute_repl_stream_ceiling_lsn(uint64_t committed_lsn,
uint64_t last_written,
uint64_t checkpoint_lsn)
{
if (last_written == checkpoint_lsn && last_written > committed_lsn)
last_written = committed_lsn;
return (last_written > committed_lsn) ? last_written : committed_lsn;
}
/* ───────────────────────────────────────────────────────────────────────
* raft-drive-async Task 3.2 — SnapshotSend slow-lane offload (bug-1198).
*
* The snapshot transfer (a fresh persist_checkpoint → fdatasync, then a
* chunked socket write of the snapshot file) is the one unbounded operation
* reachable from the repl-ack decision loop. Run inline it blocks the
* io_worker/drive thread for the whole transfer (and queues that peer's later
* messages behind it). This offloads it to the bounded slow lane, keyed per
* standby so at most one snapshot per peer is in flight (Req 1.5).
* ─────────────────────────────────────────────────────────────────────── */
typedef struct {
cluster_t *c;
endpoint_t *ep;
int peer_idx;
} snapshot_send_ctx_t;
/* ha_snapshot_do_send: the actual (blocking) snapshot work — shared by the
* inline path and the lane worker so both behave identically. In XCE_TESTING
* builds the send is replaced by ha_repl_send_snapshot_hook(peer_idx). */
static void ha_snapshot_do_send(cluster_t *c, endpoint_t *ep, int peer_idx)
{
#ifdef XCE_TESTING
if (ha_repl_send_snapshot_hook != NULL) {
ha_repl_send_snapshot_hook(peer_idx);
return;
}
#endif
if (c && c->persist.snap_path[0]) {
extern int ha_send_snapshot_to_standby(endpoint_t *, const char *,
uint64_t, uint64_t);
/* Req 4.1: create a FRESH checkpoint so the transferred snapshot
* reflects current state (also trims WAL + rotates old snaps).
* -EBUSY means another checkpoint is in flight — fall back to the
* existing checkpoint_lsn rather than skipping the cycle (at most one
* checkpoint in flight is the existing persist_checkpoint invariant). */
int ckpt_rc = persist_checkpoint(&c->persist);
if (ckpt_rc != 0 && ckpt_rc != -EBUSY)
WARN("HA repl: persist_checkpoint before snapshot send "
"failed rc=%d — sending existing checkpoint", ckpt_rc);
/* raft-ha-core-safety task 3.2: source the checkpoint's own term so
* the standby can reconcile its per-LSN term index at the boundary
* after install (Snapshot Boundary Reconcile). */
uint64_t checkpoint_term =
persist_entry_term(&c->persist, c->persist.checkpoint_lsn);
ha_send_snapshot_to_standby(ep, c->persist.snap_path,
c->persist.checkpoint_lsn,
checkpoint_term);
}
}
/*
* snapshot_lane_fn: slow-lane worker entry. Runs the bounded blocking
* snapshot send off the drive thread, then ALWAYS clears the per-peer
* in-flight flag (success or failure) so the dedup never wedges and a later
* heartbeat can retry. Frees the ctx.
*
* BORROWED-ENDPOINT LIFETIME (bug-1197 family): `ep` here is a standby
* endpoint shared with the io_worker epoll loop, NOT one this lane owns
* (contrast reconnect_lane_fn, which owns the ep it creates). ha_snapshot_
* send_dispatch takes an endpoint_ref() on `ep` before submitting (mirrors
* every vtask-bound ep — vtask_enq +1 / run_task -1) so a concurrent standby
* teardown (RDHUP -> zarray_del -> endpoint_killer -> endpoint_unref, the
* general bug-1298 refcount system) can only drop the OWNER's reference
* while this lane transfer is in flight — it cannot free `ep` out from under
* the send. This function's endpoint_unref below is the matching release. */
static void snapshot_lane_fn(void *arg)
{
snapshot_send_ctx_t *sx = (snapshot_send_ctx_t *)arg;
if (!sx) return;
cluster_t *c = sx->c;
int peer_idx = sx->peer_idx;
ha_snapshot_do_send(c, sx->ep, peer_idx);
/* Clear in-flight LAST so a concurrent heartbeat tick cannot enqueue a
* second snapshot for this peer until this one has fully finished.
* Clear the tracked LSN (Req 3.1-3.3) alongside the flag — same
* lifecycle, one more field — so rotate_snapshots never sees a stale
* in-flight LSN for a transfer that has already completed. */
atomic_store_explicit(&c->ha.snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&c->ha.snapshot_inflight[peer_idx], 0,
memory_order_release);
endpoint_unref(sx->ep);
free(sx);
}
/*
* ha_snapshot_send_dispatch: route a SEND_SNAPSHOT decision either inline
* (flag OFF, production default — runs on the caller/drive thread, unchanged
* legacy behavior) or onto the slow lane (flag ON — non-blocking). The
* per-peer single-flight CAS on ha->snapshot_inflight[] is the authoritative
* dedup; the lane's per-key dedup is a redundant backstop. Mirrors the
* reconnect-offload terminal-path handling (Task 2.1).
*/
static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c,
endpoint_t *ep, int peer_idx)
{
/* Flag OFF (default) or no cluster handle (no lane) → inline. */
if (!ha_snapshot_offload_enabled() || c == NULL) {
ha_snapshot_do_send(c, ep, peer_idx);
return;
}
/* Per-peer single-flight: CAS 0->1. Already 1 → a snapshot to this peer
* is in flight; enqueue nothing (Req 1.5). */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&ha->snapshot_inflight[peer_idx], &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return;
}
/* Record the checkpoint position this transfer references (Req 3.1-3.3)
* at the SAME point the flag above is set — before the first byte of the
* transfer goes out — so a reader that observes snapshot_inflight[peer_idx]
* == 1 can find a meaningful LSN alongside it. rotate_snapshots (task 4.3)
* consults this to avoid deleting the checkpoint file this transfer is
* still reading from. */
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx],
c->persist.checkpoint_lsn, memory_order_release);
snapshot_send_ctx_t *sx = malloc(sizeof(*sx));
if (!sx) {
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
sx->c = c; sx->ep = ep; sx->peer_idx = peer_idx;
/* Borrowed-endpoint lifetime (bug-1197 family): take our own reference
* before the lane ever sees `ep`, mirroring vtask_enq's +1 for a
* pool-bound ep. This is the ONLY ref taken across the 3 submit-outcome
* branches below — release it on every path that does NOT hand off to
* snapshot_lane_fn (which owns the matching unref on completion). */
endpoint_ref(ep);
uint64_t key = (uint64_t)(peer_idx + 1); /* +1: key 0 disables dedup */
int srv = slowlane_submit(&c->slowlane, key, snapshot_lane_fn, sx);
if (srv == -EEXIST) {
/* Lane still holds this peer's key from a prior snapshot task that has
* already cleared snapshot_inflight (its last action) but whose key the
* lane has not yet removed. Our CAS saw 0 and set 1, but the submit is
* refused — release the flag we set so no clear is lost, and let the
* next tick retry. */
endpoint_unref(ep);
free(sx);
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure (still queued, will run). Only on a real failure tear down
* and release the guard so the next tick retries; never self-terminate. */
WARN("HA repl: slowlane_submit snapshot peer[%d] failed: %d — retry "
"next cycle", peer_idx, srv);
endpoint_unref(ep);
free(sx);
atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0,
memory_order_release);
atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: enqueued exactly once; snapshot_lane_fn clears the flag
* and the tracked LSN. */
}
/* ───────────────────────────────────────────────────────────────────────
* raft-drive-async Task 3.3 — requester-less ConfigPush slow-lane offload
* (bug-1198).
*
* The two REQUESTER-LESS config-push sites — post-election promotion
* (raft_cb_become_leader) and the heartbeat-driven HA_REPL_PUSH_CONFIG
* convergence case — call push_config_transport_only, which spawns one push
* thread per standby and then BLOCKS the caller in wait_all_with_timeout up to
* CONF_PUSH_TIMEOUT_MS. Run inline on the single io_worker that drives both of
* those sites, that wait stalls every heartbeat/vote for the whole timeout.
* This offloads it to the bounded slow lane. Config push is a CLUSTER-WIDE
* fan-out, so a single cluster-wide single-flight guard (configpush_inflight) +
* a single dedup key (key=1) suffice — at most one push in flight at a time.
*
* The admin-upgrade path (push_config_to_standbys, which has a waiting
* requester / majority barrier) is NOT routed here and stays unchanged.
* ─────────────────────────────────────────────────────────────────────── */
#ifdef XCE_TESTING
extern void (*ha_configpush_probe)(void);
#endif
typedef struct {
cluster_t *c;
uint8_t *bytes; /* deep copy, owned by the ctx; NULL when blen==0 */
size_t blen;
uint8_t sha[32]; /* valid only when has_sha != 0 */
int has_sha;
uint64_t new_tag;
} configpush_ctx_t;
/* ha_config_push_do_send: the actual (blocking) config-push work — shared by
* the inline path and the lane worker so both behave identically. In
* XCE_TESTING builds a non-NULL ha_configpush_probe() is invoked right around
* the send so a test can record the running thread / park the lane worker. */
static void ha_config_push_do_send(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag)
{
#ifdef XCE_TESTING
/* XCE_TESTING does not link cluster.c and uses hook shims for the real
* transport (push_config_transport_only's extern is #ifndef XCE_TESTING).
* The probe stands in for the send and records the running thread. */
if (ha_configpush_probe != NULL)
ha_configpush_probe();
(void)c; (void)bytes; (void)blen; (void)sha256; (void)new_tag;
#else
push_config_transport_only(c, bytes, blen, sha256, new_tag, NULL);
#endif
}
/*
* configpush_lane_fn: slow-lane worker entry. Runs the bounded blocking
* config push off the drive thread, then ALWAYS clears the cluster-wide
* in-flight flag (success or failure) so the dedup never wedges and a later
* promotion/heartbeat can retry. Frees the deep-copied bytes and the ctx.
*
* ⚠ BORROWED-ENDPOINT LIFETIME HAZARD (bug-1197 family — why this path is
* gated OFF by default). push_config_transport_only here reads
* c->ha.standby_endpoints and writes to those standby endpoints, which are
* shared with the io_worker epoll loop, NOT owned by this lane (contrast
* reconnect_lane_fn, which owns the ep it creates). If a standby
* RDHUP-disconnects while this push runs, the io_worker can endpoint_fini/free
* those endpoints concurrently → UAF. Today (flag OFF) the push runs inline ON
* the io_worker, so teardown and push are serialized on one thread and cannot
* overlap. Flipping g_drive_async_configpush_offload ON is therefore only safe
* once Task 3.4 establishes the endpoint-lifetime discipline for the whole
* pool/lane dispatch model. Until then this code lands DORMANT and is
* exercised only by unit tests with a stable (or absent) endpoint set.
*/
static void configpush_lane_fn(void *arg)
{
configpush_ctx_t *cx = (configpush_ctx_t *)arg;
if (!cx) return;
cluster_t *c = cx->c;
ha_config_push_do_send(c, cx->bytes, cx->blen,
cx->has_sha ? cx->sha : NULL, cx->new_tag);
/* Clear in-flight LAST so a concurrent promotion/heartbeat tick cannot
* enqueue a second config push until this one has fully finished. */
atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release);
free(cx->bytes);
free(cx);
}
/*
* ha_config_push_offloadable: route a requester-less config push either inline
* (flag OFF, production default — runs on the caller/drive thread, unchanged
* legacy behavior; the CALLER still owns `bytes` and frees them after we
* return) or onto the slow lane (flag ON — non-blocking; we take a deep copy of
* `bytes` so the caller's free is safe). The cluster-wide single-flight CAS on
* ha.configpush_inflight is the authoritative dedup; the lane's per-key dedup
* is a redundant backstop. Mirrors the snapshot-offload terminal-path handling
* (Task 3.2). Never self-terminates: on any failure it clears the guard and
* the next cycle retries.
*/
static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag)
{
/* Flag OFF (default) or no cluster handle (no lane) → inline. */
if (!ha_configpush_offload_enabled() || c == NULL) {
ha_config_push_do_send(c, bytes, blen, sha256, new_tag);
return;
}
/* Cluster-wide single-flight: CAS 0->1. Already 1 → a config push is in
* flight; enqueue nothing. */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&c->ha.configpush_inflight, &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return;
}
configpush_ctx_t *cx = malloc(sizeof(*cx));
if (!cx) {
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
cx->c = c;
cx->new_tag = new_tag;
cx->has_sha = (sha256 != NULL) ? 1 : 0;
if (cx->has_sha)
memcpy(cx->sha, sha256, 32);
if (bytes != NULL && blen > 0) {
cx->bytes = malloc(blen);
if (!cx->bytes) {
free(cx);
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
memcpy(cx->bytes, bytes, blen);
cx->blen = blen;
} else {
cx->bytes = NULL;
cx->blen = 0;
}
uint64_t key = 1; /* single cluster-wide dedup key (one config-push stream) */
int srv = slowlane_submit(&c->slowlane, key, configpush_lane_fn, cx);
if (srv == -EEXIST) {
/* Lane still holds the key from a prior config-push task that has
* already cleared configpush_inflight (its last action) but whose key
* the lane has not yet removed. Our CAS saw 0 and set 1, but the
* submit is refused — release the flag we set so no clear is lost, and
* let the next tick retry. */
free(cx->bytes);
free(cx);
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure (still queued, will run). Only on a real failure tear down
* and release the guard so the next tick retries; never self-terminate. */
WARN("HA repl: slowlane_submit configpush failed: %d — retry next cycle",
srv);
free(cx->bytes);
free(cx);
atomic_store_explicit(&c->ha.configpush_inflight, 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: enqueued exactly once; configpush_lane_fn clears the flag. */
}
#ifdef XCE_TESTING
/* Test seam: directly drive ha_config_push_offloadable (the two production call
* sites are #ifndef XCE_TESTING). Returns 0 (the dispatch itself cannot fail
* fatally — failures clear the guard and retry). */
int ha_config_push_offloadable_for_test(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag)
{
ha_config_push_offloadable(c, bytes, blen, sha256, new_tag);
return 0;
}
#endif
/*
* ha_dispatch_repl_action_for_standby — execute the decision-loop action for
* one standby on the current heartbeat cycle (Task 3.3).
*
* Reads ha->standby_reported_tag[peer_idx] / standby_reported_lsn[peer_idx]
* (stored by ha_process_standby_state on the same heartbeat reply), calls
* ha_evaluate_repl_action, and acts on the result:
*
* HA_REPL_PUSH_CONFIG → push_config_transport_only(c, …) [or test hook] (Task 3.3)
* HA_REPL_SEND_SNAPSHOT→ stub (Task 3.5)
* HA_REPL_STREAM_WAL → ha_stream_wal_to_standby(c, ha, idx) (Task 3.4)
* HA_REPL_CONVERGED → no-op
*
* One action per heartbeat cycle per standby (design §Unified Heartbeat-Driven
* Protocol). Returns 0 on success, -EINVAL on bad peer_idx.
*
* Requirements: 1.4, 5.1–5.5 (ha-repl-hardening spec)
*/
int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
cluster_t *c,
endpoint_t *ep,
int peer_idx,
uint64_t committed_tag,
uint64_t committed_lsn,
uint64_t oldest_wal_lsn)
{
if (!ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES)
return -EINVAL;
uint64_t standby_tag = ha->standby_reported_tag[peer_idx];
uint64_t standby_lsn = ha->standby_reported_lsn[peer_idx];
/* bug-1479: a standby marked in_catch_up (via ha_raft_on_repl_nack's
* immediate trigger on log-conflict rejection, or repl_broadcast's
* MAX_PENDING_ACKS eviction) can never correct its OWN self-reported
* standby_reported_lsn — it's stuck rejecting every AppendEntries, so it
* never sends a stateReport reflecting genuine progress. Use the
* corrected join_applied_lsn instead: this heartbeat-driven cycle (runs
* every heartbeat_interval_ms regardless of ACK/reject outcome) becomes
* the actual catch-up driver for this case — the ACK-driven continuation
* (ha_walack_on_ack, config_repl.c) can only fire on a REAL ack, which a
* log-conflicted standby can never produce until streamed from the
* corrected position at least once. */
if (c) {
uint64_t join_lsn;
if (repl_get_catchup_join_lsn(&c->persist.repl, ep, &join_lsn))
standby_lsn = join_lsn;
}
/* bug-1479 (gap 3): stream (and evaluate convergence against) the
* leader's own log tail, not just committed_lsn -- see
* ha_compute_repl_stream_ceiling_lsn's doc comment (ha.h) for why capping
* at committed_lsn is a permanent deadlock whenever this exact standby's
* own ack is required to advance committed_lsn in the first place. */
uint64_t stream_ceiling_lsn = committed_lsn;
if (c) {
uint64_t last_written = (c->persist.current_lsn > 0)
? c->persist.current_lsn - 1 : 0;
stream_ceiling_lsn = ha_compute_repl_stream_ceiling_lsn(
committed_lsn, last_written, c->persist.checkpoint_lsn);
}
ha_repl_action_t action = ha_evaluate_repl_action(standby_tag, standby_lsn,
committed_tag, stream_ceiling_lsn,
oldest_wal_lsn);
switch (action) {
case HA_REPL_PUSH_CONFIG:
INFO("HA repl: standby[%d] tag=%" PRIu64 " < committed_tag=%" PRIu64
" → push config", peer_idx, standby_tag, committed_tag);
#ifdef XCE_TESTING
if (ha_repl_push_config_for_standby_hook != NULL) {
ha_repl_push_config_for_standby_hook(c, peer_idx);
break;
}
#endif
if (c) {
#ifndef XCE_TESTING
extern int conf_read_raw(const char *, uint8_t **, size_t *);
uint8_t *cfg_bytes = NULL;
size_t cfg_len = 0;
if (c->myconf && c->myconf->conffile &&
conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) {
/* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON).
* The offload deep-copies cfg_bytes, so this free is still
* correct regardless of routing. */
ha_config_push_offloadable(c, cfg_bytes, cfg_len,
c->myconf->config_sha256,
c->myconf->config_version_tag);
free(cfg_bytes);
} else {
ha_config_push_offloadable(c, NULL, 0, NULL,
c->myconf ? c->myconf->config_version_tag : 0);
}
#endif
}
break;
case HA_REPL_SEND_SNAPSHOT:
INFO("HA repl: standby[%d] lsn=%" PRIu64
" < oldest_wal_lsn=%" PRIu64 " → requesting snapshot transfer",
peer_idx, standby_lsn, oldest_wal_lsn);
/* Task 3.2: route inline (flag OFF, default) or onto the slow lane
* (flag ON). The send body — fresh checkpoint + chunked transfer, or
* the XCE_TESTING hook — lives in ha_snapshot_do_send so both paths
* are identical. */
ha_snapshot_send_dispatch(ha, c, ep, peer_idx);
break;
case HA_REPL_STREAM_WAL:
/* Task 3.4: batch WAL streaming.
* Enqueues all batch OPCode_walCatchupEntry messages for the range
* [standby_lsn+1, stream_ceiling_lsn] via ha_stream_wal_to_standby()
* (bug-1479 gap 3: the ceiling extends past committed_lsn to the
* leader's own log tail — see ha_compute_repl_stream_ceiling_lsn).
* All xce_request() calls are non-blocking (enqueue-only). */
INFO("HA repl: standby[%d] lsn=%" PRIu64
" < stream_ceiling_lsn=%" PRIu64 " (committed_lsn=%" PRIu64
") → stream WAL batches",
peer_idx, standby_lsn, stream_ceiling_lsn, committed_lsn);
#ifdef XCE_TESTING
if (ha_repl_stream_wal_hook != NULL) {
const char *wal_path = (c && c->persist.wal_path[0])
? c->persist.wal_path : "";
ha_repl_stream_wal_hook(ep, wal_path, standby_lsn, stream_ceiling_lsn);
break;
}
#endif
if (c && c->persist.wal_path[0]) {
extern int ha_stream_wal_to_standby(endpoint_t *ep,
const char *wal_path,
const persist_t *p,
uint64_t leader_term,
uint64_t standby_lsn,
uint64_t committed_lsn);
/* bug-1028: pass &c->persist so streamed entry terms come from the
* authoritative per-LSN index (persist_entry_term), matching the
* standby's logOk + the library send path.
* bug-1067: pass the leader's CURRENT term so historical entries
* carry the AE header term (not their lower stored term). */
uint64_t leader_term =
atomic_load_explicit(&c->ha.term, memory_order_acquire);
ha_stream_wal_to_standby(ep, c->persist.wal_path, &c->persist,
leader_term, standby_lsn, stream_ceiling_lsn);
}
break;
case HA_REPL_CONVERGED:
/* No action needed — standby is up to date */
break;
}
return 0;
}
/* ── Dispatch gate (req 1.3, 1.4) ──────────────────────────────────────────
*
* Called from myreq_handler() in cluster.c before the opcode switch.
* Returns 0 when the operation is permitted for the current role.
*
* Primary-only ops are any op that is not one of the always-allowed set:
* nop, reply, hello, join, shutdown, heartbeat, redirect,
* steppingDown, replEntry, replAck, replSnapshot, replCatchUpDone,
* replConfig, replConfigPush, requestVote, probeConfigTag,
* walAck, upgradeConfigAck, snapshotTransfer, walCatchupEntry.
*
* For CTRL_STANDBY the check is: op is in the primary-only set → deny.
* For CTRL_INIT / CTRL_ELECTING the allowlist is tighter: only
* OP_requestVote and OP_probeConfigTag pass through.
*/
/* ha_has_write_quorum — check whether the cluster has enough connected
* standbys to achieve majority for state-changing operations.
*
* Quorum = ceil(n_peers / 2) + 1 (self-vote included).
* Example: n_peers=3 → quorum=2 → need ≥1 connected standby.
*
* Called from ha_dispatch_gate for CTRL_PRIMARY — if the cluster loses
* majority (standby failures), write ops are rejected with -XCE_ERR_NOT_READY
* to preserve the majority-consistency invariant.
*
* Read-only queries (joblist, reslist, hostlist, etc.) remain available
* regardless of majority status — they don't mutate state. */
static bool ha_has_write_quorum(ha_state_t *ha)
{
if (!ha)
return false;
/* Single-node cluster: no standbys needed — self is the majority. */
if (ha->n_peers <= 1)
return true;
/* Count standbys able to receive new log entries. When the replicator
* back-pointer is wired (task 5.3 init), use registry truth: slots with
* ep != NULL && !in_catch_up under r->lock (Req 3.1-3.3) — catch-up
* members cannot ACK new entries, so they must not count toward write
* admission. NULL back-pointer (legacy contexts) falls back to the
* standby_endpoints count populated by raft_cb_become_leader (promoted
* vote eps) and on_standby_connected (late joiners); removed by error
* handlers. */
int connected;
if (ha->repl) {
connected = repl_active_standby_count(ha->repl);
} else {
connected = ha->standby_endpoints
? ha->standby_endpoints->count : 0;
}
/* Need at least n_peers/2 connected standbys to achieve quorum.
* Integer division: n_peers/2 for n_peers=3 → 1 standby minimum. */
return connected >= (int)(ha->n_peers / 2);
}
int ha_dispatch_gate(ha_state_t *ha, int opcode)
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
switch (role) {
case CTRL_PRIMARY:
/* ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): the config
* serve-fence is the PRODUCTION READER that makes the fence real. When
* set, the latest committed WAL_CONFIG_MARKER references config bytes that
* do not hash-match the locally-held bytes (or the bytes are absent), so
* this member MUST NOT serve under the new configuration. Refuse
* config-dependent state-changing ops (loud non-serving), but keep
* replication / read-only / status ops flowing so the correct bytes can
* arrive out-of-band and re-verify clears the fence. The member STAYS
* ALIVE (returns an error code, never exits — controller-never-self-
* terminate). Checked before the quorum gate so a fenced primary never
* serves divergent config even with quorum. */
if (config_serve_fenced(ha->raft_c)) {
switch (opcode) {
/* Replication ops — let the correct config bytes arrive + re-verify */
case OPCode_heartbeat:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_upgradeConfigAck:
case OPCode_hello:
case OPCode_walAck:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replCatchUpDone:
case OPCode_walCatchupEntry:
case OPCode_snapshotTransfer:
/* Read-only queries / status — answerable while fenced */
case OPCode_get:
case OPCode_probeConfigTag:
case OPCode_ctrlstatus:
break; /* fall through to the quorum check below */
default:
/* Config-dependent state-changing op while fenced: refuse. */
return -XCE_ERR_NOT_READY;
}
}
/* config-upgrade-snapshot task 6.2 (Req 3.2, 3.3, 5.5): the Activation
* Fence is a SEPARATE, SEQUENTIAL check composed alongside the
* config-byte serve-fence above, not merged into it — the two
* conditions are logically distinct (byte-mismatch vs. this member's
* own materialization-incomplete) per design.md's Activation Fence
* component. Checked AFTER config_serve_fenced deliberately: byte
* delivery is the earlier step (the bytes must exist locally before
* this member's own checkpoint can even attempt to materialize
* against them), so checking byte-fence first surfaces the more
* fundamental blocker first in log/error ordering — but either fence
* alone is sufficient to refuse a state-changing op, so the final
* admit/refuse outcome does not depend on this ordering. Same
* allowlist as the byte-fence: replication + read-only/status ops
* stay admitted (they let materialization proceed and re-verify);
* everything else is refused with a loud non-serving error while the
* member STAYS ALIVE and keeps retrying (controller-never-self-
* terminate). */
if (upgrade_materialization_fenced(ha->raft_c)) {
switch (opcode) {
/* Replication ops — let commit/apply advance + re-verify */
case OPCode_heartbeat:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_upgradeConfigAck:
case OPCode_hello:
case OPCode_walAck:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replCatchUpDone:
case OPCode_walCatchupEntry:
case OPCode_snapshotTransfer:
/* Read-only queries / status — answerable while fenced */
case OPCode_get:
case OPCode_probeConfigTag:
case OPCode_ctrlstatus:
break; /* fall through to the quorum check below */
default:
/* State-changing op while this member's own upgrade-snapshot
* materialization is incomplete/unverified: refuse. */
return -XCE_ERR_NOT_READY;
}
}
/* State-changing operations require majority so committed entries
* are durable across failover. Read-only queries and replication
* ops are always allowed. */
if (!ha_has_write_quorum(ha)) {
/* Quorum-transition observability backstop (Req 8.1, 8.3): record the
* loss edge if a membership-change site missed it. The edge detector
* makes this silent on every call after the first — repeated rejected
* operations do NOT produce a log storm. */
ha_log_quorum_transition(ha);
switch (opcode) {
/* Replication ops — always allowed (they build majority) */
case OPCode_heartbeat:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_upgradeConfigAck:
case OPCode_hello:
case OPCode_walAck:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replCatchUpDone:
case OPCode_walCatchupEntry:
case OPCode_snapshotTransfer:
/* Read-only queries */
case OPCode_get:
case OPCode_probeConfigTag:
/* Read-only controller status (controller-status 7.1, 7.2):
* answers the local identity/health view even without quorum —
* precisely when an operator most wants it. */
case OPCode_ctrlstatus:
/* Fact-bearing ops — reports of events that already happened
* (worker job transitions/rusage, scheduler pend reasons).
* Rejecting them loses the fact forever: the reporter never
* retries (bug-396). Admit so the RETAIN path can append and
* hold them until quorum returns (Req 3.2; umbrella 2.8 —
* facts are retained, never dropped). */
case OPCode_notify:
case OPCode_update:
case OPCode_setpreason:
case OPCode_clrpreason:
return 0;
default:
return -XCE_ERR_NOT_READY;
}
}
return 0;
case CTRL_INIT:
/* Allow election RPCs AND primary→standby replication messages.
* During the startup stagger a primary may already connect and push
* config — accepting these ensures the node doesn't miss the initial
* bootstrap and can transition to CTRL_STANDBY immediately. */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_heartbeat ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
opcode == OPCode_replConfigPush ||
opcode == OPCode_walCatchupEntry ||
opcode == OPCode_snapshotTransfer ||
/* Read-only controller status — answerable in every role
* (controller-status 7.1, 7.2). */
opcode == OPCode_ctrlstatus)
return 0;
return -XCE_ERR_NOT_READY;
case CTRL_ELECTING:
/* Allow election RPCs AND primary→standby replication messages.
* A node campaigning for leadership must still accept config, WAL,
* and heartbeats from the current primary — if it wins the election
* it needs current state; if it loses, it becomes STANDBY and must
* resume replication without having missed pushes. Only job/client
* ops are rejected (we're not ready to serve). */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_heartbeat ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
opcode == OPCode_replConfigPush ||
opcode == OPCode_walCatchupEntry ||
opcode == OPCode_snapshotTransfer ||
/* Read-only controller status — answerable in every role
* (controller-status 7.1, 7.2). */
opcode == OPCode_ctrlstatus)
return 0;
return -XCE_ERR_NOT_READY;
case CTRL_STANDBY: {
/* Allow ops that standbys legitimately handle */
switch (opcode) {
case OPCode_nop:
case OPCode_reply:
case OPCode_hello:
case OPCode_join:
case OPCode_shutdown:
case OPCode_heartbeat:
case OPCode_redirect:
case OPCode_steppingDown:
case OPCode_replEntry:
case OPCode_replAck:
case OPCode_replSnapshot:
case OPCode_replCatchUpDone:
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_requestVote:
case OPCode_preVoteRequest:
case OPCode_probeConfigTag:
case OPCode_walAck:
case OPCode_upgradeConfigAck:
case OPCode_snapshotTransfer:
case OPCode_walCatchupEntry:
/* Read-only controller status — a standby answers with its local
* view (it reports the primary it knows) (controller-status 7.1, 7.2). */
case OPCode_ctrlstatus:
return 0;
default:
return -XCE_ERR_STANDBY;
}
}
default:
return 0;
}
}
/* ha_config_promotion_fenced (ha-term-safety task 4.3, R7.9): a member is
* non-promotable while config serve-fenced — its latest committed config marker
* references bytes it does not hold/match, so it must not become authoritative.
* Pure atomic read via config_serve_fenced (no I/O) — safe on the election path. */
int ha_config_promotion_fenced(const ha_state_t *ha)
{
if (!ha)
return 0;
return config_serve_fenced(ha->raft_c) ? 1 : 0;
}
/* ── Redirect helpers used by join.c ───────────────────────── */
/*
* ha_should_redirect: returns 1 when this standby knows the primary's
* address and should redirect incoming join requests there.
* Returns 0 for PRIMARY role, or for STANDBY with no known primary yet.
*/
int ha_should_redirect(const ha_state_t *ha)
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
if (role == CTRL_PRIMARY)
return 0;
if (ha->primary_host[0] == '\0' || ha->primary_port == 0)
return 0;
return 1;
}
/*
* ha_no_primary_known: returns 1 when this node is a STANDBY but does not
* yet know the primary's address. Callers should reply E_XCE_NOMASTER so
* the client retries rather than treating the error as permanent.
*/
int ha_no_primary_known(const ha_state_t *ha)
{
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire);
if (role == CTRL_PRIMARY)
return 0;
return (ha->primary_host[0] == '\0' || ha->primary_port == 0) ? 1 : 0;
}
/*
* ha_register_persist_hooks: register the three persistence hooks on the
* cluster's hook system, guarded by the hooks_registered idempotency flag.
*
* ha-single-leader-role task 2.2: extracted from ha_promote_to_primary so
* BOTH roads to PRIMARY (the raft become-leader callback and the non-raft
* ha_promote_to_primary path) share one hook-registration sequence and
* cannot diverge. Safe on a zero-init cluster_t (zeroed hooksys →
* xce_add_hook returns -EINVAL harmlessly; hooks_registered still set to 1).
* Requires non-NULL c (dereferences &c->hooksys).
*/
static void ha_register_persist_hooks(cluster_t *c)
{
if (c->ha.hooks_registered)
return;
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun, &c->persist);
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun, &c->persist);
xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun, &c->persist);
c->ha.hooks_registered = 1;
}
/*
* ha_primary_log_append (ha-term-safety task 3.1, R1.1/R1.2/R2.1): the SINGLE
* primary append entry point. Routes a local primary append (job-state entry or
* the own-term promotion no-op) through the raft library's raft_log_append, so
* the LIBRARY stamps the live election term (s->term) and assigns the LSN
* (s->log_len+1), then drives on_persist_entry — which (primary role) performs
* the actual WAL write via persist_wal_async. This makes the library index the
* authority for both the per-entry term (R1) and the LSN, retiring the
* hand-rolled bypass that assigned the LSN directly through persist_wal_async.
*
* Co-tracking invariant (closes the task 2.1 deferral): because raft_log_append
* is the only primary append path, every primary append advances BOTH the
* library index (s->log_len) and the persist write cursor (p->current_lsn) by
* exactly one, in lockstep — so p->current_lsn == s->log_len + 1 always holds and
* the LSN the library assigns equals the LSN on_persist_entry writes. raft_init
* (task 2.5) seeds s->log_len from the recovered durable floor, which equals
* p->current_lsn - 1 at boot, establishing the invariant before the first append.
*
* Persist-before-act: raft_log_append calls on_persist_entry BEFORE advancing
* s->log_len and propagates a non-zero persist rc without mutating the log (no
* phantom entry) — preserving persist-before-ACK and the bug-850 contiguity
* guard beneath the WAL write.
*
* Legacy fallback (non-raft road): when the raft library is not in RAFT_LEADER
* state (raft_log_append returns -EPERM) — the init-time non-raft promotion road
* (ha_promote_to_primary) and engine-less unit-test contexts — the library is
* not tracking the log, so there is no co-tracking hazard. We append locally
* through the legacy persist_wal_async path (which still records the live term
* via the persist_set_term seam set on the prior promotion) so those roads keep
* working unchanged. Once raft drives leadership in production, the -EPERM
* branch is never taken and the library is the sole LSN/term source.
*
* out_lsn (optional) receives the assigned 1-based LSN on success.
* Returns 0 on success; propagates a non-zero persist rc on append failure.
*/
int ha_primary_log_append(cluster_t *c, uint32_t type,
const void *payload, uint32_t len, uint64_t *out_lsn)
{
if (!c)
return -EINVAL;
/* Pack the in-flight adapter envelope [raft_entry_blob_t{type,len}][payload]
* — the SAME format on_persist_entry (ha_cb_persist_entry_stub) unpacks, so
* the encoder and decoder share the single wal_types.h definition. */
size_t blob_len = sizeof(raft_entry_blob_t) + len;
uint8_t stackbuf[256];
uint8_t *blob = (blob_len <= sizeof(stackbuf)) ? stackbuf : malloc(blob_len);
if (!blob)
return -ENOMEM;
raft_entry_blob_t *bh = (raft_entry_blob_t *)blob;
bh->type = type;
bh->payload_len = len;
if (payload && len > 0)
memcpy(blob + sizeof(*bh), payload, len);
uint64_t idx = 0;
int rc = raft_log_append(&c->ha.raft, blob, blob_len, &idx);
if (rc == -EPERM) {
/* Non-raft road: the library is not LEADER and is not tracking the log.
* Append locally through the legacy path (no library index to co-track).
* Stamp the live election term (ha->term) so the entry still records the
* leader's term (R1.1) even off the raft road — matching on_persist_entry's
* persist_set_term on the raft path. */
persist_set_term(&c->persist,
atomic_load_explicit(&c->ha.term, memory_order_acquire));
rc = persist_wal_async(&c->persist, type, payload, len);
if (rc == 0) {
pthread_mutex_lock(&c->persist.lock);
idx = c->persist.current_lsn - 1; /* the LSN just assigned */
pthread_mutex_unlock(&c->persist.lock);
}
}
if (blob != stackbuf)
free(blob);
if (rc == 0 && out_lsn)
*out_lsn = idx;
return rc;
}
/* ha_repl_ack_abs_peer_idx (ha-term-safety task 3.3 remediation): resolve the
* ABSOLUTE library peer index from an ACK endpoint's stored peer_idx, branching
* on endpoint type EXACTLY as the standby_endpoints scan at ha.c:6187-6189.
*
* Two endpoint types reach the walAck branch of repl_ack_decode_and_dispatch,
* and their stored peer_idx has DIFFERENT meaning:
* - Promoted vote ep (vote_ep_ctx_t, ep_is_vote != 0): peer_idx is a
* self-EXCLUDED "votes" index (relative). Map it back to the absolute index
* — the inverse of raft_cb_send_vote_request's votes_idx remap (self_idx ==
* my_priority): abs = votes_idx < self_idx ? votes_idx : votes_idx + 1.
* - Heartbeat standby ep (standby_ep_ctx_t, ctx==NULL, ep_is_vote == 0):
* peer_idx is ALREADY the absolute candidate index — pass it through.
*
* The earlier unconditional remap corrupted the heartbeat path: a standby ep's
* already-absolute index was remapped a second time, crediting the WRONG peer
* (false quorum / stuck commit) or, near the array bound, an OUT-OF-BOUNDS
* match_index[] write. This resolver is the single source of truth so the
* production call site and the unit test exercise identical logic. */
int ha_repl_ack_abs_peer_idx(ha_state_t *ha, int ep_is_vote, int peer_idx)
{
int self_idx = ha->my_priority; /* self_idx == my_priority by convention */
if (ep_is_vote)
return (peer_idx < self_idx) ? peer_idx : peer_idx + 1;
return peer_idx; /* heartbeat standby ep: already absolute */
}
/* ha-term-safety task 3.3 (R4.1/R4.5): drive the raft library on a replication
* ACK. Route the reply through raft_on_append_entries_reply (updating
* match_index[]/next_index[]) then raft_advance_commit, so the library
* recomputes the current-term-gated commit (Figure-8) and MAY fire
* on_commit_advanced — the task-2.4 callback feeding the synchronous barrier.
*
* Peer-index contract: `abs_peer_idx` is the ABSOLUTE library peer index
* (j != self_idx), the same index space the library uses for
* match_index[]/next_index[]. The CALLER resolves it per endpoint type via
* ha_repl_ack_abs_peer_idx (promoted-vote eps carry a relative votes index;
* heartbeat standby eps carry an already-absolute index) — this seam must NOT
* remap again or it would corrupt the heartbeat path.
*
* Defense-in-depth: a peer index outside [0, raft.n_peers) (and never self) is
* rejected loudly rather than passed into raft_on_append_entries_reply, whose
* match_index[] write is unchecked. This fences an OOB / wrong-peer write at the
* controller boundary (no assert inside src/lib/raft/) and self-heals — the
* caller simply skips this ACK's library accounting.
*
* The library drops a stale-term reply (reply_term != s->term) inside
* raft_on_append_entries_reply, so a stale ACK never touches match_index[] and
* never advances commit. raft_advance_commit is leader-only (-EPERM otherwise);
* on a non-leader the call is a harmless no-op. Neither library call performs
* I/O — safe on the iomgr ACK path. */
int ha_raft_on_repl_ack(cluster_t *c, int abs_peer_idx, uint64_t acked_lsn,
uint64_t reply_term)
{
if (!c)
return -EINVAL;
/* Defense-in-depth bounds/self check: the library's match_index[] write is
* unchecked, so reject a bogus index here (fail loud, never corrupt). */
int self_idx = c->ha.my_priority;
int n_peers = c->ha.raft.n_peers;
if (abs_peer_idx < 0 || abs_peer_idx >= n_peers || abs_peer_idx == self_idx) {
ERROR("ha_raft_on_repl_ack: rejecting out-of-range peer index %d "
"(n_peers=%d self_idx=%d) — skipping library ACK accounting",
abs_peer_idx, n_peers, self_idx);
return -EINVAL;
}
/* A successful WalAck reports the standby's matched index (acked_lsn). The
* library applies it only when reply_term == s->term (stale-term drop). */
(void)raft_on_append_entries_reply(&c->ha.raft, abs_peer_idx, /*success*/1,
acked_lsn, reply_term);
/* Recompute the current-term-gated commit; fires on_commit_advanced exactly
* once on a strict advance of a current-term entry (else holds). Leader-only:
* -EPERM off the leader leaves commit_index unchanged with no callback. */
(void)raft_advance_commit(&c->ha.raft);
return 0;
}
int ha_raft_on_repl_nack(cluster_t *c, int abs_peer_idx, uint64_t reply_term,
uint64_t *corrected_lsn_out)
{
if (!c)
return -EINVAL;
/* Same defense-in-depth bounds/self check as ha_raft_on_repl_ack. */
int self_idx = c->ha.my_priority;
int n_peers = c->ha.raft.n_peers;
if (abs_peer_idx < 0 || abs_peer_idx >= n_peers || abs_peer_idx == self_idx) {
ERROR("ha_raft_on_repl_nack: rejecting out-of-range peer index %d "
"(n_peers=%d self_idx=%d) — skipping library NACK accounting",
abs_peer_idx, n_peers, self_idx);
return -EINVAL;
}
/* A rejected AppendEntries reports a log conflict at the sent
* prevLogIndex/prevLogTerm. match_index is unused on the failure branch
* (raft.c: "failure: next_index[peer] = Max(next_index[peer]-1, 1). Does
* NOT advance commit.") -- pass 0. The library applies the backoff only
* when reply_term == s->term (stale-term drop), same as the ack path. */
(void)raft_on_append_entries_reply(&c->ha.raft, abs_peer_idx, /*success*/0,
/*match_index unused*/0, reply_term);
if (corrected_lsn_out) {
uint64_t next = c->ha.raft.next_index[abs_peer_idx];
*corrected_lsn_out = (next > 0) ? next - 1 : 0;
}
return 0;
}
/*
* ha_arm_own_term_gate: append the own-term WAL_NOOP and arm the commit
* engine's writable_lsn gate (ha-single-leader-role task 2.2, Req 1.4/2.1/2.2).
*
* The shared promotion tail both roads to PRIMARY run. Sequence:
* 1. persist_wal_async(WAL_NOOP) — appends a header-only entry and
* broadcasts it to standbys via repl_broadcast (one call does both).
* 2. derive noop_lsn = current_lsn - 1 under p->lock (the append
* post-incremented current_lsn; persist.c:1019-1021 pattern).
* 3. commit_submit(noop_lsn, COMMIT_MODE_RETAIN) — lock-free, non-blocking,
* registration-free observability bookkeeping (records nothing).
* 4. commit_set_writable_lsn(noop_lsn) — arms the own-term gate so the
* engine withholds WAIT completion until committed_lsn >= noop_lsn.
*
* Non-blocking by construction: every call here returns without waiting for
* the no-op to commit (Req 1.4 — status honesty does not wait).
*
* Degraded-but-safe (design §Error Strategy): if the no-op append fails,
* no LSN exists, so the gate stays UNSET; promotion STILL COMPLETES. The
* universal commit barrier still requires per-entry majority of a new-term
* entry, so the first state-changing entry of this term becomes the de-facto
* own-term vehicle. We do NOT block and do NOT abort promotion.
*/
static void ha_arm_own_term_gate(cluster_t *c)
{
/* ha-term-safety task 3.1: route the own-term promotion no-op through the
* library (ha_primary_log_append → raft_log_append) so the WAL_NOOP records
* the newly established term (R1.2) and the library assigns its LSN. On the
* non-raft promotion road this falls back to the legacy append; either way
* the returned LSN is the no-op's WAL LSN. */
uint64_t noop_lsn = 0;
int rc = ha_primary_log_append(c, WAL_NOOP, NULL, 0, &noop_lsn);
if (rc < 0) {
ERROR("HA: own-term WAL_NOOP append failed: %d", rc);
WARN("HA: own-term gate left UNSET (degraded-but-safe); first "
"new-term entry becomes the own-term commit vehicle");
return;
}
/* Observability-only bookkeeping (lock-free, non-blocking, records
* nothing per entry — commit state derives from registry watermarks). */
(void)commit_submit(&c->commit, noop_lsn, COMMIT_MODE_RETAIN, 0);
/* Arm the own-term gate: completion of any WAIT is withheld until
* committed_lsn >= noop_lsn. */
commit_set_writable_lsn(&c->commit, noop_lsn);
INFO("HA: own-term noop appended lsn=%" PRIu64 ", writable_lsn gate armed",
noop_lsn);
}
/*
* ha_promote_to_primary: the NON-RAFT promotion road (cluster.c init-time,
* legacy XCE_TESTING evaluation path, tests). This path OWNS its term, so it
* calls ha_do_promote (which increments the term + sets role=PRIMARY). It
* then runs the SAME shared sequence the raft road runs: register hooks,
* start the heartbeat, and arm the own-term gate (noop + writable_lsn).
*
* Every road to PRIMARY arms the own-term gate. The noop+gate tail is
* guarded on c != NULL (callers always pass a real cluster here).
*/
int ha_promote_to_primary(cluster_t *c)
{
ha_do_promote(&c->ha); /* non-raft path owns its term increment */
/* Shared promotion sequence — must match the raft become-leader tail:
* hooks live BEFORE the no-op append so the persistence seam is active
* before any racing client write. */
ha_register_persist_hooks(c);
/* Arm periodic heartbeat to standbys — keeps their watchdog alive when
* the cluster is idle (no WAL entries or config pushes). */
ha_start_primary_heartbeat(c);
/* Own-term no-op + writable_lsn gate (Req 1.4, 2.1, 2.2). */
ha_arm_own_term_gate(c);
return 0;
}
/*
* ha_demote_to_standby: disable persistence hooks, then set role=STANDBY
* and reset the hooks_registered idempotency guard (via ha_do_demote).
*/
int ha_demote_to_standby(cluster_t *c)
{
/* Disable persistence hooks before changing role */
if (c->ha.hooks_registered) {
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB,
(hook_fn_t)persist_job_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE,
(hook_fn_t)persist_queue_hook_fun);
xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER,
(hook_fn_t)persist_worker_hook_fun);
}
ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */
return 0;
}
/* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */
/*
* ha_wait_for_heartbeat_or_timeout: called from the startup path after
* ha_init() when the node needs to discover whether a primary already exists.
*
* Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where
* deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter
*
* The heartbeat handler (cluster.c) posts heartbeat_seen when it receives
* OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits
* OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler.
*
* Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived);
* ha_do_demote() was called (role = CTRL_STANDBY).
* Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT);
* caller is responsible for driving the election.
*/
ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha)
{
/* Seed jitter PRNG on first use */
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
/* Compute timeout: election_timeout + priority_step * priority + random jitter.
* Higher-priority (lower index) nodes time out sooner, giving them a head
* start in the election before lower-priority nodes begin campaigning. */
long to_ms = (long)ha->config.election_timeout_ms;
to_ms += (long)ha->my_priority * (long)ha->config.election_priority_step_ms;
if (ha->config.election_jitter_max_ms > 0) {
to_ms += (long)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)ha->config.election_jitter_max_ms + 1));
}
/* sem_timedwait requires an absolute CLOCK_REALTIME deadline */
struct timespec deadline;
clock_gettime(CLOCK_REALTIME, &deadline);
deadline.tv_sec += to_ms / 1000;
deadline.tv_nsec += (to_ms % 1000) * 1000000L;
if (deadline.tv_nsec >= 1000000000L) {
deadline.tv_sec++;
deadline.tv_nsec -= 1000000000L;
}
int rc = sem_timedwait(&ha->heartbeat_seen, &deadline);
if (rc == 0) {
/* Heartbeat arrived before the deadline — demote to standby */
ha_do_demote(ha);
return CTRL_STANDBY;
}
/* ETIMEDOUT (or any other error): treat as election timeout */
return CTRL_ELECTING;
}
/* ── Term Durability (Req 9) ────────────────────────────────────────────────
*
* ha_term_file_t: on-disk format for ha-term.bin.
* Fields are in host byte order (single-host durability, no cross-endian migration).
* CRC32 covers all preceding bytes via zlib crc32().
*/
#define HA_TERM_MAGIC 0xDEADBEEFCAFEBABEULL
typedef struct {
uint64_t magic;
uint64_t term;
uint64_t voted_for_term;
char voted_for_name[XCE_MAXNAME_LEN];
uint32_t crc32;
} ha_term_file_t;
/*
* ha_persist_term: atomically write ha_term_file_t to {workdir}/ha-term.bin.
* Uses write-to-tmp + fsync + rename for durability (existing project pattern).
* If workdir is NULL, the persist is skipped (test mode or no workdir configured).
* Req 9.1: must be called before any vote grant or term increment is reported.
*/
void ha_persist_term(ha_state_t *ha, const char *workdir)
{
if (!workdir)
return; /* test mode: skip file I/O */
ha_term_file_t f;
memset(&f, 0, sizeof(f));
f.magic = HA_TERM_MAGIC;
f.term = atomic_load_explicit(&ha->term, memory_order_acquire);
f.voted_for_term = ha->voted_for_term;
strncpy(f.voted_for_name, ha->voted_for_name, sizeof(f.voted_for_name) - 1);
f.voted_for_name[sizeof(f.voted_for_name) - 1] = '\0';
/* CRC32 covers all bytes before the crc32 field itself */
size_t crc_len = offsetof(ha_term_file_t, crc32);
f.crc32 = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len);
/* Write to tmp then rename — atomic from filesystem perspective */
char path[1024], tmp[1040];
snprintf(path, sizeof(path), "%s/ha-term.bin", workdir);
snprintf(tmp, sizeof(tmp), "%s/ha-term.bin.tmp", workdir);
int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (fd < 0)
return; /* best-effort; caller must not rely on this for correctness */
ssize_t written = write(fd, &f, sizeof(f));
if (written == (ssize_t)sizeof(f))
fsync(fd);
close(fd);
if (written == (ssize_t)sizeof(f))
rename(tmp, path);
}
/*
* ha_restore_term: read {workdir}/ha-term.bin at startup and restore
* ha->term and ha->voted_for_* into ha_state_t.
*
* On missing file (ENOENT): no-op — term stays 0. Req 9.3.
* On CRC/magic mismatch: log WARN, zero ha->term and ha->voted_for_*. Req 9.2.
* On valid file: atomically restore ha->term; copy voted_for fields. Req 9.2.
*
* Returns 0 on success (including gracefully-handled file-not-found and CRC bad),
* -errno on unexpected I/O error.
* If workdir is NULL, function is a no-op (safe for unit tests without a workdir).
*/
int ha_restore_term(ha_state_t *ha, const char *workdir)
{
if (!workdir)
return 0;
char path[1024];
snprintf(path, sizeof(path), "%s/ha-term.bin", workdir);
int fd = open(path, O_RDONLY);
if (fd < 0) {
if (errno == ENOENT)
return 0; /* first boot — no file yet, term stays 0 (Req 9.3) */
return -errno;
}
ha_term_file_t f;
memset(&f, 0, sizeof(f));
ssize_t n = read(fd, &f, sizeof(f));
close(fd);
if (n != (ssize_t)sizeof(f)) {
WARN("HA: ha-term.bin short read (%zd/%zu) — ignoring, starting at term 0",
n, sizeof(f));
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
return 0;
}
/* Verify magic */
if (f.magic != HA_TERM_MAGIC) {
WARN("HA: ha-term.bin bad magic (0x%016" PRIx64 ") — ignoring, starting at term 0",
f.magic);
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
return 0;
}
/* Verify CRC32 */
size_t crc_len = offsetof(ha_term_file_t, crc32);
uint32_t computed = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len);
if (computed != f.crc32) {
WARN("HA: ha-term.bin CRC mismatch (expected 0x%08x got 0x%08x) — ignoring, starting at term 0",
computed, f.crc32);
atomic_store_explicit(&ha->term, 0ULL, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
return 0;
}
/* Valid file — restore state (Req 9.2) */
atomic_store_explicit(&ha->term, f.term, memory_order_release);
ha->voted_for_term = f.voted_for_term;
strncpy(ha->voted_for_name, f.voted_for_name, sizeof(ha->voted_for_name) - 1);
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
return 0;
}
/* ── ha_heartbeat_is_fresh (Req 6.2) ────────────────────────────────────────
*
* Returns 1 if the elapsed time since the last primary message
* (ha->last_activity, written by ha_on_heartbeat_received) is strictly less
* than ha->config.heartbeat_timeout_ms (i.e. the primary is still alive).
* Returns 0 if no primary message has been seen yet (last_activity == 0) or
* the timeout has expired.
*
* Uses CLOCK_MONOTONIC seconds to avoid wall-clock skew; whole-second
* granularity is sufficient for the 10 s default timeout.
*/
int ha_heartbeat_is_fresh(ha_state_t *ha)
{
time_t last = ha->last_activity; /* time_t load is atomic on LP64 */
if (last == 0)
return 0; /* no primary message seen yet */
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
long ms = (long)(now.tv_sec - last) * 1000L;
return ms < (long)ha->config.heartbeat_timeout_ms;
}
/* ── Voter freshness floor (election safety) ───────────────────────────────
*
* Compute the voter's effective freshness floor as the per-field max of its
* local state and its learned-committed cluster state. Outdated candidates
* (below this floor) are denied so the cluster can never be rolled back below
* what it has already committed. See ha.h for full contract.
*/
void ha_vote_self_freshness(ha_state_t *ha,
uint64_t local_config_tag,
uint64_t local_applied_lsn,
uint64_t *out_tag,
uint64_t *out_lsn)
{
uint64_t learned_tag =
atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire);
uint64_t learned_lsn =
atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire);
if (out_tag)
*out_tag = (local_config_tag > learned_tag) ? local_config_tag : learned_tag;
if (out_lsn)
*out_lsn = (local_applied_lsn > learned_lsn) ? local_applied_lsn : learned_lsn;
}
/* ── Candidate advertised LSN (election safety, ha-commit-durability 4.2) ───
*
* A candidate advertises EXACTLY its durable WAL tip on the LSN axis — the
* bare `persist_durable_floor(p)`, NOT the folded
* max(durable_floor, last_known_committed_lsn) that ha_vote_self_freshness
* computes for the VOTER floor.
*
* Why bare, not folded: ha_vote_self_freshness unconditionally returns
* max(local_lsn, last_known_committed_lsn). Feeding it durable_floor would let
* a node restarted with a TRUNCATED log (durable_floor low) but a stale
* volatile learned_committed (high) advertise the stale-high value and WIN —
* exactly the truncated-candidate-wins hole this task closes (design §1a/§5a).
* A candidate may only claim a log position it can durably back.
*
* The candidate's TAG axis is unchanged: it still uses
* max(local_config_tag, learned_tag) (the config epoch is orthogonal to log
* durability — design §1a). Only the LSN axis bypasses the fold.
*
* NULL-safe (persist_durable_floor guards NULL → 0) for the test/seam paths.
*/
uint64_t ha_candidate_advertise_lsn(persist_t *p)
{
return persist_durable_floor(p);
}
/* bug-848: see ha.h. After local WAL recovery, advertise the recovered durable
* tip for catch-up so the primary streams only the missing suffix instead of
* re-streaming + overwriting consistent records. Monotonic: never lowers a
* higher applied_lsn (e.g. if catch-up already advanced it). */
void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p)
{
if (!ha || !p)
return;
uint64_t recovered = persist_durable_floor(p);
uint64_t cur = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
if (recovered > cur)
atomic_store_explicit(&ha->applied_lsn, recovered,
memory_order_release);
}
/* bug-1042: re-sync the raft library's log_len / commit_index / term from the
* recovered WAL AFTER persist_recover.
*
* Init-ordering hazard: ha_init_raft (myapp.c, before the election) sources
* raft.initial_log_len from persist_durable_floor — but on a STANDBY-REJOIN boot
* persist_recover has not run yet at that point, so the library comes up with
* log_len=0. After cluster_init_phase2 runs persist_recover (recovering e.g.
* durable_floor=7), the library still thinks its log is EMPTY. Every incoming
* AppendEntries then fails logOk (prevLogIndex>=1 against log_len=0 → reject),
* so the rejoined member is PERMANENTLY stuck as a non-converging standby and
* the primary re-streams [tip+1..committed] every heartbeat forever.
*
* Fix: after recovery, raise the library's log_len to the recovered WAL tip and
* its commit_index to the recovered durable floor (clamped to log_len, honoring
* the commit_index<=log_len invariant), and align raft.term with the persisted
* term so the per-LSN (term,index) reconcile reads consistent state. Monotonic:
* never lowers an already-higher library position (e.g. if catch-up advanced it
* before this runs). The library stores no entry bytes — on_get_entry_term reads
* the recovered per-LSN term index — so raising log_len is sufficient for logOk. */
void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p)
{
if (!ha || ha->raft_c == NULL || !p)
return;
uint64_t recovered_tip = persist_durable_floor(p);
if (recovered_tip > ha->raft.log_len)
ha->raft.log_len = recovered_tip;
/* commit_index advances to the recovered durable floor but never past the
* log tip (library invariant commit_index <= log_len), and never decreases. */
uint64_t recovered_commit = recovered_tip;
if (recovered_commit > ha->raft.log_len)
recovered_commit = ha->raft.log_len;
if (recovered_commit > ha->raft.commit_index)
ha->raft.commit_index = recovered_commit;
/* Align the library term with the persisted/restored HA term so the
* (term,lsn) reconcile and own-term gate read a consistent value after a
* rejoin (ha_restore_term already loaded ha->term before ha_init_raft). */
uint64_t ha_term = atomic_load_explicit(&ha->term, memory_order_acquire);
if (ha_term > ha->raft.term)
ha->raft.term = ha_term;
}
/* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ────────────────────────
*
* handle_request_vote: process an incoming OP_requestVote RPC.
*
* Decision order (fail-fast):
* 0. role gate (Req 6.1, 6.2):
* - CTRL_PRIMARY → deny
* - CTRL_STANDBY with fresh heartbeat → deny
* 1. in_catch_up → deny (Req 2.5)
* 2. req term < my term → deny (Req 2.2)
* 3. req term > my term → update term, clear voted_for (Req 2.3)
* 4. already voted for a DIFFERENT candidate in this term → deny (Req 2.6)
* 5. freshness gate: candidate not at least as fresh → deny (Req 2.4)
* 6. grant vote, record voted_for, persist (Req 9.1)
*
* Returns 1 = granted, 0 = denied, <0 = error.
*
* TEST-ONLY (XCE_TESTING): production vote dispatch goes through
* cluster.c → raft_on_vote_request. This function is the legacy
* entry point used by test_ha_vote_gate.c and test_ha_election.c
* to exercise role-gate logic in isolation without a full raft state.
*/
#ifdef XCE_TESTING
int handle_request_vote(ha_state_t *ha,
uint64_t term,
const char *candidate_id,
uint64_t config_tag,
uint64_t last_lsn,
uint64_t my_config_tag,
uint64_t my_last_lsn,
int in_catch_up_flag)
{
/* Req 2.3: higher term → step down BEFORE any role gate.
*
* Raft §5.1: "If RPC request or response contains term T > currentTerm:
* set currentTerm = T, convert to follower."
*
* Without this ordering, a CTRL_PRIMARY node at term=N would deny a vote
* request at term=N+1 without ever seeing the higher term, leaving two
* nodes both believing they lead → split-brain. */
uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire);
if (term > my_term) {
atomic_store_explicit(&ha->term, term, memory_order_release);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
/* Step down from leader: a higher-term election is in progress and we
* must not block it with the role gate below. */
ctrl_role_t cur_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (cur_role == CTRL_PRIMARY) {
ha_do_demote(ha);
INFO("HA: stepping down (term %" PRIu64 " → %" PRIu64
") — higher-term election in progress", my_term, term);
}
my_term = term;
}
/* Req 2.2: stale term → deny, do not update any state */
if (term < my_term) {
return 0;
}
/* Req 6.1, 6.2: role-based deny gate.
* Evaluated AFTER the term catch-up above: if we just stepped down due to
* a higher term, role is now CTRL_STANDBY and we fall through to voting.
* PRIMARY at the current term still denies votes (protecting the live
* primary from same-term challengers). */
ctrl_role_t my_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (my_role == CTRL_PRIMARY)
return 0; /* deny — I am primary; protect against same-term challengers */
if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(ha))
return 0; /* deny — my primary is alive (active primary connection) */
/* Req 2.5: catching-up standbys deny all votes */
if (in_catch_up_flag)
return 0;
/* Protect voted_for check+set from concurrent vote requests on different vtasks */
pthread_mutex_lock(&ha->lock);
/* Req 2.6: already voted for a DIFFERENT candidate in this term → deny.
* Same-candidate re-grant is idempotent (Raft §5.2: grant iff votedFor ∈
* {Nil, candidateId}) — matches the duplicate-vote check in raft.c. */
if (ha->voted_for_term == term && ha->voted_for_name[0] != '\0' &&
strcmp(ha->voted_for_name, candidate_id) != 0) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
/* Req 2.4 + election-safety fix #2: freshness gate against the EFFECTIVE
* floor — the max of the voter's local state and its learned-committed
* cluster state. A candidate that is below what the cluster has already
* committed (config_tag or, at equal tag, LSN) would roll the cluster back
* and must be denied even when this voter's local state is colder.
* Deny if candidate has a lower config_tag, or equal config_tag with lower
* LSN; grant if strictly fresher or equally fresh. */
uint64_t eff_tag = 0, eff_lsn = 0;
ha_vote_self_freshness(ha, my_config_tag, my_last_lsn, &eff_tag, &eff_lsn);
if (config_tag < eff_tag) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
if (config_tag == eff_tag && last_lsn < eff_lsn) {
pthread_mutex_unlock(&ha->lock);
return 0;
}
/* Grant the vote: record voted_for and persist before responding (Req 9.1) */
ha->voted_for_term = term;
strncpy(ha->voted_for_name, candidate_id, sizeof(ha->voted_for_name) - 1);
ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0';
pthread_mutex_unlock(&ha->lock);
/* workdir is not available here — persist with NULL (test mode or caller
* wraps this function with workdir before sending the network reply).
* Production dispatch path calls ha_persist_term(ha, workdir) explicitly
* before this function writes its network reply. */
ha_persist_term(ha, NULL);
return 1;
}
#endif /* XCE_TESTING */
/* ── raft-drive-async Task 2.1: reconnect-offload feature flag (ALL builds) ──
*
* drive_async_reconnect_offload gates the peer reconnect path:
* ON (production default) → primary_heartbeat_fn submits the peer reconnect
* to the slow lane and returns immediately; the delaymgr thread NEVER
* performs the blocking connect inline (ha-async-io.md thread-role
* contract — a blocking connect on the heartbeat/election-timeout thread
* stalls liveness to healthy peers and triggers spurious elections).
* OFF → legacy inline blocking connect on the delaymgr thread; retained only
* so the TDD RED test can demonstrate the pre-change stall.
*
* Compiled in ALL builds (production reads it on every heartbeat tick); the
* setter is the test seam to flip RED↔GREEN. */
static _Atomic int g_drive_async_reconnect_offload = 1;
void ha_set_reconnect_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_reconnect_offload, on ? 1 : 0,
memory_order_release);
}
int ha_reconnect_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_reconnect_offload,
memory_order_acquire);
}
/* raft-drive-async task 3.2: SnapshotSend feature flag.
*
* DEFAULT ON (1): the snapshot transfer runs on a slow-lane worker, off the
* caller (io_worker/drive) thread. The bug-1197-family borrowed-endpoint
* hazard (snapshot_lane_fn's `ep` is shared with the io_worker epoll loop,
* not lane-owned) is fixed: ha_snapshot_send_dispatch takes an endpoint_ref()
* before submit and snapshot_lane_fn releases it via endpoint_unref() on
* completion — the general bug-1298 refcount system (every heap-ep teardown
* site already routes through endpoint_unref, see cluster.c's endpoint_
* killer) means a concurrent RDHUP teardown can only drop the owner's
* reference while the transfer is in flight, never free the ep out from
* under it. Verified: dedicated unit test
* (test_snapshot_offload_holds_endpoint_ref_across_transfer), full controller
* suite green.
*
* Coupled to g_drive_async_peerctrl_dispatch (bug-1198 requires both ON
* together — see that flag's comment) and flips ON with it. The cold-start
* pool-wiring self-deadlock (bug-1376) found flipping both ON together in a
* live force-snapshot soak is root-caused and fixed — see
* g_drive_async_peerctrl_dispatch_enabled's comment.
*
* Keyed per standby (Req 1.5) so at most one snapshot per peer is in flight;
* at most one persist_checkpoint in flight (existing atomic guard). Compiled
* in ALL builds; the setter is the test seam to flip RED↔GREEN. */
static _Atomic int g_drive_async_snapshot_offload = 1;
void ha_set_snapshot_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_snapshot_offload, on ? 1 : 0,
memory_order_release);
}
int ha_snapshot_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_snapshot_offload,
memory_order_acquire);
}
/* raft-drive-async task 3.3: ConfigPush feature flag.
*
* DEFAULT OFF (0): the requester-less config push runs INLINE on the caller
* (io_worker/drive) thread — byte-for-byte the legacy behavior, i.e.
* push_config_transport_only's fan-out followed by its blocking
* wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS).
*
* Landed DORMANT alongside the Task 3.1/3.2 flags because the offload borrows
* standby endpoints it does not own (bug-1197 family — push_config_transport_only
* reads c->ha.standby_endpoints): moving the push to a lane thread while the
* io_worker may RDHUP-free those endpoints would be a cross-thread UAF.
* push_config_transport_only's hazard was DISTINCT from the snapshot (3.2)
* lane's borrowed-ep hazard — it grabbed raw endpoint_t* pointers for ALL
* standbys under c->ha.lock, released the lock, then spawned one pthread per
* standby that read/wrote those endpoints unprotected for up to
* CONF_PUSH_TIMEOUT_MS, a race that existed regardless of this flag's value
* (the pthread-per-standby fan-out is not the io_worker, so "runs inline on
* io_worker" never actually serialized it against a concurrent RDHUP
* teardown). Logged as bug-1381, **fixed 2026-07-02**: push_config_transport_only
* (config_repl.c) now takes its own endpoint_ref() on each standby ep at the
* same point the pointer is snapshotted under c->ha.lock, with a matching
* endpoint_unref() from push_thread (or from this function's own cleanup if a
* thread never started) — mirroring this same file's snapshot-lane fix.
*
* **Flip attempted 2026-07-02, REVERTED to OFF**: bug-1381 alone did not clear
* the bar. Flag-ON force-snapshot soak went 0/3 pass ("no snapshot install" /
* "did not reconverge to 3-of-3" on the freshly-rebuilt xce.build binary — NOT
* a stale-binary artifact); a flag-OFF control on the identical build went
* 1/2 pass with the SAME failure signature, so the underlying defect is
* PRE-EXISTING and orthogonal to this flag (root cause: `ha_send_snapshot_to_standby`,
* config_repl.c, never checks `xce_snapshot_chunk_req_raw`'s return in its
* chunk loop, so a chunk that fails to enqueue — e.g. ECONNRESET during the
* empty-rejoin reconnect window — is silently dropped and the sender still
* logs "transfer complete" and returns 0; logged as a new bug, see buglog).
* Even so, flag-ON's 0/3 vs flag-OFF's 1/2 is a small enough sample that a
* TIMING interaction (configpush lane traffic narrowing the window the
* snapshot lane has to complete before reconnect) can't be ruled out either.
* Flag stayed OFF until the chunk-loss bug (bug-1386) AND a second,
* independently-discovered bug (bug-1389) were fixed and re-validated.
*
* **bug-1386 FIXED 2026-07-02** (config_repl.c): the send-side chunk loop now
* checks xce_snapshot_chunk_req_raw's return and stops+propagates on failure
* instead of silently continuing and reporting "complete".
*
* **bug-1389 FIXED 2026-07-02** (this file + ha.h): re-validating bug-1386's
* fix surfaced a SECOND, independent defect — force-snapshot chunks are
* dispatched across the vtask pool with no ordering guarantee (a live soak
* showed arrival order 0,1,3,4,2,5,... across cpu#0-3 workers), but
* ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in
* ARRIVAL order instead of at its own chunk_offset — reordered chunks landed
* at the wrong file position (same total size, wrong content). A live crc32
* correlation between send and receive proved every chunk's CONTENT survived
* the wire intact, ruling out capnp/transport entirely. Fixed via
* pwrite()-at-chunk_offset (order-independent) plus a
* snapshot_recv_count/snapshot_recv_total completion tracker (guarded by
* ha->lock) so finalize fires only once every chunk has actually arrived,
* not merely because the current call happens to carry is_last.
*
* **Flag re-flipped to ON 2026-07-02** after both bugs closed: full controller
* suite green, live-validated 3/3 --force-snapshot PASS (genuine snapshot
* installs confirmed) + 6-round --chaotic PASS on the flag-ON binary.
*
* ONLY the requester-less call sites (post-election promotion, heartbeat-driven
* HA_REPL_PUSH_CONFIG) route through this flag; the admin-upgrade barrier path
* (push_config_to_standbys) is unaffected. Compiled in ALL builds; the setter
* is the test seam to flip RED<->GREEN. */
static _Atomic int g_drive_async_configpush_offload = 1;
void ha_set_configpush_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_configpush_offload, on ? 1 : 0,
memory_order_release);
}
int ha_configpush_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_configpush_offload,
memory_order_acquire);
}
/* raft-drive-async task 3.1: PeerCtrlDispatch feature flag.
*
* DEFAULT ON (1): the registered peer-control handler hands the decoded
* packet to the vtask pool and returns promptly, instead of running its
* routing body inline on the iomgr io_worker thread.
*
* Gate items closed: the task 3.4 deadlock-freedom proof (LivenessAdmission,
* task 1.6), the bug-1197 endpoint-quiescence fix (3.4a), bug-1198 (closed by
* the snapshot-offload flag's endpoint_ref/endpoint_unref fix — see that
* flag's comment), and bug-1376 (below).
*
* bug-1376 (self-deadlock, root-caused + fixed): a live --force-snapshot soak
* (a genuinely EMPTY node cold-starting and rejoining — the one scenario
* chaotic mode never exercises, since it restarts nodes with WAL intact) hung
* with both flags ON. Root cause: peer_ctrl_msg_handler held ep->dispatch_
* inflight across the ENTIRE peer_ctrl_dispatch_to_pool call, including its
* inline fallback when ha->pool is not yet wired (the common cold-start
* shape — a fresh node's own self-election can resolve, via an inbound vote
* reply, before cluster_init_phase2 finishes wiring the pool). That inline
* body can reach raft_cb_become_follower, which — now that cluster_init_
* phase2 has run synchronously earlier in the SAME call chain and wired the
* pool — tears down its own loser vote-eps via the deferred path
* (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting
* for dispatch_inflight==0 on THIS SAME ep. That count is held by this exact
* call, on this exact thread, and cannot reach 0 until the spin itself
* returns — a permanent self-deadlock. Fixed: peer_ctrl_dispatch_to_pool now
* releases dispatch_inflight at the exact right point for each of its 3 exit
* paths — before calling the inline body on every fallback path (pool==NULL,
* OOM, enqueue failure), since io_worker is single-threaded and the inline
* path never enqueues anything, so there is no pool-worker race left to
* protect against once inline execution is committed to; only after the
* (non-blocking) enqueue completes on the success path, preserving the
* original protection for the genuinely concurrent pool-worker case. Unit
* test: test_vep_self_teardown_does_not_deadlock (bounded-timeout join,
* RED->GREEN). Live-verified: --force-snapshot soak (the exact repro shape)
* now completes without hanging, confirmed a real snapshot install,
* committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak
* with both flags ON also passed clean.
*
* Compiled in ALL builds (production reads it on every peer-control message);
* the setter is the test seam to flip OFF for a regression comparison. */
static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 1;
void ha_set_peerctrl_dispatch_for_test(int on)
{
atomic_store_explicit(&g_drive_async_peerctrl_dispatch_enabled, on ? 1 : 0,
memory_order_release);
}
int ha_peerctrl_dispatch_enabled(void)
{
return atomic_load_explicit(&g_drive_async_peerctrl_dispatch_enabled,
memory_order_acquire);
}
/* raft-drive-async task 3.1: test-only observation seam (plain global, NULL in
* production — zero cost, same idiom as ha_nop_req_hook). Invoked at the top
* of peer_ctrl_msg_handler_inline before any role routing. */
void (*ha_peerctrl_body_probe)(endpoint_t *ep) = NULL;
/* ── Test-only hook shims (XCE_TESTING only) ──────────────────────────────
* Type definitions (election_timeout_ctx_t), extern declarations
* (cluster_init_phase2, push_config_transport_only,
* ha_send_ctrlhello_to_all_standbys), and static forward declarations
* (ha_heartbeat_is_fresh) have been moved to the forward-declarations block
* near the top of this file so that the raft callback implementations
* (Task 6.2) can reference them.
*/
#ifdef XCE_TESTING
/*
* ha_phase2_hook: test-only seam.
* When non-NULL, ha_elect_evaluate_and_signal() calls this INSTEAD OF the real
* cluster_init_phase2() so tests can count phase2 calls without a live cluster_t.
*/
int (*ha_phase2_hook)(void *c) = NULL;
/*
* ha_send_ctrlhello_hook: test-only seam for ha_send_ctrlhello_to_all_standbys.
* When non-NULL, the post-election path calls this INSTEAD OF the real function
* (cluster.c is not linked in unit-test binary). Req 7.2.
*/
int (*ha_send_ctrlhello_hook)(void *c) = NULL;
/*
* ha_push_config_hook: test-only seam for push_config_transport_only.
* When non-NULL, the post-election path calls this INSTEAD OF the real function
* (config_repl.c's push_config_transport_only needs a real cluster). Req 7.1.
*/
int (*ha_push_config_hook)(void *c) = NULL;
/*
* ha_nop_req_hook: test-only seam for xce_nop_req in primary_heartbeat_fn.
* When non-NULL, primary_heartbeat_fn calls this INSTEAD OF xce_nop_req so
* tests can count NOP sends without a live endpoint. Req 3.2, task 2.2.
*/
int (*ha_nop_req_hook)(endpoint_t *ep) = NULL;
/*
* ha_heartbeat_req_hook: test-only seam for the HA heartbeat send in
* primary_heartbeat_fn. When non-NULL, primary_heartbeat_fn calls this INSTEAD
* OF xce_ha_heartbeat_req with the SAME (term, committed_lsn,
* committed_config_tag) it stamps onto the wire, so tests can assert the
* heartbeat carries the primary's own current committed state. Req 1.1, 3.4.
*/
int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term,
uint64_t committed_lsn,
uint64_t committed_config_tag) = NULL;
/*
* ha_reconnect_connect_hook: test-only seam for the BLOCKING half of the
* offloaded peer reconnect (raft-drive-async Task 2.1). When non-NULL, the
* slow-lane reconnect worker calls this INSTEAD OF the real bounded
* endpoint_connect + iomgr_monitor, so a test can drive the reconnect path
* deterministically (e.g. park the worker to simulate an unreachable peer).
* Returns 0 to simulate success, <0 to simulate failure. Req 2.1, 2.2, 2.5.
*/
int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx) = NULL;
#ifdef XCE_TESTING
/*
* ha_standby_evicted_count_for_test: counts how many stale same-peer outbound
* channels on_standby_connected evicted while resolving the inbound-vs-outbound
* connection-ownership race (raft-drive-async Task 2.2, Req 2.6). Test-only
* observability — never read on the production path.
*/
_Atomic int ha_standby_evicted_count_for_test = 0;
#endif
/*
* ha_repl_push_config_for_standby_hook: test-only seam for the per-standby
* config push triggered by the heartbeat-reply decision loop (Task 3.3).
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* push_config_transport_only() when the decision is HA_REPL_PUSH_CONFIG.
* Requirements: 1.4 (ha-repl-hardening)
*/
int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx) = NULL;
/*
* ha_repl_stream_wal_hook: test-only seam for ha_stream_wal_to_standby.
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* ha_stream_wal_to_standby() when the decision is HA_REPL_STREAM_WAL.
* Requirements: 5.1–5.5 (ha-repl-hardening)
*/
int (*ha_repl_stream_wal_hook)(void *ep, const char *wal_path,
uint64_t standby_lsn,
uint64_t committed_lsn) = NULL;
/*
* ha_repl_send_snapshot_hook: test-only seam for ha_send_snapshot_to_standby.
* When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF
* ha_send_snapshot_to_standby() when the decision is HA_REPL_SEND_SNAPSHOT.
* Requirements: 4.1–4.5 (ha-repl-hardening)
*/
int (*ha_repl_send_snapshot_hook)(int peer_idx) = NULL;
/*
* ha_configpush_probe: test-only observation seam for the requester-less
* config-push offload (raft-drive-async Task 3.3). When non-NULL,
* ha_config_push_do_send() invokes it right around push_config_transport_only
* (on BOTH the inline and the lane path), so a test can record the running
* thread and optionally park the lane worker in flight to prove the
* cluster-wide single-flight dedup. NULL in production (zero cost).
*/
void (*ha_configpush_probe)(void) = NULL;
#endif /* XCE_TESTING */
/* ── Election Evaluation (Req 2.1, 2.7, 2.8) ───────────────────────────────
*
* ha_elect_evaluate_and_signal: evaluate accumulated votes; set ctx->result
* and post ctx->done when quorum is reached or all peers replied.
*
* Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size.
* ctx->votes_granted starts at 1 (self-vote); ctx->n_peers is n_other_peers.
* Quorum = ceil((n_peers + 1) / 2) = (n_peers + 1 + 1) / 2 = (n_peers + 2) / 2.
*
* Yield gate (Req 2.7): if any responding peer has a higher config_tag, or
* equal config_tag with a higher LSN than the candidate → yield (CTRL_STANDBY).
* Candidate's own tag/lsn is stored in ctx->votes[0].my_tag / my_lsn.
*
* Fatal gate (Req 2.8): if candidate has config_tag=0 and any peer has tag>0
* → CTRL_STANDBY (never PRIMARY with no config when peer has config).
*
* Returns CTRL_PRIMARY or CTRL_STANDBY; also sets ctx->result.
*/
ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx)
{
/* Guard: called from handle_request_vote_reply under ctx->lock; called from
* unit-test path (peers==NULL) without threads — done_posted prevents double
* execution if two threads both hit quorum simultaneously. */
if (ctx->done_posted)
return (ctx->result == 1) ? CTRL_PRIMARY : CTRL_STANDBY;
ctrl_role_t role_result;
/* Fix 2 (CRITICAL): zero-peer guard — single-node shortcircuit.
* n_peers==0 means no other nodes; votes[] flexible array has 0 entries,
* so accessing votes[0] would be out-of-bounds. Self-vote (votes_granted=1)
* always satisfies quorum=1. No fresher-peer or fatal-gate checks apply. */
if (ctx->n_peers == 0) {
ctx->result = 1;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_PRIMARY;
goto apply_role;
}
{
int total_nodes = ctx->n_peers + 1; /* include self */
int quorum = (total_nodes + 1) / 2; /* ceil(total/2) */
/* Req 2.8: fatal gate — candidate has no config but a peer does */
uint64_t my_tag = ctx->votes[0].my_tag;
uint64_t my_lsn = ctx->votes[0].my_lsn;
if (my_tag == 0 && ctx->max_resp_tag > 0) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
/* Check quorum */
if (ctx->votes_granted < quorum) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
/* Req 2.7: yield gate — fresher responding peer */
if (ctx->max_resp_tag > my_tag) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
if (ctx->max_resp_tag == my_tag && ctx->max_resp_lsn > my_lsn) {
ctx->result = 0;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_STANDBY;
goto apply_role;
}
/* Quorum met and no fresher peer → PRIMARY */
ctx->result = 1;
ctx->done_posted = 1;
sem_post(&ctx->done);
role_result = CTRL_PRIMARY;
}
apply_role:
/* ── Task 4.1: apply role and call cluster_init_phase2 ─────────────────
*
* After determining the election result, update ha->role and (on first boot)
* call cluster_init_phase2. This runs once per process lifetime — the
* phase2_done flag prevents re-entry on subsequent elections (RDHUP failover).
*
* The cluster_t pointer is carried in ctx->c (NULL in unit tests).
*
* Cancel the election timeout watchdog if we won (it would be a no-op now
* but the explicit cancel avoids a spurious second evaluate call).
*/
{
cluster_t *c = ctx->c;
int is_first_boot = !ha->phase2_done;
if (role_result == CTRL_PRIMARY) {
/* Cancel election timeout watchdog (quorum already reached) */
if (ctx->timeout_ctx) {
election_timeout_ctx_t *etc =
(election_timeout_ctx_t *)ctx->timeout_ctx;
atomic_store(&etc->cancelled, 1);
}
/* First boot: call cluster_init_phase2 to load config+data (Req 9.2) */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
if (ha_phase2_hook != NULL)
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
p2rc = cluster_init_phase2(c);
#endif
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure. Unlike
* raft_cb_become_leader, this legacy evaluation path has
* ALREADY signaled the caller's ctx->done semaphore with
* result=1 above (before "apply_role:") — refusing
* promotion here would leave that signal inconsistent
* with never becoming CTRL_PRIMARY. Proceed to
* CTRL_PRIMARY as before; ha_schedule_phase2_retry wires
* the commit engine shortly after instead of never. */
WARN("HA FSM: cluster_init_phase2 failed on promotion "
"(rc=%d) — proceeding to CTRL_PRIMARY, retrying "
"phase2 (bug-1477)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
}
}
/* ha-single-leader-role task 3.1 (Req 1.1, 1.5): enter CTRL_PRIMARY
* DIRECTLY — symmetric with raft_cb_become_leader's rewrite. There
* is no transitional COMMITTING state and no committing-timeout
* watchdog; new-leader safety comes from the own-term WAL_NOOP +
* writable_lsn gate armed below (ha_arm_own_term_gate), not from a
* fourth role value. The legacy XCE_TESTING election path OWNS its
* term (no raft library in test mode); it historically did NOT
* increment the term in this branch, so we preserve that — no
* ha_do_promote, no new term store (research.md §4 finding 2). */
atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY,
memory_order_release);
INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64
", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire),
is_first_boot);
/* Push config to standbys with lower config_tag (Req 7.1).
* Task 3.3: requester-less promotion push → offloadable (dormant
* flag OFF = inline, unchanged; flag ON = slow lane). */
#ifdef XCE_TESTING
if (ha_push_config_hook != NULL)
ha_push_config_hook(c);
#else
if (c != NULL)
ha_config_push_offloadable(c, NULL, 0, NULL, 0);
#endif
/* Send CtrlHelloRequest to all standbys to trigger WAL catch-up (Req 7.2) */
#ifdef XCE_TESTING
if (ha_send_ctrlhello_hook != NULL)
ha_send_ctrlhello_hook(c);
#else
if (c != NULL)
ha_send_ctrlhello_to_all_standbys(c);
#endif
/* ha-single-leader-role task 3.1: route promotion side effects
* through the SAME shared sequence raft_cb_become_leader runs so the
* test-driven (legacy) and raft-driven promotion paths cannot diverge
* (design §Legacy Path Symmetry). Guarded on a real cluster_t — the
* helpers dereference &c->hooksys / &c->persist / &c->commit, and the
* unit-test evaluation path passes c == NULL. Identical role / hook /
* heartbeat / no-watchdog / own-term-gate state results either road. */
if (c) {
/* Register the persistence hooks BEFORE the own-term no-op append,
* so the persistence seam is live before any racing client write.
* Idempotent via the hooks_registered guard. */
ha_register_persist_hooks(c);
/* Periodic heartbeat to standbys (idempotency guard prevents
* double-arming if already started). */
ha_start_primary_heartbeat(c);
/* Own-term WAL_NOOP + writable_lsn gate (Req 1.4, 2.1, 2.2) —
* non-blocking; append failure degrades safe (gate unset,
* promotion still completes). */
ha_arm_own_term_gate(c);
}
} else {
/* STANDBY result */
/* Cancel election timeout watchdog — it is now a no-op but explicit
* cancellation matches spec wording and avoids a spurious lock-acquire
* if the timeout fires after STANDBY is decided. */
if (ctx->timeout_ctx) {
election_timeout_ctx_t *etc =
(election_timeout_ctx_t *)ctx->timeout_ctx;
atomic_store(&etc->cancelled, 1);
}
/* First boot: call cluster_init_phase2 so structures are ready;
* config arrives via primary's replication push (Req 9.2, 7.3) */
if (is_first_boot) {
int p2rc = 0;
#ifdef XCE_TESTING
/* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */
if (ha_phase2_hook != NULL)
p2rc = ha_phase2_hook(c);
#else
if (c != NULL)
p2rc = cluster_init_phase2(c);
#endif
if (p2rc == 0) {
ha->phase2_done = 1;
} else {
/* bug-1477: do NOT latch phase2_done on failure — a
* standby can safely stay passive while
* ha_schedule_phase2_retry retries. */
WARN("HA FSM: cluster_init_phase2 failed on "
"become-standby (rc=%d) — member stays alive, will "
"retry (bug-1477)", p2rc);
if (c != NULL)
ha_schedule_phase2_retry(c);
}
}
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)",
atomic_load_explicit(&ha->term, memory_order_acquire),
is_first_boot);
/* Primary connects outbound and pushes config via ctrlhello.
* Standby is fully passive here — no fetch_config_from_primary. */
}
}
return role_result;
}
/* ── Vote Reply Handler (Req 2.2–2.6) ──────────────────────────────────────
*
* handle_request_vote_reply: process a reply to an OP_requestVote RPC.
*
* Called by each per-peer vote-request thread once the peer replies.
* Updates ctx->votes[peer_idx] with the reply fields, tracks the freshest
* responding peer in ctx->max_resp_tag/max_resp_lsn, increments counters,
* and evaluates whether to signal ctx->done.
*
* Quorum = ceil((n_peers + 1) / 2) = (n_peers + 2) / 2
* (n_peers = number of other peers, not counting self; votes_granted starts at 1)
*
* Posts ctx->done via ha_elect_evaluate_and_signal when:
* - votes_granted reaches quorum (fast path — no need to wait for all replies), OR
* - n_replied reaches n_peers (all peers accounted for, win or lose)
*/
/*
* ha_record_vote_reply: record a decoded vote reply into ctx->votes[peer_idx]
* WITHOUT signalling the legacy promotion path (bug-828).
*
* Records the granting flag, peer config_tag, and peer LSN under ctx->lock with
* the same idempotency guard handle_request_vote_reply uses, and updates the
* vote tally + freshest-responder tracking. It does NOT call
* ha_elect_evaluate_and_signal — so it never posts ctx->done and never runs the
* legacy apply_role promotion side-effects.
*
* This is the seam the PRODUCTION vote_reply_dispatch path calls so the granting
* voter's advertised LSN reaches ha_compute_safe_committed_lsn (via ctx->votes[])
* while the raft library remains the SOLE production promotion driver
* (raft_on_vote_reply → on_become_leader). Routing the record through here (not
* handle_request_vote_reply) avoids a second, racing promotion signal.
*
* Returns non-zero if this reply newly reached the legacy quorum/all-replied
* threshold (so handle_request_vote_reply knows to signal); 0 otherwise.
* Must be called WITHOUT ctx->lock held (it takes the lock internally).
*/
static int ha_record_vote_reply_locked(ha_elect_ctx_t *ctx,
int peer_idx,
int vote_granted,
uint64_t peer_config_tag,
uint64_t peer_lsn)
{
/* Guard against out-of-range index */
if (peer_idx < 0 || peer_idx >= ctx->n_peers)
return 0;
pthread_mutex_lock(&ctx->lock);
ha_vote_entry_t *entry = &ctx->votes[peer_idx];
/* Idempotency guard: ignore duplicate replies */
if (entry->replied) {
pthread_mutex_unlock(&ctx->lock);
return 0;
}
/* Record reply fields */
entry->replied = 1;
entry->vote_granted = vote_granted ? 1 : 0;
entry->peer_tag = peer_config_tag;
entry->peer_lsn = peer_lsn;
/* Update counters */
if (vote_granted)
ctx->votes_granted++;
ctx->n_replied++;
/* Track the freshest responding peer for the yield gate (Req 2.7) */
if (peer_config_tag > ctx->max_resp_tag) {
ctx->max_resp_tag = peer_config_tag;
ctx->max_resp_lsn = peer_lsn;
} else if (peer_config_tag == ctx->max_resp_tag && peer_lsn > ctx->max_resp_lsn) {
ctx->max_resp_lsn = peer_lsn;
}
/* Compute whether the legacy quorum/all-replied threshold is now reached.
* Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. */
int total_nodes = ctx->n_peers + 1;
int quorum = (total_nodes + 1) / 2;
int reached = (ctx->votes_granted >= quorum || ctx->n_replied >= ctx->n_peers);
pthread_mutex_unlock(&ctx->lock);
return reached;
}
/* Public record-only seam (bug-828): records the reply but never signals the
* legacy promotion path. Used by the production vote_reply_dispatch path. */
void ha_record_vote_reply(ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t term,
int vote_granted,
uint64_t peer_config_tag,
uint64_t peer_lsn)
{
(void)ha; /* fencing: caller bumps ha->term before calling here */
(void)term;
if (!ctx)
return;
(void)ha_record_vote_reply_locked(ctx, peer_idx, vote_granted,
peer_config_tag, peer_lsn);
}
void handle_request_vote_reply(ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t term,
int vote_granted,
uint64_t peer_config_tag,
uint64_t peer_lsn)
{
(void)term; /* fencing: callers bump ha->term if peer's term > ours, before calling here */
if (peer_idx < 0 || peer_idx >= ctx->n_peers)
return;
/* Record the reply (shared with the production record-only seam) and learn
* whether the legacy quorum/all-replied threshold is now reached. */
int reached = ha_record_vote_reply_locked(ctx, peer_idx, vote_granted,
peer_config_tag, peer_lsn);
/* Decide whether to signal the waiting thread.
* ha_elect_evaluate_and_signal takes ctx->lock; its done_posted guard
* prevents double sem_post if two threads reach quorum simultaneously.
* Skip if the index was out of range / a duplicate reply (reached==0 and
* nothing recorded) — the duplicate case is already idempotent in
* ha_record_vote_reply_locked, which returns 0 without re-counting. */
if (reached) {
pthread_mutex_lock(&ctx->lock);
ha_elect_evaluate_and_signal(ha, ctx);
pthread_mutex_unlock(&ctx->lock);
}
}
/* ── Task 2.2: iomgr-native vote send path ──────────────────────────────────
*
* vote_ep_ctx_t: per-peer context stored in ep->data for each outgoing vote
* endpoint. on_vote_connected fires after the TLS handshake completes (when
* endpoint_t.ssl_handshake_done transitions to 1 in io_handler).
*/
/* vote_ep_ctx_t: typedef moved to Task 6.2 forward-declaration block above */
/*
* peer_ctrl_msg_handler / vote_reply_decode_and_dispatch /
* repl_ack_decode_and_dispatch: forward declarations.
* peer_ctrl_msg_handler is the single iomgr handler set in on_vote_connected
* and never reassigned — it routes by ha->role at dispatch time.
*/
static int peer_ctrl_msg_handler(void *ep_raw);
static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt);
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx,
io_packet_t *pkt);
/* ha-prevote task 5.1: preVoteReply decode/route; defined alongside the
* real-vote decoder, feeds the later-defined pre_vote_reply_dispatch. */
static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx,
io_packet_t *pkt);
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted);
#ifdef XCE_TESTING
/*
* xce_vote_request_hook: test-only seam.
*
* When non-NULL, on_vote_connected() calls this INSTEAD OF the real
* xce_request() so tests can capture the call without requiring a live
* iomgr or SSL endpoint. The hook receives the fully built
* xce_request_t and may inspect all fields.
*
* Pattern mirrors xce_retry_send_hook in request.c.
*/
int (*xce_vote_request_hook)(endpoint_t *ep, xce_request_t *req) = NULL;
#endif /* XCE_TESTING */
/*
* on_vote_connected: iomgr callback fired after TLS handshake completes on an
* outgoing vote endpoint.
*
* Steps:
* 1. Cast ep->data to vote_ep_ctx_t.
* 2. Build request_vote_req_t from ctx fields.
* 3. Build xce_request_t{opcode=OPCode_requestVote, body.requestvotemsg=&vrq}.
* 4. Call xce_vote_request_hook (test seam, XCE_TESTING only) — then
* call xce_request() to queue the packet on the iomgr send path.
* 5. Set ep->msg_handler = vote_reply_msg_handler so the next inbound
* packet is decoded as a RequestVoteReply.
*
* Req 2.3: send vote after TLS handshake.
* Req 1.2: vote RPC via iomgr path (xce_request enqueues to endpoint_enq).
*/
static void on_vote_connected(endpoint_t *ep)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
/* Build the RequestVoteRequest payload */
request_vote_req_t vrq;
memset(&vrq, 0, sizeof(vrq));
vrq.term = vctx->my_term;
vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
vrq.config_tag = vctx->my_tag;
vrq.last_log_lsn = vctx->my_lsn;
/* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */
vrq.last_log_term = vctx->my_last_log_term;
vrq.last_log_index = vctx->my_last_log_index;
/* Wrap in xce_request_t */
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_requestVote;
req.body_which = Request_body_requestvotemsg;
req.body.requestvotemsg = &vrq;
#ifdef XCE_TESTING
if (xce_vote_request_hook != NULL)
xce_vote_request_hook(ep, &req);
else
xce_request(ep, &req);
#else
xce_request(ep, &req);
#endif
/* Set state-aware dispatcher — routes by ha->role at message-arrival time.
* Never reassigned: ELECTING → vote reply; PRIMARY → repl ack. */
ep->msg_handler = peer_ctrl_msg_handler;
}
/*
* on_pre_vote_connected: iomgr callback fired after the TLS handshake completes
* on an outgoing PreVote endpoint (ha-prevote task 4.1).
*
* Mirrors on_vote_connected but emits OPCode_preVoteRequest (reusing the
* RequestVoteRequest payload struct + union arm, per the wire schema in task
* 1.1). prospective_term (current+1) rides in the term field; nothing is
* persisted. The reply-routing handler for preVoteReply is wired in task 5.1;
* here we set the same state-aware dispatcher used by the real-vote path.
*/
static void on_pre_vote_connected(endpoint_t *ep)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
request_vote_req_t vrq;
memset(&vrq, 0, sizeof(vrq));
vrq.term = vctx->my_term; /* prospective term = current + 1 */
vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : "");
vrq.config_tag = vctx->my_tag;
vrq.last_log_lsn = vctx->my_lsn;
/* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */
vrq.last_log_term = vctx->my_last_log_term;
vrq.last_log_index = vctx->my_last_log_index;
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_preVoteRequest;
req.body_which = Request_body_requestvotemsg;
req.body.requestvotemsg = &vrq;
#ifdef XCE_TESTING
if (xce_vote_request_hook != NULL)
xce_vote_request_hook(ep, &req);
else
xce_request(ep, &req);
#else
xce_request(ep, &req);
#endif
ep->msg_handler = peer_ctrl_msg_handler;
}
/*
* xce_vote_reply_hook: test-only seam for vote_reply_msg_handler.
*
* Forward-declared here (before vote_reply_msg_handler) so the handler body
* can reference it. Defined only under XCE_TESTING; guarded below.
*
* When non-NULL, vote_reply_msg_handler() calls this INSTEAD OF the real
* handle_request_vote_reply() so tests can capture decoded fields without
* a live iomgr. Receives the decoded scalar fields — no capnp pointers.
*
* Pattern mirrors xce_vote_request_hook used in task 2.2.
*/
#ifdef XCE_TESTING
int (*xce_vote_reply_hook)(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx,
uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn) = NULL;
#endif /* XCE_TESTING */
/*
* vote_reply_dispatch: internal helper — delegates to the raft library or
* the test hook (when XCE_TESTING and hook is non-NULL).
*
* Production path: raft_on_vote_reply owns quorum tracking and fires
* on_become_leader/on_become_follower via the raft callbacks.
* Test path: xce_vote_reply_hook captures decoded fields without real iomgr.
*/
static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx,
uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn)
{
#ifdef XCE_TESTING
if (xce_vote_reply_hook != NULL) {
xce_vote_reply_hook(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn);
return;
}
#endif
/* bug-828 (PRIMARY fix): route the decoded reply into the election context
* BEFORE driving the raft library. Without this, the granting voter's
* advertised peer_lsn was discarded — ctx->votes[idx].vote_granted stayed 0,
* so a clean failover's ha_compute_safe_committed_lsn (read in
* raft_cb_become_leader / on_become_leader) saw only the self-vote
* (n_granted=1) and truncated committed entries down to self_lsn.
*
* Record-only: ha_record_vote_reply records ctx->votes[idx] under ctx->lock
* but does NOT post ctx->done / run the legacy apply_role promotion path.
* raft_on_vote_reply remains the SOLE production promotion driver (it owns
* the quorum tally and fires on_become_leader) — so there is no double
* promotion. `idx` is vctx->peer_idx, the local ctx->votes[] index (set in
* raft_cb_send_vote_request via the absolute→local mapping), which is exactly
* the index ha_compute_safe_committed_lsn and the test hook use. */
ha_record_vote_reply(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn);
/* bug-1417: raft_on_vote_reply's from_idx must be the ABSOLUTE cluster
* index (the space vote_replies[]/self_idx live in — raft_start_election
* pre-marks vote_replies[self_idx] and sends to every absolute peer !=
* self_idx, raft.c:113-153), NOT the compressed self-excluded votes_idx
* `idx` carries for ctx->votes[] above. Passing the compressed idx
* straight through collides with vote_replies[self_idx] whenever a
* peer's compressed votes_idx numerically equals self_idx (true for any
* peer at or above self_idx in a >2-node cluster, e.g. self_idx=1's
* absolute peer 2 compresses to votes_idx=1) — raft_on_vote_reply's
* per-peer dedup guard then silently treats the genuine reply as an
* already-counted duplicate and drops it, so a middle-priority node can
* never reach quorum on that peer's vote alone. ha_repl_ack_abs_peer_idx
* (ha.c:3779) already implements the exact inverse of
* raft_cb_send_vote_request's votes_idx remap and is unit-tested; reuse
* it here (ep_is_vote=1) rather than duplicating the arithmetic.
*
* bug-1435: raft_on_vote_reply mutates raft_state_t's plain (non-atomic)
* votes_granted/replies_received/election_decided fields and can fire
* on_become_leader/on_become_follower synchronously. With raft-drive-async
* pool dispatch (default ON), a DIFFERENT peer's vote reply can be
* processed on a DIFFERENT pool worker concurrently — without raft_lock,
* two concurrent calls can race: one reaches quorum and promotes to
* PRIMARY, while the other, racing on the pre-increment vote count,
* independently satisfies the "all replied, no quorum" branch and
* immediately fires on_become_follower for the same term, silently
* reverting role to STANDBY right after promotion (verified data race —
* every request thereafter is correctly, faithfully rejected as STANDBY
* by ha_dispatch_gate, matching the observed symptom exactly). Hold
* raft_lock across the whole call so the state-machine evaluation and any
* callback it fires are atomic with respect to other raft_state_t
* entry points. */
pthread_mutex_lock(&ha->raft_lock);
raft_on_vote_reply(&ha->raft, ha_repl_ack_abs_peer_idx(ha, /*ep_is_vote=*/1, idx),
vote_granted, term);
pthread_mutex_unlock(&ha->raft_lock);
}
/*
* vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch
* to vote_reply_dispatch. Factored from vote_reply_msg_handler so that
* peer_ctrl_msg_handler can invoke the same logic without code duplication.
* Packet ownership (raft-drive-async task 3.1): the caller
* (peer_ctrl_msg_handler) has ALREADY taken `pkt` off ep->fsm_in.pkt and
* cleared ep->fsm_in.pkt to NULL — `pkt` is passed in explicitly and this
* function frees it on every path. Clearing in the caller (not here) is what
* makes the off-iomgr dispatch path race-free: the body never touches
* ep->fsm_in.pkt, so a concurrent io_worker read on the same endpoint cannot
* collide with the in-flight packet. vote_reply_dispatch can still reach
* raft_cb_become_follower → endpoint_fini(ep) (loser path) safely because
* ep->fsm_in.pkt is already NULL.
*/
static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt)
{
ha_state_t *ha = vctx->ha;
ha_elect_ctx_t *ctx = vctx->ctx;
int idx = vctx->peer_idx;
int ok = 0;
uint64_t vr_term = 0;
int vr_granted = 0;
uint64_t vr_tag = 0;
uint64_t vr_lsn = 0;
if (!pkt || !pkt->data || pkt->sz == 0)
goto done;
{
struct capn capn_ctx;
struct Request outer_rq;
Request_ptr outer_rp;
struct RequestReply rep_s;
struct RequestVoteReply vr_raw;
int capn_ok;
capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
(size_t)pkt->sz, 0);
if (capn_ok != 0)
goto done;
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
read_Request(&outer_rq, outer_rp);
if (outer_rq.body_which != Request_body_replymsg ||
!outer_rq.body.replymsg.p.data) {
capn_free(&capn_ctx);
goto done;
}
read_RequestReply(&rep_s, outer_rq.body.replymsg);
if (rep_s.data_which != RequestReply_data_requestvotereplymsg) {
capn_free(&capn_ctx);
goto done;
}
memset(&vr_raw, 0, sizeof(vr_raw));
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
capn_free(&capn_ctx);
vr_term = (uint64_t)vr_raw.term;
vr_granted = vr_raw.voteGranted ? 1 : 0;
vr_tag = (uint64_t)vr_raw.peerConfigTag;
vr_lsn = (uint64_t)vr_raw.peerLastLogLsn;
ok = 1;
}
done:
if (pkt) { free_packet(pkt); free(pkt); }
if (ok) {
uint64_t cur = atomic_load_explicit(&ha->term, memory_order_acquire);
while (vr_term > cur) {
if (atomic_compare_exchange_weak_explicit(&ha->term, &cur, vr_term,
memory_order_acq_rel,
memory_order_acquire))
break;
}
vote_reply_dispatch(ha, ctx, idx, vr_term, vr_granted, vr_tag, vr_lsn);
} else {
vote_reply_dispatch(ha, ctx, idx,
atomic_load_explicit(&ha->term, memory_order_acquire),
0, 0, 0);
}
return 0;
}
/*
* pre_vote_reply_decode_and_dispatch: decode a preVoteReply packet on an
* outbound PreVote endpoint and feed it to the PreVote tally (ha-prevote task
* 5.1). Mirrors vote_reply_decode_and_dispatch exactly — the PreVote reply
* reuses the identical RequestVoteReply payload / union arm — but routes to
* pre_vote_reply_dispatch (the raft-library PreVote tally) instead of the
* real-vote tally, and deliberately performs NO term adoption (a PreVote reply
* never changes our term; Req 1.2). vctx->peer_idx is the ABSOLUTE cluster
* index that raft_on_pre_vote_reply expects (set in ha_send_pre_vote_request).
* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly
* (already taken off ep->fsm_in.pkt by peer_ctrl_msg_handler); this function
* frees it on every path and never touches ep->fsm_in.pkt.
*/
static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx,
io_packet_t *pkt)
{
ha_state_t *ha = vctx->ha;
cluster_t *c = vctx->c;
int idx = vctx->peer_idx; /* ABSOLUTE cluster index */
int ok = 0;
int vr_granted = 0;
if (!pkt || !pkt->data || pkt->sz == 0)
goto done;
{
struct capn capn_ctx;
struct Request outer_rq;
Request_ptr outer_rp;
struct RequestReply rep_s;
struct RequestVoteReply vr_raw;
int capn_ok;
capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
(size_t)pkt->sz, 0);
if (capn_ok != 0)
goto done;
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
read_Request(&outer_rq, outer_rp);
if (outer_rq.body_which != Request_body_replymsg ||
!outer_rq.body.replymsg.p.data) {
capn_free(&capn_ctx);
goto done;
}
read_RequestReply(&rep_s, outer_rq.body.replymsg);
if (rep_s.data_which != RequestReply_data_requestvotereplymsg) {
capn_free(&capn_ctx);
goto done;
}
memset(&vr_raw, 0, sizeof(vr_raw));
read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg);
capn_free(&capn_ctx);
vr_granted = vr_raw.voteGranted ? 1 : 0;
ok = 1;
}
done:
if (pkt) { free_packet(pkt); free(pkt); }
/* A malformed / wrong-body reply counts as a non-grant (best-effort
* delivery, per design Event Contract). No term adoption either way. */
pre_vote_reply_dispatch(ha, c, idx, ok ? vr_granted : 0);
return 0;
}
/*
* repl_ack_decode_and_dispatch: decode an inbound packet on a promoted vote
* endpoint while role == CTRL_PRIMARY.
*
* - OPCode_replCatchUpDone → INFO no-op (ack-counting removed in task 4.1)
* - OPCode_reply with stateReportMsg → ha_process_standby_state (Task 3.1)
* - OPCode_reply with walackreplymsg → repl_on_ack (Task 4.1)
* - OPCode_reply with replconfigpushreplymsg → repl_on_config_ack
* (barrier-rollout task 5.2 r1 / ha-config-bootstrap task 8.1)
* - OPCode_nop or OPCode_reply with any other body → silently ignored
* - anything else → WARN + drop
*/
static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx,
io_packet_t *pkt)
{
ha_state_t *ha = vctx->ha;
cluster_t *c = vctx->c;
/* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in
* explicitly (peer_ctrl_msg_handler already cleared ep->fsm_in.pkt); this
* function frees it and never touches ep->fsm_in.pkt.
*
* raft-drive-async tasks 3.2/3.3 (FUTURE): the long ops reachable from the
* stateReport branch below — ha_dispatch_repl_action_for_standby's
* snapshot send / config push — are where the SlowLane offload lands. For
* task 3.1 the body only relocates onto the pool and still runs bounded
* decode/route work inline in the vtask entry. */
if (!pkt || !pkt->data || pkt->sz == 0) {
if (pkt) { free_packet(pkt); free(pkt); }
return 0;
}
struct capn capn_ctx;
struct Request outer_rq;
Request_ptr outer_rp;
int capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data,
(size_t)pkt->sz, 0);
free_packet(pkt);
free(pkt);
if (capn_ok != 0)
return 0;
outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1);
read_Request(&outer_rq, outer_rp);
int opcode = (int)outer_rq.opcode;
if (opcode == OPCode_replCatchUpDone) {
capn_free(&capn_ctx);
/* ha-single-leader-role task 4.1: the force-promotion ack-counting
* machinery has been deleted — promotion is driven solely by the
* heartbeat-driven decision loop and per-entry majority ack. Standbys
* still send OPCode_replCatchUpDone and the dispatch gate still admits
* it; it is now an INFO-logged no-op. */
(void)ha;
(void)c;
INFO("HA: replCatchUpDone received (no-op; ack-counting machinery removed in task 4.1)");
} else if (opcode == OPCode_reply &&
outer_rq.body_which == Request_body_replymsg &&
outer_rq.body.replymsg.p.data) {
/* Heartbeat NOP reply — decode inner RequestReply for stateReport */
struct RequestReply rep_s;
memset(&rep_s, 0, sizeof(rep_s));
read_RequestReply(&rep_s, outer_rq.body.replymsg);
if (rep_s.data_which == RequestReply_data_stateReportMsg) {
/* Decode stateReport and call convergence tracker (Task 3.1).
* read_StateReport reads scalar fields directly from the capnp
* pointer without heap allocation — safe to call before capn_free. */
struct StateReport sr_raw;
memset(&sr_raw, 0, sizeof(sr_raw));
read_StateReport(&sr_raw, rep_s.data.stateReportMsg);
state_report_t sr;
sr.applied_lsn = (uint64_t)sr_raw.appliedLsn;
sr.config_tag = (uint64_t)sr_raw.configTag;
sr.term = (uint64_t)sr_raw.term;
capn_free(&capn_ctx);
/* bug-1421: a primary never learns of a follower's higher term via
* the AE-response path (an AE reject is dropped before it reaches
* raft_on_append_entries_reply — see ha_raft_on_repl_ack). The
* heartbeat-reply state report is the fallback term-discovery
* vector: it fires on every convergence cycle regardless of
* replication outcome, so a stale-term primary steps down here
* instead of retrying AppendEntries against a follower that will
* reject every entry forever. raft_on_higher_term_observed only
* touches (term, voted_for, role) — never peer-indexed
* bookkeeping — so it is safe to call before peer_idx is resolved.
* bug-1435: serialize against concurrent vote-reply/vote-request
* processing (see ha.h raft_lock doc) — this call can also fire
* on_become_follower, racing the same fields. */
pthread_mutex_lock(&ha->raft_lock);
(void)raft_on_higher_term_observed(&ha->raft, sr.term);
pthread_mutex_unlock(&ha->raft_lock);
/* peer_idx from standby_ep_ctx_t (layout-compatible with vote_ep_ctx_t) */
int peer_idx = vctx->peer_idx;
uint64_t committed_lsn = 0;
uint64_t committed_tag = 0;
uint64_t last_written = 0; /* latest WAL LSN — may be > committed */
#ifndef XCE_TESTING
if (c) {
committed_lsn = atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
committed_tag = atomic_load_explicit(&ha->committed_config_tag,
memory_order_acquire);
/* current_lsn is the next LSN to write; last written = current - 1.
* Read without lock — monotonic, approximate value is sufficient
* for the replication decision. */
last_written = (c->persist.current_lsn > 0)
? c->persist.current_lsn - 1 : 0;
}
#else
/* In test mode, c may be NULL; caller sets ha fields directly */
committed_lsn = atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire);
committed_tag = atomic_load_explicit(&ha->committed_config_tag,
memory_order_acquire);
(void)c;
#endif
ha_process_standby_state(ha, peer_idx,
sr.config_tag, sr.applied_lsn,
committed_tag, committed_lsn);
/* Derive oldest_wal_lsn from the WAL directory.
* The oldest retained segment's start LSN is the boundary —
* standbys below it need a snapshot (Task 3.5b). */
uint64_t oldest_wal_lsn = 0;
#ifndef XCE_TESTING
if (c && c->persist.wal_path[0]) {
extern uint64_t ha_compute_oldest_wal_lsn(const char *wal_path);
oldest_wal_lsn = ha_compute_oldest_wal_lsn(c->persist.wal_path);
}
#endif
/* Replication target (bug-1478): committed_lsn, with the
* last_written bootstrap fallback ONLY when committed_lsn is
* genuinely stuck at 0 — see ha_compute_repl_target_lsn's own
* doc comment for why an unconditional max() permanently wedges
* catch-up once a local-only checkpoint trails a real,
* already-advancing committed_lsn. */
uint64_t target_lsn =
ha_compute_repl_target_lsn(committed_lsn, last_written);
ha_dispatch_repl_action_for_standby(ha, c, ep, peer_idx,
committed_tag, target_lsn,
oldest_wal_lsn);
} else if (rep_s.data_which == RequestReply_data_walackreplymsg) {
/* Task 4.1: WAL ACK reply from standby — extract lsn + success,
* call repl_on_ack to update acked_lsn and signal majority waiter. */
struct WalAckReply war_raw;
memset(&war_raw, 0, sizeof(war_raw));
read_WalAckReply(&war_raw, rep_s.data.walackreplymsg);
uint64_t acked_lsn = (uint64_t)war_raw.lsn;
uint64_t reply_term = (uint64_t)war_raw.term;
int success = war_raw.success ? 1 : 0;
capn_free(&capn_ctx);
if (success && c) {
repl_on_ack(&c->persist.repl, ep, acked_lsn);
TRACE("repl_ack_decode_and_dispatch: WAL ACK from standby"
" lsn=%lu success=1", (unsigned long)acked_lsn);
/* ha-term-safety task 3.3 (R4.1/R4.5): route the ACK through the
* raft library so it updates match_index[]/next_index[] and
* recomputes the current-term-gated commit (raft_advance_commit).
* This fires on_commit_advanced (task 2.4) ONLY when a quorum
* agrees on a CURRENT-TERM entry — making the library the live
* commit source alongside the durable barrier; a stale-term reply
* is dropped inside the library. Driven on the same iomgr ACK
* path; neither library call performs I/O.
*
* Resolve the ABSOLUTE library peer index per endpoint type FIRST
* (remediation): this branch fires for BOTH promoted-vote eps
* (vctx->ctx != NULL, peer_idx is a relative votes index) AND
* heartbeat standby eps (vctx->ctx == NULL, peer_idx is already
* absolute) — see the authoritative scan at ha.c:6187-6189. A
* blind remap would double-map the heartbeat path onto the wrong
* (or out-of-range) match_index[] slot. */
int ack_abs_idx = ha_repl_ack_abs_peer_idx(
&c->ha, /*ep_is_vote*/ vctx->ctx != NULL, vctx->peer_idx);
ha_raft_on_repl_ack(c, ack_abs_idx, acked_lsn, reply_term);
/* ha-convergence-recovery task 4.2 (D6, Req 5.1/5.2/7.1):
* ACK-driven catch-up — continue streaming to a standby still
* behind target, or re-include it at parity. Shares the exact
* decision the unit test drives via ha_walack_continuation_for_test.
* Scheduling is O(1) on iomgr; all WAL file I/O runs in the
* deferred continuation task. */
ha_walack_on_ack(c, ep, vctx->peer_idx, acked_lsn);
} else if (!success) {
/* bug-1421: a WAL NACK can also carry a higher term (the
* standby rejected this entry as stale) — route it through
* the same term-discovery path as the heartbeat state report
* so the primary steps down instead of retrying forever.
* bug-1435: serialize against concurrent raft_state_t access
* (see ha.h raft_lock doc). */
pthread_mutex_lock(&ha->raft_lock);
(void)raft_on_higher_term_observed(&ha->raft, reply_term);
pthread_mutex_unlock(&ha->raft_lock);
/* bug-1479: a same-or-lower-term NACK means an ordinary Raft
* log conflict (ae_reject_logmismatch), NOT a stale-term
* primary — the case above doesn't cover it. Before this fix
* the reject was silently dropped: the library's own,
* already-correct nextIndex backoff (raft_on_append_entries_
* reply) was never invoked, so the primary kept re-
* broadcasting NEW entries assuming agreement at an index
* the standby had already rejected, forever — the standby
* would eventually get silently evicted via
* MAX_PENDING_ACKS, but nothing would ever stream it back
* into sync (repl_start_catchup's own driver, the ACK-driven
* continuation, needs a real ACK to fire, and a log-
* conflicted standby can never send one while stuck).
* Immediately (not after 16 silent failures) back off via
* the library and mark catch-up at the corrected position —
* ha_dispatch_repl_action_for_standby's heartbeat-driven
* decision (below) is what actually streams to it. */
if (c) {
int ack_abs_idx = ha_repl_ack_abs_peer_idx(
&c->ha, /*ep_is_vote*/ vctx->ctx != NULL, vctx->peer_idx);
uint64_t corrected_lsn = 0;
if (ha_raft_on_repl_nack(c, ack_abs_idx, reply_term,
&corrected_lsn) == 0) {
repl_start_catchup(&c->persist.repl, ep, corrected_lsn);
}
}
WARN("repl_ack_decode_and_dispatch: WAL NACK from standby"
" lsn=%lu success=0 — marking catch-up",
(unsigned long)acked_lsn);
}
} else if (rep_s.data_which == RequestReply_data_replconfigpushreplymsg) {
/* Config-push ACK from standby (ha-commit-barrier-rollout task
* 5.2 remediation r1; completes the unlanded half of
* ha-config-bootstrap task 8.1): the standby replies to
* OP_replConfigPush with replconfigpushreplymsg carrying
* {accepted, versionTag} (config_repl.c:744-752). Decode the
* acked tag and apply it to the replicator so the configAck
* barrier (repl_wait_majority_ack) can observe the covering
* ACK — without this branch every multi-member config upgrade
* would time out. read_ReplConfigPushReply reads scalars only
* — safe to call before capn_free. */
struct ReplConfigPushReply rcp_raw;
memset(&rcp_raw, 0, sizeof(rcp_raw));
read_ReplConfigPushReply(&rcp_raw, rep_s.data.replconfigpushreplymsg);
uint64_t acked_tag = (uint64_t)rcp_raw.versionTag;
int accepted = rcp_raw.accepted ? 1 : 0;
capn_free(&capn_ctx);
if (accepted && c) {
repl_on_config_ack(&c->persist.repl, ep, acked_tag);
TRACE("repl_ack_decode_and_dispatch: config ACK from standby"
" tag=%016" PRIx64 " accepted=1", acked_tag);
} else if (!accepted) {
WARN("repl_ack_decode_and_dispatch: config push rejected by"
" standby (tag=%016" PRIx64 ") — ignoring", acked_tag);
}
} else {
capn_free(&capn_ctx);
}
} else if (opcode == OPCode_nop) {
/* Plain NOP with no body — silently ignore */
capn_free(&capn_ctx);
} else {
capn_free(&capn_ctx);
WARN("repl_ack_decode_and_dispatch: unexpected opcode %d on promoted ep",
opcode);
}
return 0;
}
/*
* peer_ctrl_msg_handler_inline: the state-aware peer-control routing body.
* Routes by ha->role at dispatch time — no explicit handler reassignment needed.
* CTRL_ELECTING → vote_reply_decode_and_dispatch
* CTRL_PRIMARY → repl_ack_decode_and_dispatch
* CTRL_STANDBY + PreVote → pre_vote_reply_decode_and_dispatch (task 5.1)
* any other role → drop packet (WARN)
*
* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly —
* the caller (peer_ctrl_msg_handler) has already taken it off ep->fsm_in.pkt and
* cleared that slot to NULL. This body and the routers it calls free `pkt` on
* every path and NEVER touch ep->fsm_in.pkt, which is what lets this body run
* off the iomgr thread (on the vtask pool) without colliding with a concurrent
* io_worker read on the same endpoint.
*
* This body is run either INLINE on the io_worker thread (dispatch flag OFF —
* the production default; identical to the pre-task behavior) or on a vtask
* pool worker (flag ON; via peer_ctrl_vtask_entry).
*
* PreVote note (ha-prevote task 5.1): a preVoteReply arrives on the candidate's
* OUTBOUND PreVote endpoint while the candidate's role is still CTRL_STANDBY
* (a PreVote round performs no role change — Req 1.1). Without this branch the
* reply would hit the default case and be dropped, stranding the round until it
* times out. We gate on raft.pre_vote_in_progress so a STANDBY node with no
* active round still drops unexpected traffic.
*/
static int peer_ctrl_msg_handler_inline(endpoint_t *ep, io_packet_t *pkt)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
ha_state_t *ha = vctx->ha;
/* raft-drive-async task 3.1: test-only observation seam — fires before any
* routing so a test can count/timestamp/park the body. NULL in production. */
if (ha_peerctrl_body_probe)
ha_peerctrl_body_probe(ep);
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
/* PreVote replies land while CTRL_STANDBY with a round in progress. */
if (role == CTRL_STANDBY && ha->raft.pre_vote_in_progress)
return pre_vote_reply_decode_and_dispatch(vctx, pkt);
switch (role) {
case CTRL_ELECTING:
return vote_reply_decode_and_dispatch(vctx, pkt);
case CTRL_PRIMARY:
return repl_ack_decode_and_dispatch(ep, vctx, pkt);
default:
if (pkt) { free_packet(pkt); free(pkt); }
WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role);
return 0;
}
}
/*
* peer_ctrl_vtask_entry (raft-drive-async task 3.1): vtask pool entry that runs
* the peer-control routing body off the iomgr thread. Extracts (ep, pkt) from
* the VTASK_IO task data — the SAME io_task_data_t shape the lib vtask path
* uses — takes ownership of pkt out of the task data (so vtask_fini, which only
* frees the data struct, cannot reach it) and hands it to the body, which frees
* it. No double-free: pkt is freed exactly once inside the body; run_task then
* frees the task + data struct. endpoint_unblock_tasks (per-endpoint
* serialization, Req 4.1/4.2) is driven by run_task after this returns.
*/
static int peer_ctrl_vtask_entry(vtask_t *task)
{
io_task_data_t *iod = &task->data->u.iodata;
endpoint_t *ep = iod->ep;
io_packet_t *pkt = iod->pkt;
iod->pkt = NULL; /* body owns pkt now; nothing else may reference it */
if (ep == NULL) {
if (pkt) { free_packet(pkt); free(pkt); }
return -EINVAL;
}
return peer_ctrl_msg_handler_inline(ep, pkt);
}
/*
* peer_ctrl_classify (raft-drive-async task 3.1): assign the LivenessAdmission
* task class for an inbound peer-control message. This handler decodes vote
* replies, PreVote replies and WAL/config ACKs — all RECEIPT decode + commit
* eval, which is bounded liveness work. Per the design's deadlock-freedom
* argument (R4.5), ack/commit-advance decode MUST be VTASK_LIVENESS so it is
* NEVER admission-gated: a quorum of acks can always find a reserved worker to
* advance committed_lsn even when the pool is saturated by *_BLOCK tasks.
*
* (The vote-GRANT term fsync — VTASK_LIVE_BLOCK, R4.6 — is handled on the
* INBOUND RequestVote handler, not on these reply/ack endpoints; and the long
* config-push/snapshot ops are offloaded to the SlowLane in tasks 3.2/3.3. So
* everything this handler dispatches is liveness work today.)
*
* ┌─ FLAG-ON HAZARD CLOSED (raft-drive-async finding-2; bug-1198) ───────────┐
* │ LIVENESS is correct only once the snapshot send runs OFF the pool. The │
* │ CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT → │
* │ persist_checkpoint (a BLOCKING fdatasync) in │
* │ repl_ack_decode_and_dispatch → ha_dispatch_repl_action_for_standby. │
* │ Running that blocking op while tagged VTASK_LIVENESS (never gated) would │
* │ break the deadlock-freedom reservation — a burst of snapshot sends could │
* │ occupy every liveness worker and starve the acks that advance │
* │ committed_lsn. │
* │ │
* │ CLOSED: the borrowed-endpoint lifetime hazard (bug-1197) is fixed by an │
* │ endpoint_ref/endpoint_unref bracket around the lane dispatch (see │
* │ ha_snapshot_send_dispatch / snapshot_lane_fn), so WHENEVER both flags are │
* │ ON together the repl-ack path's snapshot send is ALWAYS offloaded before │
* │ peer_ctrl_classify ever tags it VTASK_LIVENESS on the pool — bug-1198 │
* │ cannot recur. Both flags now default ON (bug-1376, the cold-start │
* │ pool-wiring self-deadlock found flipping both ON, is root-caused + fixed │
* │ — see g_drive_async_peerctrl_dispatch_enabled's comment). │
* └──────────────────────────────────────────────────────────────────────────┘
*/
static vtask_class_t peer_ctrl_classify(ha_state_t *ha, io_packet_t *pkt)
{
(void)ha;
(void)pkt;
return VTASK_LIVENESS;
}
/* ── bug-1197: deferred vote-ep teardown ──────────────────────────────────────
*
* When g_drive_async_peerctrl_dispatch_enabled is ON, raft_cb_become_follower
* runs on a pool worker and must endpoint_fini() every loser vote-ep. But
* another pool worker can be mid-vtask (or have a queued vtask) on one of those
* same eps reading ep->data (a vote_ep_ctx_t) — freeing ep->data underneath it
* is a UAF. Instead of freeing inline we enqueue a teardown vtask BOUND to the
* ep (VTASK_IO): the pool's per-ep busy serialization guarantees it runs after
* any in-flight or earlier-queued same-ep vtask completes. We also mark the ep
* tearing_down + iomgr_unmonitor it BEFORE enqueue so the io_worker drops any
* late same-batch event instead of dispatching a new vtask onto a doomed ep. */
#ifdef XCE_TESTING
/* test-only ordering probe: fired (with the real ep) at the top of the teardown
* vtask body, before endpoint_fini, so a test can assert teardown ran strictly
* after the last live same-ep use. NULL in production. */
void (*ha_vep_teardown_probe)(endpoint_t *) = NULL;
/* test-only drain probe: fired in peer_ctrl_msg_handler AFTER the tearing_down
* check passes but BEFORE the pool enqueue (inside the dispatch_inflight
* bracket), so a test can park a dispatch in the drained window and prove
* endpoint_mark_tearing_down blocks until the enqueue completes. NULL in
* production. */
void (*ha_peerctrl_predispatch_probe)(endpoint_t *) = NULL;
#endif
/* Runs on a pool worker, bound VTASK_IO to ep, strictly AFTER any in-flight or
* queued same-ep peer-control vtask (per-ep busy FIFO). On the loser path
* ectx->votes[i].ep is the sole reference to ep, so this owns the free. */
static int vote_ep_teardown_vtask_entry(vtask_t *task)
{
io_task_data_t *iod = &task->data->u.iodata;
endpoint_t *ep = iod->ep;
if (ep != NULL) {
#ifdef XCE_TESTING
if (ha_vep_teardown_probe)
ha_vep_teardown_probe(ep);
#endif
/* bug-1298: drop the creator/owner reference. We DON'T fini/free here
* and DON'T NULL iod->ep: this vtask still holds the per-vtask reference
* (taken in vtask_enq), so the ep stays alive through run_task's post-body
* endpoint_unblock_tasks; run_task then drops the per-vtask ref, and that
* last endpoint_unref performs the fini+free. No UAF, no NULL hack. */
endpoint_unref(ep);
}
return 0;
}
/* Build + enqueue the teardown vtask. Caller has ALREADY
* endpoint_mark_tearing_down(ep) + iomgr_unmonitor(ep). Controllers never
* self-terminate: on any alloc/enqueue failure we fail loud (WARN) and tear the
* ep down inline — with the pool unwired or the task rejected there is no other
* worker that could be racing this ep, so an inline endpoint_fini+free is safe. */
static void vote_ep_teardown_enqueue(ha_state_t *ha, endpoint_t *ep)
{
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = calloc(1, sizeof(*task));
if (pool == NULL || data == NULL || task == NULL) {
WARN("bug-1197: vote-ep teardown enqueue unavailable (pool=%p data=%p "
"task=%p) — tearing down inline (stay alive)",
(void *)pool, (void *)data, (void *)task);
free(data);
free(task);
endpoint_unref(ep); /* bug-1298: no vtask enqueued — drop owner ref → free */
return;
}
data->vtype = VTASK_IO;
data->u.iodata.ep = ep; /* drives per-ep serialization in bind_vtask */
data->u.iodata.pkt = NULL;
vtask_init(task, (vtask_func_t)vote_ep_teardown_vtask_entry, data);
/* VTASK_LIVENESS is never admission-gated, so the teardown can always be
* enqueued even when the pool is saturated by blocking tasks. */
if (vtask_enq_classified(pool, task, VTASK_LIVENESS) < 0) {
WARN("bug-1197: vote-ep teardown enqueue failed — tearing down inline "
"(stay alive)");
free(data);
free(task);
/* bug-1298: vtask_enq already dropped the per-vtask ref it took on the
* failed enqueue, so refcnt is back to the owner ref — drop it → free. */
endpoint_unref(ep);
}
}
/* The flag-ON loser-path teardown for one vote-ep: mark tearing_down, stop epoll
* wakeups, then defer the free onto the ep's per-ep FIFO. ORDERING (bug-1197):
* - endpoint_mark_tearing_down sets the flag under ep->lock; the io_worker's
* late-event gate (peer_ctrl_msg_handler, ON branch) reads it under ep->lock
* via endpoint_is_tearing_down, so any dispatch DECISION taken after the
* mark observes it and drops — nothing new is enqueued for this ep.
* - iomgr_unmonitor stops FUTURE epoll events (an event already in the current
* batch is handled by the tearing_down gate above).
* - the teardown vtask is itself VTASK_IO bound to ep, so the pool's per-ep
* busy serialization runs it strictly after any in-flight / earlier-queued
* same-ep vtask completes.
* NOTE: we cannot hold ep->lock ACROSS the dispatch enqueue (vtask_enq_classified
* → dataq_enq takes the taskq lock, while the pool dispatcher takes ep->lock
* UNDER the taskq lock in vtask_eligible → endpoint_is_task_blocked — an AB/BA
* inversion). The check is therefore done under ep->lock and the enqueue
* outside it; see the io_worker gate for the residual-window note. */
static void vote_ep_teardown_async(ha_state_t *ha, endpoint_t *ep)
{
endpoint_mark_tearing_down(ep);
if (ep->iomgr != NULL)
iomgr_unmonitor(ep->iomgr, ep);
vote_ep_teardown_enqueue(ha, ep);
}
#ifdef XCE_TESTING
/* ha_vote_ep_teardown_for_test: drive the flag-ON per-ep async teardown branch
* for a single ep without staging a full election/become_follower. */
void ha_vote_ep_teardown_for_test(ha_state_t *ha, endpoint_t *ep)
{
vote_ep_teardown_async(ha, ep);
}
#endif
/*
* peer_ctrl_dispatch_to_pool (raft-drive-async task 3.1): hand the decoded
* peer-control packet to the vtask pool and return promptly. Per-endpoint
* arrival-order serialization is automatic — bind_vtask calls
* endpoint_block_tasks(ep) and the dispatcher won't bind a second task for the
* same endpoint until this one completes (Req 4.1/4.2).
*
* Controllers NEVER self-terminate (CLAUDE.md): on any failure to enqueue
* (pool unwired, OOM, or admission reject) we fail loud and run the body INLINE
* rather than drop a consensus message — staying alive and correct.
*
* ┌─ FLAG-ON HAZARD CLOSED (raft-drive-async finding-1; bug-1197, task 3.4a) ─┐
* │ With this dispatch enabled, raft_cb_become_follower (pool worker) tearing │
* │ down peer vote-eps could race the single io_worker thread reading the │
* │ same ep's data for a same-batch queued event — a freed-vctx UAF. Fixed by │
* │ deferred teardown-as-vtask: become_follower marks each loser vote-ep │
* │ tearing_down + iomgr_unmonitor's it BEFORE enqueuing a per-ep-bound │
* │ teardown vtask (vote_ep_teardown_vtask_entry), so the pool's per-ep busy │
* │ serialization guarantees it runs strictly after any in-flight/queued │
* │ same-ep peer-control vtask. An RCU-like dispatch_inflight drain closes the │
* │ TOCTOU window between the flag-check and enqueue. See the bug-1197 │
* │ deferred vote-ep teardown block below and vote_ep_teardown_async. │
* └──────────────────────────────────────────────────────────────────────────┘
*/
/*
* bug-1376 (self-deadlock, found investigating the raft-drive-async flag
* flip): the caller (peer_ctrl_msg_handler) holds ep->dispatch_inflight
* incremented across this whole call. That counter exists to stop a POOL
* WORKER thread from touching a queued vtask's ep concurrently with a
* teardown — it protects the brief window between the tearing_down check
* and a (non-blocking) enqueue. It must NOT still be held while running the
* INLINE fallback body: peer_ctrl_msg_handler_inline can recursively reach
* raft_cb_become_follower -> vote_ep_teardown_async -> endpoint_mark_
* tearing_down on THIS SAME ep (a node's own vote-request ep receiving the
* very reply that makes it step down), which spins waiting for
* dispatch_inflight==0 — a count only this call itself holds, on the same
* thread, unable to ever reach 0 until this call returns. Permanent hang
* (io_worker is single-threaded, so nothing else could ever be racing this
* ep while we run inline — dispatch_inflight has nothing left to protect
* once we've committed to the inline path).
*
* Fix: release dispatch_inflight at the exact right point for whichever of
* the 3 exit paths fires below, instead of the caller holding it across the
* whole call. The ENQUEUE-success path still releases only after the
* (non-blocking) enqueue completes, preserving the original protection for
* the genuinely concurrent pool-worker case; every INLINE-fallback path
* releases BEFORE calling the inline body, since no pool worker can ever be
* racing an ep whose message we are about to process synchronously
* ourselves. */
static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
ha_state_t *ha = vctx->ha;
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
if (pool == NULL) {
WARN("peer_ctrl_dispatch_to_pool: vtask pool not wired — running "
"inline (stay alive, never drop a peer-control message)");
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
return peer_ctrl_msg_handler_inline(ep, pkt);
}
vtask_data_t *data = (vtask_data_t *)calloc(1, sizeof(*data));
vtask_t *task = (vtask_t *)calloc(1, sizeof(*task));
if (data == NULL || task == NULL) {
WARN("peer_ctrl_dispatch_to_pool: OOM allocating vtask — running "
"inline (stay alive)");
free(data);
free(task);
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
return peer_ctrl_msg_handler_inline(ep, pkt);
}
data->vtype = VTASK_IO;
data->u.iodata.ep = ep; /* drives per-endpoint serialization in bind_vtask */
data->u.iodata.pkt = pkt; /* ownership transferred into the task */
vtask_init(task, (vtask_func_t)peer_ctrl_vtask_entry, data);
/* raft-drive-async task 3.1: the task is the SOLE owner of pkt (the shim
* already cleared ep->fsm_in.pkt). Mark it so taskq_item_killer frees pkt
* if this task is destroyed while still queued (never executed) at
* vtaskmgr_fini — without this the pkt would leak on shutdown. The
* executed path frees pkt inside peer_ctrl_vtask_entry (and NULLs
* iod->pkt), so there is no double-free. */
task->owns_io_pkt = 1;
vtask_class_t cls = peer_ctrl_classify(ha, pkt);
int rc = vtask_enq_classified(pool, task, cls);
if (rc < 0) {
/* Task NOT enqueued — we still own task/data/pkt. LIVENESS is never
* gated so this should not happen; if it does, fail loud and run inline
* (never drop). Free the wrapper structs only — pkt is freed by the
* inline body below. */
WARN("peer_ctrl_dispatch_to_pool: vtask_enq_classified failed (rc=%d) "
"— running inline (stay alive)", rc);
free(data);
free(task);
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
return peer_ctrl_msg_handler_inline(ep, pkt);
}
/* Enqueued: release now that the (non-blocking) enqueue itself has
* completed. The queued vtask is what a pool worker will process; a
* concurrent teardown drain now correctly waits on the per-ep busy/FIFO
* gate (bind_vtask) for that vtask, not on this call. */
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
return 0; /* returns promptly; the body runs on a pool worker */
}
/*
* peer_ctrl_msg_handler: the registered iomgr msg_handler for peer-control
* endpoints (set in on_vote_connected / on_pre_vote_connected / on_standby_
* connected). Runs on the io_worker thread.
*
* raft-drive-async task 3.1: takes ownership of the inbound packet off
* ep->fsm_in.pkt and clears that slot IMMEDIATELY (so a subsequent io_worker
* read on this endpoint allocates a fresh packet and cannot collide with the
* one we are about to process), then either runs the routing body inline
* (dispatch flag OFF — production default, byte-for-byte the prior behavior) or
* hands it to the vtask pool and returns promptly (flag ON).
*/
static int peer_ctrl_msg_handler(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
io_packet_t *pkt = ep->fsm_in.pkt;
ep->fsm_in.pkt = NULL; /* take ownership before any routing/dispatch */
if (!ha_peerctrl_dispatch_enabled())
return peer_ctrl_msg_handler_inline(ep, pkt);
/* bug-1197 late-event gate (ON path only): if this ep is being torn down,
* a same-batch event may still be delivered by the io_worker after the
* teardown thread called iomgr_unmonitor. Dropping it here keeps the doomed
* ep off the vtask pool.
*
* The window between the tearing_down check and the pool enqueue is closed
* by ep->dispatch_inflight: we increment it (seq_cst) BEFORE reading the
* flag and decrement it AFTER the enqueue completes. endpoint_mark_tearing_
* down sets the flag (under ep->lock) and then DRAINS this counter to 0
* before returning. So once the teardown caller's mark returns, every
* dispatch that read the flag as false has already finished its enqueue;
* because the teardown vtask is enqueued only AFTER mark returns, that
* peer-control vtask is strictly FIFO-before the teardown vtask (and gates
* on the same ep busy bit). Any dispatch starting after the flag is set
* reads it true here and drops. No same-ep vtask is ever enqueued after the
* teardown vtask — the window is fully closed. We must NOT hold ep->lock
* across the enqueue (it would invert with the taskq lock, see
* vote_ep_teardown_async); the counter is the lock-free handshake instead. */
atomic_fetch_add_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst);
if (endpoint_is_tearing_down(ep)) {
if (pkt) { free_packet(pkt); free(pkt); }
atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1,
memory_order_seq_cst);
return 0;
}
#ifdef XCE_TESTING
/* test-only seam: fires AFTER the flag-check passed but BEFORE the enqueue,
* so a test can park a dispatch in exactly the drained window and prove
* endpoint_mark_tearing_down blocks until the enqueue completes. */
if (ha_peerctrl_predispatch_probe)
ha_peerctrl_predispatch_probe(ep);
#endif
/* bug-1376: peer_ctrl_dispatch_to_pool owns the matching fetch_sub now
* (released at the exact right point for whichever of its 3 exit paths
* fires — see its own comment) instead of a single unconditional
* decrement here after it returns. See that function for why. */
return peer_ctrl_dispatch_to_pool(ep, pkt);
}
#ifdef XCE_TESTING
/*
* vote_reply_build_pkt_for_test: test helper that encodes a RequestVoteReply
* into a newly allocated io_packet_t that vote_reply_msg_handler_for_test can decode.
*
* Wire format mirrors the real peer reply:
* Request{opcode=reply, body.replymsg=RequestReply{
* data_which=requestvotereplymsg,
* data.requestvotereplymsg=RequestVoteReply{term, voteGranted, ...}
* }}
*
* Returns a freshly allocated io_packet_t* on success, NULL on failure.
* Handler takes ownership: it calls free_packet()+free() when done.
* Tests set ep->fsm_in.pkt to the returned pointer before calling the handler.
*/
io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
uint64_t peer_tag, uint64_t peer_lsn)
{
struct capn cs;
struct capn_segment *seg;
struct RequestVoteReply vr_capn;
RequestVoteReply_ptr vr_ptr;
struct RequestReply rr_capn;
RequestReply_ptr rr_ptr;
struct Request rq_capn;
Request_ptr rq_ptr;
io_packet_t *pkt;
int rc;
capn_init_malloc(&cs);
seg = capn_root(&cs).seg;
/* RequestVoteReply — all scalar fields */
memset(&vr_capn, 0, sizeof(vr_capn));
vr_capn.term = term;
vr_capn.voteGranted = vote_granted ? 1 : 0;
vr_capn.peerConfigTag = peer_tag;
vr_capn.peerLastLogLsn = peer_lsn;
vr_ptr = new_RequestVoteReply(seg);
write_RequestVoteReply(&vr_capn, vr_ptr);
/* RequestReply envelope */
memset(&rr_capn, 0, sizeof(rr_capn));
rr_capn.seq = 1;
rr_capn.opcode = OPCode_requestVote;
rr_capn.status = 0;
rr_capn.data_which = RequestReply_data_requestvotereplymsg;
rr_capn.data.requestvotereplymsg = vr_ptr;
rr_ptr = new_RequestReply(seg);
write_RequestReply(&rr_capn, rr_ptr);
/* Outer Request{opcode=reply} */
memset(&rq_capn, 0, sizeof(rq_capn));
rq_capn.opcode = OPCode_reply;
rq_capn.seq = 1;
rq_capn.body_which = Request_body_replymsg;
rq_capn.body.replymsg = rr_ptr;
rq_ptr = new_Request(seg);
write_Request(&rq_capn, rq_ptr);
capn_setp(capn_root(&cs), 0, rq_ptr.p);
pkt = calloc(1, sizeof(*pkt));
if (!pkt) { capn_free(&cs); return NULL; }
pkt->sz = capn_size(&cs);
rc = alloc_packet(pkt);
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
capn_free(&cs);
return pkt;
}
/*
* vote_reply_msg_handler_for_test: test entry point for vote-reply decode path.
* Temporarily forces role to CTRL_ELECTING so peer_ctrl_msg_handler routes to
* vote_reply_decode_and_dispatch, then restores the original role.
* Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
*/
int vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
ha_state_t *ha = vctx ? vctx->ha : NULL;
int32_t saved_role = ha ? atomic_load_explicit(&ha->role, memory_order_acquire) : 0;
if (ha)
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
ep->fsm_in.pkt = pkt;
int rc = peer_ctrl_msg_handler(ep);
if (ha)
atomic_store_explicit(&ha->role, saved_role, memory_order_release);
return rc;
}
/*
* vote_ep_ctx_build_for_test: test helper that allocates a vote_ep_ctx_t,
* fills it from the supplied parameters, and wires up ep->data and
* ep->on_connected so the test can drive on_vote_connected indirectly.
*
* Returns ep (same pointer passed in) for call-chain convenience.
* Caller is responsible for free(ep->data) after use.
*/
endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t my_term,
uint64_t my_tag,
uint64_t my_lsn,
const char *my_name)
{
vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx));
if (!vctx) return ep; /* test will fail when on_connected is NULL */
vctx->ha = ha;
vctx->c = NULL;
vctx->ctx = ctx;
vctx->peer_idx = peer_idx;
vctx->my_term = my_term;
vctx->my_tag = my_tag;
vctx->my_lsn = my_lsn;
vctx->my_name = my_name;
ep->data = vctx;
ep->on_connected = on_vote_connected;
return ep;
}
#ifdef XCE_TESTING
/*
* vote_ep_wire_on_fini_for_test (ha-elect-ctx-lifecycle task 3.3): wire the
* real vote_ep_on_fini hook onto ep->on_fini, exactly as raft_cb_send_vote_request
* / raft_cb_send_pre_vote_request do in production. vote_ep_on_fini is static
* to this TU, so tests that build an endpoint via vote_ep_ctx_build_for_test
* and need to drive a REAL endpoint_fini() (to prove the completion hook
* fires and releases the carrier reference) call this to wire it first.
*/
void vote_ep_wire_on_fini_for_test(endpoint_t *ep)
{
endpoint_set_on_fini(ep, vote_ep_on_fini);
}
#endif
/*
* election_timeout_fn_for_test: test helper that exercises the election timeout
* callback logic with an explicit cancelled_val instead of an atomic field.
* Allows testing the cancelled-gate without setting up a real delaymgr.
*
* Returns 0 in all cases (mirrors the real callback's return value).
*/
int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx,
int cancelled_val)
{
if (cancelled_val)
return 0;
pthread_mutex_lock(&ctx->lock);
ha_elect_evaluate_and_signal(ha, ctx);
pthread_mutex_unlock(&ctx->lock);
return 0;
}
/*
* repl_catchup_done_pkt_for_test: build a minimal io_packet_t encoding
* Request{opcode=replCatchUpDone, body_which=nodata}.
* Used by peer_ctrl_msg_handler tests to verify the repl-ack dispatch path.
*/
io_packet_t *repl_catchup_done_pkt_for_test(void)
{
struct capn cs;
struct capn_segment *seg;
struct Request rq_capn;
Request_ptr rq_ptr;
io_packet_t *pkt;
int rc;
capn_init_malloc(&cs);
seg = capn_root(&cs).seg;
memset(&rq_capn, 0, sizeof(rq_capn));
rq_capn.opcode = OPCode_replCatchUpDone;
rq_capn.seq = 1;
rq_capn.body_which = Request_body_nomsg;
rq_ptr = new_Request(seg);
write_Request(&rq_capn, rq_ptr);
capn_setp(capn_root(&cs), 0, rq_ptr.p);
pkt = calloc(1, sizeof(*pkt));
if (!pkt) { capn_free(&cs); return NULL; }
pkt->sz = capn_size(&cs);
rc = alloc_packet(pkt);
if (rc < 0) { capn_free(&cs); free(pkt); return NULL; }
capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0);
capn_free(&cs);
return pkt;
}
/*
* vote_ep_err_handler_for_test: call the static vote_ep_err_handler on ep.
* Allows tests to invoke the err_handler without going through iomgr/endpoint machinery.
*/
int vote_ep_err_handler_for_test(endpoint_t *ep)
{
return vote_ep_err_handler((void *)ep);
}
/*
* peer_ctrl_msg_handler_for_test: expose peer_ctrl_msg_handler for tests.
* Sets ep->fsm_in.pkt then invokes the handler. Packet ownership transferred.
*/
int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
{
ep->fsm_in.pkt = pkt;
return peer_ctrl_msg_handler(ep);
}
/*
* pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL
* decode/route path (peer_ctrl_msg_handler) while the candidate is CTRL_STANDBY
* with a PreVote round in progress — the exact production condition under which
* a PreVote reply arrives. Unlike vote_reply_msg_handler_for_test it does NOT
* force the role: it proves the STANDBY+pre_vote_in_progress branch added in
* task 5.1 actually routes the reply to the PreVote tally (instead of dropping
* it). Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred.
*/
int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt)
{
ep->fsm_in.pkt = pkt;
return peer_ctrl_msg_handler(ep);
}
/*
* vote_ep_ctx_set_cluster_for_test: set the cluster_t pointer inside the
* vote_ep_ctx_t stored at ep->data. Tests that need the promoted-endpoint
* dispatch path to reach ha_promote_to_primary(c) with a live cluster pointer
* must call this after vote_ep_ctx_build_for_test (which sets vctx->c = NULL
* by default).
* Requirements: task 7.3 (ha-peer-connection-reuse).
*/
void vote_ep_ctx_set_cluster_for_test(endpoint_t *ep, cluster_t *c)
{
if (!ep || !ep->data) return;
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
vctx->c = c;
}
/*
* raft_cb_become_leader_for_test: drive the leader promotion path without
* a live cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress
* production-only calls (phase2, push_config, ctrlhello).
*/
void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term)
{
ha->raft_c = NULL;
raft_cb_become_leader((void *)ha, term);
}
#ifdef XCE_TESTING
/* ha_arm_promotion_fence_timer_for_test (ha-commit-durability task 4.4):
* expose the file-static bounded-window arm path so the durability suite can
* drive the exact deferral wired into raft_cb_become_leader without staging a
* full election. Production never references this symbol (XCE_TESTING only). */
void ha_arm_promotion_fence_timer_for_test(cluster_t *c, bool truncated,
uint64_t durable_floor,
uint64_t target_hwm)
{
ha_arm_promotion_fence_timer(&c->ha, c, truncated, durable_floor,
target_hwm);
}
#endif
/*
* raft_cb_become_follower_for_test: drive the follower path without a live
* cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production
* calls (phase2, fetch_config); ha_arm_heartbeat_timeout_vtask is a no-op
* when c=NULL so no watchdog thread is spawned.
*/
void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term)
{
ha->raft_c = NULL;
raft_cb_become_follower((void *)ha, term, -1);
}
/*
* raft_cb_become_follower_with_cluster_for_test (ha-convergence-recovery
* task 2.1): drive the follower path with a LIVE cluster pointer so the
* cluster-dependent demotion-convergence branch (task 2.2) is exercised
* against a real persist/hooksys/delaymgr fixture. Sets ha->raft_c = c so
* the become-follower callback sees a real cluster (unlike the NULL-cluster
* shim above). raft_cb_become_follower is file-static, so this seam must
* live here in ha.c.
*/
void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term)
{
c->ha.raft_c = c;
raft_cb_become_follower((void *)&c->ha, term, -1);
}
/*
* raft_cb_become_leader_with_cluster_for_test (fix-promotion-standby-registration):
* drive the leader promotion path with a LIVE cluster pointer so the
* cluster-dependent standby-registration branch (repl_add_standby for each
* surviving vote endpoint whose votes[i].peer is set) is exercised against a
* real persist/repl/commit/delaymgr fixture. Sets ha->raft_c = c so the
* become-leader callback sees a real cluster (unlike the NULL-cluster shim
* raft_cb_become_leader_for_test above, which suppresses the c-dependent
* registration block entirely). raft_cb_become_leader is file-static, so this
* seam must live here in ha.c. Symmetric with
* raft_cb_become_follower_with_cluster_for_test.
*/
void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, uint64_t term)
{
c->ha.raft_c = c;
raft_cb_become_leader((void *)&c->ha, term);
}
#endif /* XCE_TESTING */
/* ── Task 2.4 / raft-adapter-conformance task 3.1: vote endpoint err handlers ──
*
* vote_ep_handle_promoted_disconnect: shared CTRL_PRIMARY-role cleanup for a
* dropped promoted connection (standby-endpoint deregistration, WAL
* replicator deregistration, quorum-transition logging). Extracted
* byte-for-byte from the formerly-single vote_ep_err_handler so both the
* real-vote and PreVote on_err handlers apply the identical cleanup — this
* branch's behavior is unchanged by the split.
*/
static void vote_ep_handle_promoted_disconnect(ha_state_t *ha, endpoint_t *ep,
vote_ep_ctx_t *vctx)
{
/* Promoted connection dropped — remove from standby_endpoints */
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ha->standby_endpoints->count; i++) {
if (zarray_get(ha->standby_endpoints, i) == ep) {
zarray_del(ha->standby_endpoints, i);
break;
}
}
pthread_mutex_unlock(&ha->lock);
}
/* Task 2.2: deregister from WAL replicator.
* Req 1.2: when a standby disconnects, the HA layer shall deregister
* the endpoint from the replicator via repl_remove_standby.
* vctx->c may be NULL in test mode — guard accordingly. */
if (vctx->c)
repl_remove_standby(&vctx->c->persist.repl, ep);
/* Quorum-transition observability (Req 8.1): a standby just dropped — the
* write-quorum count changed. Record the loss edge once (silent if
* quorum is still held after the removal). */
ha_log_quorum_transition(ha);
}
/*
* vote_ep_err_handler: on_err callback for outgoing REAL-VOTE endpoints
* (registered in raft_cb_send_vote_request). Called by iomgr when the
* endpoint encounters an error (connect failure, SSL error, remote close
* before reply).
*
* Election-phase branch counts a post-connect failure through
* vote_reply_dispatch — the SAME production tally path a connect-time
* failure (raft_cb_send_vote_request's own rc<0 branches, bug-1423) or a
* delivered denial (vote_reply_msg_handler) already uses — instead of the
* legacy handle_request_vote_reply, which bypasses raft_on_vote_reply
* (raft-adapter-conformance Requirement 4.2).
*/
static int vote_ep_err_handler(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
if (!vctx) return 0;
ha_state_t *ha = vctx->ha;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_PRIMARY) {
vote_ep_handle_promoted_disconnect(ha, ep, vctx);
} else {
/* Election phase: count as a denied vote through the production
* tally, same path a connect-time failure or a delivered denial
* uses (raft-adapter-conformance Requirement 4.2). */
vote_reply_dispatch(ha, vctx->ctx, vctx->peer_idx, vctx->my_term,
/*vote_granted=*/0, 0, 0);
}
/* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data.
* vctx is freed in ha_election_cleanup() when all per-peer contexts release. */
return 0;
}
/*
* pre_vote_ep_err_handler: on_err callback for outgoing PREVOTE endpoints
* (registered in raft_cb_send_pre_vote_request). Mirrors vote_ep_err_handler
* above for the PreVote endpoint kind — the two kinds share vote_ep_ctx_t but
* are already registered at distinct call sites, so the split (rather than a
* new discriminant field on vote_ep_ctx_t) mirrors an existing seam.
*
* Election-phase branch counts a post-connect failure through
* pre_vote_reply_dispatch — the SAME production tally path a connect-time
* failure (raft_cb_send_pre_vote_request's own rc<0 branches, bug-1423 PreVote
* twin) already uses — instead of the legacy handle_request_vote_reply
* (raft-adapter-conformance Requirement 4.3).
*/
static int pre_vote_ep_err_handler(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
if (!vctx) return 0;
ha_state_t *ha = vctx->ha;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_PRIMARY) {
vote_ep_handle_promoted_disconnect(ha, ep, vctx);
} else {
/* Election phase: count as a not-granted PreVote reply through the
* production tally, same path a connect-time failure uses
* (raft-adapter-conformance Requirement 4.3). */
pre_vote_reply_dispatch(ha, vctx->c, vctx->peer_idx, /*granted=*/0);
}
/* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data.
* vctx is freed in ha_election_cleanup() when all per-peer contexts release. */
return 0;
}
#ifdef XCE_TESTING
/*
* pre_vote_ep_err_handler_for_test: call the static pre_vote_ep_err_handler on
* ep. Allows tests to invoke the PreVote err_handler without going through
* iomgr/endpoint machinery — the PreVote twin of vote_ep_err_handler_for_test
* (raft-adapter-conformance task 3.2).
*/
int pre_vote_ep_err_handler_for_test(endpoint_t *ep)
{
return pre_vote_ep_err_handler((void *)ep);
}
#endif /* XCE_TESTING */
/* ── ha_init_raft: one-time raft state machine initialization ────────────────
*
* Called once at startup after ha_restore_term() to wire the raft state
* machine with the correct cluster size and callbacks. Subsequent elections
* use raft_start_election() / raft_on_leader_lost() on this same state.
*
* n_peers: total candidates_count from cluster config (includes self).
*/
void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers)
{
/* Map persisted voted_for_name back to a peer index for initial_voted_for.
* This lets raft_check_and_start_election (RDHUP path) see the prior vote
* without re-reading the file. */
int restored_voted_for = -1;
if (ha->voted_for_name[0] != '\0' && c && c->myconf) {
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
for (int i = 0; i < pcfg->candidates_count; i++) {
if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) {
restored_voted_for = i;
break;
}
}
}
/* ha-term-safety task 2.5 (R1.5, R2.1): restore the durable log/commit floor
* from the recovered WAL so the library boots at the recovered tip — not an
* empty log at LSN 0. The library never reads storage; the caller restores
* initial_log_len / initial_commit_index here (same discipline as
* initial_term / initial_voted_for above).
*
* initial_log_len = recovered durable last-log index
* (persist_durable_floor: re-derived from the WAL tip
* by persist_recover; 0 on empty first boot).
* initial_commit_index = recovered committed LSN (committed_lsn). Recovery
* re-derives the durable tip but the post-election
* safe-commit point governs the committed position;
* we pass whatever committed prefix is established at
* init (0 on a fresh boot). Clamp to log_len to honor
* the library invariant commit_index <= log_len.
*
* The recovered last-log TERM is surfaced on demand via on_get_entry_term(
* log_len) (the library's logOk freshness gate reads it) — there is no
* separate initial last-log-term field. */
uint64_t initial_log_len = c ? persist_durable_floor(&c->persist) : 0;
uint64_t initial_commit_index = c
? atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire)
: 0;
if (initial_commit_index > initial_log_len)
initial_commit_index = initial_log_len;
raft_config_t rcfg;
memset(&rcfg, 0, sizeof(rcfg));
rcfg.n_peers = n_peers;
rcfg.self_idx = ha->my_priority;
rcfg.election_timeout_ms = ha->config.election_timeout_ms;
rcfg.my_priority = ha->my_priority;
rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire);
rcfg.initial_voted_for = restored_voted_for;
rcfg.initial_log_len = initial_log_len;
rcfg.initial_commit_index = initial_commit_index;
rcfg.cbs = &s_real_cbs;
rcfg.ctx = ha;
raft_init(&ha->raft, &rcfg);
ha->n_peers = n_peers;
ha->raft_c = c;
INFO("HA: raft initialized (n_peers=%d, self=%d, term=%" PRIu64
", log_len=%" PRIu64 ", commit_index=%" PRIu64 ")",
n_peers, ha->my_priority,
atomic_load_explicit(&ha->term, memory_order_acquire),
initial_log_len, initial_commit_index);
}
/* ── ha-prevote task 4.1: candidate-side PreVote orchestration ─────────────
*
* The PreVote round is a side-effect-free majority probe interposed BEFORE the
* destructive truncate + CTRL_ELECTING + real-election sequence. During the
* round the node stays CTRL_STANDBY and mutates no persistent state (term /
* voted_for / role). Only on a WIN does it enter the unchanged real-election
* path; on a LOSS / round-timeout it stays STANDBY, re-arms the liveness
* watchdog, and logs campaign abandonment.
*
* This task implements the round machinery + the WON/LOST branches. The
* failover triggers (ha_on_primary_ctrl_fd_rdhup / liveness_poll_fn /
* ha_on_heartbeat_received) are rewired to call this in task 4.2 — until then
* the round is INERT in production.
*/
/*
* prevote_round_ctx_t: delaymgr context for the PreVote round timeout.
* cancelled is set when the round is decided (WON or LOST) before the timeout
* fires, so a late timeout task is a no-op.
*/
typedef struct {
_Atomic int cancelled;
ha_state_t *ha;
cluster_t *c;
} prevote_round_ctx_t;
/* Active round-timeout context. _Atomic so arm (vtask) vs timeout-fire
* (delaymgr) vs cancel-on-decision is well-defined. */
static prevote_round_ctx_t * _Atomic s_prevote_round_ctx = NULL;
#ifdef XCE_TESTING
/*
* ha_prevote_rearm_liveness_hook: test-only seam. When non-NULL, the LOST /
* timeout branch calls this INSTEAD OF ha_arm_heartbeat_timeout_vtask so a unit
* test can count re-arms without a live delaymgr (Req 5.4).
*/
int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c) = NULL;
#endif
/* ha_prevote_rearm_liveness: re-arm the standby liveness watchdog after a lost
* round so a later genuine failure still triggers a new PreVote round (the
* watchdog does not re-arm itself once it has fired). */
static void ha_prevote_rearm_liveness(ha_state_t *ha, cluster_t *c)
{
#ifdef XCE_TESTING
if (ha_prevote_rearm_liveness_hook != NULL) {
ha_prevote_rearm_liveness_hook(ha, c);
return;
}
#endif
ha_arm_heartbeat_timeout_vtask(ha, c);
}
/* ha_prevote_cancel_round_timeout: mark the active round-timeout context
* cancelled (the delaymgr task frees it on fire) and clear the static pointer
* so a later decision/timeout sees no active round. */
static void ha_prevote_cancel_round_timeout(void)
{
prevote_round_ctx_t *old = atomic_load_explicit(&s_prevote_round_ctx,
memory_order_acquire);
if (old) {
atomic_store_explicit(&old->cancelled, 1, memory_order_release);
atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release);
}
}
/*
* ha_enter_real_election: the WON-branch sequence — truncate uncommitted WAL
* above the committed boundary, then enter the UNCHANGED real-election path
* (ha_start_election_async transitions to CTRL_ELECTING and delegates the term
* bump + self-vote to raft_start_election). Factored here so both the PreVote
* WON branch (task 4.1) and the rewired trigger (task 4.2) can call it; this
* task does NOT change ha_on_primary_ctrl_fd_rdhup's own behaviour.
*
* c == NULL (unit-test mode): skip truncate + the live election; just transition
* the role to CTRL_ELECTING so the WON transition is observable.
*/
static void ha_enter_real_election(ha_state_t *ha, cluster_t *c)
{
if (!c) {
/* Test mode: the real election needs iomgr/delaymgr; just mark the
* destructive transition so the WON branch is observable. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
memory_order_release);
return;
}
/* bug-828 final slice (Leader Completeness): do NOT pre-truncate the WAL
* here. The earlier code truncated above ha->last_known_committed_lsn
* BEFORE the vote — but that watermark is the LOCALLY-LEARNED commit point,
* which LAGS the true cluster commit point (the primary commits an entry on
* a durable majority, then dies before a heartbeat carries that committed
* LSN to this standby). persist_truncate_wal_above clamps durable_lsn down
* to the cutoff, so a standby that had DURABLY persisted committed entries
* [committed+1 .. tip] would discard them and then advertise the stale-low
* cutoff as its durable floor. Every voter doing the same made
* safe-committed-from-votes collapse to the stale watermark, and the
* AUTHORITATIVE post-election truncate (raft_cb_become_leader,
* persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale
* prefix — losing committed jobs on a clean failover (bug-828/795).
*
* Raft model: a candidate does NOT truncate its log before campaigning. It
* keeps its full durable log and advertises its real durable tip
* (ha_candidate_advertise_lsn == persist_durable_floor), so voters holding
* more committed entries deny a colder candidate and the winner has every
* committed entry (Leader Completeness). Divergent UNCOMMITTED tails are
* removed ONLY by the authoritative post-election safe-commit truncate
* (above the majority-held point) and by the library's per-entry (term,lsn)
* reconcile (on_truncate_from) during subsequent AppendEntries — never by a
* stale-watermark pre-truncate that can erase committed state. */
/* Candidate freshness for the RequestVote payload (ha-commit-durability
* task 4.2). The TAG axis uses the same config-epoch fold the voter side
* computes (ha_vote_self_freshness), but the LSN axis advertises the BARE
* durable WAL tip — a candidate may only claim a log position it can
* durably back, so a truncated-restart node cannot over-advertise a stale
* volatile learned-committed value and win (design §1a/§5a). */
uint64_t cand_tag = 0, cand_lsn = 0;
if (c->myconf) {
uint64_t local_tag = c->myconf->config_version_tag;
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL);
cand_lsn = ha_candidate_advertise_lsn(&c->persist);
}
int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
const char *my_name = (c->myconf) ? c->myconf->name : "";
ha->raft_c = c;
#ifdef XCE_TESTING
/* Test seam: the WON-branch real election is the same step the pre-rewire
* rdhup body fired, so the existing rdhup-integration tests intercept it
* here (capturing my_tag/my_lsn / counting election starts) — they now
* drive it through the PreVote→WON path (ha-prevote task 4.2).
*
* Mark the role CTRL_ELECTING before invoking the hook: the real
* ha_start_election_async transitions to CTRL_ELECTING as its first step,
* so a test that intercepts the election must still observe the destructive
* role transition the WON branch performs (the spy hook does not start a
* real election that would set it). */
if (ha_rdhup_election_hook != NULL) {
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
memory_order_release);
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
int np = 0;
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
if (strcmp(cfg->candidates[i].name,
c->myhost ? c->myhost->name : "") != 0)
peer_ptrs[np++] = &cfg->candidates[i];
}
ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn,
my_name);
}
return;
}
#endif
/* ha_start_election_async transitions to CTRL_ELECTING and drives the
* UNCHANGED raft_start_election (term++ + self-vote). Peer connections are
* driven by raft_cb_send_vote_request; the peers[] arg is unused. */
ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name);
}
/*
* ha_start_prevote_async: start a PreVote round (ha-prevote task 4.1).
*
* Snapshots self tag/lsn into the round metadata carrier (ha->elect_ctx, read
* by raft_cb_send_pre_vote_request), calls raft_begin_pre_vote (self-grant +
* per-peer send via the callback), and arms a round-timeout bounded by
* election_timeout_ms (Req 5.4 — no unbounded delay). The node stays
* CTRL_STANDBY; NOTHING persistent is mutated (no term/voted_for/role change).
*
* If raft_begin_pre_vote returns RAFT_PRE_VOTE_WON immediately (single-node
* cluster: self-grant is already a majority), the WON branch runs at once.
*
* c == NULL (unit-test mode): the send callback returns early and the round
* timeout is a no-op; the round state lives entirely in raft_state_t so the
* reply-driven branches are still fully exercisable via the test seam.
*/
void ha_start_prevote_async(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
ha->raft_c = c;
/* Snapshot self freshness for the PreVote request payload (same floor as a
* real vote). Stored in the metadata carrier ha->elect_ctx so
* raft_cb_send_pre_vote_request can read my_tag/my_lsn. */
uint64_t cand_tag = 0, cand_lsn = 0;
int n_peers = ha->n_peers > 0 ? ha->n_peers : ha->raft.n_peers;
if (c && c->myconf) {
uint64_t local_tag = c->myconf->config_version_tag;
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
/* Task 4.2: TAG via the config-epoch fold, LSN as the bare durable WAL
* tip (PreVote must not over-advertise either — design §1a). */
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL);
cand_lsn = ha_candidate_advertise_lsn(&c->persist);
n_peers = c->myconf->peers_cfg.candidates_count;
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
ha_elect_ctx_t *mc = calloc(1, sizeof(*mc) +
(size_t)n_other * sizeof(mc->votes[0]));
if (mc) {
mc->n_peers = n_other;
mc->result = -1;
mc->c = c;
mc->self_tag = cand_tag;
mc->self_lsn = cand_lsn;
pthread_mutex_init(&mc->lock, NULL);
sem_init(&mc->done, 0, 0);
atomic_init(&mc->refcount, 1); /* ha-elect-ctx-lifecycle task 2.4: bias reference */
if (n_other > 0) {
mc->votes[0].my_tag = cand_tag;
mc->votes[0].my_lsn = cand_lsn;
}
/* ha-elect-ctx-lifecycle task 2.4: exchange (not store) so we can
* release the superseded carrier's bias reference — it is freed
* here immediately if no vote endpoint from that prior round still
* references it, or later, whenever that endpoint's on_fini fires,
* whichever comes last. Replaces the old deliberate leak (see
* design.md's Architecture §Mechanism). */
ha_elect_ctx_t *old_ctx = (ha_elect_ctx_t *)atomic_exchange_explicit(
&ha->elect_ctx, mc, memory_order_acq_rel);
ha_elect_ctx_release(old_ctx);
}
ha->n_peers = n_peers;
}
INFO("HA: starting PreVote round (prospective term=%" PRIu64 ", n_peers=%d) "
"— staying CTRL_STANDBY, no persist",
ha->raft.term + 1, n_peers);
/* raft_begin_pre_vote: self-grant + per-peer send; mutates only the
* pre_vote_* tally in raft_state_t (no term/voted_for/role change).
* bug-1435: serialize against concurrent raft_state_t access (see ha.h
* raft_lock doc) — pre_votes_granted/pre_vote_replies[] are just as
* unprotected as the real-vote tally. */
pthread_mutex_lock(&ha->raft_lock);
int rc = raft_begin_pre_vote(&ha->raft);
pthread_mutex_unlock(&ha->raft_lock);
/* Arm the round timeout (bounded by election_timeout_ms). Production only;
* the round state is in raft_state_t so tests exercise the branches via the
* reply seam without the timer. */
ha_prevote_cancel_round_timeout();
if (c && c->delaymgr.running) {
prevote_round_ctx_t *tctx = calloc(1, sizeof(*tctx));
if (tctx) {
atomic_init(&tctx->cancelled, 0);
tctx->ha = ha;
tctx->c = c;
atomic_store_explicit(&s_prevote_round_ctx, tctx,
memory_order_release);
extern int ha_prevote_round_timeout_fn(void *arg);
int trc = delay_task_ms(&c->delaymgr,
(int)ha->config.election_timeout_ms, 1,
(delay_task_entry_fn_t)ha_prevote_round_timeout_fn,
tctx);
if (trc < 0) {
WARN("HA: ha_start_prevote_async: round-timeout schedule failed: %d",
trc);
atomic_store_explicit(&s_prevote_round_ctx, NULL,
memory_order_release);
free(tctx);
}
}
}
/* Single-node: the self-grant is already a majority — win immediately. */
if (rc == RAFT_PRE_VOTE_WON) {
ha_prevote_cancel_round_timeout();
INFO("HA: PreVote won immediately (single-node) — entering real election");
ha_enter_real_election(ha, c);
}
}
/*
* pre_vote_reply_dispatch: feed a PreVote reply into the raft tally and branch
* on its outcome (ha-prevote task 4.1). peer_idx is the ABSOLUTE cluster index
* of the replying peer (the same index raft_begin_pre_vote sent to).
*
* RAFT_PRE_VOTE_WON → cancel round timeout, run the real-election sequence
* (truncate + CTRL_ELECTING + ha_start_election_async).
* RAFT_PRE_VOTE_LOST → stay CTRL_STANDBY, re-arm liveness, log abandonment.
* RAFT_PRE_VOTE_PENDING → no-op (await more replies).
*
* Mirrors vote_reply_dispatch → raft_on_vote_reply, but the raft library makes
* NO role/term change here — the controller acts on the returned status.
*
* Production caller (task 5.1): pre_vote_reply_decode_and_dispatch, reached from
* peer_ctrl_msg_handler when a preVoteReply arrives on the outbound PreVote
* endpoint. Also driven by the XCE_TESTING reply seam.
*/
static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted)
{
/* bug-1435: serialize against concurrent raft_state_t access (see ha.h
* raft_lock doc). raft_lock is RECURSIVE: the WON branch below calls
* ha_enter_real_election -> ... -> raft_start_election synchronously on
* this same thread, which re-acquires raft_lock at its own call site —
* safe by design (same-thread nesting), never blocks a legitimate chain. */
pthread_mutex_lock(&ha->raft_lock);
int outcome = raft_on_pre_vote_reply(&ha->raft, peer_idx, granted);
pthread_mutex_unlock(&ha->raft_lock);
if (outcome == RAFT_PRE_VOTE_WON) {
ha_prevote_cancel_round_timeout();
INFO("HA: PreVote round WON — entering real election (term will be %"
PRIu64 ")", ha->raft.term + 1);
ha_enter_real_election(ha, c);
return;
}
if (outcome == RAFT_PRE_VOTE_LOST) {
ha_prevote_cancel_round_timeout();
/* Req 7.3: log campaign abandonment distinctly from a genuine failover
* so operators can tell a suppressed disruptive election apart. */
INFO("HA: PreVote round LOST — abandoning campaign, staying CTRL_STANDBY "
"(no term increment); re-arming liveness watchdog");
/* Stay CTRL_STANDBY — no persist, no role change (Req 1.4). */
ha_prevote_rearm_liveness(ha, c);
return;
}
/* RAFT_PRE_VOTE_PENDING — await more replies, do nothing. */
}
/*
* ha_prevote_round_timeout_fn: delaymgr task body for the PreVote round timeout.
* A timeout that fires before a decision is treated as a LOSS (Req 5.4).
*/
int ha_prevote_round_timeout_fn(void *arg)
{
prevote_round_ctx_t *ctx = (prevote_round_ctx_t *)arg;
/* bug (2026-07-10 chaotic-soak ASAN finding): clear s_prevote_round_ctx
* for THIS ctx BEFORE freeing it, unconditionally -- regardless of which
* branch below runs. The prior code only cleared the static pointer
* inside the "round still in progress" branch; when pre_vote_in_progress
* was ALREADY false here (the round was decided via a reply arriving on
* a different path, one that doesn't itself clear this pointer) this
* function fell straight through to free(ctx) below with
* s_prevote_round_ctx still pointing at the about-to-be-freed memory.
* The NEXT PreVote round's ha_start_prevote_async then calls
* ha_prevote_cancel_round_timeout() defensively before arming its own
* timer, reads that dangling pointer, and writes through it --
* heap-use-after-free (confirmed via ASAN on a live chaotic HA soak,
* both crashing nodes wedged with an identical ha.c:6712 UAF). CAS so a
* newer round's context (if one has already replaced this one by the
* time this delaymgr task runs) is never incorrectly cleared. */
{
prevote_round_ctx_t *expected = ctx;
atomic_compare_exchange_strong_explicit(
&s_prevote_round_ctx, &expected, NULL,
memory_order_acq_rel, memory_order_relaxed);
}
if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) {
#ifndef XCE_TESTING
free(ctx);
#endif
return 0;
}
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
/* Only act if the round is still in progress (not already decided). */
if (ha->raft.pre_vote_in_progress) {
ha->raft.pre_vote_in_progress = 0;
WARN("HA: PreVote round timed out before a decision — treating as LOSS, "
"staying CTRL_STANDBY; re-arming liveness watchdog (Req 5.4)");
ha_prevote_rearm_liveness(ha, c);
}
#ifndef XCE_TESTING
free(ctx);
#endif
return 0;
}
#ifdef XCE_TESTING
/*
* pre_vote_reply_dispatch_for_test: drive the real pre_vote_reply_dispatch with
* decoded (peer_idx, granted) fields, bypassing iomgr (ha-prevote task 4.1).
*/
void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted)
{
pre_vote_reply_dispatch(ha, c, peer_idx, granted);
}
/*
* ha_prevote_round_timeout_fire_for_test: simulate the round timeout firing
* before a decision, driving the LOST branch (Req 5.4).
*/
void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c)
{
prevote_round_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = ha;
ctx.c = c;
atomic_init(&ctx.cancelled, 0);
ha_prevote_round_timeout_fn(&ctx);
}
#endif /* XCE_TESTING */
/* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ────
*
* Non-blocking election initiator. Sets up the raft state machine and
* delegates all election mechanics to raft_start_election().
*
* raft_start_election() handles:
* - term increment and self-vote (via on_persist callback)
* - single-node shortcircuit (fires on_become_leader immediately)
* - multi-node RPC fan-out (via on_send_vote_request callback per peer)
* - election timeout scheduling (via on_schedule_timeout callback)
*
* ha_elect_ctx_t is still allocated here because raft_cb_send_vote_request
* reads ha->elect_ctx to retrieve my_tag/my_lsn for the capnp VoteRequest
* payload. Vote counting and quorum decisions are now owned by the raft
* library (raft_on_vote_reply fires on_become_leader/follower callbacks).
*
* Requirements: 1.3, 2.1 (shortcircuit), 2.4 (async vote send), 9.2
*/
void ha_start_election_async(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name)
{
(void)peers; /* peer connections driven by raft_cb_send_vote_request */
(void)my_name; /* self-vote name stored in on_persist via raft_start_election */
/* Step 0: raft is initialized once at startup by ha_init_raft(); nothing
* to do here. ha->raft_c and ha->n_peers are already set. */
/* Step 1: transition to CTRL_ELECTING before firing RPCs.
* raft_start_election fires on_become_leader / on_send_vote_request below;
* the role must already be ELECTING so ha_dispatch_gate() admits vote RPCs. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release);
INFO("HA FSM: → CTRL_ELECTING (async, n_peers=%d)", n_peers);
/* Step 2: pre-allocate ha_elect_ctx_t so raft_cb_send_vote_request can
* read my_tag/my_lsn from it when building the capnp VoteRequest payload.
* Vote counting is now owned by the raft library; this ctx is only a
* metadata carrier for tag/lsn. For multi-node elections it is kept alive
* until all per-peer vote_ep_ctx_t endpoints finish (iomgr lifetime).
* For single-node, raft_start_election fires on_become_leader immediately
* so no ctx reads occur after return.
*
* n_peers = total cluster size (including self), matching raft convention.
* standby_converged[] is indexed by votes_idx (0 .. n_peers-2), so
* convergence loops iterate i < n_peers - 1 (skip self). */
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
/* Store total cluster size (including self) — matches raft_config_t.n_peers */
ha->n_peers = n_peers;
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
(size_t)n_other * sizeof(ctx->votes[0]));
if (!ctx) {
WARN("ha_start_election_async: calloc ctx failed");
return;
}
ctx->n_peers = n_other;
ctx->votes_granted = 1; /* self-vote placeholder (raft tracks actual count) */
ctx->n_replied = 0;
ctx->max_resp_tag = 0;
ctx->max_resp_lsn = 0;
ctx->result = -1;
ctx->done_posted = 0;
ctx->c = c;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
atomic_init(&ctx->refcount, 1); /* ha-elect-ctx-lifecycle task 2.4: bias reference */
/* Store candidate's own tag/lsn unconditionally in self_lsn/self_tag (task 6.3):
* ha_compute_safe_committed_lsn uses these for the majority computation,
* including the single-node case where votes[] is empty. */
ctx->self_lsn = my_lsn;
ctx->self_tag = my_config_tag;
/* Also store in votes[0] for raft_cb_send_vote_request (multi-node only) */
if (n_other > 0) {
ctx->votes[0].my_tag = my_config_tag;
ctx->votes[0].my_lsn = my_lsn;
}
/* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx.
* ha-elect-ctx-lifecycle task 2.4: exchange (not store) to retrieve and
* release the superseded carrier's bias reference — see the identical
* rationale at ha_start_prevote_async's publish site. */
ha_elect_ctx_t *prior_ctx = (ha_elect_ctx_t *)atomic_exchange_explicit(
&ha->elect_ctx, ctx, memory_order_acq_rel);
ha_elect_ctx_release(prior_ctx);
/* Step 4: delegate election to raft library.
*
* A node with no valid local config (my_config_tag == 0) must not start an
* election — any peer with a real config tag will beat it in the freshness
* gate and the node would waste a term increment. It stays in CTRL_ELECTING,
* accepts incoming vote requests (granting to fresher peers), and will
* become STANDBY after the winner sends it a heartbeat/ctrlhello.
* n_peers and raft state are already initialised above so RDHUP retries work.
*
* raft_start_election when allowed:
* - increments ha->raft.term and records self-vote (fires on_persist)
* - single-node: fires on_become_leader → raft_cb_become_leader
* - multi-node: fires on_send_vote_request per peer → raft_cb_send_vote_request
* then fires on_schedule_timeout → raft_cb_schedule_timeout
*
* No direct ha->term++ or ha->voted_for_* writes happen in this function
* after this call; the raft library owns those via on_persist. */
/* Req 4.3 / ha-config-bootstrap Req 2.8:
* A node with no local config (my_config_tag == 0) must not start an
* election if any peer has a valid config — it cannot win (freshness gate
* denies it) and it would waste a term increment. It stays in CTRL_ELECTING,
* accepts incoming vote requests (granting to fresher peers), and becomes
* STANDBY after the winner sends a heartbeat.
*
* Exception: if ALL peers also have config_tag=0 (cluster cold-start with no
* configured node), someone must win — allow election normally.
*
* my_config_tag comes from myconf->config_version_tag set by load_primary_config;
* n_peers and the peer list from --peers are always available via the parameters.
* We check if any peer returned a non-zero tag via the xce_probe_config_tag path
* already called in myapp_run before ha_start_election_async. */
if (my_config_tag == 0 && n_peers > 1) {
INFO("ha_start_election_async: no local config (tag=0, n_peers=%d) — "
"participating as voter only, not starting own election", n_peers);
/* Transition to CTRL_STANDBY so myapp_run schedules fetch_config_from_primary.
* The primary will push config via ctrlhello once it becomes CTRL_PRIMARY. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
/* ha-elect-ctx-lifecycle task 2.4: no vote sends will happen from this
* node this round — ctx was never captured by any vote endpoint, so
* this release (refcount 1->0) frees it immediately, same outcome as
* the old direct destroy+free, now routed through the single release
* helper for consistency. */
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
ha_elect_ctx_release(ctx);
return;
}
/* bug-1435: serialize against concurrent raft_state_t access (see ha.h
* raft_lock doc). This call can fire on_become_leader synchronously
* (single-node case) or on_send_vote_request per peer + on_schedule_
* timeout (multi-node) — none of which re-enter a raft_* call, so the
* lock is held only across this one call. */
pthread_mutex_lock(&ha->raft_lock);
raft_start_election(&ha->raft);
pthread_mutex_unlock(&ha->raft_lock);
/* For single-node, raft_start_election fires on_become_leader synchronously.
* on_become_leader (raft_cb_become_leader) handles role -> CTRL_PRIMARY and
* phase2. Release the ctx here since no raft callbacks
* will reference it after raft_start_election returns for single-node
* (no vote endpoint is ever created — votes[] is empty). */
if (n_peers == 1) {
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
ha_elect_ctx_release(ctx);
return;
}
INFO("ha_start_election_async: returning (n_peers=%d, raft election started)",
n_peers);
/* Returns immediately — election result delivered via raft on_become_leader/follower */
}
#ifdef XCE_TESTING
/*
* ha_rdhup_election_hook: test-only seam for ha_start_election_async in the
* RDHUP failover path. When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls
* this instead of the real ha_start_election_async(). Req 6.3, 6.5, 9.3.
*/
void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name) = NULL;
#endif
/* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ───────────────────
*
* Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback),
* 6.4 (no heartbeat within timeout → elect), 6.8 (role flip)
* 10.5 (INFO logs on FSM transitions)
*/
/*
* ha_begin_failover: the single failover entry point (ha-prevote task 4.2).
*
* Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and
* the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route
* here AFTER their connection-teardown bookkeeping and the config-less
* voter-only guard. It starts a side-effect-free PreVote round and returns;
* it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those
* destructive steps are deferred to PreVote success (ha_enter_real_election,
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
* and term untouched (Req 1.1, 4.3, 4.4).
*/
/* ha_election_stagger_ms: see ha.h. Same formula as the startup stagger
* (myapp.c) — priority baseline plus fresh random jitter, re-randomized each
* call so repeated split-vote collisions converge (Raft §5.2; bug-606). */
uint32_t ha_election_stagger_ms(ha_state_t *ha)
{
if (!ha) return 0;
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
uint32_t step = ha->config.election_priority_step_ms;
uint32_t jmax = ha->config.election_jitter_max_ms;
uint32_t delay = (uint32_t)ha->my_priority * step;
if (jmax > 0)
delay += (uint32_t)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)jmax + 1));
return delay;
}
/* ha_jittered_election_delay: see ha.h. Canonical Raft randomized election
* timeout — [base, 2*base], re-drawn each campaign — applied to the per-campaign
* watchdog so a constant cold-connection grant RTT cannot resonate just past a
* fixed deadline forever (task 5.7). Independent of election_jitter_max_ms and
* never below the configured floor. */
uint32_t ha_jittered_election_delay(ha_state_t *ha, uint32_t base)
{
if (!ha || base == 0)
return base;
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
uint32_t extra = (uint32_t)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)base + 1));
return base + extra;
}
/* failover_stagger_ctx_t: carries ha/c into the delaymgr-scheduled PreVote start.
* Freed by ha_failover_stagger_fn (one-shot). */
typedef struct {
ha_state_t *ha;
cluster_t *c;
} failover_stagger_ctx_t;
/* ha_failover_stagger_fn: delaymgr body that starts the PreVote round after the
* per-node stagger (bug-606). Guard: if a primary connected during the stagger
* (we are a STANDBY with a fresh heartbeat), the cluster already has a leader —
* skip the PreVote entirely instead of needlessly probing a live primary. */
static int ha_failover_stagger_fn(void *arg)
{
failover_stagger_ctx_t *ctx = (failover_stagger_ctx_t *)arg;
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
INFO("HA: failover stagger elapsed but a primary is alive — skipping "
"PreVote (bug-606)");
free(ctx);
return 0;
}
ha_start_prevote_async(ha, c);
free(ctx);
return 0;
}
/* CLOCK_MONOTONIC milliseconds — for the RDHUP failover debounce (5.10b). */
static uint64_t ha_mono_ms(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L);
}
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
/* Stamp the failover-trigger time so a rapid RDHUP cadence from a dead
* primary's fd cannot re-trigger a fresh failover on every event (5.10b
* debounce in ha_on_primary_ctrl_fd_rdhup). Any trigger source (RDHUP or the
* deferred heartbeat-timeout stagger) records here. */
atomic_store_explicit(&ha->last_failover_ms, ha_mono_ms(), memory_order_release);
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
"PreVote wins) — staying CTRL_STANDBY");
/* Stagger the PreVote start per node so two survivors that lost the primary
* simultaneously do not campaign in lock-step and split the vote forever
* (Raft §5.2; bug-606). Production only: in unit-test mode (c == NULL) or
* with no running delaymgr, start synchronously so the existing failover /
* PreVote tests observe the unchanged immediate behaviour. */
uint32_t stagger = ha_election_stagger_ms(ha);
if (c && c->delaymgr.running && stagger > 0) {
failover_stagger_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (sctx) {
sctx->ha = ha;
sctx->c = c;
INFO("HA: staggering PreVote start by %ums (priority=%d) to avoid "
"split vote (bug-606)", stagger, ha->my_priority);
int rc = delay_task_ms(&c->delaymgr, (int)stagger, 1,
ha_failover_stagger_fn, sctx);
if (rc >= 0)
return;
/* Scheduling failed — fall through to a synchronous start. */
WARN("HA: ha_begin_failover: stagger schedule failed (%d) — "
"starting PreVote immediately", rc);
free(sctx);
}
}
ha_start_prevote_async(ha, c);
}
void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
/* raft-ha-core-safety task 5.6: do NOT restart failover while an election is
* already in flight. A won PreVote enters the real election (CTRL_ELECTING),
* or a PreVote round may be mid-decision (pre_vote_in_progress). The dead
* primary's control fd keeps RDHUP'ing on a deterministic ~15s cadence — if
* each RDHUP re-triggers a fresh ha_begin_failover it PREEMPTS the in-progress
* real election before it can complete, bumping the term every cycle and
* keeping two surviving candidates dueling in lockstep (split vote, unbounded
* term escalation; research.md §"Task 5.5 fix VERIFIED"). Drop the RDHUP:
* close the dead control fd and let the in-progress round finish or time out —
* its election timeout is jittered, which desynchronizes the duel so one
* survivor wins. A node NOT currently electing falls through to the normal
* 5.2 suppress / genuine-loss failover path below. */
{
ctrl_role_t cur = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (cur == CTRL_ELECTING || ha->raft.pre_vote_in_progress) {
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
INFO("HA: primary_ctrl_fd RDHUP but an election is already in progress "
"(role=%d, prevote=%d) — dropping the RDHUP, not restarting "
"failover (task 5.6: don't preempt the in-progress election / "
"churn the term)", (int)cur, ha->raft.pre_vote_in_progress);
return;
}
}
/* raft-ha-core-safety 5.10b: RDHUP failover DEBOUNCE. The 5.6 guard above
* only catches an RDHUP while the role is INSTANTANEOUSLY CTRL_ELECTING /
* pre_vote_in_progress. But a dead primary's fd RDHUPs faster than an
* election settles, and each RDHUP that lands in the gap AFTER a real
* election loses (role briefly back at CTRL_STANDDBY before the next attempt)
* re-triggers a fresh failover → a term bump per RDHUP (observed: 48
* begin_failovers, terms 6→24 in one round; research.md §"5.10b DEEPER").
* Suppress an RDHUP-triggered failover for up to 2× the election timeout
* after the last trigger — long enough to cover the jittered [T,2T] election
* watchdog (5.7) so the election's OWN retry runs without the external RDHUP
* cadence piling on extra term bumps. A genuinely-dead primary is still
* failed over — at most once per window, not once per RDHUP. First trigger
* (last_failover_ms == 0) is never debounced. */
{
uint64_t now_ms = ha_mono_ms();
uint64_t last = atomic_load_explicit(&ha->last_failover_ms,
memory_order_acquire);
uint64_t cooldown_ms = 2u * (uint64_t)ha->config.election_timeout_ms;
if (last != 0 && now_ms >= last && (now_ms - last) < cooldown_ms) {
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
INFO("HA: primary_ctrl_fd RDHUP within failover cooldown (%lums since "
"last trigger, cooldown=%lums) — debouncing, not restarting "
"failover (5.10b: a sustained RDHUP cadence must not ladder the "
"term; the in-flight election's own jittered retry handles it)",
(unsigned long)(now_ms - last), (unsigned long)cooldown_ms);
return;
}
}
/* Cancel the liveness poll task first (Req 5.4): prevents a concurrent
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
ha_disarm_heartbeat_timeout_vtask(ha);
/* Close and null the primary control fd. Connection-teardown bookkeeping
* stays at the trigger regardless of the PreVote outcome (ha-prevote task
* 4.2). */
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
/* raft-ha-core-safety task 5.2: SPURIOUS-FAILOVER SUPPRESSION.
*
* A TRANSIENT control-channel RDHUP while the current-term primary is still
* alive (a heartbeat arrived within heartbeat_timeout_ms) must NOT launch an
* unwinnable PreVote failover. The live stall (research.md §"Phase 4/5"): a
* recovered standby whose channel to the live primary flaps treats each
* RDHUP as "primary died" → PreVote → peers see the primary alive and deny →
* round times out → reconnect → repeat ~15s, so the node prevote-spins
* forever instead of rejoining. Mirror the guard ha_failover_stagger_fn
* already applies on the DEFERRED path (a flap = the primary re-established
* the channel, so ha_heartbeat_is_fresh is the right liveness signal —
* the same one handle_request_vote uses to deny same-term challengers).
*
* Stay CTRL_STANDBY and await the primary's reconnect (handle_ctrlhello);
* re-arm the liveness watchdog so a GENUINE subsequent loss is still caught
* (the watchdog resets last_activity to now, granting a fresh grace window —
* if the primary truly never heartbeats again it fires a real failover after
* heartbeat_timeout_ms). GENUINE loss (stale heartbeat) falls through to the
* failover path below unchanged; Election Safety is unaffected (no term bump,
* no role change here). */
ctrl_role_t rdhup_role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (rdhup_role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
INFO("HA: primary_ctrl_fd RDHUP but the primary is alive (fresh "
"heartbeat) — suppressing PreVote failover, staying CTRL_STANDBY "
"and awaiting reconnect (spurious-failover suppression, "
"raft-ha-core-safety 5.2)");
if (c)
ha_arm_heartbeat_timeout_vtask(ha, c);
return;
}
/* ha-prevote task 4.2: the failover trigger NO LONGER transitions to
* CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and
* STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is
* deferred to PreVote success (ha_enter_real_election). The node only ever
* inflates its term / unseats a healthy primary if a PreVote majority is
* first obtained (Req 1.1, 4.3, 4.4). */
INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying "
"CTRL_STANDBY until PreVote wins) (Req 1.1)");
if (!c) {
/* Unit-test mode: no persist / no live transport. Still begin the
* round so the deferred-destructive behaviour is observable (the round
* state lives in raft_state_t; ha_enter_real_election with c==NULL only
* flips the role on a WIN). */
ha_begin_failover(ha, c);
return;
}
/* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening
* Req 10.5): a config-less node (config_version_tag == 0) must NEVER start
* an RDHUP re-election while a configured peer exists. Two such nodes would
* grant each other (the freshness gate only denies a LOWER tag; tag0==tag0
* with equal lsn is granted), forming a majority and electing a config-less
* primary over the node that actually holds the config — which then pushes
* an empty config and breaks replication.
*
* This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a
* config-less node can never win a PreVote (the freshness gate denies it),
* so there is no point starting a round — and starting one would needlessly
* probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote
* still GRANTS incoming vote requests to the fresher configured peer
* regardless of role, and this node transitions/fetches config when the
* configured winner connects and pushes config via ctrlhello.
*
* Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false,
* so a lone node still self-elects (its PreVote trivially wins). */
uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0;
int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
if (my_config_tag == 0 && n_peers_guard > 1) {
INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — "
"voter only, not starting PreVote/RDHUP re-election; awaiting "
"configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
return;
}
/* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async
* (it snapshots self freshness into ha->elect_ctx so the PreVote request
* payload — and, on a WIN, the deferred real election — advertise the
* effective freshness floor, NOT the stale persist.committed_lsn: see the
* bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */
ha_begin_failover(ha, c);
}
/* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */
/*
* liveness_poll_ctx_t: context for the delaymgr-based standby liveness task.
*
* Requirements: 1.2, 1.4, 1.5, 1.6, 5.4
*
* Replaces hb_vtask_t + hb_timeout_thread. No pthread, no semaphore.
* The poll fn reads ha->last_activity (written by ha_on_heartbeat_received)
* and fires ha_on_primary_ctrl_fd_rdhup if elapsed >= heartbeat_timeout_ms.
*
* cancelled is _Atomic so that writes from ha_disarm_heartbeat_timeout_vtask
* (main/vtask context) are visible to liveness_poll_fn (delaymgr worker thread)
* without a data race.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
_Atomic int cancelled;
} liveness_poll_ctx_t;
/* Static pointer to the active context. _Atomic so concurrent arm/disarm
* vs poll-fn execution is well-defined. Access pattern:
* arm — allocates and stores (single writer, vtask context)
* disarm — sets cancelled, frees (single writer, vtask context)
* poll_fn — reads via passed arg only; never dereferences s_liveness_ctx
*/
static liveness_poll_ctx_t * _Atomic s_liveness_ctx = NULL;
/* rdhup_failover_vtask_entry / ha_dispatch_rdhup_failover: offload
* ha_on_primary_ctrl_fd_rdhup (which chains into ha_start_election_async ->
* raft_start_election -> on_persist -> ha_persist_term's synchronous fsync)
* off the delaymgr thread onto a VTASK_LIVE_BLOCK vtask, mirroring
* ha_dispatch_election_start. This is the real, common production trigger for
* a standby's failover election (heartbeat-timeout watchdog) -- delaymgr must
* only detect the timeout and hand off, never run the election itself. */
typedef struct {
ha_state_t *ha;
cluster_t *c;
} rdhup_failover_dispatch_t;
static int rdhup_failover_vtask_entry(vtask_t *task)
{
rdhup_failover_dispatch_t *d =
(rdhup_failover_dispatch_t *)task->data->u.generic;
ha_on_primary_ctrl_fd_rdhup(d->ha, d->c);
free(d);
return 0;
}
static void ha_dispatch_rdhup_failover(ha_state_t *ha, cluster_t *c)
{
vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool;
vtask_data_t *data = calloc(1, sizeof(*data));
vtask_t *task = vtask_alloc();
rdhup_failover_dispatch_t *d = calloc(1, sizeof(*d));
if (pool == NULL || data == NULL || task == NULL || d == NULL) {
WARN("ha_dispatch_rdhup_failover: vtask unavailable (pool=%p data=%p "
"task=%p d=%p) — failing over inline (stay alive)",
(void *)pool, (void *)data, (void *)task, (void *)d);
free(data);
free(task);
free(d);
ha_on_primary_ctrl_fd_rdhup(ha, c);
return;
}
d->ha = ha;
d->c = c;
data->vtype = VTASK_GENERIC;
data->u.generic = d;
vtask_init(task, (vtask_func_t)rdhup_failover_vtask_entry, data);
if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) {
WARN("ha_dispatch_rdhup_failover: vtask enqueue failed — failing over "
"inline (stay alive)");
free(data);
free(task);
free(d);
ha_on_primary_ctrl_fd_rdhup(ha, c);
}
}
/*
* liveness_poll_fn: delaymgr task replacing hb_timeout_thread.
*
* Checks whether the primary has been silent for >= heartbeat_timeout_ms.
* If so, calls ha_on_primary_ctrl_fd_rdhup and does not re-arm.
* If not, re-arms itself for the next poll interval.
* If cancelled, returns immediately without any action.
*
* Also skips if role != CTRL_STANDBY (Req 5.4): prevents duplicate failover
* if EPOLLRDHUP already transitioned the role to CTRL_ELECTING.
*/
static int liveness_poll_fn(void *arg)
{
liveness_poll_ctx_t *ctx = (liveness_poll_ctx_t *)arg;
/* Check cancelled flag first — set by ha_arm_heartbeat_timeout_vtask when
* re-arming replaces this ctx. When cancelled, free the ctx here (the
* re-armer cannot safely free it because this task may already be running
* on a delaymgr thread). Do NOT re-arm.
*
* In XCE_TESTING builds callers pass stack-allocated ctxs, so we must
* not free in that path. Production builds always heap-allocate via
* ha_arm_heartbeat_timeout_vtask. */
if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) {
#ifndef XCE_TESTING
free(ctx);
#endif
return 0;
}
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
/* Skip if not standby — EPOLLRDHUP may have already changed the role */
if (atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY)
return 0;
/* Compute elapsed time since last primary message */
struct timespec now_ts;
clock_gettime(CLOCK_MONOTONIC, &now_ts);
time_t now = now_ts.tv_sec;
time_t last = ha->last_activity; /* time_t write is atomic on LP64 */
time_t elapsed = (last > 0) ? (now - last) : 0;
uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000;
if (last > 0 && (uint32_t)elapsed >= timeout_s) {
/* Silence exceeded timeout — trigger failover */
WARN("HA: liveness poll: no primary activity for %lus (timeout=%us) → failover",
(unsigned long)elapsed, timeout_s);
ha_dispatch_rdhup_failover(ha, c);
/* Do NOT re-arm — failover handles re-election */
return 0;
}
/* Still fresh — re-arm for next poll interval */
if (c && c->delaymgr.running) {
uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2;
if (poll_ms < 100) poll_ms = 100;
delay_task_ms(&c->delaymgr, (int)poll_ms, 1,
(delay_task_entry_fn_t)liveness_poll_fn, ctx);
}
return 0;
}
#ifdef XCE_TESTING
/*
* liveness_poll_fn_test_entry: test-only entry point that calls liveness_poll_fn
* directly with a pre-constructed ctx, bypassing delaymgr.
* Only compiled when XCE_TESTING is defined.
*/
void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx)
{
liveness_poll_fn(ctx);
}
/*
* primary_heartbeat_fn_for_test: call primary_heartbeat_fn directly with c.
* Bypasses delaymgr scheduling so tests can drive the idle-NOP logic without
* a live timer. Only compiled when XCE_TESTING is defined. (Task 2.2)
*/
int primary_heartbeat_fn_for_test(cluster_t *c)
{
return primary_heartbeat_fn(c);
}
#endif
void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
/* Disarm any existing poll ctx by setting cancelled=1.
* Do NOT free it here — the delaymgr task may already be running on
* another thread and will free the ctx itself when it sees cancelled=1.
* Clearing s_liveness_ctx prevents a second re-arm attempt from operating
* on the stale pointer. */
liveness_poll_ctx_t *old = atomic_load_explicit(&s_liveness_ctx,
memory_order_acquire);
if (old) {
atomic_store_explicit(&old->cancelled, 1, memory_order_release);
atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release);
}
liveness_poll_ctx_t *ctx = calloc(1, sizeof(*ctx));
if (!ctx) return;
ctx->ha = ha;
ctx->c = c;
atomic_init(&ctx->cancelled, 0);
/* Initialise last_activity to now so the first poll computes elapsed from
* the arm time rather than from zero. */
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
ha->last_activity = ts.tv_sec;
atomic_store_explicit(&s_liveness_ctx, ctx, memory_order_release);
uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2;
if (poll_ms < 100) poll_ms = 100;
INFO("HA: standby liveness poll armed (timeout=%ums, poll=%ums)",
ha->config.heartbeat_timeout_ms, poll_ms);
if (c && c->delaymgr.running) {
int rc = delay_task_ms(&c->delaymgr, (int)poll_ms, 1,
(delay_task_entry_fn_t)liveness_poll_fn, ctx);
if (rc < 0)
WARN("HA: ha_arm_heartbeat_timeout_vtask: delay_task_ms failed: %d", rc);
}
}
/* ha_discover_primary: if CTRL_ELECTING, step down to STANDBY on receiving
* a valid primary message. A campaigning node that sees a live primary must
* defer — continuing the election risks split-brain (bug-353). */
void ha_discover_primary(ha_state_t *ha, cluster_t *c)
{
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role != CTRL_ELECTING)
return;
uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire);
INFO("HA: discovered live primary while ELECTING (term=%" PRIu64 ") — stepping down",
term);
raft_cb_become_follower(ha, term, -1);
}
bool ha_msg_term_is_stale(ha_state_t *ha, uint64_t msg_term)
{
if (!ha || msg_term == 0)
return false;
uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire);
return msg_term < term;
}
void ha_on_heartbeat_received(ha_state_t *ha)
{
if (!ha) return;
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
ha->last_activity = ts.tv_sec;
tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec);
/* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader
* is alive — a PreVote round started by a transient blip must be ABANDONED.
* Cancel the in-progress round (clear the raft round state + the controller
* round-timeout handle) and re-attach as a standby. No persistent term /
* voted_for / role change happens here: the node never left CTRL_STANDBY
* during the round, so re-attaching is a no-op on role/term — exactly the
* non-disruptive rejoin the spec requires. */
if (ha->raft.pre_vote_in_progress) {
ha->raft.pre_vote_in_progress = 0;
ha->raft.pre_votes_granted = 0;
ha_prevote_cancel_round_timeout();
INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round "
"and re-attaching as CTRL_STANDBY (leader alive, no term change) "
"(Req 4.2)");
}
}
/*
* ha_observe_heartbeat_committed: reconcile-free monotonic learn of the
* primary's committed position from a heartbeat (ha-heartbeat-committed-
* propagation, Req 2.1, 2.2, 2.3, 3.1, 3.3).
*
* Single-receiver-thread path (controller request handling), so a relaxed
* compare-then-store is sufficient. The config tag is written BEFORE the LSN
* (both release-ordered) so a concurrent acquire-load reader that observes the
* advanced LSN never pairs it with a stale tag.
*
* Advisory only: NO ha_rejoin_reconcile / NO WAL truncation / NO change to any
* produced/persisted committed, role, term, or applied state. This is the
* explicit distinction from cluster_apply_ctrlhello_reply (which truncates).
*/
void ha_observe_heartbeat_committed(cluster_t *c, uint64_t committed_lsn,
uint64_t committed_config_tag)
{
if (!c) return;
ha_state_t *ha = &c->ha;
uint64_t cur = atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire);
/* Monotonic no-op: a zero (absent/liveness-only) or non-advancing LSN must
* leave the learned position — and its config tag — untouched (Req 2.2). */
if (committed_lsn == 0 || committed_lsn <= cur)
return;
/* Tag moves with the LSN, never independently regressed (Req 2.3): publish
* the tag first so the LSN's release store carries the matching tag. */
atomic_store_explicit(&ha->last_known_committed_config_tag,
committed_config_tag, memory_order_release);
atomic_store_explicit(&ha->last_known_committed_lsn, committed_lsn,
memory_order_release);
/* Reconcile-free: nothing else is touched (Req 3.1, 3.3). */
tlog(TLOG_DEBUG3,
"HA: learned committed_lsn=%" PRIu64 " config_tag=%" PRIu64
" from heartbeat (was %" PRIu64 ")",
committed_lsn, committed_config_tag, cur);
}
/*
* ha_standby_advance_committed_and_apply (2026-07-10, controller-invariant
* audit finding — commit/apply decoupling extended to the standby): drive
* this node's OWN p->committed_lsn from a learned primary committed position,
* clamped to what this node has actually durably received, then kick the
* SAME apply pump (persist_apply_pump_kick -> apply_committed_entry()) the
* primary's own online-apply and restart replay use.
*
* This is the missing half of the standby ingest path's commit-then-apply
* story: ha_apply_single_entry / ha_apply_term_entry_via_library persist +
* ACK a replicated entry unconditionally (required for durability and the
* primary's commit majority) but no longer apply it — entity materialization
* now waits for THIS function to observe that the entry is actually
* committed, never merely durable-on-receipt. Safe against a lagging
* standby: the clamp to local_tip means committed_lsn here can never run
* ahead of what this node has itself received, so persist_apply_pump_run's
* apply_pump_read_entry can never be asked to read an LSN this node doesn't
* have on disk yet.
*
* Race-safe against a CONCURRENT writer (2026-07-10, controller-invariant-
* audit finding): this function is gated to non-primary roles by its
* callers, but raft_cb_become_leader's own persist_promotion_advance_committed
* call happens BEFORE role flips to CTRL_PRIMARY (ha.c, the role store sits
* ~170 lines after the advance call) -- so a still-in-flight WAL catch-up
* entry (this function is also called from ha_apply_wal_catchup_entry's exit
* points) can legitimately run on another thread during that window, racing
* this function against the promotion path's own advance. Both now go
* through persist_committed_lsn_advance_atomic's CAS loop, which is safe
* against that race by construction (never loses an update, never
* regresses) -- no shared lock needed across the two call sites.
*/
static void ha_standby_advance_committed_and_apply(cluster_t *c,
uint64_t learned_committed_lsn)
{
if (!c || learned_committed_lsn == 0)
return;
persist_t *p = &c->persist;
pthread_mutex_lock(&p->lock);
uint64_t local_tip = p->current_lsn > 0 ? p->current_lsn - 1 : 0;
pthread_mutex_unlock(&p->lock);
uint64_t target = learned_committed_lsn < local_tip
? learned_committed_lsn : local_tip;
persist_committed_lsn_advance_atomic(p, target);
persist_apply_pump_kick(p, c->ha.pool);
}
/*
* ha_consume_heartbeat_committed: standby OP_nop consume branch — the
* non-primary gate (Req 3.4) + the stale-term guard (Req 3.2), then the
* monotonic observe (Req 1.2). Reconcile-free; updates nothing on a NULL body,
* a failed gate, or a zero committed value (Req 4.2). Pure additive helper —
* the caller runs the unchanged liveness/reply path first (Req 4.1).
*/
void ha_consume_heartbeat_committed(cluster_t *c, const heartbeat_msg_t *hb)
{
if (!c || !hb)
return;
/* (3.4) A primary uses its own authoritative committed — never an inbound
* heartbeat value. Only non-primary nodes consume. */
ctrl_role_t role =
(ctrl_role_t)atomic_load_explicit(&c->ha.role, memory_order_acquire);
if (role == CTRL_PRIMARY)
return;
/* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat through the raft
* library as a 0-entry AppendEntries so the LIBRARY performs UpdateTerm — a
* heartbeat at a strictly-higher term makes this node adopt the term and step
* down. This REPLACES the prior hand-rolled term-guard that only DROPPED a
* stale heartbeat (hb->term < myterm) and never adopted a higher term.
*
* The heartbeat carries no entries and no prev tuple, so we present the
* node's own log tip as (prevLogIndex,prevLogTerm) — logOk holds (already-
* have at the tip) for an equal/higher-term heartbeat, and UpdateTerm runs
* first regardless. raft_init must have wired the raft state (production
* boot); guard for the pre-init placeholder where raft_c is unset. */
if (c->ha.raft_c == c) {
uint64_t my_log_len = c->ha.raft.log_len;
uint64_t my_tip_term = (my_log_len > 0)
? persist_entry_term(&c->persist, my_log_len) : 0;
wal_catchup_entry_t hbae;
memset(&hbae, 0, sizeof(hbae));
hbae.n_entries = 0; /* heartbeat: zero entries */
hbae.term = hb->term; /* leader's live term (UpdateTerm) */
hbae.prev_log_index = my_log_len; /* already-have at our tip */
hbae.prev_log_term = my_tip_term;
hbae.leader_commit = hb->committed_lsn;
(void)ha_consume_append_entries(c, &hbae, /*leader_idx*/-1);
}
/* (3.2) Stale-term guard for the committed LEARN below: after UpdateTerm a
* heartbeat from a primary whose term is behind ours is stale/partitioned —
* do not let its committed inflate the vote-freshness floor. Re-read our term
* (it may have advanced via the library UpdateTerm above). */
uint64_t myterm = atomic_load_explicit(&c->ha.term, memory_order_acquire);
if (hb->term < myterm)
return;
/* (1.2) Monotonic, reconcile-free learn of the committed position. Zero/
* non-advancing LSN no-ops inside the observe (Req 4.2). */
ha_observe_heartbeat_committed(c, hb->committed_lsn,
hb->committed_config_tag);
/* 2026-07-10: drive this node's own committed_lsn + apply pump from the
* position just learned (see ha_standby_advance_committed_and_apply) —
* the commit-then-apply half of the standby ingest path. */
ha_standby_advance_committed_and_apply(c, hb->committed_lsn);
}
void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)
{
(void)ha;
liveness_poll_ctx_t *ctx = atomic_load_explicit(&s_liveness_ctx,
memory_order_acquire);
if (!ctx) return;
atomic_store_explicit(&ctx->cancelled, 1, memory_order_release);
atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release);
}
/* ── Primary heartbeat sender + late-joiner outbound connect ─────────────── */
/*
* standby_ep_ctx_t: per-peer context for outbound standby connections opened
* by the primary during the heartbeat task.
*
* Layout deliberately matches vote_ep_ctx_t for the first 4 fields so that
* peer_ctrl_msg_handler (which casts ep->data to vote_ep_ctx_t*) can safely
* read ha, c, and peer_idx. ctx is always NULL here (no active election).
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
ha_elect_ctx_t *ctx; /* always NULL — not a vote endpoint */
int peer_idx; /* index into c->myconf->peers_cfg.candidates[] */
/* Task 1.1 (ha-heartbeat-liveness): last time a push/nop was sent to this
* standby (CLOCK_MONOTONIC seconds). Initialized to 0 at allocation;
* written by on_standby_connected and primary_heartbeat_fn (task 2.2). */
time_t last_push_time;
} standby_ep_ctx_t;
#ifdef XCE_TESTING
/*
* standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data,
* and return ep. last_push_time is initialised to 0 (calloc).
* Caller must free(ep->data) after the test. (Task 2.2)
*/
endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
cluster_t *c,
int peer_idx)
{
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (!sctx) return NULL;
sctx->ha = ha;
sctx->c = c;
sctx->ctx = NULL;
sctx->peer_idx = peer_idx;
/* last_push_time = 0 from calloc — tests set it explicitly */
ep->data = sctx;
return ep;
}
/*
* on_standby_connected_for_test: call on_standby_connected directly.
* ep->data must be a valid standby_ep_ctx_t (use standby_ep_ctx_build_for_test).
* xce_request will fail on a null fd — tests only check last_push_time. (Task 2.2)
*/
int on_standby_connected_for_test(endpoint_t *ep)
{
on_standby_connected(ep);
return 0;
}
/*
* standby_ep_err_handler_primary_for_test: expose the outbound-standby error
* handler so a test can tear down a heap endpoint the production way (remove
* from standby_endpoints + replicator, free ep->data) before reclaiming the
* endpoint itself. (raft-drive-async Task 2.2)
*/
int standby_ep_err_handler_primary_for_test(endpoint_t *ep)
{
return standby_ep_err_handler_primary(ep);
}
#endif /* XCE_TESTING */
/*
* on_standby_connected: iomgr on_connected callback for outbound standby
* endpoints created by the heartbeat task.
*
* Sends OPCode_hello (CtrlHelloRequest) so the receiving standby runs
* handle_ctrlhello and triggers WAL catch-up + config push. Registers the
* endpoint in ha->standby_endpoints so future heartbeats reach it.
*/
static void on_standby_connected(endpoint_t *ep)
{
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
ha_state_t *ha = sctx->ha;
cluster_t *c = sctx->c;
/*
* Register in standby_endpoints, resolving the inbound-vs-outbound
* connection-ownership race (raft-drive-async Task 2.2, Req 2.6/4.4;
* ha-async-io.md connection ownership).
*
* The hazard: BOTH raft_cb_become_leader (promotes surviving vote eps) and
* this path (late-joiner / reconnect) zarray_store into standby_endpoints
* keyed only by ep-POINTER identity. So while an outbound reconnect to
* peer X is in flight, a DIFFERENT ep for the same peer X (a just-promoted
* vote-ep, or a stale prior channel whose RDHUP has not yet been processed)
* may already be registered. Leaving both would create two live outbound
* state-push channels to one peer — heartbeats/WAL fan out to a dead/stale
* duplicate (lost/reordered messages, Req 2.6) and the older endpoint is
* orphaned in the replicator.
*
* Resolution (deterministic, per-peer): the freshly-connected channel wins.
* Under ha->lock collect any DIFFERENT ep with the same peer_idx and drop
* it from the array; then register `ep`. The actual teardown of the stale
* ep runs AFTER releasing ha->lock because standby_ep_err_handler_primary
* re-takes ha->lock (and touches the replicator) — avoiding self-deadlock
* and holding the lock across no I/O (R4.4: iomgr/registration state mutated
* only via the lock-guarded interface, never racing the event loop).
*/
endpoint_t *stale_same_peer = NULL;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
int already = 0;
for (int i = 0; i < ha->standby_endpoints->count; i++) {
endpoint_t *cur = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
if (cur == ep) { already = 1; continue; }
/* Different ep, same peer → stale duplicate to evict. */
standby_ep_ctx_t *cur_sctx =
cur ? (standby_ep_ctx_t *)cur->data : NULL;
if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx) {
stale_same_peer = cur;
zarray_del(ha->standby_endpoints, i);
/* count shrank by one; the surviving scan continues but we
* only ever expect a single same-peer duplicate. */
break;
}
}
if (!already)
zarray_store(ha->standby_endpoints, ep);
pthread_mutex_unlock(&ha->lock);
}
/* Tear down the evicted stale channel OUTSIDE ha->lock. Mirror the library
* disconnect path (handle_error → iomgr_unmonitor → err_handler) so we
* never free an endpoint whose fd is still in the epoll set (R4.4: mutate
* iomgr/registration state ONLY via the lock-guarded interface, never
* racing the event loop). err-handler removes it from standby_endpoints +
* the replicator and frees ep->data; we then reclaim the endpoint itself.
* Never self-terminate — this is routine ownership reconciliation. */
if (stale_same_peer) {
WARN("HA: peer[%d] reconnected — evicting stale duplicate outbound "
"channel to keep exactly one live connection (Req 2.6)",
sctx->peer_idx);
if (stale_same_peer->iomgr)
iomgr_unmonitor((iomgr_t *)stale_same_peer->iomgr, stale_same_peer);
(void)standby_ep_err_handler_primary(stale_same_peer);
/* err-handler set ep->data = NULL and freed sctx; bug-1298: drop the
* owner ref — endpoint_unref reclaims outq/ssl/packets and frees the
* calloc'd struct when the last reference goes (deferred if a pool vtask
* still references this just-unmonitored ep). */
endpoint_unref(stale_same_peer);
#ifdef XCE_TESTING
atomic_fetch_add(&ha_standby_evicted_count_for_test, 1);
#endif
}
/* Reset failure counter so the next disconnect restarts logging from attempt 1. */
ha->peer_connect_fail_count[sctx->peer_idx] = 0;
/* Record last_push_time before any send so the first heartbeat tick after
* connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */
{
struct timespec _ts;
clock_gettime(CLOCK_MONOTONIC, &_ts);
sctx->last_push_time = _ts.tv_sec;
}
/* Send ctrlhello — triggers handle_ctrlhello on the standby side */
uint64_t lsn = atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
uint64_t tag = c->myconf ? c->myconf->config_version_tag : 0;
char *name = c->myconf ? c->myconf->name : "";
ctrl_hello_req_t hello = { .name = name, .key = NULL,
.applied_lsn = lsn, .config_tag = tag };
xce_request_t req;
memset(&req, 0, sizeof(req));
req.opcode = OPCode_hello;
req.body_which = Request_body_ctrlhellomsg;
req.body.ctrlhellomsg = &hello;
int rc = xce_request(ep, &req);
if (rc < 0)
WARN("HA: ctrlhello to peer[%d] failed: %d", sctx->peer_idx, rc);
else
INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent",
sctx->peer_idx);
/* Task 2.1: register late-joiner with WAL replicator.
* Req 1.1: primary must register standby with repl_add_standby when
* establishing the standby control channel (late-joiner outbound connect).
* Host/port from candidates[peer_idx] — the peer we connected to. */
if (c && c->myconf &&
sctx->peer_idx >= 0 &&
sctx->peer_idx < c->myconf->peers_cfg.candidates_count) {
const xce_candidate_t *peer =
&c->myconf->peers_cfg.candidates[sctx->peer_idx];
int rc = repl_add_standby(&c->persist.repl, ep,
peer->hostname, (uint32_t)peer->port);
if (rc >= 0)
INFO("HA: repl registered late-joiner standby peer[%d] (slot %d, %s:%u)",
sctx->peer_idx, rc, peer->hostname, (uint32_t)peer->port);
else
WARN("HA: repl_add_standby late-joiner peer[%d] failed: %d",
sctx->peer_idx, rc);
}
/* Quorum-transition observability (Req 8.2): a late-joiner standby connected —
* the write-quorum count changed. Record the restore edge once (silent if
* quorum was already held). */
ha_log_quorum_transition(ha);
/* Set state-aware dispatcher for subsequent messages on this endpoint */
ep->msg_handler = peer_ctrl_msg_handler;
}
/*
* standby_ep_err_handler_primary: error handler for outbound standby
* endpoints created by the heartbeat task. Removes from standby_endpoints
* and frees the context — the next heartbeat tick will attempt reconnect.
*/
static int standby_ep_err_handler_primary(void *ep_raw)
{
endpoint_t *ep = (endpoint_t *)ep_raw;
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
if (!sctx) return 0;
ha_state_t *ha = sctx->ha;
cluster_t *c = sctx->c;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ha->standby_endpoints->count; i++) {
if (zarray_get(ha->standby_endpoints, i) == ep) {
zarray_del(ha->standby_endpoints, i);
break;
}
}
pthread_mutex_unlock(&ha->lock);
}
/* Task 2.2: deregister late-joiner standby from WAL replicator before
* freeing the context. Req 1.2: RDHUP or connection error triggers
* repl_remove_standby so the replicator stops sending WAL entries to it. */
if (c)
repl_remove_standby(&c->persist.repl, ep);
free(sctx);
ep->data = NULL;
return 0;
}
/*
* reconnect_ctx_t: owns the prepared endpoint across the in-flight reconnect
* window (raft-drive-async Task 2.1). Built on the delaymgr tick by the
* non-blocking prep (ha_connect_prep_peer), consumed once on a slow-lane
* worker by reconnect_lane_fn, then freed. The ctx keeps `ep`/`sctx` valid
* for the whole window so the lane worker never touches freed memory while the
* delaymgr thread races ahead to the next tick (Req 2.3 lifetime).
*/
typedef struct {
cluster_t *c;
int peer_idx; /* ABSOLUTE candidate index */
endpoint_t *ep; /* prepared, not yet connected */
standby_ep_ctx_t *sctx; /* == ep->data */
int should_log;
} reconnect_ctx_t;
/*
* ha_connect_prep_peer: the NON-BLOCKING half of the outbound peer connect.
* Allocates the endpoint context, initialises the endpoint, sets the SSL
* client state and the on-connected / err handlers — all of which are
* non-blocking and safe to run on the delaymgr tick. Does NOT perform the
* blocking connect (that is ha_connect_finish_peer, run on the slow lane).
*
* Returns a heap reconnect_ctx_t* (caller owns) on success, or NULL on an
* allocation failure (nothing leaked).
*/
static reconnect_ctx_t *ha_connect_prep_peer(cluster_t *c, int peer_idx)
{
ha_state_t *ha = &c->ha;
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (!sctx) return NULL;
sctx->ha = ha;
sctx->c = c;
sctx->peer_idx = peer_idx;
endpoint_t *ep = endpoint_get();
if (!ep) { free(sctx); return NULL; }
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
endpoint_set_peer(ep, c->myhost);
ep->data = sctx;
ep->iomgr = &c->iomgr;
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
if (ep->ssl)
SSL_set_connect_state(ep->ssl);
}
endpoint_set_on_connected(ep, on_standby_connected);
endpoint_set_err_handler(ep, standby_ep_err_handler_primary);
int n = ++ha->peer_connect_fail_count[peer_idx];
int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0);
reconnect_ctx_t *rc_ctx = calloc(1, sizeof(*rc_ctx));
if (!rc_ctx) {
ep->data = NULL;
endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (not monitored) */
free(sctx);
return NULL;
}
rc_ctx->c = c;
rc_ctx->peer_idx = peer_idx;
rc_ctx->ep = ep;
rc_ctx->sctx = sctx;
rc_ctx->should_log = should_log;
return rc_ctx;
}
/*
* ha_connect_finish_peer: the BLOCKING half of the outbound peer connect —
* the bounded connect + lock-guarded iomgr registration. MUST run on a
* slow-lane worker, NEVER on the delaymgr tick (Req 2.1, 2.2; ha-async-io.md
* thread-role contract).
*
* On success the endpoint is connecting and registered with iomgr; on_connected
* (on_standby_connected) fires later from the iomgr loop. On failure the
* endpoint is torn down here and the controller simply retries on the next
* heartbeat tick (controllers never self-terminate).
*
* Returns 0 on a successful connect+register, <0 otherwise.
*
* NOTE (Task 2.2 seam): this implements the happy-path connect+register +
* teardown-on-failure. The inbound-vs-outbound connection-ownership race
* resolution, the no-lost/no-reordered-message guarantee on success, and the
* richer failure/retry hardening are owned by Task 2.2 — they layer on top of
* this happy path without changing this function's contract (return 0/<0,
* caller clears the per-peer in-flight flag).
*/
static int ha_connect_finish_peer(reconnect_ctx_t *rc_ctx)
{
cluster_t *c = rc_ctx->c;
endpoint_t *ep = rc_ctx->ep;
standby_ep_ctx_t *sctx = rc_ctx->sctx;
int peer_idx = rc_ctx->peer_idx;
const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[peer_idx];
#ifdef XCE_TESTING
if (ha_reconnect_connect_hook) {
/* Deterministic test path: the hook stands in for the real bounded
* connect + register so a test can park the worker (unreachable peer)
* or return a synthetic success/failure without a live socket. On
* the test path we do NOT touch the real endpoint fd. */
int hrc = ha_reconnect_connect_hook(c, peer_idx);
/* The hook stands in for the real connect+iomgr_monitor: it NEVER
* registers the synthetic endpoint with iomgr, so the prepped ep/sctx
* are unowned on BOTH success and failure here — reclaim them in either
* case (test-only path; production reclaims via iomgr/err-handler).
* Reverting this to free only on failure re-introduces the 2.1
* GREEN-test endpoint leak masked by detect_leaks=0. */
ep->data = NULL;
endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (test path) */
free(sctx);
return hrc;
}
#endif
if (rc_ctx->should_log)
INFO("HA: peer[%d] (%s) not connected — initiating outbound connect "
"(attempt %d)", peer_idx, peer->hostname,
c->ha.peer_connect_fail_count[peer_idx]);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) {
if (rc_ctx->should_log)
WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
peer_idx, peer->hostname, peer->port, rc);
/* bug-1298: ep->data cleared (sctx freed separately below) then drop the
* owner ref — endpoint_unref does fini+free of the calloc'd struct. */
ep->data = NULL;
endpoint_unref(ep);
free(sctx);
return rc;
}
rc = iomgr_monitor(ep); /* lock-guarded epoll registration (Req 4.4) */
if (rc < 0) {
ep->data = NULL;
endpoint_unref(ep); /* bug-1298: fini+free (deferred if a vtask refs it) */
free(sctx);
return rc;
}
INFO("HA: connecting outbound to peer[%d] (%s:%u)",
peer_idx, peer->hostname, peer->port);
return 0;
}
/*
* reconnect_lane_fn: the slow-lane worker entry (raft-drive-async Task 2.1).
* Runs the bounded blocking connect off the delaymgr thread, then ALWAYS
* clears the per-peer in-flight flag (success or failure) so the dedup never
* wedges and the next heartbeat tick can retry an unreachable peer (Req 2.5,
* 2.4 — never self-terminates). Frees the reconnect ctx.
*/
static void reconnect_lane_fn(void *arg)
{
reconnect_ctx_t *rc_ctx = (reconnect_ctx_t *)arg;
if (!rc_ctx) return;
cluster_t *c = rc_ctx->c;
int peer_idx = rc_ctx->peer_idx;
(void)ha_connect_finish_peer(rc_ctx);
/* Clear in-flight LAST, after the endpoint is fully connected/registered
* or torn down — so a concurrent heartbeat tick cannot enqueue a second
* reconnect that races this one's endpoint setup (Req 2.5). */
atomic_store_explicit(&c->ha.reconnect_inflight[peer_idx], 0,
memory_order_release);
free(rc_ctx);
}
/*
* ha_connect_to_peer: open a new outbound TLS connection to a peer candidate
* and register it as a standby channel. Called from primary_heartbeat_fn for
* peers that are absent from standby_endpoints.
*
* Offload flag ON (default, Task 2.1): prepares the endpoint (non-blocking)
* and submits the bounded connect to the slow lane with a per-peer dedup key,
* then returns immediately — the delaymgr thread NEVER blocks on connect.
*
* Offload flag OFF (legacy / TDD RED only): runs the blocking connect inline
* on the calling (delaymgr) thread. Retained solely so the RED test can show
* the pre-change stall; production runs with the flag ON.
*/
static void ha_connect_to_peer(cluster_t *c, int peer_idx)
{
ha_state_t *ha = &c->ha;
if (!ha_reconnect_offload_enabled()) {
/* ── Legacy inline path (flag OFF) — blocks the caller. ── */
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
if (!rc_ctx) return;
(void)ha_connect_finish_peer(rc_ctx);
free(rc_ctx);
return;
}
/* ── Offload path (flag ON) — non-blocking on the delaymgr thread. ──
*
* Per-peer single-flight: CAS the in-flight flag 0->1. If it was already
* 1, a reconnect to this peer is already in flight on the lane — enqueue
* nothing (Req 2.5). This guard is the authoritative dedup; the lane's
* own per-key dedup is a redundant backstop. */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&ha->reconnect_inflight[peer_idx], &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return; /* already reconnecting this peer */
}
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
if (!rc_ctx) {
/* prep failed (alloc) — release the in-flight guard so the next tick
* retries; controller stays alive (Req 2.4). */
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* Submit the blocking connect to the slow lane. Dedup key is the
* per-peer absolute index (+1 so it is non-zero; key 0 disables dedup). */
uint64_t key = (uint64_t)(peer_idx + 1);
int srv = slowlane_submit(&c->slowlane, key, reconnect_lane_fn, rc_ctx);
if (srv == -EEXIST) {
/* Lane still holds this peer's dedup key from a PRIOR reconnect task.
* This is the narrow window where that task already cleared our
* reconnect_inflight flag (its last action) but the lane has not yet
* removed its key. Our CAS therefore succeeded (flag was 0) even
* though the lane refuses the submit. We MUST release the flag we just
* set — otherwise no task owns its clear and dedup wedges permanently.
* Tear down our duplicate prep; the next tick retries cleanly. */
rc_ctx->ep->data = NULL;
endpoint_unref(rc_ctx->ep); /* bug-1298: fini+free the calloc'd struct */
free(rc_ctx->sctx);
free(rc_ctx);
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure: the task is still queued and will run, so we keep the flag
* set and let reconnect_lane_fn clear it. Only on a true failure do
* we tear down and release the guard so the next tick retries. */
WARN("HA: slowlane_submit reconnect peer[%d] failed: %d — retry next tick",
peer_idx, srv);
rc_ctx->ep->data = NULL;
endpoint_unref(rc_ctx->ep); /* bug-1298: fini+free the calloc'd struct */
free(rc_ctx->sctx);
free(rc_ctx);
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: the task is enqueued and will run exactly once on a lane
* worker; reconnect_lane_fn clears the in-flight flag on completion. */
}
/*
* primary_heartbeat_fn: self-perpetuating one-shot delaymgr task.
*
* On each tick:
* 1. Sends OPCode_nop to all connected standbys (keeps watchdog alive).
* 2. Scans the peers list; for any peer NOT in standby_endpoints, opens a
* fresh outbound TLS connection so late joiners are bootstrapped without
* the standby ever initiating an outbound connect.
*
* Stops naturally on demotion: the role check prevents re-arming.
*/
static int primary_heartbeat_fn(void *arg)
{
cluster_t *c = (cluster_t *)arg;
ha_state_t *ha = &c->ha;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role != CTRL_PRIMARY)
return 0; /* demoted — stop sending NOPs */
/* 1. Heartbeat all connected standbys — idle-triggered NOP (Req 3.1-3.4).
*
* For each standby ep, compute elapsed time since the last outbound push.
* Send NOP only when elapsed >= heartbeat_interval_ms so that WAL entries,
* config pushes, and ctrlhello suppress redundant NOPs (Req 3.4).
*/
{
struct timespec _hb_ts;
clock_gettime(CLOCK_MONOTONIC, &_hb_ts);
time_t now = _hb_ts.tv_sec;
int interval_s = (int)(ha->config.heartbeat_interval_ms / 1000);
if (interval_s <= 0)
interval_s = HA_DEFAULT_HEARTBEAT_INTERVAL_MS / 1000;
if (ha->standby_endpoints) {
/* Liveness eviction threshold (wall clock, matches ep->ts_last_in). */
time_t now_wall = time(NULL);
uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000;
if (timeout_s == 0)
timeout_s = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS / 1000;
/* Stamp the primary's OWN current committed position + term onto
* every heartbeat this tick (Req 1.1, 3.4). Read atomically here,
* once per tick, so every standby sees a consistent snapshot. Use
* the primary's authoritative state — NOT the standby-side
* last_known_* fields. */
uint64_t hb_term =
atomic_load_explicit(&ha->term, memory_order_acquire);
uint64_t hb_committed_lsn =
atomic_load_explicit(&c->persist.committed_lsn,
memory_order_acquire);
uint64_t hb_committed_config_tag =
atomic_load_explicit(&ha->committed_config_tag,
memory_order_acquire);
/* Endpoints evicted this tick; their repl deregistration runs AFTER
* ha->lock is released (repl lock-ordering — mirrors the err handlers). */
endpoint_t *evicted[MAX_HA_CANDIDATES];
int n_evicted = 0;
pthread_mutex_lock(&ha->lock);
int n = ha->standby_endpoints->count;
for (int i = 0; i < n; i++) {
endpoint_t *ep = (endpoint_t *)zarray_get(ha->standby_endpoints, i);
if (!ep) continue;
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data;
if (!sctx) continue;
/* Liveness eviction (symmetric with the standby's liveness poll):
* a standby that was responsive at least once (ts_last_in > 0) but
* has sent nothing for longer than heartbeat_timeout is dead or was
* restarted. A wedged/dead endpoint left in the array makes the
* reconnect scan below treat the peer as still-connected, so the
* restarted node never receives a fresh ctrlhello and loops
* elections forever (terms climbing unbounded). Evict it here; the
* reconnect scan re-establishes a fresh channel on this/next tick.
* Removal from standby_endpoints is the load-bearing action — the
* endpoint object itself is reclaimed by its err handler when the
* dead connection finally surfaces an epoll event (bounded). */
if (ep->ts_last_in > 0 &&
(now_wall - ep->ts_last_in) > (time_t)timeout_s) {
WARN("HA: standby[%d] silent for %lds (timeout=%us) — evicting "
"dead endpoint, will reconnect",
i, (long)(now_wall - ep->ts_last_in), timeout_s);
zarray_del(ha->standby_endpoints, i);
if (n_evicted < MAX_HA_CANDIDATES)
evicted[n_evicted++] = ep;
i--; n--;
continue;
}
time_t elapsed = now - sctx->last_push_time;
if (elapsed >= (time_t)interval_s) {
/* Heartbeat carries the primary's own committed + term
* (Req 1.1, 3.4). */
#ifdef XCE_TESTING
int rc;
if (ha_heartbeat_req_hook)
rc = ha_heartbeat_req_hook(ep, hb_term,
hb_committed_lsn,
hb_committed_config_tag);
else if (ha_nop_req_hook)
rc = ha_nop_req_hook(ep);
else
rc = xce_ha_heartbeat_req(ep, hb_term,
hb_committed_lsn,
hb_committed_config_tag);
#else
int rc = xce_ha_heartbeat_req(ep, hb_term,
hb_committed_lsn,
hb_committed_config_tag);
#endif
if (rc < 0)
WARN("HA: heartbeat nop to standby[%d] failed: %d", i, rc);
else
sctx->last_push_time = now;
}
}
pthread_mutex_unlock(&ha->lock);
for (int k = 0; k < n_evicted; k++)
repl_remove_standby(&c->persist.repl, evicted[k]);
}
}
/* 2. Reconnect to any peer missing from standby_endpoints */
if (c->myconf) {
const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg;
for (int i = 0; i < pcfg->candidates_count; i++) {
if (i == ha->my_priority) continue; /* skip self */
/* Check if this peer already has a standby endpoint.
*
* Two endpoint types can be in standby_endpoints:
* - Promoted vote eps: ep->data = vote_ep_ctx_t, peer_idx is
* votes_idx (relative, self excluded). Convert absolute i to
* votes_idx using the same formula as raft_cb_send_vote_request.
* - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL),
* peer_idx is absolute candidate index. Compare directly with i.
*/
int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1;
int found = 0;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int j = 0; j < ha->standby_endpoints->count; j++) {
endpoint_t *sep = (endpoint_t *)zarray_get(
ha->standby_endpoints, j);
if (!sep || !sep->data) continue;
standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)sep->data;
int match_idx = (sctx->ctx != NULL)
? votes_idx_for_i /* promoted vote ep: relative */
: i; /* standby ep: absolute */
if (sctx->peer_idx == match_idx) { found = 1; break; }
}
pthread_mutex_unlock(&ha->lock);
}
if (!found)
ha_connect_to_peer(c, i);
}
}
/* runcount=-1 (infinite repeat): the delaymgr re-enqueues this task
* automatically. The role check at entry stops work on demotion. */
return 0;
}
void ha_start_primary_heartbeat(cluster_t *c)
{
if (!c) return;
ha_state_t *ha = &c->ha;
int interval_ms = (int)ha->config.heartbeat_interval_ms;
if (interval_ms <= 0)
interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS;
/* Idempotency: only arm once per process lifetime. With runcount=-1 the
* delaymgr re-enqueues the task automatically; duplicate tasks would stack
* on every re-election. */
{
static _Atomic int s_armed = 0;
int expected = 0;
if (!atomic_compare_exchange_strong(&s_armed, &expected, 1))
return; /* already armed — task is still running */
}
/* Guard: skip if delaymgr is not running (e.g. unit tests with zeroed cluster_t).
* In production the delaymgr is always started before any promotion. */
if (!c->delaymgr.running) {
INFO("HA: heartbeat skipped (delaymgr not running)");
return;
}
/* 2026-07-10 (chaotic-soak finding): fire the FIRST heartbeat synchronously,
* right now, instead of only arming the periodic delay_task_ms below.
* delay_task_ms's first invocation fires only AFTER interval_ms elapses
* (it is a periodic timer, not "run now then repeat") -- so despite this
* function's own long-standing comment/doc ("start primary heartbeat
* immediately so standbys receive NOP"), a freshly-promoted leader was
* actually waiting a full heartbeat_interval_ms (default 3000ms) before a
* reconnected/catching-up standby ever got a heartbeat to reply to. That
* reply (stateReport) is what lets the primary learn the standby's true
* applied_lsn (ha_process_standby_state) and start catch-up streaming --
* which gates the per-entry majority-ack that advances committed_lsn past
* this election's conservative safe_committed_lsn floor. A live chaotic
* soak showed jobs above safe_lsn (legitimately majority-committed before
* the old primary died, per Leader Completeness) invisible via
* expect-job/joblist for the full 3s wait even after the
* persist_promotion_advance_committed fix above -- because nothing
* corroborated them was ever ASKED for that long. primary_heartbeat_fn
* only performs non-blocking async sends (xce_ha_heartbeat_req), so
* calling it inline here carries the same non-blocking-I/O contract as
* the rest of promotion and needs no vtask/thread-role change. */
primary_heartbeat_fn(c);
/* runcount=-1 (infinite): delaymgr re-enqueues automatically.
* The role check in primary_heartbeat_fn no-ops on demotion. */
int rc = delay_task_ms(&c->delaymgr, interval_ms, -1,
(delay_task_entry_fn_t)primary_heartbeat_fn, c);
if (rc < 0)
WARN("ha_start_primary_heartbeat: delay_task_ms failed: %d", rc);
else
INFO("HA: primary heartbeat armed (interval=%dms, rc=%d)", interval_ms, rc);
}
View git blame Copy permalink