#define _GNU_SOURCE #include "ha.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Circular-include safety: cluster_def.h includes ha.h (include-guarded), * and ha.h only forward-declares cluster_t — no cycle. */ #include "xce/cluster_def.h" #include "xce/errnox.h" #include "controller.capnp.h" #include "persist.h" #include "request.h" #include "utils.h" /* ── HA dispatch gate error codes (defined here; declared extern in errnox.h) ── */ int XCE_ERR_NOT_READY = E_XCE_NOT_READY; int XCE_ERR_STANDBY = E_XCE_STANDBY; /* No-op stubs removed: s_real_cbs (below) is used exclusively since task 6.2. */ /* ── Forward declarations needed by real raft callbacks (Task 6.2) ──────── * These static functions and types are defined later in this file; declared * here so the callback implementations can reference them. */ /* vote_ep_ctx_t: per-peer context for outgoing vote endpoints (full def below) */ typedef struct { ha_state_t *ha; cluster_t *c; ha_elect_ctx_t *ctx; int peer_idx; uint64_t my_term; uint64_t my_tag; uint64_t my_lsn; /* ha-term-safety task 1.2: candidate's own Raft (lastLogTerm, lastLogIndex) * advertised in its RequestVote/PreVote so a voter's library logOk gate can * compare. my_last_log_index == candidate WAL log_len; my_last_log_term == * term of that last entry (recovered/live WAL term). */ uint64_t my_last_log_term; uint64_t my_last_log_index; const char *my_name; } vote_ep_ctx_t; /* election_timeout_ctx_t: delaymgr callback context for raft election timeout */ typedef struct { _Atomic int cancelled; ha_state_t *ha; cluster_t *c; ha_elect_ctx_t *ctx; } election_timeout_ctx_t; /* cluster_init_phase2 / push_config_transport_only / * ha_send_ctrlhello_to_all_standbys: defined in cluster.c / config_repl.c; * only referenced in production builds. * (XCE_TESTING uses hook shims and does not link cluster.c.) * ha.c uses push_config_transport_only ONLY (task 5.2 remediation r1): * every ha.c push site is reachable from peer_ctrl_msg_handler / io_worker * (vote replies, heartbeat replies), where the push_config_to_standbys * barrier wait would block the single iomgr epoll thread — and the config * ACK it waits for would arrive on that same blocked thread (structural * deadlock). Convergence is owned by the OP_replConfig anti-entropy pull * and the heartbeat decision loop, not by these best-effort pushes. */ #ifndef XCE_TESTING extern int cluster_init_phase2(cluster_t *c); extern void ha_send_ctrlhello_to_all_standbys(cluster_t *c); extern int push_config_transport_only(cluster_t *c, const uint8_t *bytes, size_t blen, const uint8_t sha256[32], uint64_t new_tag, int *n_out); #endif /* raft-drive-async Task 3.3: requester-less config-push offload router (defined * below, near the snapshot helpers). Forward-declared because * raft_cb_become_leader (earlier) is one of its two call sites. */ static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes, size_t blen, const uint8_t sha256[32], uint64_t new_tag); /* config_repl.c is linked in BOTH production and XCE_TESTING builds. * ha-term-safety task 4.3 (remediation r1): the commit-advance hook activates a * newly-committed config marker (Finding B); ha_dispatch_gate / the pre-promotion * gate read the serve-fence (Finding C). */ extern int config_activate_committed(cluster_t *c); extern int config_serve_fenced(const cluster_t *c); /* task 4.3 (remediation r2, Finding D): the commit-advance path schedules an * off-iomgr retry that re-activates a newly-committed config marker and clears a * stale serve-fence (non-blocking ordq_put only). */ extern int config_schedule_activate_retry(cluster_t *c); /* ── config-upgrade-snapshot task 6.1: Activation Fence ─────────────────── * upgrade_materialization_verify (below) is a SEPARATE fence from the * config-byte serve-fence above — see ha_state_t's upgrade_materialization_ * fence field doc comment (ha.h) for the full "why a second fence" rationale. * It reads the SAME committed-marker authority config_activate_committed * itself reads. */ extern int config_committed_marker(const cluster_t *c, uint64_t *version_tag, uint8_t sha256_out[32]); /* upgrade_materialization_read_live_config_sha256 (config-upgrade-snapshot * task 6.1, Req 3.2/3.4): resolve and hash THIS MEMBER'S CURRENTLY-ACTIVE * config bytes fresh, right now — never a cached/frozen copy (in particular, * never c->persist.cfgref_sha, which is only a snapshot-time REFERENCE * captured at whatever moment the last checkpoint ran, not proof the bytes * are still current this instant). * * DELIBERATE DESIGN CHOICE: this calls the same two already-PUBLIC * config_repl.c entry points (conf_read_current_filename, conf_read_raw) * that config_activate_committed's own local_config_sha256 helper uses * internally, rather than re-deriving the //conf/current -> * cluster.bin. resolution a third time (persist.c's * read_active_config_bytes already duplicates that resolution inline once, * with its own documented rationale for why persist.c specifically avoids a * persist.c -> config_repl.c call edge). ha.c carries no such constraint — * it already depends on config_repl.c directly for config_activate_ * committed/config_serve_fenced/config_schedule_activate_retry/conf_read_raw * above, so calling these two existing, already-public accessors here is * in-boundary, ordinary integration along an established edge, not a new * layering violation or a third duplicate implementation of path validation. * * Returns 0 with out_sha populated on success; -ENOENT when no local config * bytes exist yet (benign — e.g. c->myconf unset, or no conf/current pointer * written yet); another negative errno surfaces a genuine read/validation * failure from the underlying calls. */ static int upgrade_materialization_read_live_config_sha256(cluster_t *c, uint8_t out_sha[32]) { extern int conf_read_current_filename(const char *workdir, char *out, size_t outsz); extern int conf_read_raw(const char *conffile, uint8_t **bytes, size_t *len); myconf_t *mc = c->myconf; if (!mc || !mc->workdir || !mc->name) return -ENOENT; /* effective_dir = workdir/name — must match conf_save_config's own path * construction (config_repl.c) exactly, mirroring local_config_sha256. */ char conf_dir[4096]; snprintf(conf_dir, sizeof(conf_dir), "%s/%s", mc->workdir, mc->name); char fname[256] = {0}; int rc = conf_read_current_filename(conf_dir, fname, sizeof(fname)); if (rc != 0) return -ENOENT; /* no local bytes yet — caller fences + retries */ char path[4352]; snprintf(path, sizeof(path), "%s/conf/%s", conf_dir, fname); uint8_t *bytes = NULL; size_t blen = 0; rc = conf_read_raw(path, &bytes, &blen); if (rc != 0) return -ENOENT; SHA256(bytes, blen, out_sha); free(bytes); return 0; } /* upgrade_materialization_verify (config-upgrade-snapshot task 6.1, Req 1.8, * 3.2, 3.4): the Activation Fence's own verify function — mirrors * config_activate_committed's exact shape (config_repl.c:304-377) for a * DIFFERENT condition. config_serve_fence (config_repl.c) asks "do the * locally-held config BYTES hash-match the committed marker?"; THIS fence * asks "has THIS MEMBER'S OWN materialization (the upgrade-snapshot * checkpoint, config-upgrade-snapshot task 5) actually completed for the * CURRENT committed marker, with matching config bytes?" — a distinct, * independently-trackable condition: bytes can already hash-match * (config_serve_fence clear) while this member's own checkpoint for that * exact marker has not run yet, e.g. immediately after a marker commits but * before apply_committed_entry's WAL_CONFIG_MARKER case has actually fired * and written a checkpoint. * * "Materialization complete" is determined via c->persist.cfgref_present / * cfgref_tag / cfgref_sha — persist.c's scan_latest_committed_config_ref, * invoked from EVERY checkpoint (persist_checkpoint_write_locked), stamps * these fields with the latest committed WAL_CONFIG_MARKER as of that * checkpoint. cfgref_present && cfgref_tag == marker_tag && cfgref_sha == * marker_sha therefore means "some checkpoint on THIS member has already * captured a reference to this exact committed marker" — materialization for * it has happened. This is the SAME field config_committed_marker's own * compaction fallback already reads (config_repl.c), so this is a read of an * existing, already-correct signal, not a new one. * * That alone is not sufficient: the checkpoint's cfgref_sha is a REFERENCE * captured AT CHECKPOINT TIME, not proof the bytes are STILL current right * now (the WAL marker's raft-commit and the R7 config-push byte delivery are * two independent channels with no strict ordering guarantee). So this * function ALSO re-reads the live on-disk config bytes fresh on every call * (never c->persist.cfgref_sha, never any other cached/embedded copy) via * upgrade_materialization_read_live_config_sha256 (above), and hash-verifies * that fresh read against the committed marker too. Both must hold for the * fence to clear. * * Never exits/aborts on this condition (controller/CLAUDE.md's never-self- * terminate invariant) — always fail loud + fence + stay alive + retriable, * mirroring config_activate_committed exactly: logs loud only on the 0->1 * fence transition (never spams on repeated failed retries), returns * -EAGAIN on failure so the caller retries, clears + returns 0 on success. */ int upgrade_materialization_verify(cluster_t *c) { if (!c) return -EINVAL; /* 1. The committed marker is the authority for the target upgrade. An * absent marker (no configuration change has ever committed on this * cluster, or the marker has not committed yet) is NOT a divergence — * clear any stale fence and succeed, mirroring config_activate_ * committed's own -ENOENT handling exactly. */ uint64_t marker_tag = 0; uint8_t marker_sha[32]; int rc = config_committed_marker(c, &marker_tag, marker_sha); if (rc == -ENOENT) { atomic_store_explicit(&c->ha.upgrade_materialization_fence, 0, memory_order_release); return 0; } if (rc != 0) return rc; /* -EINVAL / -EIO surfaced to the caller */ /* 2. Materialization-complete signal: has ANY checkpoint on THIS member * already captured a reference to this EXACT committed marker? */ bool materialized = c->persist.cfgref_present && c->persist.cfgref_tag == marker_tag && memcmp(c->persist.cfgref_sha, marker_sha, 32) == 0; /* 3. Hash-verify: re-read the LIVE, currently-active config bytes fresh * on every attempt and compare against the committed marker. */ uint8_t live_sha[32]; int live_rc = upgrade_materialization_read_live_config_sha256(c, live_sha); bool bytes_verified = (live_rc == 0) && (memcmp(live_sha, marker_sha, 32) == 0); if (!materialized || !bytes_verified) { int was = atomic_exchange_explicit(&c->ha.upgrade_materialization_fence, 1, memory_order_acq_rel); if (!was) ERROR("upgrade materialization fence: committed config marker " "tag=%016" PRIx64 " -- materialization %s " "(cfgref_present=%d cfgref_tag=%016" PRIx64 "), live config " "bytes %s (rc=%d) -- member's OWN upgrade-snapshot " "materialization for this marker is INCOMPLETE or " "MISMATCHED; member NON-SERVING and NON-PROMOTABLE under the " "new configuration, staying alive and retrying " "(config-upgrade-snapshot R1.8/3.2/3.4)", marker_tag, materialized ? "captured" : "NOT YET captured", c->persist.cfgref_present, c->persist.cfgref_tag, bytes_verified ? "hash-match" : "do NOT hash-match", live_rc); return -EAGAIN; } atomic_store_explicit(&c->ha.upgrade_materialization_fence, 0, memory_order_release); INFO("upgrade materialization VERIFIED for committed config marker " "tag=%016" PRIx64 " -- materialization complete, live config bytes " "hash-verified", marker_tag); return 0; } /* upgrade_materialization_fenced: trivial atomic-read accessor — mirrors * config_serve_fenced (config_repl.c:262-268) exactly, for the new * independent fence. Pure atomic read (no I/O), safe on the election/ * dispatch path. */ int upgrade_materialization_fenced(const cluster_t *c) { if (!c) return 0; return atomic_load_explicit(&c->ha.upgrade_materialization_fence, memory_order_acquire) ? 1 : 0; } /* ── config-upgrade-snapshot task 6.2: Activation Fence self-triggering retry ── * Mirrors config_schedule_activate_retry's exact four-function shape * (config_repl.c:409-507) for the upgrade_materialization_fence instead of * config_serve_fence: a one-shot NON-BLOCKING delaymgr task offloads the * actual blocking re-verify work (upgrade_materialization_verify's disk * reads) onto a VTASK_LIVE_BLOCK vtask, so the delaymgr thread itself never * blocks (ha-async-io.md thread-role contract). */ /* upgrade_materialization_retry_fn: re-runs upgrade_materialization_verify so * a member whose materialization was incomplete or hash-mismatched re-checks * and clears the fence once conditions allow. Does the actual blocking disk * I/O; callers must invoke it off the delaymgr thread (see * upgrade_materialization_retry_vtask_entry / * schedule_upgrade_materialization_retry below). Public so tests can drive it * directly without a vtask pool — mirrors config_activate_retry_fn. */ int upgrade_materialization_retry_fn(void *arg) { cluster_t *c = (cluster_t *)arg; if (!c) return -EINVAL; if (!upgrade_materialization_fenced(c)) return 0; /* already cleared by another path — nothing to retry */ int rc = upgrade_materialization_verify(c); if (rc == 0) INFO("upgrade materialization retry: fence CLEARED — this member's " "own materialization for the committed config marker now " "verified"); else if (rc == -EAGAIN) TRACE("upgrade materialization retry: still fenced (materialization " "incomplete or hash-mismatched) — will retry on the next " "commit advance"); else WARN("upgrade materialization retry: upgrade_materialization_verify " "rc=%d", rc); return rc; } /* upgrade_materialization_retry_vtask_entry: vtask entry wrapping * upgrade_materialization_retry_fn so its blocking disk I/O runs on a vtask * worker, never inline on the single delaymgr thread. Mirrors * config_activate_retry_vtask_entry. */ static int upgrade_materialization_retry_vtask_entry(vtask_t *task) { cluster_t *c = (cluster_t *)task->data->u.generic; upgrade_materialization_retry_fn(c); return 0; } /* upgrade_materialization_retry_delaymgr_trampoline: the delaymgr task body. * Delaymgr must only trigger + enqueue, never block — dispatch the actual * retry work to a vtask (VTASK_LIVE_BLOCK, the same class * config_activate_retry_delaymgr_trampoline uses) and return immediately. * Fails loud and runs inline (stay alive, never silently drop a scheduled * retry) if the pool is unavailable. Mirrors * config_activate_retry_delaymgr_trampoline. */ static int upgrade_materialization_retry_delaymgr_trampoline(void *arg) { cluster_t *c = (cluster_t *)arg; if (!c) return -EINVAL; vtaskmgr_t *pool = &c->vtaskmgr; vtask_data_t *data = calloc(1, sizeof(*data)); vtask_t *task = vtask_alloc(); if (data == NULL || task == NULL) { WARN("upgrade materialization retry: vtask alloc failed — running " "inline (stay alive)"); free(data); free(task); return upgrade_materialization_retry_fn(c); } data->vtype = VTASK_GENERIC; data->u.generic = c; vtask_init(task, (vtask_func_t)upgrade_materialization_retry_vtask_entry, data); if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { WARN("upgrade materialization retry: vtask enqueue failed — running " "inline (stay alive)"); free(data); free(task); return upgrade_materialization_retry_fn(c); } return 0; } /* schedule_upgrade_materialization_retry: enqueue the retry trampoline on the * delaymgr (NON-BLOCKING — delay_task_ms only does ordq_put; the trampoline * itself offloads the blocking work to a vtask, see above). Called from the * commit-advance path (iomgr ACK thread) so the retry's disk I/O runs OFF * that thread (ha-async-io invariant). No-op when the member is NOT * upgrade-materialization fenced (steady state schedules nothing) or when * the delaymgr is not running (the unit-test fixture path, which never inits * delaymgr — avoids leaking a task into an un-drained queue). Mirrors * config_schedule_activate_retry exactly. Returns 0 on enqueue/no-op, * negative on enqueue failure. */ int schedule_upgrade_materialization_retry(cluster_t *c) { if (!c) return -EINVAL; if (!upgrade_materialization_fenced(c)) return 0; /* steady state: nothing to retry, schedule nothing */ if (!c->delaymgr.running) return 0; /* delaymgr not running (test fixture) — caller drives directly */ /* One-shot, minimal delay: run promptly but off the iomgr ACK thread. */ int rc = delay_task_ms(&c->delaymgr, 1, 1, (delay_task_entry_fn_t)upgrade_materialization_retry_delaymgr_trampoline, c); if (rc < 0) WARN("upgrade materialization retry: delay_task_ms failed: %d — " "fence will clear on a later commit advance", rc); return rc; } /* ha_phase2_retry_fn / ha_phase2_retry_vtask_entry / * ha_phase2_retry_delaymgr_trampoline / ha_schedule_phase2_retry (bug-1477): * see the doc comments in ha.h. Structurally identical to * upgrade_materialization_retry_fn's own 3-layer shape (blocking retry_fn -> * vtask entry -> delaymgr trampoline) just above, except the trampoline * re-arms itself on failure since nothing else recurring would otherwise * ever retry cluster_init_phase2 for a node stuck on myapp_run's passive * skip_election path. */ int ha_phase2_retry_fn(void *arg) { cluster_t *c = (cluster_t *)arg; if (!c) return -EINVAL; if (c->ha.phase2_done) return 0; /* already done by another path -- nothing to retry */ int rc; #ifdef XCE_TESTING rc = (ha_phase2_hook != NULL) ? ha_phase2_hook(c) : 0; #else rc = cluster_init_phase2(c); #endif if (rc == 0) { c->ha.phase2_done = 1; INFO("HA: cluster_init_phase2 retry succeeded — commit engine now " "wired (bug-1477)"); } else { WARN("HA: cluster_init_phase2 retry rc=%d — member stays alive and " "will retry again (bug-1477, never self-terminate)", rc); ha_schedule_phase2_retry(c); } return rc; } static int ha_phase2_retry_vtask_entry(vtask_t *task) { cluster_t *c = (cluster_t *)task->data->u.generic; ha_phase2_retry_fn(c); return 0; } static int ha_phase2_retry_delaymgr_trampoline(void *arg) { cluster_t *c = (cluster_t *)arg; if (!c) return -EINVAL; vtaskmgr_t *pool = &c->vtaskmgr; vtask_data_t *data = calloc(1, sizeof(*data)); vtask_t *task = vtask_alloc(); if (data == NULL || task == NULL) { WARN("HA: cluster_init_phase2 retry: vtask alloc failed — running " "inline (stay alive)"); free(data); free(task); return ha_phase2_retry_fn(c); } data->vtype = VTASK_GENERIC; data->u.generic = c; vtask_init(task, (vtask_func_t)ha_phase2_retry_vtask_entry, data); if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { WARN("HA: cluster_init_phase2 retry: vtask enqueue failed — running " "inline (stay alive)"); free(data); free(task); return ha_phase2_retry_fn(c); } return 0; } int ha_schedule_phase2_retry(cluster_t *c) { if (!c) return -EINVAL; if (c->ha.phase2_done) return 0; /* steady state: nothing to retry, schedule nothing */ if (!c->delaymgr.running) return 0; /* delaymgr not running (test fixture) — caller drives directly */ uint32_t interval_ms = c->ha.config.heartbeat_interval_ms; if (interval_ms == 0) interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS; int rc = delay_task_ms(&c->delaymgr, (int)interval_ms, 1, (delay_task_entry_fn_t)ha_phase2_retry_delaymgr_trampoline, c); if (rc < 0) WARN("HA: cluster_init_phase2 retry: delay_task_ms failed: %d — " "will retry on the next natural re-trigger", rc); return rc; } static void on_vote_connected(endpoint_t *ep); static void on_pre_vote_connected(endpoint_t *ep); static int vote_ep_err_handler(void *ep_raw); /* raft-adapter-conformance task 3.1: PreVote's own on_err handler, registered * at raft_cb_send_pre_vote_request's endpoint-setup call site (mirrors the * real-vote vote_ep_err_handler forward-declared above). */ static int pre_vote_ep_err_handler(void *ep_raw); /* bug-1423: forward-declared so raft_cb_send_vote_request's connect/monitor * failure branches can route a synthetic "no vote" reply through the SAME * production path a real reply uses (see vote_reply_dispatch's own comment, * below, for why this must never be the legacy handle_request_vote_reply). */ static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx, uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn); /* bug-1423 (PreVote twin): same rationale as vote_reply_dispatch above, for * raft_cb_send_pre_vote_request's connect/monitor failure branches. */ static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c, int peer_idx, int granted); /* bug-1197: defer a vote-ep teardown onto the per-ep FIFO of the vtask pool so * it runs strictly after the last live use of the ep (flag-ON loser path). */ static void vote_ep_teardown_async(ha_state_t *ha, endpoint_t *ep); /* 2026-07-10 controller-invariant audit: forward-declared so the catch-up * ingest path (ha_apply_wal_catchup_entry, earlier in this file) can kick the * standby apply pump against the already-known commit position without * waiting for the next heartbeat. Defined near ha_observe_heartbeat_committed. */ static void ha_standby_advance_committed_and_apply(cluster_t *c, uint64_t learned_committed_lsn); /* ha-single-leader-role task 2.2: shared promotion seam — every road to * PRIMARY registers the persist hooks and arms the own-term gate. */ static void ha_register_persist_hooks(cluster_t *c); static void ha_arm_own_term_gate(cluster_t *c); int ha_heartbeat_is_fresh(ha_state_t *ha); /* exposed for PreVote denial-reason logging (cluster.c) */ static int primary_heartbeat_fn(void *arg); static void on_standby_connected(endpoint_t *ep); static int standby_ep_err_handler_primary(void *ep_raw); /* ── ha-elect-ctx-lifecycle task 2.1: election carrier reference-counted * lifetime ────────────────────────────────────────────────────────────── * * ha_elect_ctx_release: decrement ctx->refcount; on reaching zero, destroy * the carrier's mutex/semaphore and free it. This is the ONLY place * ha_elect_ctx_t is destroyed/freed — every release path (vote-endpoint * teardown completion via vote_ep_on_fini, and round supersession in * ha_start_election_async / ha_start_prevote_async) routes through here, so * the carrier is freed exactly once, from whichever release is last. */ static void ha_elect_ctx_release(ha_elect_ctx_t *ctx) { if (ctx == NULL) return; if (atomic_fetch_sub_explicit(&ctx->refcount, 1, memory_order_acq_rel) != 1) return; /* not the last reference */ pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); } #ifdef XCE_TESTING /* ha_elect_ctx_release_for_test: drive ha_elect_ctx_release directly so unit * tests can verify the release/free mechanism without staging a full * election round. */ void ha_elect_ctx_release_for_test(ha_elect_ctx_t *ctx) { ha_elect_ctx_release(ctx); } #endif /* * vote_ep_on_fini: endpoint_t.on_fini completion hook wired onto every vote * endpoint at capture time (raft_cb_send_vote_request / * raft_cb_send_pre_vote_request). Fires from inside endpoint_fini() * immediately before ep->data (the vote_ep_ctx_t) is freed — releases the * carrier reference this endpoint held. */ static void vote_ep_on_fini(endpoint_t *ep) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; if (vctx == NULL) return; ha_elect_ctx_release(vctx->ctx); } /* ── Real raft callback implementations (Task 6.2) ───────────── * * These are registered as s_real_cbs in ha_init, replacing the no-op stubs. * They are wired into the active election path in task 6.3. * * All callbacks receive ctx = ha_state_t*. * ha->raft_c is set by ha_start_election_async before any raft_* calls. * When ha->raft_c is NULL (unit test mode), callbacks that require cluster_t * are no-ops. */ /* * raft_cb_send_vote_request: encode and send a capnp VoteRequest to peer. * * peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[]. * Uses the existing iomgr vote-send path (on_vote_connected / vote_ep_ctx_t). * Reads the current election context from ha->elect_ctx (set before raft_start_election). * Returns 0 on success; non-zero if no cluster context or alloc failure. */ static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c || !c->myconf) return 1; /* test mode */ const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; if (peer_idx < 0 || peer_idx >= pcfg->candidates_count) return 1; ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (!ectx) return 1; /* no election in progress */ xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx]; /* Compute the local index of peer_idx within ectx->votes[]: * votes[] has n_peers - 1 = n_other entries for peers excluding self. * Map: votes_idx = peer_idx if peer_idx < self_idx, else peer_idx - 1. */ int self_idx = ha->my_priority; /* self_idx == my_priority by convention */ int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1; if (votes_idx < 0 || votes_idx >= ectx->n_peers) return 1; /* Allocate per-peer context */ vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx)); if (!vctx) { handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); return 1; } vctx->ha = ha; vctx->c = c; vctx->ctx = ectx; vctx->peer_idx = votes_idx; vctx->my_term = term; /* tag and lsn come from votes[0] which holds our own values */ vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0; vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0; /* ha-term-safety task 1.2: advertise our Raft (lastLogTerm, lastLogIndex). * lastLogIndex == our WAL log length (the library's log_len == WAL tip LSN); * lastLogTerm == the term STAMPED ON THAT LAST ENTRY. * * bug-851 / task 5.5: lastLogTerm MUST be the term of the entry at log_len — * sourced via persist_entry_term, the SAME accessor voters use to compute * their own last-log term (on_get_entry_term → persist_entry_term, ha.c:1556). * The old stopgap advertised c->persist.term (the persist layer's RUNNING * term), which is stale/0 after a restart and does NOT equal the last entry's * stored term. That made a freshly-restarted but freshest survivor advertise * a too-low lastLogTerm, so every voter's logOk rejected it (asymmetric with * how the voter computes its own term) and the election stalled. */ vctx->my_last_log_index = ha->raft.log_len; vctx->my_last_log_term = (c && ha->raft.log_len > 0) ? persist_entry_term(&c->persist, ha->raft.log_len) : 0; vctx->my_name = c->myconf ? c->myconf->name : ""; /* Get a fresh endpoint */ endpoint_t *ep = endpoint_get(); if (!ep) { free(vctx); handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); return 1; } endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0); if (c) endpoint_set_peer(ep, c->myhost); ep->data = vctx; ep->iomgr = &c->iomgr; /* ha-elect-ctx-lifecycle task 2.2: this endpoint now references ectx via * vctx->ctx — take a reference before any path that could reach * endpoint_fini (connect failure below, and beyond). Released exactly * once by vote_ep_on_fini (wired below) when this endpoint's teardown * completes, regardless of which teardown path gets there. */ atomic_fetch_add_explicit(&ectx->refcount, 1, memory_order_relaxed); if (ha->client_ssl_ctx) { SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx; ep->ssl = SSL_new(client_ctx); if (ep->ssl) SSL_set_connect_state(ep->ssl); } endpoint_set_on_connected(ep, on_vote_connected); endpoint_set_err_handler(ep, vote_ep_err_handler); endpoint_set_on_fini(ep, vote_ep_on_fini); int rc = endpoint_connect(ep, peer->hostname, peer->port, DEFAULT_CONNECT_TIMEOUT_MS); if (rc < 0) { endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (not monitored) */ /* bug-1423: a connect failure (e.g. peer dead/unreachable) must still * register as a "no vote" reply through the SAME path a real reply * uses. Without this, raft_state_t.replies_received never reaches * n_peers-1 whenever any peer is unreachable, so the library's * "everyone's replied, decide now" fast path (raft.c "Step 6: * all-replied-no-quorum") is structurally unreachable — recovery then * depends solely on the raw election timeout, observed to leave a * candidate stuck in CTRL_ELECTING far longer than expected in a * 3-node cluster with one node down. Test mode (c==NULL) keeps the * legacy handle_request_vote_reply call that existing unit tests * depend on; production routes through vote_reply_dispatch, the same * call a genuine reply makes (ha.c vote_reply_dispatch). */ if (!c) handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); else vote_reply_dispatch(ha, ectx, votes_idx, term, /*vote_granted=*/0, 0, 0); return 1; } rc = iomgr_monitor(ep); if (rc < 0) { endpoint_unref(ep); /* bug-1298: fini frees ep->data (vctx), then frees ep */ /* bug-1423: same fix as the connect-failure branch above — this was * unconditionally calling the legacy handle_request_vote_reply even in * production, bypassing raft_on_vote_reply (the sole production * promotion driver per vote_reply_dispatch's own comment). */ if (!c) handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); else vote_reply_dispatch(ha, ectx, votes_idx, term, /*vote_granted=*/0, 0, 0); return 1; } /* Track endpoint for promotion/close after election result. * Also record the candidate pointer: raft_cb_become_leader uses * votes[i].peer->hostname / ->port to register the surviving vote * endpoint with the WAL replicator (repl_add_standby). Without this, * the promotion registration block (guarded by votes[i].peer != NULL) * is skipped — the standbys never enter c->persist.repl.standbys[], * commit/quorum see zero standbys, and every write fails NOT_READY. * `peer` points into c->myconf->peers_cfg.candidates[] (persistent; * outlives the election — repl_add_standby copies host/port). * (fix-promotion-standby-registration, Req 1.1) */ ectx->votes[votes_idx].ep = ep; ectx->votes[votes_idx].peer = peer; return 0; } /* * raft_cb_send_pre_vote_request: encode and send a capnp PreVote request to a * peer (ha-prevote task 4.1). * * Mirrors raft_cb_send_vote_request exactly — same iomgr connect path, same * vote_ep_ctx_t carrier, same per-peer endpoint tracking — but the connected * handler (on_pre_vote_connected) emits OPCode_preVoteRequest instead of * OPCode_requestVote. prospective_term is the candidate's CURRENT+1 term; it * is NEVER persisted (PreVote is a side-effect-free straw poll). * * peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[]. * Returns 0 on success; non-zero in test mode (no cluster) or on alloc failure. * * Transport-side reply routing for preVoteReply is wired separately in task 5.1; * this callback only emits the request. */ static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx, uint64_t prospective_term) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c || !c->myconf) return 1; /* test mode — caller owns transport */ const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; if (peer_idx < 0 || peer_idx >= pcfg->candidates_count) return 1; ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (!ectx) return 1; /* no round metadata carrier */ xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx]; /* Map absolute peer_idx → votes[] local index (same convention as the * real-vote send callback). */ int self_idx = ha->my_priority; int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1; if (votes_idx < 0 || votes_idx >= ectx->n_peers) return 1; vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx)); if (!vctx) return 1; vctx->ha = ha; vctx->c = c; vctx->ctx = ectx; /* Carry the ABSOLUTE peer index: the PreVote tally in raft_state_t is keyed * by absolute cluster index (raft_begin_pre_vote loops over i in * [0, n_peers) and raft_on_pre_vote_reply dedups by that same index). */ vctx->peer_idx = peer_idx; vctx->my_term = prospective_term; vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0; vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0; /* ha-term-safety task 1.2 + bug-851/task 5.5: advertise lastLogTerm as the * term of the entry at log_len (persist_entry_term — the same source voters * use), NOT the stale c->persist.term. Identical sourcing to the real-vote * send path above; see the full rationale there. */ vctx->my_last_log_index = ha->raft.log_len; vctx->my_last_log_term = (c && ha->raft.log_len > 0) ? persist_entry_term(&c->persist, ha->raft.log_len) : 0; vctx->my_name = c->myconf ? c->myconf->name : ""; endpoint_t *ep = endpoint_get(); if (!ep) { free(vctx); return 1; } endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0); if (c) endpoint_set_peer(ep, c->myhost); ep->data = vctx; ep->iomgr = &c->iomgr; /* ha-elect-ctx-lifecycle task 2.3: same mechanism as the real-vote send * path above — take a reference before any path that could reach * endpoint_fini. */ atomic_fetch_add_explicit(&ectx->refcount, 1, memory_order_relaxed); if (ha->client_ssl_ctx) { SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx; ep->ssl = SSL_new(client_ctx); if (ep->ssl) SSL_set_connect_state(ep->ssl); } endpoint_set_on_connected(ep, on_pre_vote_connected); endpoint_set_err_handler(ep, pre_vote_ep_err_handler); endpoint_set_on_fini(ep, vote_ep_on_fini); int rc = endpoint_connect(ep, peer->hostname, peer->port, DEFAULT_CONNECT_TIMEOUT_MS); if (rc < 0) { endpoint_unref(ep); /* bug-1298: fini+free */ /* bug-1423 (PreVote twin): a connect failure (dead/unreachable peer) * must still register as a "not granted" reply with the raft library's * PreVote tally, same as the real-vote send path. PreVote's own round * timeout (ha_prevote_round_timeout_fn) is an unconditional fallback * that always resolves the round regardless, so this gap is not a * permanent-deadlock class bug like the real-vote one was — but * leaving it means every round against a down peer wastes the full * round timeout instead of concluding as soon as the live peer * replies. */ if (c) pre_vote_reply_dispatch(ha, c, peer_idx, /*granted=*/0); return 1; } rc = iomgr_monitor(ep); if (rc < 0) { endpoint_unref(ep); /* bug-1298: fini+free */ /* bug-1423 (PreVote twin): same fix as the connect-failure branch above. */ if (c) pre_vote_reply_dispatch(ha, c, peer_idx, /*granted=*/0); return 1; } return 0; } /* * raft_cb_vote_eligible: freshness gate delegated from raft library. * * from_idx is the absolute candidate index of the requester. * Applies the same checks as handle_request_vote: * - role gate: PRIMARY → deny * - fresh heartbeat on STANDBY → deny * - in_catch_up → deny * - freshness: candidate config_tag/lsn must be ≥ ours * Returns 1 to grant, 0 to deny. */ static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term) { (void)from_idx; (void)term; ha_state_t *ha = (ha_state_t *)ctx; /* Role gate */ ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) return 0; /* Fresh heartbeat on STANDBY WITH a live primary connection → primary is * alive, deny. BOTH conditions are required (mirrors the handle_request_vote * gate at ~ha.c:2205). Without the primary_ctrl_fd>=0 guard a node whose * primary has DIED keeps a falsely-"fresh" status: ha_heartbeat_is_fresh * derives from last_activity, which ha_arm_heartbeat_timeout_vtask re-arms to * now on every failover retry — so the node would deny every legitimate * higher-term PreVote for minutes, preventing convergence (bug-660). * primary_ctrl_fd is -1 after the primary RDHUPs / the liveness watchdog * fires / on a fresh boot, so a dead-or-absent primary correctly does NOT * deny. */ if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 && ha_heartbeat_is_fresh(ha)) return 0; /* In catch-up WHILE a live primary is feeding us → deny. * * raft-ha-core-safety task 5.3 (bug-1272): this deny used to be * UNCONDITIONAL, which deadlocks the election under overlapping churn — after * several rapid kills the only fully-up-to-date node is killed and the two * survivors are both restarted-and-lagging (in_catch_up=1) with NO live * primary left to catch up FROM (so the flag never clears). Each holds the * full committed prefix and is a SAFE candidate, yet they mutually deny every * PreVote → permanent stall (round-N election stall, 7.2 soak). * * Gate the deny on a LIVE primary, mirroring the heartbeat-fresh deny above: * the purpose of the in_catch_up deny is to avoid disrupting a primary that is * ACTIVELY replicating to us — not to fence a survivor whose primary is gone. * Once the primary is unreachable, raft_log_ok (already applied in Step 2 of * raft_on_pre_vote_request / raft_on_vote_request, BEFORE this callback) * guarantees the candidate's (lastLogTerm,lastLogIndex) >= ours, so granting a * fresher candidate is safe (Leader Completeness preserved). */ if (atomic_load_explicit(&ha->in_catch_up, memory_order_acquire) && ha->primary_ctrl_fd >= 0 && ha_heartbeat_is_fresh(ha)) return 0; /* Freshness gate requires knowing the candidate's config_tag/lsn. * The raft library does not pass these; they are carried in the capnp * RequestVoteRequest fields. For the adapter, freshness is checked in * handle_request_vote (called from the iomgr decode path in task 6.3). * Here we return 1 (eligible) so the raft library proceeds; the full * freshness gate is enforced by the caller before invoking raft_on_vote_request. * This is safe because the caller (task 6.3 dispatcher) already applies * the freshness check from handle_request_vote before delegating here. */ return 1; } /* raft_election_timeout_fn: delaymgr callback for raft election timeout. * Fires raft_on_election_timeout after delay_ms if not cancelled. * Owns the election_timeout_ctx_t — frees it on exit. */ static int raft_election_timeout_fn(void *arg) { election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg; ha_state_t *ha = t->ha; /* Clear the stored pointer before checking cancelled so on_cancel_timeout * is a safe no-op if it races with this callback. */ ha->raft_election_timeout_ctx = NULL; if (atomic_load(&t->cancelled)) { free(t); return 0; } /* bug-1435: serialize against concurrent vote-reply/vote-request * processing on the pool (see ha.h raft_lock doc). */ pthread_mutex_lock(&ha->raft_lock); raft_on_election_timeout(&ha->raft); pthread_mutex_unlock(&ha->raft_lock); free(t); return 0; } /* * raft_cb_schedule_timeout: schedule election timeout via delaymgr. * * Creates an election_timeout_ctx_t and calls delay_task_ms. * The pointer is stored in ha->raft_election_timeout_ctx for cancellation. */ static void raft_cb_schedule_timeout(void *ctx, uint32_t delay_ms) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c) return; /* test mode */ election_timeout_ctx_t *t = calloc(1, sizeof(*t)); if (!t) return; atomic_init(&t->cancelled, 0); t->ha = ha; t->c = c; t->ctx = NULL; /* raft path does not use ha_elect_ctx_t */ /* task 5.7: randomize the per-campaign watchdog to [delay_ms, 2*delay_ms] * (Ongaro §3.4). The library passes a FIXED election_timeout_ms; arming the * watchdog at exactly that value lets a constant cold-connection grant RTT * land just past every deadline forever (resonance → unbounded term climb). * Re-randomizing here breaks the resonance without touching the pure * (sim-tested) library. */ delay_ms = ha_jittered_election_delay(ha, delay_ms); int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1, raft_election_timeout_fn, t); if (rc < 0) { WARN("raft_cb_schedule_timeout: delay_task_ms failed: %d", rc); free(t); return; } ha->raft_election_timeout_ctx = t; } /* * raft_cb_cancel_timeout: cancel the pending election timeout. * Sets cancelled=1 on the stored election_timeout_ctx_t. */ static void raft_cb_cancel_timeout(void *ctx) { ha_state_t *ha = (ha_state_t *)ctx; election_timeout_ctx_t *t = (election_timeout_ctx_t *)ha->raft_election_timeout_ctx; if (t) atomic_store(&t->cancelled, 1); ha->raft_election_timeout_ctx = NULL; } /* raft_election_start_vtask_entry: vtask entry wrapping raft_check_and_start_ * election. raft_start_election's on_persist callback (raft_cb_persist -> * ha_persist_term) does a synchronous fsync -- must never run inline on the * delaymgr thread (ha-async-io.md thread-role contract: delaymgr only fires * timers and triggers work, then returns; it never blocks, since the same * thread also drives heartbeats and every other election timeout). */ static int raft_election_start_vtask_entry(vtask_t *task) { ha_state_t *ha = (ha_state_t *)task->data->u.generic; /* bug-1435: serialize against concurrent vote-reply/vote-request * processing on the pool (see ha.h raft_lock doc). */ pthread_mutex_lock(&ha->raft_lock); raft_check_and_start_election(&ha->raft); pthread_mutex_unlock(&ha->raft_lock); return 0; } /* ha_dispatch_election_start: offload raft_check_and_start_election onto a * vtask (VTASK_LIVE_BLOCK -- the same class already used for the vote-GRANT * term fsync, R4.6) so its persist-before-campaign fsync never blocks the * delaymgr thread. On pool unavailability/enqueue failure, fail loud and run * inline anyway (stay alive, never silently drop a scheduled election -- * mirrors persist_apply_pump_kick's and vote_ep_teardown_enqueue's fallback). */ static void ha_dispatch_election_start(ha_state_t *ha) { vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool; vtask_data_t *data = calloc(1, sizeof(*data)); vtask_t *task = vtask_alloc(); if (pool == NULL || data == NULL || task == NULL) { WARN("ha_dispatch_election_start: vtask unavailable (pool=%p data=%p " "task=%p) — starting election inline (stay alive)", (void *)pool, (void *)data, (void *)task); free(data); free(task); pthread_mutex_lock(&ha->raft_lock); raft_check_and_start_election(&ha->raft); pthread_mutex_unlock(&ha->raft_lock); return; } data->vtype = VTASK_GENERIC; data->u.generic = ha; vtask_init(task, (vtask_func_t)raft_election_start_vtask_entry, data); if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { WARN("ha_dispatch_election_start: vtask enqueue failed — starting " "election inline (stay alive)"); free(data); free(task); pthread_mutex_lock(&ha->raft_lock); raft_check_and_start_election(&ha->raft); pthread_mutex_unlock(&ha->raft_lock); } } /* raft_rdhup_delay_fn: delaymgr callback for RDHUP-triggered delayed election. * Owns its ctx (election_timeout_ctx_t) — frees it on exit. * Dispatches the actual election start off the delaymgr thread (see * ha_dispatch_election_start) only if this node hasn't already voted. */ static int raft_rdhup_delay_fn(void *arg) { election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg; ha_state_t *ha = t->ha; if (!atomic_load(&t->cancelled)) ha_dispatch_election_start(ha); free(t); return 0; } /* * raft_cb_start_election_after_delay: schedule RDHUP retry timer. * After delay_ms, calls raft_check_and_start_election to start election * only if this node hasn't already voted for another candidate. */ static void raft_cb_start_election_after_delay(void *ctx, uint32_t delay_ms) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c) return; /* test mode */ /* Priority 0 → delay_ms == 0: highest-priority node starts immediately. * Dispatched off the calling thread (see ha_dispatch_election_start) since * this callback's caller is not guaranteed to be vtask-safe. */ if (delay_ms == 0) { ha_dispatch_election_start(ha); return; } election_timeout_ctx_t *t = calloc(1, sizeof(*t)); if (!t) return; atomic_init(&t->cancelled, 0); t->ha = ha; t->c = c; t->ctx = NULL; int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1, raft_rdhup_delay_fn, t); if (rc < 0) { WARN("raft_cb_start_election_after_delay: delay_task_ms failed: %d", rc); free(t); } } /* ── Task 4.1 (ha-repl-hardening): safe commit point computation ───────────── * * ha_compute_safe_committed_lsn: determine the safe committed LSN and config * tag after winning an election, using the vote replies collected in ctx. * * Algorithm (design §Flow 0: Post-Election — Determine Safe Commit Point): * 1. Collect LSNs from all granted voters: * - Self always granted: self_lsn = ctx->self_lsn (stored unconditionally) * - Each peer with vote_granted=1: lsn = votes[i].peer_lsn * 2. Sort granted LSNs descending. * 3. safe_committed_lsn = sorted[floor(n_granted / 2)] * (the majority-threshold element — the highest LSN held by a majority) * 4. Same logic for config tags (ctx->self_tag / votes[i].peer_tag). * * Single-node (n_peers == 0): votes[] is empty; safe_committed_lsn = ctx->self_lsn. * This prevents WAL truncation to 0 on re-election in a single-node cluster. * * Requirements: 2.3 (ha-repl-hardening spec) * Boundary: Controller FSM (ha.c) */ static int _cmp_u64_desc(const void *a, const void *b) { uint64_t ua = *(const uint64_t *)a; uint64_t ub = *(const uint64_t *)b; if (ua > ub) return -1; if (ua < ub) return 1; return 0; } void ha_compute_safe_committed_lsn(ha_elect_ctx_t *ctx, uint64_t *safe_lsn_out, uint64_t *safe_tag_out) { if (!ctx || !safe_lsn_out || !safe_tag_out) return; /* Single-node: no votes[] entries — self is the only majority. * Return self_lsn so WAL is not truncated to 0 on re-election. */ if (ctx->n_peers == 0) { *safe_lsn_out = ctx->self_lsn; *safe_tag_out = ctx->self_tag; return; } /* Collect granted voter LSNs and tags. * Maximum granted voters = n_peers + 1 (self + all other peers). */ int max_voters = ctx->n_peers + 1; uint64_t *lsns = calloc((size_t)max_voters, sizeof(uint64_t)); uint64_t *tags = calloc((size_t)max_voters, sizeof(uint64_t)); if (!lsns || !tags) { free(lsns); free(tags); *safe_lsn_out = 0; *safe_tag_out = 0; return; } int n_granted = 0; /* Self always granted: self's lsn/tag from the dedicated self_lsn/self_tag fields */ lsns[n_granted] = ctx->self_lsn; tags[n_granted] = ctx->self_tag; n_granted++; /* Each peer with vote_granted=1 */ for (int i = 0; i < ctx->n_peers; i++) { if (ctx->votes[i].vote_granted) { lsns[n_granted] = ctx->votes[i].peer_lsn; tags[n_granted] = ctx->votes[i].peer_tag; n_granted++; } } /* Sort descending */ qsort(lsns, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc); qsort(tags, (size_t)n_granted, sizeof(uint64_t), _cmp_u64_desc); /* Majority-threshold element: index = floor(n_granted / 2) * For n_granted=3: index=1 (2nd element, 0-based) — majority=2 out of 3. * For n_granted=1: index=0 (self only). */ int idx = n_granted / 2; *safe_lsn_out = lsns[idx]; *safe_tag_out = tags[idx]; free(lsns); free(tags); INFO("HA: safe_committed_lsn=%" PRIu64 " safe_config_tag=%" PRIu64 " (from %d granted voters)", *safe_lsn_out, *safe_tag_out, n_granted); } /* ── bug-1060: never truncate the new leader's own durable log ─────────────── * * The post-election WAL truncation cutoff must never fall below the leader's own * durable tip, or it destroys entries committed by a prior leader (which this * node had acked as a standby, and so are committed) — a Raft §5.4 Leader * Completeness violation that diverges WALs and deadlocks the next election. * See ha.h for the full rationale. */ uint64_t ha_promotion_truncate_floor(uint64_t safe_lsn, uint64_t durable_tip) { return safe_lsn > durable_tip ? safe_lsn : durable_tip; } /* ── bug-1462: never floor committed_config_tag below the leader's own * loaded config — see ha.h for full rationale. */ uint64_t ha_promotion_config_tag_floor(uint64_t safe_tag, uint64_t self_tag) { return safe_tag > self_tag ? safe_tag : self_tag; } /* ── raft-drive-async task 1.1: peer-scaled liveness reservation ───────────── * * Pure: reserved = max(1, min(n_peers - 1, n_vcpus - 1)). The floor of 1 is * applied LAST so deadlock-freedom (>=1 guaranteed-free liveness worker) holds * even when the n_vcpus ceiling would otherwise drive the value to 0 (e.g. * n_vcpus == 1). See ha.h for the full rationale; the caller WARNs on the * under-provisioned (n_vcpus < n_peers) clamp — this helper never logs. */ int drive_async_liveness_reserved(int n_peers, int n_vcpus) { int desired = n_peers - 1; /* worst-case concurrent liveness work */ int ceiling = n_vcpus - 1; /* keep >=1 commit (REQ_BLOCK) slot */ int reserved = desired < ceiling ? desired : ceiling; /* min(desired, ceiling) */ if (reserved < 1) /* floor applied LAST — always wins */ reserved = 1; return reserved; } /* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ─────────── * * ha_promotion_gate_decision: pure decision (Req 2.2, 2.5, 2.6). Build the Boot * Policy Decision Table inputs from the promoting member's recovery/durability * state and delegate to persist_recover_decision so the 3.2/3.3 table is the * single source of truth (DRY). "Behind the HWM" is durable_floor < safe_hwm; * a truncated log is treated as behind regardless of its volatile position * (design.md note). A hollow node (truncated OR behind) can never be ELIGIBLE * until it catches up — independent of quorum size (Req 2.5). */ recover_decision_t ha_promotion_gate_decision(bool recovery_truncated, uint64_t durable_floor, uint64_t safe_hwm, bool peer_reachable, bool catch_up_succeeded) { recover_policy_inputs_t in = { .verdict = recovery_truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN, .durable_below_hwm = (durable_floor < safe_hwm), .healthy_peer_reachable = peer_reachable, .catch_up_succeeded = catch_up_succeeded, .durable_floor_lsn = durable_floor, .cluster_hwm_lsn = safe_hwm, }; return persist_recover_decision(&in); } /* ha_evaluate_promotion_fence: live gate over the existing in_catch_up fence * (Req 2.2, 2.6). Reuses ha->in_catch_up — does NOT invent a parallel fence. * - ELIGIBLE → clear the fence, return ELIGIBLE (a member that caught up past * the committed boundary regains eligibility, Req 2.6). * - FENCE → set the fence, return FENCE (refuse promotion; catch up from a * healthy peer). * - FENCE_UNRECOVERABLE → run the 3.3 loud-fence action (logs LOUDLY + sets * the sticky non-promotable flag) and ALSO set the in_catch_up * fence. The member STAYS ALIVE, non-promotable, retrying * catch-up — it NEVER self-exits (Req 2.4). */ recover_decision_t ha_evaluate_promotion_fence(ha_state_t *ha, bool recovery_truncated, uint64_t durable_floor, uint64_t safe_hwm, bool peer_reachable, bool catch_up_succeeded) { recover_decision_t d = ha_promotion_gate_decision( recovery_truncated, durable_floor, safe_hwm, peer_reachable, catch_up_succeeded); switch (d) { case RECOVER_DECISION_ELIGIBLE: /* Release the fence: catch-up brought the member to a * committed-consistent state at/above the cluster boundary (Req 2.6). */ if (ha != NULL) atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release); break; case RECOVER_DECISION_FENCE: /* Refuse promotion: extend the catch-up fence so the member catches up * from a healthy peer before it can lead (Req 2.2). */ if (ha != NULL) atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release); INFO("HA: promotion FENCED — recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64 " (catch up before leading; " "ha-commit-durability Req 2.2)", recovery_truncated, durable_floor, safe_hwm); break; case RECOVER_DECISION_FENCE_UNRECOVERABLE: { /* Unrecoverable: route through the 3.3 loud-fence action — logs LOUDLY * and sets the sticky non-promotable flag. Also hold the in_catch_up * fence so the member refuses promotion. The process STAYS ALIVE and * keeps retrying catch-up; it NEVER self-exits (Req 2.4). */ recover_policy_inputs_t in = { .verdict = recovery_truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN, .durable_below_hwm = (durable_floor < safe_hwm), .healthy_peer_reachable = peer_reachable, .catch_up_succeeded = catch_up_succeeded, .durable_floor_lsn = durable_floor, .cluster_hwm_lsn = safe_hwm, }; persist_t *p = (ha != NULL && ha->raft_c != NULL) ? &ha->raft_c->persist : NULL; persist_recover_loud_fence_if_unrecoverable(p, &in); if (ha != NULL) atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release); break; } } return d; } /* ── ha-commit-durability task 4.4: bounded-window catch-up re-alert ───────── * * (never-self-exit rework 2026-06-18.) A member that wins an election while * recovery-truncated / behind the HWM with no peer reachable AT THE PROMOTION * INSTANT is FENCED (in_catch_up set, non-promotable) and STAYS ALIVE, retrying * catch-up indefinitely. This one-shot timer exists ONLY for operator * VISIBILITY: when it fires, if the member is STILL unrecoverable (still * truncated/behind AND no peer AND no progress) it RE-EMITS the loud, * operator-visible alert via persist_recover_loud_fence_if_unrecoverable (which * also keeps the sticky non-promotable flag set) — it NEVER terminates the * process. If the member has caught up (not truncated AND durable_floor >= * HWM) the timer RELEASES the fence (Req 2.6). If progress is possible (peer * back, or durable floor advanced) it leaves the member fenced under the * existing catch-up machinery. * * Earlier this timer DEFERRED-then-ABORTED. The abort is GONE: a controller * MUST NEVER self-terminate as a designed response to an operational condition * (user invariant, 2026-06-18). The task-5.2 soak quorum collapse was caused * by that self-abort; staying alive + retrying self-heals when a peer returns. * * The window is a settable seam so the timer test runs fast. Default = * DEFAULT_RECOVERY_CATCHUP_WINDOW_MS (persist.h). */ static _Atomic uint32_t s_promotion_fence_window_ms = DEFAULT_RECOVERY_CATCHUP_WINDOW_MS; uint32_t ha_promotion_fence_window_ms(void) { uint32_t w = atomic_load_explicit(&s_promotion_fence_window_ms, memory_order_acquire); return w ? w : DEFAULT_RECOVERY_CATCHUP_WINDOW_MS; } #ifdef XCE_TESTING void ha_set_promotion_fence_window_ms(uint32_t ms) { atomic_store_explicit(&s_promotion_fence_window_ms, ms, memory_order_release); } #endif /* promotion_fence_ctx_t: delaymgr context for the bounded catch-up window. * Captures the cluster committed high-water mark (target_hwm) and the durable * floor AT SCHEDULE TIME (baseline_durable_floor) so the timer can detect * catch-up progress (durable_floor advanced toward the HWM). `cancelled` is set * by ha_cancel_promotion_fence_timer (recovery / shutdown) so a fired-but- * cancelled callback is a safe no-op. */ typedef struct { _Atomic int cancelled; ha_state_t *ha; cluster_t *c; bool was_truncated; /* recovery-truncated at schedule time */ uint64_t target_hwm; /* cluster committed high-water mark */ uint64_t baseline_durable_floor; /* durable floor at schedule time */ } promotion_fence_ctx_t; /* ha_cancel_promotion_fence_timer: cancel a pending bounded-window FATAL timer. * Idempotent; safe to call when no timer is armed. Used when the member catches * up (fence released) or when the controller is shutting down — so neither a * caught-up nor a stopped member ever reaches the deferred FATAL. */ void ha_cancel_promotion_fence_timer(ha_state_t *ha) { if (!ha) return; promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)atomic_exchange_explicit( &ha->promotion_fence_ctx, NULL, memory_order_acq_rel); if (t) atomic_store_explicit(&t->cancelled, 1, memory_order_release); } /* Forward decl: the timeout callback re-arms itself for periodic re-alert. */ static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c, bool truncated, uint64_t durable_floor, uint64_t target_hwm); /* promotion_fence_timeout_fn: the bounded-window re-alert timer callback. Owns * its ctx — frees it on exit. Re-evaluates the LIVE recovery/durability state. * It NEVER terminates the process (never-self-exit invariant, 2026-06-18): * - caught up (NOT truncated AND durable_floor >= target_hwm) → release the * in_catch_up fence + clear the sticky unrecoverable flag; eligible again * (Req 2.6). * - a healthy peer is reachable now, OR catch-up is making progress → leave * the member fenced under the existing catch-up machinery (it will release * via ha_evaluate_promotion_fence → ELIGIBLE when it completes); do not * re-arm the re-alert while progress is happening. * - STILL unrecoverable (truncated/behind AND no peer AND no progress) → * RE-EMIT the loud operator alert via persist_recover_loud_fence_if_- * unrecoverable (keeps the sticky flag + in_catch_up set) and RE-ARM the * timer so the alert keeps repeating until a peer returns. The process * STAYS ALIVE and keeps retrying — it NEVER aborts. */ static int promotion_fence_timeout_fn(void *arg) { promotion_fence_ctx_t *t = (promotion_fence_ctx_t *)arg; ha_state_t *ha = t->ha; cluster_t *c = t->c; /* Clear the stored handle before checking cancelled so a concurrent * ha_cancel_promotion_fence_timer is a safe no-op (matches the * raft_election_timeout_fn convention). */ promotion_fence_ctx_t *cur = (promotion_fence_ctx_t *)atomic_load_explicit( &ha->promotion_fence_ctx, memory_order_acquire); if (cur == t) atomic_store_explicit(&ha->promotion_fence_ctx, NULL, memory_order_release); if (atomic_load_explicit(&t->cancelled, memory_order_acquire)) { free(t); return 0; } if (!c) { free(t); return 0; } /* defensive: no persist to evaluate */ /* Re-read the LIVE state after the window. */ bool truncated = persist_recovery_truncated(&c->persist); uint64_t durable_floor = persist_durable_floor(&c->persist); bool below_hwm = durable_floor < t->target_hwm; bool made_progress = durable_floor > t->baseline_durable_floor; bool peer_reachable = (ha->repl != NULL) ? (repl_active_standby_count(ha->repl) > 0) : false; /* raft-apply-decoupling task 5.3 (round-2 remediation, Req 8.4): a fence * set for an APPLY_DIVERGENT reason must NEVER be live-cleared by this * purely-structural re-check. apply_verdict is only ever recomputed by * persist_recover() (a boot-time-once replay, cluster.c) — more WAL * bytes arriving or a healthy peer appearing cannot prove a local * apply-dispatch defect is fixed, so this periodic timer must treat an * apply-divergent fence as untouchable until the next full restart * replays cleanly (persist_recover_decision fences apply_divergent * unconditionally for the same reason — see persist.c). */ bool apply_divergent = persist_apply_verdict(&c->persist) == APPLY_DIVERGENT; /* self_restart_pending: same unconditional, never-live-cleared refusal as * apply_divergent — see the field's doc comment in ha.h. This purely * structural re-check (durable floor vs HWM) must not release a fence set * because a demoted ex-primary's execv() failed; that node's WAL state can * look fully caught-up while its in-memory state is still stale. */ bool self_restart_pending = atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire); /* apply_resource_fenced (job-array-single-wal-record task 4.3, Req * 9.4/9.5): same unconditional, never-live-cleared-by-structure refusal * as apply_divergent/self_restart_pending above — a node stuck on a * resource-exhaustion apply failure (task 4.1/4.2's Live Apply Resource * Fence) can have a perfectly healthy WAL (not truncated, durable_floor * >= HWM) by construction, since apply success and WAL durability are * orthogonal. This purely structural re-check must not release the fence * on WAL health alone — only the apply pump's own successful retry of the * stuck LSN clears it (persist_clear_apply_resource_fence, called from * inside persist_apply_pump_run itself). */ bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist); /* Caught up: not truncated AND durable position reached the boundary AND * the current apply verdict is not DIVERGENT AND no self-restart is * pending AND no live apply resource fence is engaged → release the * fence + clear the sticky flag; the member is eligible on its next * election (Req 2.6, Req 9.5). */ if (!truncated && !below_hwm && !apply_divergent && !self_restart_pending && !apply_resource_fenced) { atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release); persist_clear_recovery_unrecoverable(&c->persist); INFO("HA: promotion fence RELEASED after catch-up — durable_floor=%" PRIu64 " >= cluster HWM=%" PRIu64 " (ha-commit-durability Req 2.6)", durable_floor, t->target_hwm); free(t); return 0; } /* Still hollow, but a peer is reachable or catch-up advanced the durable * floor: progress is possible. Leave the member fenced (in_catch_up stays * set) under the existing catch-up machinery; do not re-arm the re-alert * while progress is happening. */ if (peer_reachable || made_progress) { INFO("HA: promotion fence catch-up window elapsed but progress is " "possible (peer_reachable=%d durable_floor=%" PRIu64 " baseline=%" PRIu64 " HWM=%" PRIu64 ") — remaining fenced, catching up " "(ha-commit-durability Req 2.2)", peer_reachable, durable_floor, t->baseline_durable_floor, t->target_hwm); free(t); return 0; } /* STILL unrecoverable: truncated/behind AND no healthy peer AND no catch-up * progress. RE-EMIT the loud operator alert and keep the member fenced + * non-promotable + ALIVE (Req 2.4) — NEVER terminate. Re-arm the re-alert * timer so the operator keeps hearing about it until a peer returns. */ uint64_t target_hwm = t->target_hwm; /* capture before free (UAF guard) */ { recover_policy_inputs_t in = { .verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN, .durable_below_hwm = below_hwm, .healthy_peer_reachable = false, .catch_up_succeeded = false, .durable_floor_lsn = durable_floor, .cluster_hwm_lsn = target_hwm, }; /* Loud, operator-visible alert + sticky non-promotable flag. STAYS * ALIVE: no abort/exit anywhere on this path. */ persist_recover_loud_fence_if_unrecoverable(&c->persist, &in); } free(t); /* Re-arm so the loud alert repeats periodically while the member remains * unrecoverable. The member keeps retrying catch-up; the next firing * re-evaluates and releases the fence once a peer returns and it catches up. */ ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, target_hwm); return 0; } /* ha_arm_promotion_fence_timer: fence the member (set in_catch_up) and schedule * the one-shot bounded-window RE-ALERT timer. Cancels any previously-armed * timer first so a re-evaluation never leaks or double-fires. On a failure to * allocate/schedule the timer it emits the loud unrecoverable fence ONCE (stays * alive, non-promotable) — it NEVER aborts (never-self-exit invariant). */ static void ha_arm_promotion_fence_timer(ha_state_t *ha, cluster_t *c, bool truncated, uint64_t durable_floor, uint64_t target_hwm) { /* Fence first (Req 2.2): refuse promotion while hollow. */ atomic_store_explicit(&ha->in_catch_up, 1, memory_order_release); /* Cancel a stale timer (e.g. a prior promotion attempt). */ ha_cancel_promotion_fence_timer(ha); promotion_fence_ctx_t *t = calloc(1, sizeof(*t)); if (!t) goto loud_fence_no_timer; atomic_init(&t->cancelled, 0); t->ha = ha; t->c = c; t->was_truncated = truncated; t->target_hwm = target_hwm; t->baseline_durable_floor = durable_floor; uint32_t window_ms = ha_promotion_fence_window_ms(); int rc = delay_task_ms(&c->delaymgr, (int)window_ms, 1, promotion_fence_timeout_fn, t); if (rc < 0) { WARN("ha_arm_promotion_fence_timer: delay_task_ms failed: %d", rc); free(t); goto loud_fence_no_timer; } atomic_store_explicit(&ha->promotion_fence_ctx, t, memory_order_release); INFO("HA: promotion FENCED + catch-up re-alert window armed (%u ms) — " "recovery_truncated=%d durable_floor=%" PRIu64 " < cluster HWM=%" PRIu64 "; staying ALIVE + non-promotable, retrying catch-up " "(ha-commit-durability Req 2.2/2.4)", window_ms, truncated, durable_floor, target_hwm); return; loud_fence_no_timer: { /* Could not arm the re-alert timer — still emit the loud unrecoverable * fence once so the operator sees it; the member stays fenced * (in_catch_up set above) + non-promotable + ALIVE. NO abort. */ recover_policy_inputs_t in = { .verdict = truncated ? RECOVER_TRUNCATED : RECOVER_CLEAN, .durable_below_hwm = (durable_floor < target_hwm), .healthy_peer_reachable = false, .catch_up_succeeded = false, .durable_floor_lsn = durable_floor, .cluster_hwm_lsn = target_hwm, }; persist_recover_loud_fence_if_unrecoverable(&c->persist, &in); } } /* * raft_cb_become_leader: node won the election — enter CTRL_PRIMARY directly. * * Equivalent of ha_elect_evaluate_and_signal CTRL_PRIMARY path: * - enter CTRL_PRIMARY (no transitional state) * - call cluster_init_phase2 on first boot * - push config, send ctrlhello * * ha->raft_c must be non-NULL (production path only; test mode uses noop stubs). */ static void raft_cb_become_leader(void *ctx, uint64_t term) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; /* Cancel any pending election timeout */ raft_cb_cancel_timeout(ctx); /* Sync ha->term from raft term (raft owns term increments) */ atomic_store_explicit(&ha->term, term, memory_order_release); /* The cluster committed high-water mark for this promotion — derived from * the voter replies (NOT a stale ctrlhello). The task-4.3 pre-promotion * eligibility gate (below) compares the local CLAMPED durable position * against it. Lifted to function scope so the gate can read it after the * truncate-clamp. */ uint64_t promotion_safe_lsn = 0; /* Task 4.1 (ha-repl-hardening): compute safe_committed_lsn from vote replies. * Uses voter LSNs collected during election (ha->elect_ctx->votes[]) to find * the highest LSN held by a majority without an extra round-trip. * Design §Flow 0: Post-Election — Determine Safe Commit Point. */ { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (ectx) { uint64_t safe_lsn = 0; uint64_t safe_tag = 0; ha_compute_safe_committed_lsn(ectx, &safe_lsn, &safe_tag); promotion_safe_lsn = safe_lsn; /* Store results for use in WAL truncation and post-election commit. * safe_lsn is the highest LSN on a majority; entries above it * may not have been committed on all nodes and must be truncated. * * bug-1060 (Leader Completeness, Raft §5.4): the truncation cutoff is * clamped to the leader's OWN durable tip so it never destroys entries * this node holds durably — those were acked by this node as a standby * and so were committed by the prior leader. Truncating to the bare * majority element (which can sit BELOW the leader's durable tip when * a slower peer is in the granting majority) diverged WALs and dead- * locked the next election. With the clamp, persist_truncate_wal_above * is a no-op on the leader's durable log; safe_lsn only governs removal * of a non-durable divergent tail strictly above the durable tip. */ if (c) { uint64_t durable_tip = persist_durable_floor(&c->persist); uint64_t trunc_floor = ha_promotion_truncate_floor(safe_lsn, durable_tip); persist_truncate_wal_above(&c->persist, trunc_floor); /* 2026-07-10 (chaotic-soak finding): persist_truncate_wal_above * only advances persist->committed_lsn on the branches that * perform a REAL truncation; its no-op path (this leader's own * log already covers safe_lsn -- the common case) leaves * committed_lsn untouched at its pre-election value, stalling * commit-gated apply until the normal ack-driven commit path * organically catches up seconds later. See * persist_promotion_advance_committed's doc comment. */ persist_promotion_advance_committed(&c->persist, safe_lsn); } /* Task 4.2 (ha-repl-hardening): set committed_lsn = safe_lsn. * persist_promotion_advance_committed (above) advances * c->persist.committed_lsn in the production path (c != NULL), * durable-capped and monotonic. Also update * ha->last_known_committed_lsn so the test-mode decision loop * (XCE_TESTING path in standby heartbeat reply handler) and any * code reading ha->last_known_committed_lsn see the safe commit * point immediately after promotion. * Design §Flow 0 step 5: committed_lsn = safe_committed_lsn. * Req 2.3 (ha-repl-hardening). */ atomic_store_explicit(&ha->last_known_committed_lsn, safe_lsn, memory_order_release); /* Update committed config tag baseline for the post-election decision * loop. bug-1462: clamp to the leader's OWN loaded config tag — * unlike WAL LSN, a node's actively-loaded config is never * speculative, so the bare majority floor (correct for LSN) must * not be allowed to strand a real config behind an all-zero floor * reported by voters that never received a live config push. * See ha_promotion_config_tag_floor's doc comment (ha.h). */ uint64_t committed_tag_floor = ha_promotion_config_tag_floor(safe_tag, ectx->self_tag); atomic_store_explicit(&ha->committed_config_tag, committed_tag_floor, memory_order_release); } } /* ── ha-commit-durability task 4.3: pre-promotion eligibility gate ─────── * Refuse promotion BEFORE the node becomes authoritative CTRL_PRIMARY if it * carries the recovery-truncated flag (task 3.1 — a structurally corrupt log * must never lead) OR its CLAMPED durable position is below the cluster * committed high-water mark (promotion_safe_lsn, from the voter replies). * The durable position is read AFTER persist_truncate_wal_above above, so it * reflects the 2.2 truncate-clamp (a node whose durable tip was clamped down * is correctly seen as behind). A hollow node (truncated OR behind) is * refused regardless of quorum size — combined with the 4.2 freshness floor * it can neither win the vote nor self-promote (Req 2.5). On refusal we set * the in_catch_up fence and return WITHOUT becoming primary; the member * catches up from a healthy peer and a later promotion attempt passes the * gate (Req 2.2/2.6). Only the production path (real persist) is gated; the * c==NULL unit shim has no persist and skips it. * * Belt-and-suspenders (never-self-exit rework 2026-06-18): a member that has * been marked sticky-unrecoverable (persist_recovery_unrecoverable) must * NEVER become primary while that flag is set — it stays alive, fenced, and * keeps retrying catch-up. The flag clears automatically on catch-up. */ if (c != NULL) { bool truncated = persist_recovery_truncated(&c->persist); uint64_t durable_floor = persist_durable_floor(&c->persist); bool unrecoverable = persist_recovery_unrecoverable(&c->persist); /* raft-apply-decoupling task 5.3 (round-2 remediation, Req 8.4): * same guard as promotion_fence_timeout_fn above — a fence set for * an APPLY_DIVERGENT reason must never be live-cleared, and this * gate must never promote while the current apply verdict is * divergent, REGARDLESS of what `unrecoverable` currently reads * (defense-in-depth: it must not rely on the other call site alone * having kept `unrecoverable` accurate). apply_verdict is only ever * recomputed by a fresh boot-time persist_recover() replay. */ bool apply_divergent = persist_apply_verdict(&c->persist) == APPLY_DIVERGENT; /* self_restart_pending (2026-07-10, ha-convergence-recovery Req 1.4 * supersession): a demoted ex-primary whose execv() re-exec failed * must never become primary again while carrying its stale pre- * demotion in-memory state — same unconditional-refuse treatment as * apply_divergent, and for the same reason: this node's WAL/durability * state can look perfectly ELIGIBLE while the in-memory state the * restart exists to discard is still live. No live-clear path exists; * only a successful restart (a fresh process) clears it. */ bool self_restart_pending = atomic_load_explicit(&ha->self_restart_pending, memory_order_acquire); /* apply_resource_fenced (job-array-single-wal-record task 4.3, Req * 9.4/9.5): same unconditional-refuse treatment as apply_divergent/ * self_restart_pending above — a node stuck on a resource-exhaustion * apply failure (task 4.1/4.2's Live Apply Resource Fence) must never * become primary while the fence is engaged, regardless of how * structurally ELIGIBLE its WAL looks (truncated/durable_floor are * orthogonal to apply success). Only the apply pump's own successful * retry of the stuck LSN clears it — no election/HWM condition can. */ bool apply_resource_fenced = persist_apply_resource_fenced(&c->persist); /* A healthy peer is reachable for catch-up iff some voter granted with a * live endpoint survives this election (these become the standby * channels promoted below). */ bool peer_reachable = false; { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (ectx) { for (int i = 0; i < ectx->n_peers; i++) { if (ectx->votes[i].vote_granted && ectx->votes[i].ep != NULL) { peer_reachable = true; break; } } } } /* catch_up_succeeded is false at the promotion-eligibility evaluation: * a hollow member never silently proceeds to leadership. We evaluate * the PURE 3.2/3.3 decision here. A FENCE or an UNRECOVERABLE * (truncated/behind + no peer right now) both fence the member * (in_catch_up) and arm the catch-up re-alert timer; the member STAYS * ALIVE and keeps retrying — it NEVER self-exits (never-self-exit * invariant, 2026-06-18; this is what fixes the task-5.2 soak collapse * where a self-aborting recovering winner repeatedly dropped quorum). */ recover_decision_t gate = ha_promotion_gate_decision( truncated, durable_floor, promotion_safe_lsn, peer_reachable, /*catch_up_succeeded=*/false); if (gate == RECOVER_DECISION_ELIGIBLE && !unrecoverable && !apply_divergent && !self_restart_pending && !apply_resource_fenced) { /* ELIGIBLE, not sticky-unrecoverable, not apply-divergent, no * pending self-restart, AND no live apply resource fence engaged: * release the fence (a caught-up member regains eligibility, Req * 2.6), clear the sticky flag if set, and cancel any pending * re-alert timer. Proceed to authoritative promotion below. */ atomic_store_explicit(&ha->in_catch_up, 0, memory_order_release); persist_clear_recovery_unrecoverable(&c->persist); ha_cancel_promotion_fence_timer(ha); } else { /* FENCE, UNRECOVERABLE, sticky-unrecoverable, apply-divergent, * self-restart-pending, or apply-resource-fenced: refuse * promotion — do NOT become CTRL_PRIMARY. Fence the member and * arm the catch-up re-alert timer. The member STAYS ALIVE, * non-promotable, retrying catch-up (Req 2.2/2.4). NEVER * self-exit, never fence silently. An apply-divergent, * self-restart-pending, or apply-resource-fenced member is * refused unconditionally (Req 8.4; ha-convergence-recovery Req * 1.4; job-array-single-wal-record Req 9.4/9.5) — no peer/HWM * condition can clear any of these triggers; apply-resource- * fenced clears only from inside the apply pump itself on a * successful retry (task 4.2), the others only on a fresh boot * (clean restart replay, or a successful re-exec). */ WARN("HA: promotion REFUSED (decision=%d unrecoverable=%d " "apply_divergent=%d self_restart_pending=%d " "apply_resource_fenced=%d) — " "recovery_truncated=%d durable_floor=%" PRIu64 " cluster_hwm=%" PRIu64 " peer_reachable=%d; fencing + " "arming catch-up re-alert window, staying ALIVE " "(ha-commit-durability Req 2.2/2.4; raft-apply-decoupling " "Req 8.4; ha-convergence-recovery Req 1.4; " "job-array-single-wal-record Req 9.4/9.5)", (int)gate, unrecoverable, apply_divergent, self_restart_pending, apply_resource_fenced, truncated, durable_floor, promotion_safe_lsn, peer_reachable); ha_arm_promotion_fence_timer(ha, c, truncated, durable_floor, promotion_safe_lsn); return; } } /* ── ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): config * serve-fence promotion gate ────────────────────────────────────────────── * A member whose latest committed config marker references bytes it does not * hold (config_serve_fenced) MUST NOT become authoritative CTRL_PRIMARY — it * cannot serve under the committed configuration. Refuse promotion and STAY * ALIVE (controller-never-self-terminate): the member remains non-promotable * and keeps retrying; once the correct bytes arrive out-of-band and re-verify * clears the fence, a later promotion attempt passes this gate. The check is a * pure atomic read (no I/O) — safe on the election path. */ if (config_serve_fenced(c)) { WARN("HA: promotion REFUSED — config serve-FENCED (committed config " "marker references bytes not present/matching locally); member " "stays ALIVE, non-promotable, awaiting out-of-band config bytes " "(ha-term-safety R7.9)"); return; /* do NOT become CTRL_PRIMARY; never self-exit */ } /* ── config-upgrade-snapshot task 6.2 (Req 1.8, 3.2, 3.3, 4.3, 5.5): * Activation Fence promotion gate ────────────────────────────────────── * A member whose OWN upgrade-snapshot materialization for the current * committed config marker has not completed and hash-verified must not * become authoritative CTRL_PRIMARY — doing so would let it win an * election (Raft's log-freshness gate only proves it has the committed * MARKER, not that it has finished MATERIALIZING under it) and then * immediately violate the serving gate above the moment it starts * accepting requests. This is a SEPARATE, SEQUENTIAL check from the * config serve-fence directly above (same rationale as the dispatch-gate * composition: byte-mismatch and materialization-incomplete are distinct * conditions) — never a new promotion-eligibility mechanism for WAL/log * completeness (Req 3.1 is explicit that Raft's own freshness gate * already owns that; this fence is layered on top, purely about * materialization completeness). Refuse promotion and STAY ALIVE * (controller-never-self-terminate): the member remains non-promotable * and keeps retrying (schedule_upgrade_materialization_retry, driven off * the commit-advance path); once its own materialization completes and * re-verify clears the fence, a later promotion attempt passes this * gate. Pure atomic read (no I/O) — safe on the election path. */ if (upgrade_materialization_fenced(c)) { WARN("HA: promotion REFUSED — upgrade materialization FENCED (this " "member's own upgrade-snapshot materialization for the " "committed config marker is incomplete or hash-mismatched); " "member stays ALIVE, non-promotable, retrying materialization " "(config-upgrade-snapshot Req 1.8/3.2/3.3/4.3/5.5)"); return; /* do NOT become CTRL_PRIMARY; never self-exit */ } int is_first_boot = !ha->phase2_done; /* First boot: call cluster_init_phase2 */ if (is_first_boot) { int p2rc = 0; #ifdef XCE_TESTING if (ha_phase2_hook != NULL) p2rc = ha_phase2_hook(c); #else if (c != NULL) p2rc = cluster_init_phase2(c); #endif if (p2rc == 0) { ha->phase2_done = 1; } else { /* bug-1477: do NOT latch phase2_done on failure, and do NOT * proceed to CTRL_PRIMARY with an unwired commit engine — mirror * the upgrade_materialization_fenced refusal just above (stay * ALIVE, non-promotable, retry; a later promotion attempt * succeeds once ha_schedule_phase2_retry completes). Before this * fix, phase2_done was latched unconditionally, so this failure * path never existed for the code below to worry about. */ WARN("HA: promotion REFUSED — cluster_init_phase2 failed " "(rc=%d); member stays ALIVE, non-promotable, retrying " "(bug-1477, never self-terminate)", p2rc); if (c != NULL) ha_schedule_phase2_retry(c); return; /* do NOT become CTRL_PRIMARY with an unwired commit engine */ } /* bug-1445 fix: cluster_init_phase2() above just ran this process's * very FIRST persist_recover()/replay_wal() -- the ONLY thing that * can engage the Live Apply Resource Fence (job-array-single-wal- * record tasks 4.1/4.2, extended to boot replay by task 7.2) for * THIS node. The eligibility gate above (~line 1337) necessarily * read persist_apply_resource_fenced() as false: a fresh persist_t * defaults to unfenced (persist_init) and this is the first replay * that could ever have set it -- so that gate's ELIGIBLE decision * was made blind to whatever this first replay just did. Re-check * here, once, right after the only call that could have changed the * answer: if this node's own first-ever replay just fenced it, * refuse promotion now instead of falling through to CTRL_PRIMARY * below on a decision that is now stale. Every later election in * this process's lifetime is unaffected -- is_first_boot is false * by then, so the gate above already covers it correctly (this is * exactly what the sibling test, * test_replay_wal_engages_fence_on_resource_exhaustion_failure, * proves). Mirrors the existing refusal branch above verbatim (same * WARN shape, same ha_arm_promotion_fence_timer() call, same * unconditional-refuse treatment as apply_divergent/ * self_restart_pending) rather than inventing a second mechanism. */ if (c != NULL && persist_apply_resource_fenced(&c->persist)) { WARN("HA: promotion REFUSED — this node's own first-ever boot " "replay (cluster_init_phase2, just completed) engaged the " "Live Apply Resource Fence; the earlier eligibility check " "in this same election evaluation necessarily missed it " "(bug-1445) — member stays ALIVE, non-promotable, " "retrying apply (job-array-single-wal-record Req 9.4/9.5)"); ha_arm_promotion_fence_timer(ha, c, persist_recovery_truncated(&c->persist), persist_durable_floor(&c->persist), promotion_safe_lsn); return; } } /* ha-single-leader-role (Req 1.1): enter CTRL_PRIMARY directly — there is * no transitional COMMITTING state. New-leader safety comes from the * own-term commit rule (the WAL_NOOP + writable_lsn gate armed below), * not from a fourth role value. The raft-owned term was already synced * at entry (~line 463), so we do NOT call ha_do_promote here (it would * double-increment the term, research.md §4 finding 2). This store sits * OUTSIDE the c != NULL guard so the c==NULL test shim * (raft_cb_become_leader_for_test) reaches it. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); INFO("HA FSM (raft): → CTRL_PRIMARY (term=%" PRIu64 ", first_boot=%d)", term, is_first_boot); /* 2026-07-10 (chaotic-soak regression finding, bug-1043 family recurrence): * a node promoted from standby can carry an apply-lag backlog -- entries * already committed per c->persist.committed_lsn (just finalized above by * the truncate-clamp / safe_lsn assignment) but never materialized into * the job/queue table, because standby-side materialization is now * commit-gated and only kicked by heartbeat consumption or catch-up * ingest (ha_standby_advance_committed_and_apply) -- BOTH of which stop * firing the instant this node becomes CTRL_PRIMARY (ha_consume_heartbeat_ * committed explicitly skips primaries). Without draining here, that * backlog sits unapplied: expect-job/joblist on the new primary would * report a genuinely committed (WAL-consistent, walcompare-clean) job as * missing. * * SYNCHRONOUS on purpose (persist_apply_pump_run_sync, not the async * persist_apply_pump_kick): the async kick only ENQUEUES the drain onto a * vtask and returns immediately, so this function (and thus promotion) * could complete -- and the node could start answering expect-job/joblist * queries as primary -- before the vtask worker actually finished * applying the backlog. A live chaotic HA soak caught exactly that race * (2026-07-10): the async version still lost committed jobs on a freshly * promoted primary. * * THREAD-CONTEXT CORRECTION (2026-07-10, controller-invariant-audit): this * comment previously claimed promotion "already runs on a VTASK_LIVE_BLOCK * vtask" -- that is TRUE only for the single-node self-grant and RDHUP- * failover election-start paths (ha_dispatch_election_start). It is FALSE * for the common real multi-node case: a winning RequestVoteReply is * dispatched via peer_ctrl_classify, which hardcodes VTASK_LIVENESS (never * admission-gated, documented as never expected to block -- see the * bug-1198 hazard box above peer_ctrl_classify's definition, which closed * this EXACT class of bug for HA_REPL_SEND_SNAPSHOT's persist_checkpoint). * persist_apply_pump_run_sync's pread()s of recently-written WAL segments * are lower practical risk than a bare fdatasync (almost always page-cache * hot, since the entries were just appended) but are still, strictly, * blocking I/O on a task class the design says never blocks. This residual * gap is tracked (see buglog.json) rather than fixed inline here: closing * it properly means either reclassifying the winning-vote-reply case in * peer_ctrl_classify or dispatching promotion's tail onto a fresh * VTASK_LIVE_BLOCK vtask, both larger changes needing their own dedicated * soak-validation pass. Do NOT remove this call to "fix" the classification * gap -- it is what closes the bug-1043-family apply-lag/job-loss bug; * removing it reopens that regression. The backlog itself stays inherently * small (only entries committed since the last heartbeat-driven kick). */ if (c) persist_apply_pump_run_sync(&c->persist); /* Req 2.1 (ha-repl-hardening): re-initialize per-standby convergence * tracking for THIS primary term. Stale converged flags from a * previous term (demote → re-elect cycle) must not satisfy the * majority-convergence gate before standbys re-report their state. */ memset(ha->standby_reported_tag, 0, sizeof(ha->standby_reported_tag)); memset(ha->standby_reported_lsn, 0, sizeof(ha->standby_reported_lsn)); memset(ha->standby_converged, 0, sizeof(ha->standby_converged)); /* Promote surviving vote endpoints to standby replication channels. * Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to * ha->standby_endpoints so push_config and ctrlhello reach them. */ { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (ectx && ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int i = 0; i < ectx->n_peers; i++) { if (ectx->votes[i].ep != NULL) { zarray_store(ha->standby_endpoints, ectx->votes[i].ep); INFO("HA: promoted vote ep[%d] → standby channel", i); /* Task 2.1: register promoted ep with WAL replicator. * Req 1.1: primary must register standby with repl_add_standby * when establishing the standby control channel. * peer->hostname / peer->port from the vote entry's peer pointer. */ if (c && ectx->votes[i].peer != NULL) { int rc = repl_add_standby(&c->persist.repl, ectx->votes[i].ep, ectx->votes[i].peer->hostname, (uint32_t)ectx->votes[i].peer->port); if (rc >= 0) INFO("HA: repl registered standby ep[%d] (slot %d, %s:%u)", i, rc, ectx->votes[i].peer->hostname, ectx->votes[i].peer->port); else WARN("HA: repl_add_standby ep[%d] failed: %d", i, rc); } } } pthread_mutex_unlock(&ha->lock); } } /* Quorum-transition observability (Req 8.1, 8.2): promotion registered the * surviving vote endpoints as standbys — the write-quorum count just changed. * Record any loss/restore edge once (no-op when quorum is held at promotion). */ ha_log_quorum_transition(ha); if (c) { /* Push config to standbys — read raw bytes from disk so standbys * can save the file. NULL bytes are passed only in test mode. */ #ifdef XCE_TESTING if (ha_push_config_hook != NULL) ha_push_config_hook(c); #else { uint8_t *cfg_bytes = NULL; size_t cfg_len = 0; extern int conf_read_raw(const char *, uint8_t **, size_t *); if (c->myconf && c->myconf->conffile && conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) { /* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON). * The offload deep-copies cfg_bytes, so this free is still * correct regardless of routing. */ ha_config_push_offloadable(c, cfg_bytes, cfg_len, c->myconf->config_sha256, c->myconf->config_version_tag); free(cfg_bytes); } else { ha_config_push_offloadable(c, NULL, 0, NULL, c->myconf ? c->myconf->config_version_tag : 0); } } #endif /* Send CtrlHelloRequest to all standbys to trigger WAL catch-up */ #ifdef XCE_TESTING if (ha_send_ctrlhello_hook != NULL) ha_send_ctrlhello_hook(c); #else ha_send_ctrlhello_to_all_standbys(c); #endif /* ha-single-leader-role task 2.2: register the persistence hooks FIRST, * before the own-term no-op append, so the persistence seam is live * before any racing client write can reach the cluster. Idempotent * via the hooks_registered guard. */ ha_register_persist_hooks(c); /* Start primary heartbeat immediately so standbys receive NOP and their * liveness poll does not fire. ha_start_primary_heartbeat is also * called in ha_promote_to_primary; the idempotency guard (role check * at fn top) prevents double-arming. */ ha_start_primary_heartbeat(c); /* ha-single-leader-role task 2.2 (Req 1.4, 2.1, 2.2): append the * own-term WAL_NOOP and arm the commit engine's writable_lsn gate. * This is the shared promotion tail both roads to PRIMARY run. * Non-blocking: promotion never waits for the no-op to commit. */ ha_arm_own_term_gate(c); } } /* * raft_cb_become_follower: node lost election or saw higher term — enter STANDBY. * * Syncs ha->term from the raft-reported term, sets role to CTRL_STANDBY, * calls cluster_init_phase2 on first boot (standbys also need it to * receive config replication push from the new primary). */ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx) { (void)leader_idx; ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; /* Cancel any pending election timeout */ raft_cb_cancel_timeout(ctx); /* Sync ha->term */ atomic_store_explicit(&ha->term, term, memory_order_release); int is_first_boot = !ha->phase2_done; /* First boot: call cluster_init_phase2 so structures are ready */ if (is_first_boot) { int p2rc = 0; #ifdef XCE_TESTING if (ha_phase2_hook != NULL) p2rc = ha_phase2_hook(c); #else if (c != NULL) p2rc = cluster_init_phase2(c); #endif if (p2rc == 0) { ha->phase2_done = 1; } else { /* bug-1477: do NOT latch phase2_done on failure — a standby * (unlike a promoted primary) can safely stay passive while * ha_schedule_phase2_retry retries; nothing below depends on * phase2 having succeeded. */ WARN("HA: cluster_init_phase2 on become-follower failed (rc=%d) " "— member stays alive, will retry (bug-1477)", p2rc); if (c != NULL) ha_schedule_phase2_retry(c); } } /* Close all outgoing vote endpoints (loser path). * The primary will push config and ctrlhello via promoted connections; * the standby waits passively on its listener for the inbound connection. * * FLAG-ON HAZARD (raft-drive-async finding-1; bug-1197): when * g_drive_async_peerctrl_dispatch_enabled is ON this loop runs on a POOL * worker and frees ep->data while the io_worker may still deliver a queued * peer-control event on another ep → UAF. Before task 3.4 flips the flag * ON, each endpoint_fini below MUST be preceded by iomgr_unmonitor(ep). See * the full hazard note at peer_ctrl_dispatch_to_pool. Safe today (flag OFF). */ { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (ectx) { for (int i = 0; i < ectx->n_peers; i++) { endpoint_t *ep = ectx->votes[i].ep; if (ep == NULL) continue; INFO("HA: closing vote ep[%d] (loser path)", i); if (ha_peerctrl_dispatch_enabled() && ha->pool != NULL) { /* bug-1197: this callback runs on a pool worker; another * worker may be mid-vtask (or have a queued vtask) on this * same ep reading ep->data. Defer teardown onto the ep's * per-ep FIFO so endpoint_fini+free runs strictly after the * last live use, and gate late dispatch (tearing_down). */ vote_ep_teardown_async(ha, ep); } else { /* bug-1298: drop the owner ref (deferred fini+free if a pool * vtask still references this vote-ep). Also fixes the old * flag-OFF struct leak (was endpoint_fini-only). */ endpoint_unref(ep); } ectx->votes[i].ep = NULL; } } } /* ha-convergence-recovery Req 1.1-1.3, 1.5: demotion convergence. * * Read the PRIOR role BEFORE the CTRL_STANDBY store below. If this node was * the PRIMARY, it may hold an uncommitted, potentially divergent WAL suffix * and provisional in-memory state. Synchronously disable the three * persistence hooks HERE (before any further event can be persisted — Req * 1.1, reusing ha_demote_to_standby's disable sequence), then schedule a * one-shot delaymgr task that truncates above committed_lsn and self-restarts * (Req 1.2/1.3/1.5) off this callback thread (D1: no file I/O or exec on the * iomgr/raft callback thread). * * Non-PRIMARY demotions (STANDBY/INIT/ELECTING) take the unchanged path — * they have no registered hooks and no provisional suffix to discard, so * their behavior is byte-equivalent to before this spec. */ ctrl_role_t prior_role = (ctrl_role_t)atomic_load_explicit( &ha->role, memory_order_acquire); if (prior_role == CTRL_PRIMARY && c != NULL) { /* Req 1.1: disable persistence hooks SYNCHRONOUSLY, before the role * store and before scheduling, so no provisional entry can be produced * after the higher term was observed. Mirrors ha_demote_to_standby * (ha.c) — disable each hook, then clear the idempotency guard. */ if (ha->hooks_registered) { xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun); xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun); xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun); ha->hooks_registered = 0; } /* raft-apply-decoupling task 1.4 (Req 1.3, 1.4, 9.3, 9.4): a demoted * PRIMARY can never advance committed_lsn again for anything it had * pending -- resolve every currently registered commit continuation * as FAILED now, synchronously with the rest of this "stop being * primary" sequence, rather than leaving a requester parked until * the commit engine's own timeout-sweep thread (task 1.3) eventually * times it out. lsn_floor=0 resolves ALL pending continuations (lsn * 0 is never a real registration -- the free-slot sentinel). Uses * the same -XCE_ERR_STANDBY vocabulary the HA dispatch gate already * uses to reject writes to a non-primary. Reachable only when this * node was already PRIMARY, which means cluster_init_phase2 (and * therefore commit_engine_init) already ran, so c->commit is always * initialized here. */ commit_resolve_pending_above(&c->commit, 0, -XCE_ERR_STANDBY); } atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); INFO("HA FSM (raft): → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)", term, is_first_boot); /* Req 1.2/1.3/1.5: schedule the deferred truncate-above-committed-LSN + * self-restart now that hooks are disabled and the role is STANDBY. Only * the was-PRIMARY edge converges; all other edges skip this entirely. */ if (prior_role == CTRL_PRIMARY && c != NULL) { WARN("HA: PRIMARY demoted (term=%" PRIu64 ") — scheduling demotion " "convergence (truncate above committed_lsn + self-restart)", term); (void)ha_schedule_demotion_convergence(c); } /* Arm heartbeat watchdog — fires ha_on_primary_ctrl_fd_rdhup if primary * sends no ctrlhello within election_timeout_ms. (This also resets * last_activity = now, granting a full grace window before the first poll.) */ ha_arm_heartbeat_timeout_vtask(ha, c); } /* * raft_cb_persist: persist term + voted_for to disk before granting vote. * * Maps raft integer voted_for index to a candidate name using cluster config. * Updates ha->term, ha->voted_for_term, ha->voted_for_name then calls * ha_persist_term(ha, workdir). * * voted_for == -1 means "no vote"; voted_for >= 0 is the candidate index. * Returns 0 on success, non-zero on error (causes raft library to deny vote). */ static int raft_cb_persist(void *ctx, uint64_t term, int voted_for) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; /* Store ha-term.bin under {workdir}/{myname}/data/ alongside WAL/snapshot. */ char data_dir[1024] = {0}; if (c && c->myconf && c->myconf->workdir && c->myconf->name) snprintf(data_dir, sizeof(data_dir), "%s/%s/data", c->myconf->workdir, c->myconf->name); const char *workdir = data_dir[0] ? data_dir : NULL; /* Update ha->term */ atomic_store_explicit(&ha->term, term, memory_order_release); /* Map voted_for index to name */ pthread_mutex_lock(&ha->lock); if (voted_for == -1) { ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); } else { ha->voted_for_term = term; /* Look up candidate name by index */ if (c && c->myconf && voted_for < c->myconf->peers_cfg.candidates_count) { const char *name = c->myconf->peers_cfg.candidates[voted_for].name; strncpy(ha->voted_for_name, name, sizeof(ha->voted_for_name) - 1); ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0'; } else { /* Test mode or out-of-range: use "self" placeholder */ if (voted_for == ha->my_priority) { strncpy(ha->voted_for_name, "self", sizeof(ha->voted_for_name) - 1); } else { memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); } } } pthread_mutex_unlock(&ha->lock); ha_persist_term(ha, workdir); return 0; } /* raft_cb_trace: emit raft state-machine events at DEBUG3 level. * Active when controller is started with -vvv; suppressed otherwise. */ static void raft_cb_trace(void *ctx, const char *event, const char *detail) { ha_state_t *ha = (ha_state_t *)ctx; tlog(TLOG_DEBUG3, "RAFT[%s] term=%" PRIu64 " event=%s%s%s", ha->my_priority == 0 ? "p0" : ha->my_priority == 1 ? "p1" : "p2", atomic_load_explicit(&ha->term, memory_order_relaxed), event, detail ? " " : "", detail ? detail : ""); } /* ── ha-term-safety: MINIMAL log/commit callback placeholders ──────────────── * * TODO(ha-term-safety task 2.x): REPLACE these stubs with the real bodies. * The raft library's raft_init() now REQUIRES all five non-NULL, and * raft_on_vote_request() internally calls on_get_entry_term for its logOk gate. * These placeholders exist ONLY to restore the controller build (tasks 1.1/1.2); * they perform NO persistence, NO replication, and NO truncation. See CONCERNS. * * on_persist_entry → task 2.1 (durable WAL append/overwrite) — IMPLEMENTED * on_get_entry_term → task 2.2 (WAL entry-term lookup; persist_entry_term, 1.3) — IMPLEMENTED * on_send_append_entries→ task 2.3 (AppendEntries transmit) — IMPLEMENTED * on_truncate_from → task 2.2 (conflict truncation) — IMPLEMENTED * on_commit_advanced → task 2.4 (commit-barrier trigger / commit source) — IMPLEMENTED * * on_get_entry_term returns 0 (== "absent") for now: with the log-replication * path unwired, the library's local last-log term reads as 0, so the logOk gate * grants on lastLogTerm >= 0 — i.e. it does NOT yet tighten the existing * config_tag/last_log_lsn freshness gate already enforced in cluster.c. The real * accessor (task 2.2/1.3) makes the gate effective. */ /* on_persist_entry (ha-term-safety task 2.1, R1.1–R1.4): durably append/overwrite * the log entry at 1-based `index` (== WAL LSN) stamping the LEADER-supplied * `term` into hdr.term, then return the persist rc. * * Persist-before-act: this is called BEFORE the library reports the append/accept * (raft_log_append / raft_on_append_entries_request). On a non-zero (failure) rc * the library does NOT advance its log and a standby does NOT ACK (the existing * persist-before-ACK + bug-850 contiguity guard sit beneath the WAL write and are * preserved — we do not bypass them). We never false-ACK on failure. * * `entry` is the adapter envelope (raft_entry_blob_t prefix + payload) packed by * the append/replication driver (tasks 3.1/3.2); the library holds no entry bytes * and treats it as opaque. We decode (type, payload, payload_len) from it for the * existing persist write path. * * Path selection by role: * - PRIMARY own-append: persist_wal_async — assigns the contiguous next LSN * (== index here, since the library's index == log_len+1 == current_lsn) and * defers fsync to the group-commit flusher (the steady-state append path). * - STANDBY replicated apply: persist_wal_write_replicated — writes at the GIVEN * lsn=index with the contiguity guard + inline persist-before-ACK fsync. * * NOTE (deferred to 2.3/3.2): the leader's per-entry term is carried on the repl * envelope by task 2.3/3.2; until then the standby path stamps the term we set * here via persist_set_term — which IS the supplied `term` argument — so a test * that drives this callback already records the leader term. The envelope wiring * makes the live replication path source the same value. */ static int ha_cb_persist_entry_stub(void *ctx, uint64_t index, uint64_t term, const void *entry, size_t len) { ha_state_t *ha = (ha_state_t *)ctx; if (!ha || !ha->raft_c) return -EINVAL; /* test mode without a cluster: cannot persist */ persist_t *p = &ha->raft_c->persist; /* Decode the adapter entry envelope: a fixed raft_entry_blob_t prefix * carrying the WAL type + payload_len, followed by the payload bytes. */ uint32_t type = WAL_NOOP; const void *payload = NULL; uint32_t plen = 0; if (entry && len >= sizeof(raft_entry_blob_t)) { const raft_entry_blob_t *blob = (const raft_entry_blob_t *)entry; type = blob->type; plen = blob->payload_len; /* Reject a malformed envelope whose declared payload exceeds the buffer * — fail loud rather than read past the blob. */ if ((size_t)plen + sizeof(*blob) > len) return -EINVAL; if (plen > 0) payload = (const uint8_t *)entry + sizeof(*blob); } else if (entry && len > 0) { /* Defensive: an un-enveloped blob is treated as malformed. */ return -EINVAL; } /* Stamp the live election term so the WAL header records the LEADER's term * for this entry (R1.1/R1.2/R1.4). */ persist_set_term(p, term); /* Persist at LSN=index through the existing write path (persist-before-ACK * and the bug-850 contiguity guard are preserved beneath these). */ if (ha_is_primary(ha)) { /* Own-append: persist_wal_async assigns the contiguous next LSN, which * equals `index` for the leader (library index == current_lsn). */ return persist_wal_async(p, type, payload, plen); } return persist_wal_write_replicated(p, index, type, payload, plen); } /* on_get_entry_term (ha-term-safety task 2.2, R1.5/R3.1): return the recorded * Raft term for the entry at 1-based LSN `index`, sourced from the WAL term * index via persist_entry_term. Returns 0 when the LSN is absent (no entry). * The library calls this for its logOk / (term,lsn) reconcile gate. * * Legacy-0 caveat (from the task 1.3 review MEDIUM): persist_entry_term maps a * present legacy-0 entry (hdr.term == 0, written before this feature) to * WAL_LEGACY_TERM (== 1). In a mixed pre/post-feature cluster where a * post-feature leader is genuinely in term 1, a (term,lsn) EQUALITY check * cannot tell a legacy-0 entry (mapped to 1) apart from a real term-1 entry by * term alone. This is safe because the Raft log-matching invariant (same term * + same index ⇒ same entry, enforced across the prefix) plus the bounded * one-time upgrade window (every entry written after upgrade carries a real * term) mean the only collision is at the legacy/term-1 boundary, where the * prefixes already agree. We note it explicitly rather than special-casing it. */ static uint64_t ha_cb_get_entry_term_stub(void *ctx, uint64_t index) { ha_state_t *ha = (ha_state_t *)ctx; if (!ha || !ha->raft_c) return 0; /* test mode without a cluster: no log → absent */ return persist_entry_term(&ha->raft_c->persist, index); } /* ha_read_wal_entry_at: source ONE local WAL entry at LSN `lsn` for the * AppendEntries send path (ha-term-safety task 2.3). The library holds no entry * bytes, so the leader reads its own WAL segment files (seg-%016llx.wal under * wal_path) and returns the entry's type + payload. The per-LSN term is NOT read * from the on-disk header here — the caller takes it from persist_entry_term * (the authoritative in-memory term index), per the library contract that the * wire entry's term == the term recorded for that LSN. `payload` is malloc'd and * owned by the caller (freed after the send). Returns 0 on success, negative on * absence/read error. */ static int ha_read_wal_entry_at(const char *wal_path, uint64_t lsn, uint8_t *type_out, uint8_t **payload_out, uint32_t *plen_out) { *type_out = 0; *payload_out = NULL; *plen_out = 0; DIR *d = opendir(wal_path); if (!d) return -errno ? -errno : -EIO; /* Enumerate segment start-LSNs, descending, so we open the latest segment * whose start LSN is <= the target first. */ uint64_t seg_lsns[4096]; int nseg = 0; struct dirent *de; while ((de = readdir(d)) != NULL && nseg < (int)(sizeof(seg_lsns) / sizeof(*seg_lsns))) { uint64_t sl = 0; if (sscanf(de->d_name, "seg-%016" SCNx64 ".wal", &sl) == 1) seg_lsns[nseg++] = sl; } closedir(d); if (nseg == 0) return -ENOENT; /* insertion sort ascending */ for (int i = 1; i < nseg; i++) { uint64_t key = seg_lsns[i]; int j = i - 1; while (j >= 0 && seg_lsns[j] > key) { seg_lsns[j + 1] = seg_lsns[j]; j--; } seg_lsns[j + 1] = key; } int rc = -ENOENT; for (int si = 0; si < nseg; si++) { /* Skip a segment whose successor already starts past the target. */ if (si + 1 < nseg && seg_lsns[si + 1] <= lsn) continue; if (seg_lsns[si] > lsn) break; char seg_path[4096]; snprintf(seg_path, sizeof(seg_path), "%s/seg-%016" PRIx64 ".wal", wal_path, seg_lsns[si]); int fd = open(seg_path, O_RDONLY); if (fd < 0) continue; uint32_t off = 0; for (;;) { wal_entry_hdr_t hdr; ssize_t n = pread(fd, &hdr, sizeof(hdr), (off_t)off); if (n < (ssize_t)sizeof(hdr)) break; if (hdr.lsn == 0 && hdr.type == 0 && hdr.payload_len == 0) break; if (hdr.lsn == lsn) { *type_out = (uint8_t)hdr.type; if (hdr.payload_len > 0) { uint8_t *pbuf = malloc(hdr.payload_len); if (!pbuf) { rc = -ENOMEM; } else { ssize_t pr = pread(fd, pbuf, hdr.payload_len, (off_t)(off + sizeof(hdr))); if (pr == (ssize_t)hdr.payload_len) { *payload_out = pbuf; *plen_out = hdr.payload_len; rc = 0; } else { free(pbuf); rc = -EIO; } } } else { rc = 0; /* zero-payload entry (e.g. WAL_NOOP) */ } close(fd); return rc; } off += (uint32_t)(sizeof(hdr) + hdr.payload_len); } close(fd); } return rc; } /* on_send_append_entries (ha-term-safety task 2.3, R1.3/R3.1): transmit ONE * AppendEntries to the standby at slot `peer_idx`, carrying the prev tuple * (prev_log_index, prev_log_term) + leader_commit, and — for n_entries==1 — the * single entry at LSN prev_log_index+1 sourced from this leader's OWN WAL (the * library holds no entry bytes). The entry's term is taken from * persist_entry_term (the authoritative per-LSN term index), so the standby * persists the LEADER's recorded term, not a substitute. n_entries==0 emits a * heartbeat (zero entries) carrying only the prev tuple + leader_commit. * * Non-blocking (ha-async-io): delegates to repl_send_append_entries which * enqueues via the async repl transport — never blocks in the callback. * peer_idx is the library's raft peer index, mapped 1:1 to the repl standby * registry slot (the same index match_index/next_index use). * * DORMANT IN PRODUCTION (raft-adapter-conformance task 4.1, Requirement 3.3): * this callback is wired as on_send_append_entries (see s_real_cbs below) and * is fully protocol-conformant, but it is reachable ONLY via the raft * library's raft_send_append_entries (src/lib/raft/raft.c:595), which has * ZERO production call sites — every caller is either * src/lib/raft/raft_sim.c (the pure library simulation harness) or the * XCE_TESTING-only test helper ha_cb_send_append_entries_for_test (below). * Nothing in the running controller ever DRIVES raft_send_append_entries, so * this callback is registered but never invoked in production. * * The mechanism that ACTUALLY drives AppendEntries-equivalent * send/retry/backoff in production today is a different, coarser * substitute: repl_broadcast (src/controller/repl.c) fans out each * freshly-appended WAL entry to every standby as it is written (retrying via * ha_raft_on_repl_nack on a NACK), and ha_stream_wal_to_standby * (src/controller/config_repl.c) streams a standby's catch-up range once it * falls behind. Together these two are the sole production * AppendEntries-equivalent send/retry/backoff mechanism for Requirement 3.3 * — NOT this per-peer, nextIndex-driven sender. * * This is a deliberate documentation-only resolution, not an oversight: see * research.md's "Documentation-only resolution for the dead nextIndex-driven * AppendEntries sender (Requirement 3)" decision and * src/controller/CLAUDE.md's TLA+ Action-to-Code Mapping for the recorded * disposition. The dead path is kept as an intentionally-dormant, * protocol-exact reference implementation (retiring or wiring it in were * both rejected — see research.md) rather than silently left undocumented. * Its dormancy is guarded against silent regression by the XCE_TESTING-only * ha_send_append_entries_stub_call_count counter incremented immediately * below: a future change that accidentally gives raft_send_append_entries a * production caller moves this counter off zero, which task 4.2's * non-regression test (test_ha_core_safety.c) asserts against while * confirming repl_broadcast/ha_stream_wal_to_standby fire for the same * scenario. * * See also the independently-derived reachability trace at * test_ha_core_safety.c:2436-2464 and the ha_apply_single_entry comment * above (raft-ha-core-safety task 6.2), which confirm the same * zero-production-caller finding for this function's only caller * (repl_send_append_entries). */ #ifdef XCE_TESTING /* ha_send_append_entries_stub_call_count (raft-adapter-conformance task 4.1, * Requirement 3.3): non-regression guard counting invocations of * ha_cb_send_append_entries_stub (the on_send_append_entries callback). * Expected to stay at 0 across any test that drives only the production * replication path (repl_broadcast / ha_stream_wal_to_standby) — see the * DORMANT IN PRODUCTION comment above. External linkage under XCE_TESTING * only, so a test can read/reset it directly (declared extern in ha.h, * mirroring the ha_wal_ack_send_hook seam). NULL/0 semantics: always 0 in a * production build (the whole symbol does not exist outside XCE_TESTING). */ uint64_t ha_send_append_entries_stub_call_count = 0; #endif static int ha_cb_send_append_entries_stub(void *ctx, int peer_idx, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, int n_entries) { #ifdef XCE_TESTING ha_send_append_entries_stub_call_count++; #endif ha_state_t *ha = (ha_state_t *)ctx; if (!ha || !ha->raft_c) return -EINVAL; /* test mode without a cluster: cannot send */ persist_t *p = &ha->raft_c->persist; replicator_t *r = ha->repl ? ha->repl : &p->repl; if (n_entries == 0) { /* Heartbeat: prev tuple + leader_commit, no entry. */ return repl_send_append_entries(r, peer_idx, prev_log_index, prev_log_term, leader_commit, /*n_entries*/0, /*entry_lsn*/0, /*type*/0, /*term*/0, NULL, 0); } /* n_entries == 1: source the entry at prev_log_index+1 from the local WAL. */ uint64_t entry_lsn = prev_log_index + 1; uint64_t entry_term = persist_entry_term(p, entry_lsn); uint8_t type = 0; uint8_t *payload = NULL; uint32_t plen = 0; int rc = ha_read_wal_entry_at(p->wal_path, entry_lsn, &type, &payload, &plen); if (rc != 0) { WARN("on_send_append_entries: cannot source local WAL entry at lsn=%lu " "(rc=%d) — not sending to peer %d", (unsigned long)entry_lsn, rc, peer_idx); return rc; } rc = repl_send_append_entries(r, peer_idx, prev_log_index, prev_log_term, leader_commit, /*n_entries*/1, entry_lsn, type, entry_term, payload, plen); free(payload); return rc; } /* on_truncate_from (ha-term-safety task 2.2, R1.8/R3.2): delete the local log * entries at LSN `index..end` because they conflict with the leader's log at a * (term,lsn) the library detected. persist_truncate_wal_above(p, cutoff) removes * every entry with LSN > cutoff, so to drop LSN >= index we pass cutoff = * index - 1. This REPLACES the committed-boundary truncate heuristic on the * library-driven path: the library, not a watermark, decides the cut point. * * Returns the truncate rc. A non-zero rc propagates so the library/leader * retries; the member stays non-promotable per R3.5 (that fence is existing * behavior and is not touched here). persist_truncate_wal_above also shrinks * the per-LSN term index for the removed LSNs, so a later on_get_entry_term on * a truncated LSN reads back 0 (absent). */ static int ha_cb_truncate_from_stub(void *ctx, uint64_t index) { ha_state_t *ha = (ha_state_t *)ctx; if (!ha || !ha->raft_c) return -EINVAL; /* test mode without a cluster: cannot truncate */ if (index == 0) return -EINVAL; /* LSN 0 is reserved — never an entry to truncate */ return persist_truncate_wal_above(&ha->raft_c->persist, index - 1); } /* on_commit_advanced (ha-term-safety task 2.4, R4.1/R4.5/R5.4): feed the raft * library's current-term-gated `commit_index` into the existing synchronous * commit barrier as the commit SOURCE. The library fired this ONLY on a strict * advance of its Figure-8 current-term commit index (raft.c), so the barrier * now advances committed_lsn only when BOTH a durable majority agrees AND the * library has admitted the entry — the term-blind safe-commit * (ha_compute_safe_committed_lsn) is no longer the steady-state commit source. * * This callback only TRIGGERS the barrier (records the ceiling + re-evaluates); * it performs NO durability itself. The barrier mechanism — majority self-cap at * the durable floor, persist-before-ACK, monotonic committed_lsn — is unchanged * (ha-commit-durability). The own-term writable_lsn gate is RETAINED as a * complementary WAIT-completion predicate (it never caps committed_lsn, so it * cannot deadlock against this source; see ha_arm_own_term_gate). * * Non-blocking (ha-async-io): commit_set_raft_commit_index takes only ce->lock * and never performs I/O — safe on the iomgr ACK path that drove * raft_advance_commit. */ static void ha_cb_commit_advanced_stub(void *ctx, uint64_t commit_index) { ha_state_t *ha = (ha_state_t *)ctx; if (!ha || !ha->raft_c) return; /* test mode without a cluster: no barrier to trigger */ commit_set_raft_commit_index(&ha->raft_c->commit, commit_index); /* ha-term-safety task 4.3 (remediation r1, Finding B): a config marker may * have just entered the committed prefix. Activation is NOT performed HERE: * this callback runs on the iomgr ACK path, where the async-I/O invariant * forbids the disk reads config_activate_committed performs * (scan_latest_committed_marker pread + load_primary_config). The committed * marker is instead consumed off the iomgr thread by the call sites that make * §7.10 true in production: * - boot/recovery (myapp.c step 7): activates the latest committed marker, * hash-verified, after persist recovery. * - handle_repl_config_push: after the out-of-band bytes are saved, gates * activation on the committed marker (verify-before-activate). * * ha-term-safety task 4.3 (remediation r2, Finding D / R7.9): the two sites * above are NOT sufficient when the BYTES arrive (and are saved) BEFORE the * NEW marker commits — the config MARKER (WAL) and the config BYTES (out-of- * band push) travel on INDEPENDENT transports. In that ordering the push path * hash-checks the freshly-saved NEW bytes against the still-OLDER committed * marker, MISMATCHES, and sets the serve-fence; when the NEW marker later * commits HERE, nothing re-runs activation (this callback can't do disk I/O, * and the primary's config-behind re-push net does not fire because the push * already advanced this member's advertised config_version_tag) — so the * standby would stay PERMANENTLY fenced. Close the gap: while serve-fenced, * schedule an OFF-IOMGR delaymgr task (config_schedule_activate_retry, NON- * BLOCKING ordq_put only) that re-runs config_activate_committed on a thread * allowed to block on disk; once the NEW marker is in the committed prefix and * the saved bytes verify, the retry clears the fence and activates. No-op once * unfenced (no busy-spin); a still-genuine mismatch reschedules on the next * commit advance. */ if (config_serve_fenced(ha->raft_c)) config_schedule_activate_retry(ha->raft_c); /* config-upgrade-snapshot task 6.2 (Req 3.4, 5.5): symmetric commit- * advance trigger for the Activation Fence. Materialization technically * completes on APPLY (persist.c's WAL_CONFIG_MARKER case), which lags * commit under raft-apply-decoupling — this hook does not need to fire * at the PRECISE instant materialization becomes possible, only to * eventually re-check and clear the fence once conditions allow (Req * 3.4/5.5's "keep retrying" framing, not a precise event-driven * guarantee). Commit advancing is a reasonable, precedented "something * changed, worth re-checking" signal, and schedule_upgrade_materialization_ * retry itself is a no-op when unfenced, so this adds no busy-spin in the * steady state. */ if (upgrade_materialization_fenced(ha->raft_c)) schedule_upgrade_materialization_retry(ha->raft_c); } static const raft_callbacks_t s_real_cbs = { .on_send_vote_request = raft_cb_send_vote_request, .on_send_pre_vote_request = raft_cb_send_pre_vote_request, .on_vote_eligible = raft_cb_vote_eligible, .on_become_leader = raft_cb_become_leader, .on_become_follower = raft_cb_become_follower, .on_persist = raft_cb_persist, .on_schedule_timeout = raft_cb_schedule_timeout, .on_cancel_timeout = raft_cb_cancel_timeout, .on_start_election_after_delay = raft_cb_start_election_after_delay, /* ha-term-safety: MINIMAL placeholders to satisfy raft_init's non-NULL * requirement — REPLACE in task 2.x (see stubs + CONCERNS). */ .on_persist_entry = ha_cb_persist_entry_stub, .on_get_entry_term = ha_cb_get_entry_term_stub, .on_send_append_entries = ha_cb_send_append_entries_stub, .on_truncate_from = ha_cb_truncate_from_stub, .on_commit_advanced = ha_cb_commit_advanced_stub, .on_trace = raft_cb_trace, }; /* ── PRNG seeding ───────────────────────────────────────────── */ void ha_seed_jitter(uint64_t *seed, int my_priority) { uint8_t buf[8] = {0}; int fd = open("/dev/urandom", O_RDONLY); if (fd >= 0) { ssize_t n = 0; while (n < 8) { ssize_t r = read(fd, buf + n, (size_t)(8 - n)); if (r <= 0) break; n += r; } close(fd); if (n == 8) { memcpy(seed, buf, 8); if (*seed != 0) return; } } struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); *seed = (uint64_t)ts.tv_nsec ^ ((uint64_t)my_priority << 56) ^ ((uint64_t)getpid() << 16); } int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority) { if (!ha || !cfg) return -EINVAL; /* Zero first so ha_fini() is safe even if validation fails below */ memset(ha, 0, sizeof(*ha)); if (cfg->heartbeat_interval_ms < 2000) return -EINVAL; if (cfg->heartbeat_timeout_ms < cfg->heartbeat_interval_ms) return -EINVAL; if (cfg->election_timeout_ms < 500) return -EINVAL; ha->config = *cfg; ha->my_priority = my_priority; atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); atomic_store_explicit(&ha->applied_lsn, 0ULL, memory_order_release); atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed); atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed); atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed); /* failover_requested / watchdog_stop deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */ pthread_mutex_init(&ha->lock, NULL); /* bug-1435: dedicated lock serializing raft_state_t's election/vote entry * points against concurrent pool-worker dispatch (see ha.h field doc). * RECURSIVE: a WON/quorum outcome legitimately re-enters the state machine * synchronously on the SAME thread (e.g. raft_on_pre_vote_reply's WON * branch -> ha_enter_real_election -> ha_start_election_async -> * raft_start_election, all before the outer call returns) — that is a * same-thread call chain, not a race, and must not self-deadlock. A plain * mutex would deadlock on that legitimate nesting; PTHREAD_MUTEX_RECURSIVE * still fully serializes the DIFFERENT-thread case this lock exists to fix. */ { pthread_mutexattr_t raft_lock_attr; pthread_mutexattr_init(&raft_lock_attr); pthread_mutexattr_settype(&raft_lock_attr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&ha->raft_lock, &raft_lock_attr); pthread_mutexattr_destroy(&raft_lock_attr); } ha->ssl_ctx = NULL; ha->jitter_seed = 0; ha->standby_endpoints = NULL; ha->primary_ctrl_fd = -1; /* election-iomgr-refactor Task 2.1: explicit zero-init for documentation clarity * (memset above already zeroes these, but being explicit is better documentation) */ ha->client_ssl_ctx = NULL; ha->phase2_done = 0; ha->n_peers = 0; /* ha-heartbeat-liveness Task 1.1: zero-init liveness timestamp. * Written by ha_on_heartbeat_received (task 2.1); read by liveness_poll_fn. */ ha->last_activity = 0; /* ha-repl-hardening Task 1.4: per-standby convergence tracking arrays. * memset(ha, 0, sizeof(*ha)) above already zeroes these arrays; the explicit * note documents the invariant: on promotion to CTRL_PRIMARY every * standby starts as not-converged (tag=0, lsn=0, converged=false). */ /* heartbeat_seen starts at 0: no heartbeat seen yet. * ha_wait_for_heartbeat_or_timeout() calls sem_timedwait on this. */ sem_init(&ha->heartbeat_seen, 0, 0); /* ── Task 6.1/6.2: Initialize embedded raft_state_t ───────────────────── * * Wire real callbacks (s_real_cbs) so the state machine is ready when * ha_start_election_async re-initialises raft with the correct n_peers / * self_idx / initial_term after ha_restore_term(). * * n_peers = 1 and self_idx = 0 are safe placeholders here; they are * overwritten by ha_raft_reinit() (called from ha_start_election_async) * which runs after ha_restore_term() and knows the real cluster size. * * initial_term = 0 / initial_voted_for = -1 are first-boot defaults; * ha_raft_reinit() restores the persisted values before any election fires. * * ha->raft_c is NULL until ha_start_election_async sets it. */ ha->raft_c = NULL; ha->raft_election_timeout_ctx = NULL; raft_config_t raft_cfg; memset(&raft_cfg, 0, sizeof(raft_cfg)); raft_cfg.n_peers = 1; /* placeholder; overwritten by ha_raft_reinit */ raft_cfg.self_idx = my_priority; /* self_idx == priority by convention */ raft_cfg.election_timeout_ms = cfg->election_timeout_ms; raft_cfg.my_priority = my_priority; raft_cfg.initial_term = 0; /* ha_restore_term() updates ha->term after ha_init */ raft_cfg.initial_voted_for = -1; /* no vote on fresh init */ raft_cfg.cbs = &s_real_cbs; raft_cfg.ctx = ha; raft_init(&ha->raft, &raft_cfg); return 0; } /* SIGUSR1 handler / ha_install_sigusr1() deleted — failover_requested replaced by * ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */ void ha_do_promote(ha_state_t *ha) { /* Monotonically increasing fencing token — standbys reject entries from lower terms */ atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel); atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); /* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */ ha->last_received_crc = WAL_CRC_NONE; /* Req 10.5: log FSM transition to PRIMARY */ INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")", atomic_load_explicit(&ha->term, memory_order_acquire)); } void ha_do_demote(ha_state_t *ha) { atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); /* Reset so the next ha_promote_to_primary() re-registers hooks idempotently */ ha->hooks_registered = 0; /* Hook disable requires hooksys_t — done by ha_demote_to_standby(cluster_t*) */ /* Req 10.5: log FSM transition to STANDBY */ INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ")", atomic_load_explicit(&ha->term, memory_order_acquire)); } /* probe_thread() / wait_any_probe() deleted — only called by am_i_primary() (election-iomgr-refactor) */ /* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */ /* * ha_is_primary: lock-free role check. */ int ha_is_primary(const ha_state_t *ha) { return (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire) == CTRL_PRIMARY; } /* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */ /* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */ /* ── P3-6: Test-only failover shim ────────────────────────────── */ #ifdef XCE_TESTING uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha) { ha_do_demote(ha); ha_do_promote(ha); return atomic_load_explicit(&ha->term, memory_order_acquire); } /* ha-term-safety task 2.1: drive the WIRED on_persist_entry callback so a unit * test exercises the real adapter body (term stamp + WAL write) without * raft_log_append (task 3.1). Exposed only under XCE_TESTING. */ int ha_cb_persist_entry_for_test(ha_state_t *ha, uint64_t index, uint64_t term, const void *entry, size_t len) { return s_real_cbs.on_persist_entry(ha, index, term, entry, len); } /* ha-term-safety task 2.2: drive the WIRED on_get_entry_term callback so a unit * test exercises the real entry-term accessor. Exposed only under XCE_TESTING. */ uint64_t ha_cb_get_entry_term_for_test(ha_state_t *ha, uint64_t index) { return s_real_cbs.on_get_entry_term(ha, index); } /* ha-term-safety task 2.2: drive the WIRED on_truncate_from callback so a unit * test exercises the real conflict truncation. Exposed only under XCE_TESTING. */ int ha_cb_truncate_from_for_test(ha_state_t *ha, uint64_t index) { return s_real_cbs.on_truncate_from(ha, index); } /* ha-term-safety task 2.3: drive the WIRED on_send_append_entries callback so a * unit test exercises the real AppendEntries send body (peer→ep map, prev tuple * + leader_commit carry, local-WAL entry sourcing). Pair with * repl_send_ae_capture_hook to assert envelope content. Exposed only under * XCE_TESTING. */ int ha_cb_send_append_entries_for_test(ha_state_t *ha, int peer_idx, uint64_t prev_log_index, uint64_t prev_log_term, uint64_t leader_commit, int n_entries) { return s_real_cbs.on_send_append_entries(ha, peer_idx, prev_log_index, prev_log_term, leader_commit, n_entries); } /* ha-term-safety task 2.4: drive the WIRED on_commit_advanced callback so a unit * test exercises the real commit-source trigger (feed the library commit_index * into the cluster's commit barrier). Exposed only under XCE_TESTING. */ void ha_cb_commit_advanced_for_test(ha_state_t *ha, uint64_t commit_index) { s_real_cbs.on_commit_advanced(ha, commit_index); } /* ha-term-safety task 3.1: drive the (static) own-term gate arm — the WAL_NOOP * append now routed through ha_primary_log_append — so a unit test can assert * the promotion no-op records the live term via the library. Exposed only under * XCE_TESTING. */ void ha_arm_own_term_gate_for_test(cluster_t *c) { ha_arm_own_term_gate(c); } #endif /* ha_check_heartbeat_nocluster() / ha_watchdog_thread() / ha_start_watchdog() / * ha_stop_watchdog() deleted — watchdog pthread replaced by * ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */ void ha_fini(ha_state_t *ha) { if (!ha) return; /* ha-commit-durability task 4.4: cancel any pending bounded-window FATAL * timer so a stopped member never reaches the deferred FATAL. The delaymgr * task ctx is freed by promotion_fence_timeout_fn when it fires; cancelling * makes that firing a no-op (it observes cancelled=1 and frees). */ ha_cancel_promotion_fence_timer(ha); pthread_mutex_destroy(&ha->lock); pthread_mutex_destroy(&ha->raft_lock); sem_destroy(&ha->heartbeat_seen); /* Fix ASAN leak: free the SSL context if one was allocated */ if (ha->ssl_ctx) { SSL_CTX_free((SSL_CTX *)ha->ssl_ctx); ha->ssl_ctx = NULL; } } /* ── Task 2.1/2.2: WAL catch-up entry application (standby side) ───────────── * * ha_apply_wal_catchup_entry: apply WAL catch-up entries received from the * primary to local persistent storage and advance applied_lsn on success. * * applied_lsn semantic (bug-991): the LAST CONTIGUOUSLY-HELD LSN — the same * meaning recovery (= durable_floor), checkpoint (= checkpoint_lsn), and init * (= 0) assign it, and the watermark the standby advertises to the leader * (sr->applied_lsn) so the leader's catch-up streamer streams the half-open * range (applied_lsn, committed]. NEVER lsn+1 (next-to-apply): a +1 makes the * standby advertise one LSN too high, so the leader skips the real next-needed * entry and the standby's library logOk wedges forever (no WalAck → commit stall). * * Two modes (Req 5.3, 5.5): * * Batch mode (entry->n_entries > 0): * Iterate entry->entries[0..n_entries-1] in order. Persist each * sub-entry individually (entity materialization happens later, once * committed — see ha_standby_advance_committed_and_apply). NULL * sub-entries are treated as corrupt and skipped (Req 3.3). After ALL * sub-entries are processed, * advance applied_lsn to the last-successfully-applied lsn so the next * heartbeat reply reflects post-batch state (Req 5.3). * * Single-entry mode (entry->n_entries == 0): * Backward-compatible path from task 2.1. Apply entry->lsn/entry_type * directly and advance applied_lsn to lsn on success. * * In both modes: * - applied_lsn is advanced atomically and monotonically. * - On error for any individual entry: log ERROR, skip, continue (Req 3.3). * - applied_lsn is in-memory only; not persisted to disk (design §Durability). * - Standby stays in CTRL_STANDBY throughout (Req 3.4). * * Returns 0 on success (even if some batch entries were skipped due to errors). * Returns -EINVAL if ha, p, or entry is NULL. */ /* ha_advance_applied_lsn: atomically advance ha->applied_lsn to new_lsn * if new_lsn > current value (monotonic). */ static void ha_advance_applied_lsn(ha_state_t *ha, uint64_t new_lsn) { uint64_t old_lsn = (uint64_t)atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); while (new_lsn > old_lsn) { if (atomic_compare_exchange_weak_explicit( &ha->applied_lsn, &old_lsn, new_lsn, memory_order_acq_rel, memory_order_acquire)) break; } } /* ha_apply_single_entry: durably persist one decoded wal_catchup_entry_t * sub-entry (name retained for history; despite the name it no longer * materializes entity state here — see ha_standby_advance_committed_and_apply, * 2026-07-10). Returns 0 on success, non-zero on error (caller logs and skips). * * raft-ha-core-safety task 6.2 — TERM-BLIND DIRECT-PERSIST BYPASS, SAFE-BY- * GUARD (documented, not structurally retired): this function does NOT go * through raft_on_append_entries_request's logOk/(term,lsn) reconcile — it * calls persist_wal_write_replicated directly. Task 2.4 deferred deciding * this bypass's fate to 6.2; 6.2's reachability trace (see the two call * sites below and ha_apply_wal_catchup_entry) found it is UNREACHABLE with * real production data today: every real sender (repl_broadcast for live * replication; ha_stream_wal_to_standby for catch-up) always supplies * n_entries>=1 with a non-zero term (persist_entry_term's legacy-zero rule * maps a stored term of 0 to WAL_LEGACY_TERM for any present LSN — it never * returns 0 for a real entry). The ONLY function that could ever drive a * term==0 / n_entries==0 message into this path (repl_send_append_entries) * is called solely from the raft library's raft_send_append_entries, which * has zero production callers (only lib/raft/raft_sim.c and an XCE_TESTING * helper call it) — so that trigger is dormant, not live, code. Even if this * path were reached, the task-2.2 (term,lsn,payload-crc)-aware guard inside * persist_wal_write_replicated already refuses to degrade a stored non-empty * entry to empty and raises the sticky non-promotable fence on an * unrepairable below-tip conflict — pinned by * test_2_3_termblind_route_payload_guarded and * test_2_3_unrepairable_catchup_raises_nonpromotable_fence * (test_ha_core_safety.c). See ha_apply_wal_catchup_entry below and * test_6_2_real_senders_never_produce_termblind_shape for the reachability * evidence this comment summarizes. */ static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e) { /* Persist-before-apply-before-ACK: durably write the replicated entry to * this standby's own WAL BEFORE applying it in memory (and before the * caller ACKs). The ACK feeds the primary's commit majority, so it must * imply durability — a standby must not ACK an entry it could lose on * restart (Raft log[i] includes standbys; ha-async-io.md). * * persist_wal_write_replicated does NOT assign a new lsn (uses e->lsn) and * does NOT re-broadcast; it fsyncs before returning and is idempotent on * re-stream. WAL_CHECKPOINT is never streamed, but it would be rejected * by both this primitive and apply_wal_entry. */ int prc = persist_wal_write_replicated(p, e->lsn, (uint32_t)e->entry_type, e->raw_payload, (uint32_t)e->raw_payload_len); if (prc == -EAGAIN) { /* Contiguity guard (bug-850): this entry is ahead of our log tail * (a forward gap) — e.g. a live broadcast that arrived before the * in-order catch-up of the prefix. NOT an error: drop it quietly and do * NOT ack, so the catch-up continuation re-streams from current_lsn and * the log fills in order. */ TRACE("ha_apply_single_entry: lsn=%" PRIu64 " ahead of log tail — " "dropped (will arrive via in-order catch-up)", e->lsn); return prc; } if (prc != 0) { ERROR("ha_apply_single_entry: persist_wal_write_replicated failed for " "lsn=%" PRIu64 " type=%u rc=%d — not applying/acking", e->lsn, (unsigned)e->entry_type, prc); /* No apply, no ACK on persist failure (mirrors no-ACK-on-apply-fail). */ return prc; } /* commit-then-apply on the standby (2026-07-10): entity materialization * (restore_job/restore_job_state) does NOT happen here on receipt. This * function's job ends at durable persist + ACK — the entry becomes * in-memory state only once this node's OWN p->committed_lsn reaches it, * driven by ha_standby_advance_committed_and_apply (heartbeat-learned * commit position, clamped to this node's local durable tip) and applied * through apply_committed_entry() by the same pump the primary's own * online-apply and restart replay use. Applying eagerly here — before * this node (or the sender) knows the entry reached cluster majority — * could materialize a job the sender never actually committed: if a * later (term,lsn) conflict truncate removes the WAL bytes (the sender * lost leadership before reaching majority), nothing rolls back the * already-applied entity mutation, leaving a permanent phantom job with * no backing WAL entry (2026-07-10 controller-invariant audit finding). * This mirrors bug-1043's own precedent of decoupling durability/ACK from * entity materialization, extended one step further: apply now waits for * COMMIT, not just local durability. */ return 0; } #ifdef XCE_TESTING void (*ha_wal_ack_send_hook)(uint64_t term, uint64_t lsn, int success) = NULL; #endif /* ha_send_wal_ack_reply_ex: send WalAckReply {term, lsn, success} back to * the primary over ep. Skipped when ep or c is NULL (test/offline path), * unless ha_wal_ack_send_hook is set (XCE_TESTING), in which case the hook is * called with the built {term, lsn, success} INSTEAD of sending over ep -- * lets a unit test observe a NACK (success=0) without a live endpoint * (bug-1479). Task 3.1 (ha-wal-replication): ACK/NACK each applied/rejected * WAL entry. */ static void ha_send_wal_ack_reply_ex(ha_state_t *ha, struct endpoint_t *ep, cluster_t *c, uint64_t lsn, int success) { uint64_t term = (uint64_t)atomic_load_explicit(&ha->term, memory_order_acquire); #ifdef XCE_TESTING if (ha_wal_ack_send_hook) { ha_wal_ack_send_hook(term, lsn, success); return; } #endif if (!ep || !c) return; wal_ack_reply_t ack; ack.term = term; ack.lsn = lsn; ack.success = success ? 1 : 0; xce_reply_t reply; memset(&reply, 0, sizeof(reply)); reply.opcode = OPCode_walAck; reply.status = 0; reply.data_which = RequestReply_data_walackreplymsg; reply.data.walackreplymsg = &ack; xce_reply(ep, c->myhost, &reply); } /* ha_send_wal_ack_reply: send a successful WalAckReply {term, lsn, * success=1} -- the common case, every existing call site. */ static void ha_send_wal_ack_reply(ha_state_t *ha, struct endpoint_t *ep, cluster_t *c, uint64_t lsn) { ha_send_wal_ack_reply_ex(ha, ep, c, lsn, /*success*/1); } /* ha_apply_term_entry_via_library: route ONE term-carrying replicated entry * (a live broadcast or a catch-up sub-entry) through the raft library * (raft_on_append_entries_request via ha_consume_append_entries) so the LIBRARY * does logOk + (term,lsn) reconcile and on_persist_entry stamps the LEADER's * term — never the local term-0 direct-apply path (ha-term-safety task 3.4). * * `e` is the single entry (its term/prev_log_index/prev_log_term carry the AE * metadata, set by the leader's broadcast/stream path). On accept (the entry's * term is now recorded at its LSN) the caller ACKs and advances applied. Returns * 1 if the entry was accepted (recorded), 0 otherwise (reject / not yet * applicable — e.g. a logOk-failing forward gap that catch-up will backfill). */ static int ha_apply_term_entry_via_library(ha_state_t *ha, persist_t *p, cluster_t *c, struct endpoint_t *ep, wal_catchup_entry_t *e) { /* bug-1475 / bug-1466: resync the raft library's log_len from this node's * OWN recovered/durable WAL tip before consuming the AE, exactly mirroring * ha_sync_raft_log_from_recovery's boot-time computation (persist_durable_ * floor), but applied live, JUST-IN-TIME, on this same single AE-receive * call path — never touching persist.c or adding any new cross-thread * mutation of ha->raft (ha_apply_wal_catchup_entry / this function already * own that field on this thread, same as every other s->log_len write in * the AE-receive path below). * * Root cause: persist_write_checkpoint_marker (a LOCAL-only, never- * replicated WAL_CHECKPOINT entry written by ANY role — primary or * standby — whenever a committed WAL_CONFIG_MARKER triggers an immediate * checkpoint) advances persist_t's current_lsn directly, WITHOUT going * through raft_log_append — so s->log_len silently falls one behind the * true physical WAL tip for every checkpoint written since the last * resync (documented gap, bug-1466). Once behind, the very next AE for * the entry immediately following the checkpoint carries a prev_log_index * equal to the checkpoint's own LSN (config_repl.c's streamer correctly * advances its tracker across the skipped checkpoint) — but a stale * s->log_len makes logOk's `prev_log_index <= s->log_len` bound fail * forever, permanently wedging this standby's catch-up (every subsequent * heartbeat re-streams the identical doomed range with zero progress). * * persist_durable_floor is a lock-free atomic read of durable_lsn (the * flusher's own fsync-confirmed watermark, which DOES count a checkpoint's * consumed LSN slot, same as at boot) — safe to call from any thread. The * write below (raising ha->raft.log_len) happens on the same thread that * already owns every other s->log_len mutation in this call chain * (raft_on_append_entries_request), so no new lock is introduced. * Monotonic: only ever raises log_len, mirroring * ha_sync_raft_log_from_recovery exactly. Deliberately does NOT touch the * term index (on_get_entry_term(checkpoint_lsn) stays 0/absent on every * member, by design — see persist_write_checkpoint_marker's own comment) * — recording a synthetic term here would risk a cross-member mismatch * (each member's checkpoint stamps ITS OWN current term at write time, * which need not agree with a peer's), whereas leaving it absent keeps * the logOk equality check (0==0) trivially and SAFELY consistent on * every member. * * bug-1475 follow-up: the "stays absent on every member" claim above only * holds because persist.c's term index now uses a TERM_INDEX_UNSET * sentinel to keep a never-explicitly-recorded LSN (like a checkpoint's) * reading back absent PERMANENTLY. Before that fix, persist.c's array- * growth zero-fill silently reclassified the checkpoint's gap slot as * present-with-WAL_LEGACY_TERM the instant any LATER lsn was recorded — * which happens immediately on the sender (it streams after its own * later entries exist) but not yet on a receiver that hasn't appended * past its own checkpoint, so the two sides read DIFFERENT terms for the * SAME checkpoint LSN and logOk permanently rejected. See * TERM_INDEX_UNSET's comment (persist.c) and * test_bug1475_catchup_survives_midstream_checkpoint (test_config_repl.c) * for the full mechanism and live-instrumentation evidence. */ if (p) { uint64_t recovered_tip = persist_durable_floor(p); if (recovered_tip > ha->raft.log_len) ha->raft.log_len = recovered_tip; } /* Wrap the single entry in an AE container whose TOP-LEVEL fields carry the * prev tuple + term that ha_consume_append_entries reads, with entries[0] * pointing at the entry itself (the same shape repl_send_append_entries / * repl_broadcast produce on the wire). */ wal_catchup_entry_t *sub_arr[1] = { e }; wal_catchup_entry_t ae; memset(&ae, 0, sizeof(ae)); ae.n_entries = 1; ae.entries = sub_arr; ae.term = e->term; ae.prev_log_index = e->prev_log_index; ae.prev_log_term = e->prev_log_term; ae.leader_commit = e->leader_commit; uint64_t before_term = persist_entry_term(p, e->lsn); int rc = ha_consume_append_entries(c, &ae, /*leader_idx*/ -1); if (rc != 0) return 0; uint64_t after_term = persist_entry_term(p, e->lsn); if (after_term == e->term) { /* Accepted (appended / conflict-truncate-then-persist) or already-have * (idempotent re-stream): ACK + advance applied on that signal. * * bug-991: applied_lsn is the "last contiguously-held LSN" (the same * semantic recovery/checkpoint/init use, and the watermark the leader's * catch-up streamer reads via sr->applied_lsn to stream (standby_lsn, * committed]). Advance to e->lsn (the LSN we just held), NOT e->lsn + 1: * a +1 over-advertises one LSN, so the leader's next stream begins at * e->lsn + 2 and SKIPS the real next-needed entry e->lsn + 1, wedging * logOk and stalling commit. */ (void)before_term; /* commit-then-apply on the standby (2026-07-10, supersedes the prior * bug-1043 re-apply-here fix): entity materialization does NOT happen * on receipt/persist. bug-1043's original problem — a caught-up * standby held entries on disk but not in the queryable job table — * is now solved by ha_standby_advance_committed_and_apply, driven by * this node's heartbeat-learned commit position, applying through the * SAME apply_committed_entry() seam the primary's online-apply and * restart replay use. Applying here instead (right on receipt, before * this entry is known to have reached cluster majority) risks a * permanent phantom job: a later (term,lsn) conflict truncate removes * the WAL bytes with no entity rollback (2026-07-10 controller- * invariant audit finding). */ ha_advance_applied_lsn(ha, e->lsn); ha_send_wal_ack_reply(ha, ep, c, e->lsn); return 1; } /* logOk reject (e.g. forward gap before the prefix arrives, or a genuine * (term,lsn) conflict): bug-1479 -- send a NACK (success=0) so the * primary's ha_raft_on_repl_nack can back off nextIndex and immediately * mark this standby for catch-up, matching raft.tla's AppendEntries * action (the receiver ALWAYS replies, success = logOk). Before this fix * NOTHING was sent on reject: the primary's NACK-handling branch * (repl_ack_decode_and_dispatch) was correct but unreachable, since no * production path ever produced the success=0 reply it was waiting for * -- the standby would loop rejecting the identical re-broadcast entry * forever, discovered live via the config-upgrade soak's force-elect * rotation. */ ha_send_wal_ack_reply_ex(ha, ep, c, e->lsn, /*success*/0); return 0; } int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p, wal_catchup_entry_t *entry, struct endpoint_t *ep, cluster_t *c) { if (!ha || !p || !entry) return -EINVAL; /* ── ha-term-safety task 3.2: term-carrying AppendEntries → library route ── * A term-safe AppendEntries (one sent by repl_send_append_entries / the * library send path) carries a non-zero entry term stamp. Route it through * raft_on_append_entries_request so the LIBRARY performs UpdateTerm (G5), * logOk (G2), and the (term,lsn) conflict-truncate (G3) — its on_persist_entry * callback does the WAL write (persist-before-ACK + bug-850 guard beneath). * This REPLACES the direct persist_wal_write_replicated apply for term-safe * entries; the legacy term-blind broadcast (entry term 0, no prev tuple) keeps * the direct apply below until the live send path is retired (task 3.3). * * Single-entry term-safe AE only (n_entries==1 with a term). The library * computes the target index from the prev tuple, then ACK on accept. */ if (c && c->ha.raft_c == c && entry->n_entries == 1 && entry->entries && entry->entries[0] && entry->entries[0]->term != 0) { ha_apply_term_entry_via_library(ha, p, c, ep, entry->entries[0]); /* 2026-07-10: a fresh entry just landed on disk -- kick the apply * pump against the already-known commit position so a standby mid * catch-up doesn't wait for the next heartbeat tick to materialize * entries it's already known to have committed. */ ha_standby_advance_committed_and_apply( c, atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire)); return 0; } /* ── Batch mode: entry->n_entries > 0 ─────────────────────────────────── */ if (entry->n_entries > 0) { /* Apply each sub-entry in LSN order. Track the last successfully * applied LSN so we can advance applied_lsn after the full batch. * Req 5.5: applied_lsn advances only after the full batch is written. * Req 3.3: NULL or failed entries are logged and skipped; no crash. */ uint64_t last_good_lsn = 0; int had_good = 0; for (int i = 0; i < entry->n_entries; i++) { wal_catchup_entry_t *sub = entry->entries[i]; if (!sub) { ERROR("ha_apply_wal_catchup_entry: NULL sub-entry at " "batch index %d — skipping", i); continue; } /* ha-term-safety task 3.4: a term-carrying catch-up sub-entry routes * through the LIBRARY (logOk + (term,lsn) reconcile + on_persist_entry * stamps the LEADER's term). Each sub-entry carries its own prev tuple * (wal_entry_to_wire encodes prevLog{Index,Term} per element). This * retires the term-0 direct-apply path for live/catch-up replication. * Entries stream in ascending LSN order, so the library's contiguity * (prev tuple) is satisfied in sequence; a not-yet-applicable entry * (logOk reject) is left for the catch-up continuation. * * raft-ha-core-safety task 6.2: the `sub->term != 0` fallthrough * below is the term-blind direct-persist bypass (ha_apply_single_ * entry) — documented SAFE-BY-GUARD, not structurally retired. * ha_stream_wal_to_standby (the only real batch-catch-up sender) * always stamps sub->term via persist_entry_term, which maps a * legacy on-disk term of 0 to WAL_LEGACY_TERM for any present LSN, * so a real streamed sub-entry is never term==0 — this fallthrough * is unreachable with production data (see the ha_apply_single_ * entry comment for the full reachability trace + the pin test * test_6_2_real_senders_never_produce_termblind_shape). */ if (c && c->ha.raft_c == c && sub->term != 0) { if (ha_apply_term_entry_via_library(ha, p, c, ep, sub)) { last_good_lsn = sub->lsn; had_good = 1; } continue; } int rc = ha_apply_single_entry(p, sub); if (rc != 0) { /* -EAGAIN = contiguity gap (bug-850): entry ahead of our tail, * dropped quietly — not an error (catch-up will backfill). */ if (rc != -EAGAIN) ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed " "for batch index %d lsn=%" PRIu64 " type=%u rc=%d " "— skipping entry", i, sub->lsn, (unsigned)sub->entry_type, rc); /* Task 3.2: no ACK for failed/dropped entries */ continue; } /* Task 3.1: ACK each successfully applied sub-entry */ ha_send_wal_ack_reply(ha, ep, c, sub->lsn); last_good_lsn = sub->lsn; had_good = 1; } /* Advance applied_lsn to last_good_lsn after the full batch (bug-991: * last contiguously-held LSN, NOT last_good_lsn + 1 — see the * ha_apply_term_entry_via_library comment for why the +1 wedged the * leader's catch-up streamer). Only advance if at least one entry was * applied successfully. */ if (had_good) ha_advance_applied_lsn(ha, last_good_lsn); /* 2026-07-10: kick the apply pump against the already-known commit * position so a batch catch-up materializes promptly instead of * waiting for the next heartbeat tick (see the single-entry-term-safe * branch above for the full rationale). */ if (had_good) ha_standby_advance_committed_and_apply( c, atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire)); return 0; } /* ── Single-entry mode (task 2.1 backward-compatible path) ────────────── * raft-ha-core-safety task 6.2: this branch calls ha_apply_single_entry * UNCONDITIONALLY — no term check at all — so it is the other half of the * term-blind bypass documented at ha_apply_single_entry above. It is only * ever reached with entry->n_entries==0, the 0-entry AppendEntries * heartbeat shape built by repl_send_append_entries. That function is * called solely from the raft library's raft_send_append_entries, which * has ZERO production call sites (only lib/raft/raft_sim.c and an * XCE_TESTING test helper call it) — so in the CURRENT codebase this * branch is dormant/unreachable from any live sender, not a legacy- * rollout shape carrying real data. Documented SAFE-BY-GUARD (not * structurally retired): see the ha_apply_single_entry comment for the * full reachability trace and the task-2.2 guard that would still bound * the damage if this ever became reachable. */ int rc = ha_apply_single_entry(p, entry); if (rc != 0) { /* -EAGAIN = contiguity gap (bug-850): dropped quietly, backfilled by * the in-order catch-up; any other rc is a real failure. */ if (rc != -EAGAIN) ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed for " "lsn=%" PRIu64 " type=%u rc=%d — skipping entry", entry->lsn, (unsigned)entry->entry_type, rc); /* Task 3.2: no ACK on apply failure */ return rc; } /* Advance applied_lsn atomically to the last contiguously-held LSN * (bug-991: entry->lsn, NOT entry->lsn + 1 — see the * ha_apply_term_entry_via_library comment for the streamer-skip wedge a +1 * caused). This is the same last-held semantic recovery/checkpoint/init use, * and matches the (standby_lsn, committed] range the leader streams. */ ha_advance_applied_lsn(ha, entry->lsn); /* Task 3.1: ACK the successfully applied single entry */ ha_send_wal_ack_reply(ha, ep, c, entry->lsn); /* 2026-07-10: kick the apply pump against the already-known commit * position (see the batch-mode branch above for the full rationale). */ ha_standby_advance_committed_and_apply( c, atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire)); return 0; } /* ── ha-term-safety task 3.2: library-routed AppendEntries intake ─────────── * * ha_consume_append_entries: route ONE incoming heartbeat (0-entry) or * replication (1-entry) AppendEntries through the raft library * (raft_on_append_entries_request) so the LIBRARY — not hand-rolled term-blind * code — performs: * - UpdateTerm first on EVERY RPC incl. heartbeat (G5, R2.6): a higher-term * AE makes the node adopt the term + step down (via on_become_follower). * - logOk (G2, R3.1): (prevLogIndex,prevLogTerm) match against the local log * via on_get_entry_term — a mismatch is REJECTED (no false convergence). * - (term,lsn) conflict-truncate (G3, R3.2/R3.3): a divergent same-LSN tail is * truncated (on_truncate_from) then the leader's entry re-persisted * (on_persist_entry), never touching the agreed prefix below the divergence. * * This REPLACES the term-blind term handling in ha_consume_heartbeat_committed * and the committed-boundary truncate in ha_rejoin_reconcile: the library's * (term,lsn) decision, not a watermark, drives truncation. * * `entry` is a decoded wal_catchup_entry_t whose TOP-LEVEL fields carry the AE * metadata (prev_log_index, prev_log_term, leader_commit) and, for a single * entry, the entry itself (lsn, entry_type, term, raw_payload). `leader_idx` is * the raft peer index of the sending leader. * * The library callbacks own all persistence/truncation; on a REJECT (stale term * or logOk fail) the library mutates nothing and the member keeps its current * state to retry on a later catch-up (R3.5/R3.6 — never report false * convergence). The loud non-promotable fence for an unreconcilable member is * the EXISTING ha-commit-durability promotion-time fence; this intake never * advertises convergence it did not achieve. * * Single-receiver-thread model (controller request handling), matching the * raft_on_vote_request intake: no ha->lock is taken (the raft state is serial * per the receive thread). Returns 0 (no crash) on every path; the AE result is * reflected in the library/WAL state, and a future ACK reply (task 3.3) carries * the success/match_index to the leader. */ int ha_consume_append_entries(cluster_t *c, wal_catchup_entry_t *entry, int leader_idx) { if (!c || !entry) return -EINVAL; int n_entries = (entry->n_entries > 0) ? 1 : 0; /* 0 = heartbeat, 1 = entry */ /* AppendEntries metadata from the top-level container. */ uint64_t prev_log_index = entry->prev_log_index; uint64_t prev_log_term = entry->prev_log_term; uint64_t leader_commit = entry->leader_commit; /* The incoming AE's term: for a single entry, the entry's term stamp; for a * heartbeat, the top-level container's term (set by the sender to its live * term). raft_on_append_entries_request uses this for UpdateTerm (G5). */ uint64_t ae_term; const void *blob = NULL; size_t blob_len = 0; uint64_t entry_term = 0; /* For a single entry, repack the [raft_entry_blob_t{type,len}][payload] * envelope that on_persist_entry unpacks (the shared wal_types.h format), so * the library-driven persist writes the same bytes the leader sent. */ uint8_t stackbuf[512]; uint8_t *heapbuf = NULL; if (n_entries == 1) { /* Single-entry mode: the entry lives in entries[0] (batch) or on the * top-level container (single). Prefer the sub-entry when present. */ wal_catchup_entry_t *e = (entry->entries && entry->entries[0]) ? entry->entries[0] : entry; uint32_t type = e->entry_type; const void *payload = e->raw_payload; uint32_t plen = (uint32_t)e->raw_payload_len; entry_term = e->term; /* bug-1067: the AppendEntries HEADER term (UpdateTerm/stale-check) is the * leader's CURRENT term, NOT the entry's stored term. For a LIVE append * the two are equal (the leader appends at its current term), but for a * CATCH-UP entry written by a PRIOR leader the stored term is LOWER than * the current leader's term — using it as the AE term made the library * stale-reject (term < s->term) every historical entry, wedging cross-term * catch-up forever. The streamer stamps leader_current_term = ha->term; * fall back to the stored term only when it is unset (legacy sender). */ ae_term = (e->leader_current_term != 0) ? e->leader_current_term : entry_term; blob_len = sizeof(raft_entry_blob_t) + plen; uint8_t *buf = (blob_len <= sizeof(stackbuf)) ? stackbuf : (heapbuf = malloc(blob_len)); if (!buf) return -ENOMEM; raft_entry_blob_t *bh = (raft_entry_blob_t *)buf; bh->type = type; bh->payload_len = plen; if (payload && plen > 0) memcpy(buf + sizeof(*bh), payload, plen); blob = buf; } else { /* Heartbeat: the top-level container's term carries the leader's live * term (the sender stamps it). No entry to persist. */ ae_term = entry->term; } raft_append_result_t res; memset(&res, 0, sizeof(res)); int rc = raft_on_append_entries_request(&c->ha.raft, leader_idx, ae_term, prev_log_index, prev_log_term, leader_commit, blob, blob_len, entry_term, n_entries, &res); if (heapbuf) free(heapbuf); if (rc != 0) { /* Library-internal error (not a protocol reject — those return RAFT_OK * with res.success==0). Log loud; the member stays available to retry. */ WARN("ha_consume_append_entries: raft_on_append_entries_request rc=%d " "(leader_idx=%d term=%" PRIu64 " prevIdx=%" PRIu64 ")", rc, leader_idx, ae_term, prev_log_index); return 0; /* never crash the receive thread */ } TRACE("ha_consume_append_entries: leader_idx=%d term=%" PRIu64 " prevIdx=%" PRIu64 " n=%d -> success=%d match=%" PRIu64, leader_idx, ae_term, prev_log_index, n_entries, res.success, res.match_index); /* res.success / res.match_index are consumed by the ACK reply path (task * 3.3). On a reject (res.success==0) the library left the log untouched — * R3.5/R3.6: no false convergence, the member reconciles on a later AE. */ return 0; } /* ha_handle_snapshot_chunk — standby snapshot receive handler (Task 3.5c). * * Each call appends payload_bytes to snap_dir/snap.part. * When chunk->is_last is non-zero: * 1. Verify total bytes written matches chunk->total_size. * 2. Atomically rename snap.part → snap-{snapshot_lsn:016x}.bin. * 3. Load the snapshot via persist_load_snapshot(). * 4. Set ha->applied_lsn = p->checkpoint_lsn (in-memory). * * Mid-transfer reconnect (Req 4.4): the primary restarts every transfer at * chunk 0, so chunk_index == 0 discards any stale snap.part left by an * interrupted transfer before appending — no cross-chunk state needed. * Returns 0 on success, negative errno on error. * Requirements: 4.2, 4.3, 4.4 (ha-repl-hardening spec) */ int ha_handle_snapshot_chunk(ha_state_t *ha, persist_t *p, const char *snap_dir, snapshot_transfer_t *chunk, const uint8_t *payload_bytes, size_t payload_len) { if (!ha || !p || !snap_dir || !chunk) return -EINVAL; char part_path[4096 + 16]; snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir); /* Req 4.4: a transfer always restarts from chunk 0 — discard any stale * partial file from an interrupted previous transfer, and reset the * completion bookkeeping below for the fresh attempt. */ pthread_mutex_lock(&ha->lock); if (chunk->chunk_index == 0) { if (unlink(part_path) != 0 && errno != ENOENT) WARN("ha_handle_snapshot_chunk: unlink stale snap.part failed: %s", strerror(errno)); ha->snapshot_recv_count = 0; ha->snapshot_recv_total = 0; } pthread_mutex_unlock(&ha->lock); /* bug-1389 triage: crc32 of the exact bytes this chunk arrived with, * taken right after decode and before any local write — correlate * against ha_send_snapshot_to_standby's send-side crc32 log * (config_repl.c) by (snapshot_lsn, chunk_index) to localize whether a * mismatch is introduced before send, in the wire transport, or during * receive-side assembly. */ if (payload_bytes && payload_len > 0) TRACE("ha_handle_snapshot_chunk: chunk crc32 " "checkpoint_lsn=%" PRIu64 " chunk_index=%u len=%zu crc32=%08x", chunk->snapshot_lsn, chunk->chunk_index, payload_len, (unsigned)crc32(0L, (const Bytef *)payload_bytes, (uInt)payload_len)); /* bug-1389: chunks of one transfer are dispatched across the vtask pool * and are NOT guaranteed to arrive/execute in chunk_index order (a live * soak observed e.g. chunk_index 2 processed AFTER 3 and 4) — a live * crc32 correlation confirmed every chunk's CONTENT survives the wire * intact, but the OLD append-in-arrival-order write put reordered * chunks' bytes at the wrong file position. pwrite() at the chunk's own * chunk_offset makes assembly order-independent and safe for concurrent * writers (positioned writes to disjoint byte ranges of the same fd * don't race each other). */ if (payload_bytes && payload_len > 0) { int fd = open(part_path, O_CREAT | O_WRONLY, 0600); if (fd < 0) { int err = errno; ERROR("ha_handle_snapshot_chunk: cannot open snap.part: %s", strerror(err)); return -err; } ssize_t written = pwrite(fd, payload_bytes, payload_len, (off_t)chunk->chunk_offset); close(fd); if (written != (ssize_t)payload_len) { ERROR("ha_handle_snapshot_chunk: short write to snap.part " "(%zd of %zu bytes) at offset %" PRIu64, written, payload_len, chunk->chunk_offset); return -EIO; } } /* bug-1389: finalize once every chunk index 0..N-1 has actually been * written, not merely once THIS call happens to carry is_last — under * out-of-order arrival, is_last can be processed before an earlier * chunk. snapshot_recv_total is claimed (reset to 0) by whichever * thread wins the race so a late straggler chunk can never trigger a * second finalize. */ pthread_mutex_lock(&ha->lock); ha->snapshot_recv_count++; if (chunk->is_last) ha->snapshot_recv_total = chunk->chunk_index + 1; int should_finalize = (ha->snapshot_recv_total != 0 && ha->snapshot_recv_count >= ha->snapshot_recv_total); if (should_finalize) ha->snapshot_recv_total = 0; pthread_mutex_unlock(&ha->lock); if (!should_finalize) return 0; /* All chunks accounted for: verify size, rename, load, set applied_lsn */ /* Verify total file size equals chunk->total_size */ struct stat st; if (stat(part_path, &st) != 0) { int err = errno; ERROR("ha_handle_snapshot_chunk: stat(snap.part) failed: %s", strerror(err)); return -err; } if ((uint64_t)st.st_size != chunk->total_size) { ERROR("ha_handle_snapshot_chunk: size mismatch — " "written=%" PRIu64 " expected=%" PRIu64, (uint64_t)st.st_size, chunk->total_size); return -EIO; } /* Atomic rename snap.part → snap-{lsn:016x}.bin */ char final_path[4096 + 32]; snprintf(final_path, sizeof(final_path), "%s/snap-%016" PRIx64 ".bin", snap_dir, chunk->snapshot_lsn); if (rename(part_path, final_path) != 0) { int err = errno; ERROR("ha_handle_snapshot_chunk: rename snap.part → %s failed: %s", final_path, strerror(err)); return -err; } /* Load the snapshot — sets p->checkpoint_lsn and p->current_lsn. * ha-entity-snapshots task 6.1: entities_restored is consumed below, * purely for observability — it does not change the rc-gates-applied_lsn * ordering here (persist_load_snapshot/load_snapshot already gate their * own position advance on entity-restore success, task 5.2). */ bool entities_restored = false; int rc = persist_load_snapshot(p, final_path, chunk->snapshot_lsn, &entities_restored); if (rc != 0) { ERROR("ha_handle_snapshot_chunk: persist_load_snapshot failed rc=%d", rc); return rc; } /* raft-ha-core-safety task 3.2 (Req 3.1/5.1) — Snapshot Boundary * Reconcile: authoritatively overwrite the per-LSN term index at the * checkpoint LSN with the checkpoint's own term (as carried on the * wire). Establishes a verifiable (term, lsn) boundary so a divergent * pre-snapshot entry's STALE local term at this exact LSN is reconciled * rather than silently surviving the install. Applies identically to * both a legacy (position-only) and an entity-bearing checkpoint — the * term reconcile is orthogonal to entity content. chunk->checkpoint_term * == 0 is unambiguous and means the checkpoint LSN is genuinely absent * from the sender's own term index, never "legacy term 0, present" * (bug-1475 Finding 2 — see persist_reconcile_checkpoint_term's own doc * comment, persist.h). persist_reconcile_checkpoint_term itself * special-cases a wire 0 to TERM_INDEX_UNSET before recording, so the * receiver reads this LSN back as absent too, matching the sender. */ persist_reconcile_checkpoint_term(p, chunk->snapshot_lsn, chunk->checkpoint_term); /* Advance applied_lsn to checkpoint_lsn (in-memory only — Req 4.2) */ atomic_store_explicit(&ha->applied_lsn, p->checkpoint_lsn, memory_order_release); /* Observability only (Requirements 2.4, 4.1, 4.2, 5.1, 5.2) — a legacy * (position-only) checkpoint is an expected rollout state, not an error: * log it as informational and let the member continue through the * existing incremental catch-up path for the remaining history. */ if (entities_restored) { INFO("HA: snapshot loaded lsn=%" PRIu64 " with full job/queue entity " "restoration — applied_lsn set", p->checkpoint_lsn); } else { INFO("HA: snapshot lsn=%" PRIu64 " is a legacy position-only " "checkpoint (no entity content) — applied_lsn set, continuing " "via normal incremental catch-up", p->checkpoint_lsn); } return 0; } /* ha_nop_reply_state_report — populate *sr from ha state and wire into reply. * Reads ha->applied_lsn (atomic) and myconf->config_version_tag, stores them * in *sr, then sets reply_out->data_which = stateReportMsg and * reply_out->data.state_report = sr. * Caller keeps *sr alive until xce_reply() serialises the message. * Requirements: 1.1, 1.3, 1.4 (ha-repl-hardening spec) */ void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf, state_report_t *sr, xce_reply_t *reply_out) { sr->applied_lsn = atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); sr->config_tag = myconf ? myconf->config_version_tag : 0; sr->term = atomic_load_explicit(&ha->term, memory_order_acquire); reply_out->data_which = RequestReply_data_stateReportMsg; reply_out->data.state_report = sr; } /* * ha_evaluate_repl_action — pure decision function for the unified heartbeat- * driven replication protocol. * * Implements the decision tree from the ha-repl-hardening design: * * if standby_tag < committed_tag → HA_REPL_PUSH_CONFIG * elif standby_lsn + 1 < oldest_wal_lsn → HA_REPL_SEND_SNAPSHOT * elif standby_lsn < committed_lsn → HA_REPL_STREAM_WAL * else (tag >= committed AND lsn >= committed) → HA_REPL_CONVERGED * * No side effects. All callers must use the returned action to drive the * replication send; they must not cache the result beyond the current cycle. * * Requirements: 2.2, 2.3, 2.4 (ha-repl-hardening spec) * Boundary: Primary Heartbeat Handler (ha.c) */ ha_repl_action_t ha_evaluate_repl_action(uint64_t standby_tag, uint64_t standby_lsn, uint64_t committed_tag, uint64_t committed_lsn, uint64_t oldest_wal_lsn) { if (standby_tag < committed_tag) return HA_REPL_PUSH_CONFIG; /* Snapshot only when the standby's NEXT-needed entry (standby_lsn + 1) has * been garbage-collected below the oldest retained WAL entry — i.e. it can * no longer be served by streaming. An empty standby (standby_lsn=0) on a * fresh cluster whose oldest retained entry IS lsn 1 (post reserve-lsn-0) * needs lsn 1, which is retained: 0 + 1 < 1 is false → STREAM, no snapshot. * Using standby_lsn + 1 (not standby_lsn - ... ) avoids unsigned underflow * when oldest_wal_lsn == 0 (empty WAL): 0 + 1 < 0 is false → not snapshot. */ if (standby_lsn + 1 < oldest_wal_lsn) return HA_REPL_SEND_SNAPSHOT; if (standby_lsn < committed_lsn) return HA_REPL_STREAM_WAL; return HA_REPL_CONVERGED; } /* * ha_process_standby_state — update per-standby convergence tracking. * * Called on each heartbeat reply from a standby (CTRL_PRIMARY). * Stores (reported_tag, reported_lsn) into the per-standby * arrays, evaluates convergence, and logs INFO on the first * not-converged → converged transition (Req 7.2). Regression resets the flag * (Req 2.5). * * Convergence: reported_tag >= committed_tag AND reported_lsn >= committed_lsn. * * Requirements: 1.2, 2.2, 2.5 (ha-repl-hardening spec) */ void ha_process_standby_state(ha_state_t *ha, int peer_idx, uint64_t reported_tag, uint64_t reported_lsn, uint64_t committed_tag, uint64_t committed_lsn) { if (peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES) return; /* Store reported values */ ha->standby_reported_tag[peer_idx] = reported_tag; ha->standby_reported_lsn[peer_idx] = reported_lsn; /* Evaluate convergence */ bool was_converged = ha->standby_converged[peer_idx]; bool now_converged = (reported_tag >= committed_tag) && (reported_lsn >= committed_lsn); ha->standby_converged[peer_idx] = now_converged; /* Log INFO on first transition to converged (Req 7.2) */ if (now_converged && !was_converged) { INFO("HA: standby[%d] converged: config_tag=%" PRIu64 " applied_lsn=%" PRIu64 " (committed_tag=%" PRIu64 " committed_lsn=%" PRIu64 ")", peer_idx, reported_tag, reported_lsn, committed_tag, committed_lsn); } } /* * ha_compute_repl_target_lsn: the replication/convergence target that * ha_dispatch_repl_action_for_standby's heartbeat-driven decision compares * every standby's reported applied_lsn against. * * The correct target is simply the real, raft/commit-engine-authoritative * committed_lsn — the only position a standby can safely (and eventually) * be told it must catch up to. The ONE documented exception is cluster * BOOTSTRAP: when committed_lsn is genuinely stuck at 0 (no standby has ever * acked anything yet, so the commit engine's majority computation has never * had anything to advance against), last_written (the raw locally-appended * WAL tip) is used instead, so a first-ever connecting standby is told to * catch up to real data instead of being handed "nothing to stream". * * bug-1478: this fallback must apply ONLY in that committed_lsn==0 bootstrap * case — NOT unconditionally whenever last_written happens to exceed * committed_lsn. last_written (persist.current_lsn - 1) counts EVERY * locally-appended WAL entry, including a local-only, NEVER-replicated * WAL_CHECKPOINT (persist_write_checkpoint_marker bumps current_lsn * directly, bypassing raft_log_append/the commit engine entirely — the * same architectural gap flagged in bug-1466). Once such a checkpoint * trails an already-advancing (non-zero) committed_lsn, last_written * permanently sits one LSN past it — and since a checkpoint's own consumed * LSN slot is, by design, never streamed or applied by any standby * (ha_stream_wal_to_standby / persist_wal_write_replicated / * persist_wal_async all explicitly refuse WAL_CHECKPOINT), no standby's * applied_lsn can EVER legitimately reach that value. Taking the * max(committed_lsn, last_written) unconditionally turned the bootstrap * convenience into a permanent, unsatisfiable catch-up target for every * standby the instant a marker-triggered checkpoint (config-upgrade- * snapshot task 5) or a periodic WAL-threshold checkpoint interleaves ahead * of the next real committed advance — ha_evaluate_repl_action's * standby_lsn < committed_lsn check then never reaches HA_REPL_CONVERGED, * and ha_stream_wal_to_standby is re-invoked every heartbeat forever, * finding nothing new to send each time (bug-1478). * * Pure, side-effect-free — safe to unit test directly. */ uint64_t ha_compute_repl_target_lsn(uint64_t committed_lsn, uint64_t last_written) { if (committed_lsn == 0 && last_written > 0) return last_written; return committed_lsn; } /* bug-1479 (gap 3): see full doc comment in ha.h. Distinct from * ha_compute_repl_target_lsn above (the CONVERGENCE target) -- this is the * STREAM ceiling: how far a leader may actually push WAL to a standby, which * must extend past committed_lsn or a standby whose own ack is required to * advance committed_lsn can never receive the entries it needs to ack. */ uint64_t ha_compute_repl_stream_ceiling_lsn(uint64_t committed_lsn, uint64_t last_written, uint64_t checkpoint_lsn) { if (last_written == checkpoint_lsn && last_written > committed_lsn) last_written = committed_lsn; return (last_written > committed_lsn) ? last_written : committed_lsn; } /* ─────────────────────────────────────────────────────────────────────── * raft-drive-async Task 3.2 — SnapshotSend slow-lane offload (bug-1198). * * The snapshot transfer (a fresh persist_checkpoint → fdatasync, then a * chunked socket write of the snapshot file) is the one unbounded operation * reachable from the repl-ack decision loop. Run inline it blocks the * io_worker/drive thread for the whole transfer (and queues that peer's later * messages behind it). This offloads it to the bounded slow lane, keyed per * standby so at most one snapshot per peer is in flight (Req 1.5). * ─────────────────────────────────────────────────────────────────────── */ typedef struct { cluster_t *c; endpoint_t *ep; int peer_idx; } snapshot_send_ctx_t; /* ha_snapshot_do_send: the actual (blocking) snapshot work — shared by the * inline path and the lane worker so both behave identically. In XCE_TESTING * builds the send is replaced by ha_repl_send_snapshot_hook(peer_idx). */ static void ha_snapshot_do_send(cluster_t *c, endpoint_t *ep, int peer_idx) { #ifdef XCE_TESTING if (ha_repl_send_snapshot_hook != NULL) { ha_repl_send_snapshot_hook(peer_idx); return; } #endif if (c && c->persist.snap_path[0]) { extern int ha_send_snapshot_to_standby(endpoint_t *, const char *, uint64_t, uint64_t); /* Req 4.1: create a FRESH checkpoint so the transferred snapshot * reflects current state (also trims WAL + rotates old snaps). * -EBUSY means another checkpoint is in flight — fall back to the * existing checkpoint_lsn rather than skipping the cycle (at most one * checkpoint in flight is the existing persist_checkpoint invariant). */ int ckpt_rc = persist_checkpoint(&c->persist); if (ckpt_rc != 0 && ckpt_rc != -EBUSY) WARN("HA repl: persist_checkpoint before snapshot send " "failed rc=%d — sending existing checkpoint", ckpt_rc); /* raft-ha-core-safety task 3.2: source the checkpoint's own term so * the standby can reconcile its per-LSN term index at the boundary * after install (Snapshot Boundary Reconcile). */ uint64_t checkpoint_term = persist_entry_term(&c->persist, c->persist.checkpoint_lsn); ha_send_snapshot_to_standby(ep, c->persist.snap_path, c->persist.checkpoint_lsn, checkpoint_term); } } /* * snapshot_lane_fn: slow-lane worker entry. Runs the bounded blocking * snapshot send off the drive thread, then ALWAYS clears the per-peer * in-flight flag (success or failure) so the dedup never wedges and a later * heartbeat can retry. Frees the ctx. * * BORROWED-ENDPOINT LIFETIME (bug-1197 family): `ep` here is a standby * endpoint shared with the io_worker epoll loop, NOT one this lane owns * (contrast reconnect_lane_fn, which owns the ep it creates). ha_snapshot_ * send_dispatch takes an endpoint_ref() on `ep` before submitting (mirrors * every vtask-bound ep — vtask_enq +1 / run_task -1) so a concurrent standby * teardown (RDHUP -> zarray_del -> endpoint_killer -> endpoint_unref, the * general bug-1298 refcount system) can only drop the OWNER's reference * while this lane transfer is in flight — it cannot free `ep` out from under * the send. This function's endpoint_unref below is the matching release. */ static void snapshot_lane_fn(void *arg) { snapshot_send_ctx_t *sx = (snapshot_send_ctx_t *)arg; if (!sx) return; cluster_t *c = sx->c; int peer_idx = sx->peer_idx; ha_snapshot_do_send(c, sx->ep, peer_idx); /* Clear in-flight LAST so a concurrent heartbeat tick cannot enqueue a * second snapshot for this peer until this one has fully finished. * Clear the tracked LSN (Req 3.1-3.3) alongside the flag — same * lifecycle, one more field — so rotate_snapshots never sees a stale * in-flight LSN for a transfer that has already completed. */ atomic_store_explicit(&c->ha.snapshot_inflight_lsn[peer_idx], 0, memory_order_release); atomic_store_explicit(&c->ha.snapshot_inflight[peer_idx], 0, memory_order_release); endpoint_unref(sx->ep); free(sx); } /* * ha_snapshot_send_dispatch: route a SEND_SNAPSHOT decision either inline * (flag OFF, production default — runs on the caller/drive thread, unchanged * legacy behavior) or onto the slow lane (flag ON — non-blocking). The * per-peer single-flight CAS on ha->snapshot_inflight[] is the authoritative * dedup; the lane's per-key dedup is a redundant backstop. Mirrors the * reconnect-offload terminal-path handling (Task 2.1). */ static void ha_snapshot_send_dispatch(ha_state_t *ha, cluster_t *c, endpoint_t *ep, int peer_idx) { /* Flag OFF (default) or no cluster handle (no lane) → inline. */ if (!ha_snapshot_offload_enabled() || c == NULL) { ha_snapshot_do_send(c, ep, peer_idx); return; } /* Per-peer single-flight: CAS 0->1. Already 1 → a snapshot to this peer * is in flight; enqueue nothing (Req 1.5). */ int expected = 0; if (!atomic_compare_exchange_strong_explicit( &ha->snapshot_inflight[peer_idx], &expected, 1, memory_order_acq_rel, memory_order_acquire)) { return; } /* Record the checkpoint position this transfer references (Req 3.1-3.3) * at the SAME point the flag above is set — before the first byte of the * transfer goes out — so a reader that observes snapshot_inflight[peer_idx] * == 1 can find a meaningful LSN alongside it. rotate_snapshots (task 4.3) * consults this to avoid deleting the checkpoint file this transfer is * still reading from. */ atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], c->persist.checkpoint_lsn, memory_order_release); snapshot_send_ctx_t *sx = malloc(sizeof(*sx)); if (!sx) { atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0, memory_order_release); atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0, memory_order_release); return; } sx->c = c; sx->ep = ep; sx->peer_idx = peer_idx; /* Borrowed-endpoint lifetime (bug-1197 family): take our own reference * before the lane ever sees `ep`, mirroring vtask_enq's +1 for a * pool-bound ep. This is the ONLY ref taken across the 3 submit-outcome * branches below — release it on every path that does NOT hand off to * snapshot_lane_fn (which owns the matching unref on completion). */ endpoint_ref(ep); uint64_t key = (uint64_t)(peer_idx + 1); /* +1: key 0 disables dedup */ int srv = slowlane_submit(&c->slowlane, key, snapshot_lane_fn, sx); if (srv == -EEXIST) { /* Lane still holds this peer's key from a prior snapshot task that has * already cleared snapshot_inflight (its last action) but whose key the * lane has not yet removed. Our CAS saw 0 and set 1, but the submit is * refused — release the flag we set so no clear is lost, and let the * next tick retry. */ endpoint_unref(ep); free(sx); atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0, memory_order_release); atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0, memory_order_release); return; } if (srv < 0 && srv != -EAGAIN) { /* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a * failure (still queued, will run). Only on a real failure tear down * and release the guard so the next tick retries; never self-terminate. */ WARN("HA repl: slowlane_submit snapshot peer[%d] failed: %d — retry " "next cycle", peer_idx, srv); endpoint_unref(ep); free(sx); atomic_store_explicit(&ha->snapshot_inflight_lsn[peer_idx], 0, memory_order_release); atomic_store_explicit(&ha->snapshot_inflight[peer_idx], 0, memory_order_release); return; } /* 0 or -EAGAIN: enqueued exactly once; snapshot_lane_fn clears the flag * and the tracked LSN. */ } /* ─────────────────────────────────────────────────────────────────────── * raft-drive-async Task 3.3 — requester-less ConfigPush slow-lane offload * (bug-1198). * * The two REQUESTER-LESS config-push sites — post-election promotion * (raft_cb_become_leader) and the heartbeat-driven HA_REPL_PUSH_CONFIG * convergence case — call push_config_transport_only, which spawns one push * thread per standby and then BLOCKS the caller in wait_all_with_timeout up to * CONF_PUSH_TIMEOUT_MS. Run inline on the single io_worker that drives both of * those sites, that wait stalls every heartbeat/vote for the whole timeout. * This offloads it to the bounded slow lane. Config push is a CLUSTER-WIDE * fan-out, so a single cluster-wide single-flight guard (configpush_inflight) + * a single dedup key (key=1) suffice — at most one push in flight at a time. * * The admin-upgrade path (push_config_to_standbys, which has a waiting * requester / majority barrier) is NOT routed here and stays unchanged. * ─────────────────────────────────────────────────────────────────────── */ #ifdef XCE_TESTING extern void (*ha_configpush_probe)(void); #endif typedef struct { cluster_t *c; uint8_t *bytes; /* deep copy, owned by the ctx; NULL when blen==0 */ size_t blen; uint8_t sha[32]; /* valid only when has_sha != 0 */ int has_sha; uint64_t new_tag; } configpush_ctx_t; /* ha_config_push_do_send: the actual (blocking) config-push work — shared by * the inline path and the lane worker so both behave identically. In * XCE_TESTING builds a non-NULL ha_configpush_probe() is invoked right around * the send so a test can record the running thread / park the lane worker. */ static void ha_config_push_do_send(cluster_t *c, const uint8_t *bytes, size_t blen, const uint8_t sha256[32], uint64_t new_tag) { #ifdef XCE_TESTING /* XCE_TESTING does not link cluster.c and uses hook shims for the real * transport (push_config_transport_only's extern is #ifndef XCE_TESTING). * The probe stands in for the send and records the running thread. */ if (ha_configpush_probe != NULL) ha_configpush_probe(); (void)c; (void)bytes; (void)blen; (void)sha256; (void)new_tag; #else push_config_transport_only(c, bytes, blen, sha256, new_tag, NULL); #endif } /* * configpush_lane_fn: slow-lane worker entry. Runs the bounded blocking * config push off the drive thread, then ALWAYS clears the cluster-wide * in-flight flag (success or failure) so the dedup never wedges and a later * promotion/heartbeat can retry. Frees the deep-copied bytes and the ctx. * * ⚠ BORROWED-ENDPOINT LIFETIME HAZARD (bug-1197 family — why this path is * gated OFF by default). push_config_transport_only here reads * c->ha.standby_endpoints and writes to those standby endpoints, which are * shared with the io_worker epoll loop, NOT owned by this lane (contrast * reconnect_lane_fn, which owns the ep it creates). If a standby * RDHUP-disconnects while this push runs, the io_worker can endpoint_fini/free * those endpoints concurrently → UAF. Today (flag OFF) the push runs inline ON * the io_worker, so teardown and push are serialized on one thread and cannot * overlap. Flipping g_drive_async_configpush_offload ON is therefore only safe * once Task 3.4 establishes the endpoint-lifetime discipline for the whole * pool/lane dispatch model. Until then this code lands DORMANT and is * exercised only by unit tests with a stable (or absent) endpoint set. */ static void configpush_lane_fn(void *arg) { configpush_ctx_t *cx = (configpush_ctx_t *)arg; if (!cx) return; cluster_t *c = cx->c; ha_config_push_do_send(c, cx->bytes, cx->blen, cx->has_sha ? cx->sha : NULL, cx->new_tag); /* Clear in-flight LAST so a concurrent promotion/heartbeat tick cannot * enqueue a second config push until this one has fully finished. */ atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release); free(cx->bytes); free(cx); } /* * ha_config_push_offloadable: route a requester-less config push either inline * (flag OFF, production default — runs on the caller/drive thread, unchanged * legacy behavior; the CALLER still owns `bytes` and frees them after we * return) or onto the slow lane (flag ON — non-blocking; we take a deep copy of * `bytes` so the caller's free is safe). The cluster-wide single-flight CAS on * ha.configpush_inflight is the authoritative dedup; the lane's per-key dedup * is a redundant backstop. Mirrors the snapshot-offload terminal-path handling * (Task 3.2). Never self-terminates: on any failure it clears the guard and * the next cycle retries. */ static void ha_config_push_offloadable(cluster_t *c, const uint8_t *bytes, size_t blen, const uint8_t sha256[32], uint64_t new_tag) { /* Flag OFF (default) or no cluster handle (no lane) → inline. */ if (!ha_configpush_offload_enabled() || c == NULL) { ha_config_push_do_send(c, bytes, blen, sha256, new_tag); return; } /* Cluster-wide single-flight: CAS 0->1. Already 1 → a config push is in * flight; enqueue nothing. */ int expected = 0; if (!atomic_compare_exchange_strong_explicit( &c->ha.configpush_inflight, &expected, 1, memory_order_acq_rel, memory_order_acquire)) { return; } configpush_ctx_t *cx = malloc(sizeof(*cx)); if (!cx) { atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release); return; } cx->c = c; cx->new_tag = new_tag; cx->has_sha = (sha256 != NULL) ? 1 : 0; if (cx->has_sha) memcpy(cx->sha, sha256, 32); if (bytes != NULL && blen > 0) { cx->bytes = malloc(blen); if (!cx->bytes) { free(cx); atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release); return; } memcpy(cx->bytes, bytes, blen); cx->blen = blen; } else { cx->bytes = NULL; cx->blen = 0; } uint64_t key = 1; /* single cluster-wide dedup key (one config-push stream) */ int srv = slowlane_submit(&c->slowlane, key, configpush_lane_fn, cx); if (srv == -EEXIST) { /* Lane still holds the key from a prior config-push task that has * already cleared configpush_inflight (its last action) but whose key * the lane has not yet removed. Our CAS saw 0 and set 1, but the * submit is refused — release the flag we set so no clear is lost, and * let the next tick retry. */ free(cx->bytes); free(cx); atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release); return; } if (srv < 0 && srv != -EAGAIN) { /* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a * failure (still queued, will run). Only on a real failure tear down * and release the guard so the next tick retries; never self-terminate. */ WARN("HA repl: slowlane_submit configpush failed: %d — retry next cycle", srv); free(cx->bytes); free(cx); atomic_store_explicit(&c->ha.configpush_inflight, 0, memory_order_release); return; } /* 0 or -EAGAIN: enqueued exactly once; configpush_lane_fn clears the flag. */ } #ifdef XCE_TESTING /* Test seam: directly drive ha_config_push_offloadable (the two production call * sites are #ifndef XCE_TESTING). Returns 0 (the dispatch itself cannot fail * fatally — failures clear the guard and retry). */ int ha_config_push_offloadable_for_test(cluster_t *c, const uint8_t *bytes, size_t blen, const uint8_t sha256[32], uint64_t new_tag) { ha_config_push_offloadable(c, bytes, blen, sha256, new_tag); return 0; } #endif /* * ha_dispatch_repl_action_for_standby — execute the decision-loop action for * one standby on the current heartbeat cycle (Task 3.3). * * Reads ha->standby_reported_tag[peer_idx] / standby_reported_lsn[peer_idx] * (stored by ha_process_standby_state on the same heartbeat reply), calls * ha_evaluate_repl_action, and acts on the result: * * HA_REPL_PUSH_CONFIG → push_config_transport_only(c, …) [or test hook] (Task 3.3) * HA_REPL_SEND_SNAPSHOT→ stub (Task 3.5) * HA_REPL_STREAM_WAL → ha_stream_wal_to_standby(c, ha, idx) (Task 3.4) * HA_REPL_CONVERGED → no-op * * One action per heartbeat cycle per standby (design §Unified Heartbeat-Driven * Protocol). Returns 0 on success, -EINVAL on bad peer_idx. * * Requirements: 1.4, 5.1–5.5 (ha-repl-hardening spec) */ int ha_dispatch_repl_action_for_standby(ha_state_t *ha, cluster_t *c, endpoint_t *ep, int peer_idx, uint64_t committed_tag, uint64_t committed_lsn, uint64_t oldest_wal_lsn) { if (!ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES) return -EINVAL; uint64_t standby_tag = ha->standby_reported_tag[peer_idx]; uint64_t standby_lsn = ha->standby_reported_lsn[peer_idx]; /* bug-1479: a standby marked in_catch_up (via ha_raft_on_repl_nack's * immediate trigger on log-conflict rejection, or repl_broadcast's * MAX_PENDING_ACKS eviction) can never correct its OWN self-reported * standby_reported_lsn — it's stuck rejecting every AppendEntries, so it * never sends a stateReport reflecting genuine progress. Use the * corrected join_applied_lsn instead: this heartbeat-driven cycle (runs * every heartbeat_interval_ms regardless of ACK/reject outcome) becomes * the actual catch-up driver for this case — the ACK-driven continuation * (ha_walack_on_ack, config_repl.c) can only fire on a REAL ack, which a * log-conflicted standby can never produce until streamed from the * corrected position at least once. */ if (c) { uint64_t join_lsn; if (repl_get_catchup_join_lsn(&c->persist.repl, ep, &join_lsn)) standby_lsn = join_lsn; } /* bug-1479 (gap 3): stream (and evaluate convergence against) the * leader's own log tail, not just committed_lsn -- see * ha_compute_repl_stream_ceiling_lsn's doc comment (ha.h) for why capping * at committed_lsn is a permanent deadlock whenever this exact standby's * own ack is required to advance committed_lsn in the first place. */ uint64_t stream_ceiling_lsn = committed_lsn; if (c) { uint64_t last_written = (c->persist.current_lsn > 0) ? c->persist.current_lsn - 1 : 0; stream_ceiling_lsn = ha_compute_repl_stream_ceiling_lsn( committed_lsn, last_written, c->persist.checkpoint_lsn); } ha_repl_action_t action = ha_evaluate_repl_action(standby_tag, standby_lsn, committed_tag, stream_ceiling_lsn, oldest_wal_lsn); switch (action) { case HA_REPL_PUSH_CONFIG: INFO("HA repl: standby[%d] tag=%" PRIu64 " < committed_tag=%" PRIu64 " → push config", peer_idx, standby_tag, committed_tag); #ifdef XCE_TESTING if (ha_repl_push_config_for_standby_hook != NULL) { ha_repl_push_config_for_standby_hook(c, peer_idx); break; } #endif if (c) { #ifndef XCE_TESTING extern int conf_read_raw(const char *, uint8_t **, size_t *); uint8_t *cfg_bytes = NULL; size_t cfg_len = 0; if (c->myconf && c->myconf->conffile && conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) { /* Task 3.3: inline (flag OFF, default) or slow-lane (flag ON). * The offload deep-copies cfg_bytes, so this free is still * correct regardless of routing. */ ha_config_push_offloadable(c, cfg_bytes, cfg_len, c->myconf->config_sha256, c->myconf->config_version_tag); free(cfg_bytes); } else { ha_config_push_offloadable(c, NULL, 0, NULL, c->myconf ? c->myconf->config_version_tag : 0); } #endif } break; case HA_REPL_SEND_SNAPSHOT: INFO("HA repl: standby[%d] lsn=%" PRIu64 " < oldest_wal_lsn=%" PRIu64 " → requesting snapshot transfer", peer_idx, standby_lsn, oldest_wal_lsn); /* Task 3.2: route inline (flag OFF, default) or onto the slow lane * (flag ON). The send body — fresh checkpoint + chunked transfer, or * the XCE_TESTING hook — lives in ha_snapshot_do_send so both paths * are identical. */ ha_snapshot_send_dispatch(ha, c, ep, peer_idx); break; case HA_REPL_STREAM_WAL: /* Task 3.4: batch WAL streaming. * Enqueues all batch OPCode_walCatchupEntry messages for the range * [standby_lsn+1, stream_ceiling_lsn] via ha_stream_wal_to_standby() * (bug-1479 gap 3: the ceiling extends past committed_lsn to the * leader's own log tail — see ha_compute_repl_stream_ceiling_lsn). * All xce_request() calls are non-blocking (enqueue-only). */ INFO("HA repl: standby[%d] lsn=%" PRIu64 " < stream_ceiling_lsn=%" PRIu64 " (committed_lsn=%" PRIu64 ") → stream WAL batches", peer_idx, standby_lsn, stream_ceiling_lsn, committed_lsn); #ifdef XCE_TESTING if (ha_repl_stream_wal_hook != NULL) { const char *wal_path = (c && c->persist.wal_path[0]) ? c->persist.wal_path : ""; ha_repl_stream_wal_hook(ep, wal_path, standby_lsn, stream_ceiling_lsn); break; } #endif if (c && c->persist.wal_path[0]) { extern int ha_stream_wal_to_standby(endpoint_t *ep, const char *wal_path, const persist_t *p, uint64_t leader_term, uint64_t standby_lsn, uint64_t committed_lsn); /* bug-1028: pass &c->persist so streamed entry terms come from the * authoritative per-LSN index (persist_entry_term), matching the * standby's logOk + the library send path. * bug-1067: pass the leader's CURRENT term so historical entries * carry the AE header term (not their lower stored term). */ uint64_t leader_term = atomic_load_explicit(&c->ha.term, memory_order_acquire); ha_stream_wal_to_standby(ep, c->persist.wal_path, &c->persist, leader_term, standby_lsn, stream_ceiling_lsn); } break; case HA_REPL_CONVERGED: /* No action needed — standby is up to date */ break; } return 0; } /* ── Dispatch gate (req 1.3, 1.4) ────────────────────────────────────────── * * Called from myreq_handler() in cluster.c before the opcode switch. * Returns 0 when the operation is permitted for the current role. * * Primary-only ops are any op that is not one of the always-allowed set: * nop, reply, hello, join, shutdown, heartbeat, redirect, * steppingDown, replEntry, replAck, replSnapshot, replCatchUpDone, * replConfig, replConfigPush, requestVote, probeConfigTag, * walAck, upgradeConfigAck, snapshotTransfer, walCatchupEntry. * * For CTRL_STANDBY the check is: op is in the primary-only set → deny. * For CTRL_INIT / CTRL_ELECTING the allowlist is tighter: only * OP_requestVote and OP_probeConfigTag pass through. */ /* ha_has_write_quorum — check whether the cluster has enough connected * standbys to achieve majority for state-changing operations. * * Quorum = ceil(n_peers / 2) + 1 (self-vote included). * Example: n_peers=3 → quorum=2 → need ≥1 connected standby. * * Called from ha_dispatch_gate for CTRL_PRIMARY — if the cluster loses * majority (standby failures), write ops are rejected with -XCE_ERR_NOT_READY * to preserve the majority-consistency invariant. * * Read-only queries (joblist, reslist, hostlist, etc.) remain available * regardless of majority status — they don't mutate state. */ static bool ha_has_write_quorum(ha_state_t *ha) { if (!ha) return false; /* Single-node cluster: no standbys needed — self is the majority. */ if (ha->n_peers <= 1) return true; /* Count standbys able to receive new log entries. When the replicator * back-pointer is wired (task 5.3 init), use registry truth: slots with * ep != NULL && !in_catch_up under r->lock (Req 3.1-3.3) — catch-up * members cannot ACK new entries, so they must not count toward write * admission. NULL back-pointer (legacy contexts) falls back to the * standby_endpoints count populated by raft_cb_become_leader (promoted * vote eps) and on_standby_connected (late joiners); removed by error * handlers. */ int connected; if (ha->repl) { connected = repl_active_standby_count(ha->repl); } else { connected = ha->standby_endpoints ? ha->standby_endpoints->count : 0; } /* Need at least n_peers/2 connected standbys to achieve quorum. * Integer division: n_peers/2 for n_peers=3 → 1 standby minimum. */ return connected >= (int)(ha->n_peers / 2); } int ha_dispatch_gate(ha_state_t *ha, int opcode) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); switch (role) { case CTRL_PRIMARY: /* ha-term-safety task 4.3 (remediation r1, Finding C / R7.9): the config * serve-fence is the PRODUCTION READER that makes the fence real. When * set, the latest committed WAL_CONFIG_MARKER references config bytes that * do not hash-match the locally-held bytes (or the bytes are absent), so * this member MUST NOT serve under the new configuration. Refuse * config-dependent state-changing ops (loud non-serving), but keep * replication / read-only / status ops flowing so the correct bytes can * arrive out-of-band and re-verify clears the fence. The member STAYS * ALIVE (returns an error code, never exits — controller-never-self- * terminate). Checked before the quorum gate so a fenced primary never * serves divergent config even with quorum. */ if (config_serve_fenced(ha->raft_c)) { switch (opcode) { /* Replication ops — let the correct config bytes arrive + re-verify */ case OPCode_heartbeat: case OPCode_replConfig: case OPCode_replConfigPush: case OPCode_upgradeConfigAck: case OPCode_hello: case OPCode_walAck: case OPCode_replEntry: case OPCode_replAck: case OPCode_replCatchUpDone: case OPCode_walCatchupEntry: case OPCode_snapshotTransfer: /* Read-only queries / status — answerable while fenced */ case OPCode_get: case OPCode_probeConfigTag: case OPCode_ctrlstatus: break; /* fall through to the quorum check below */ default: /* Config-dependent state-changing op while fenced: refuse. */ return -XCE_ERR_NOT_READY; } } /* config-upgrade-snapshot task 6.2 (Req 3.2, 3.3, 5.5): the Activation * Fence is a SEPARATE, SEQUENTIAL check composed alongside the * config-byte serve-fence above, not merged into it — the two * conditions are logically distinct (byte-mismatch vs. this member's * own materialization-incomplete) per design.md's Activation Fence * component. Checked AFTER config_serve_fenced deliberately: byte * delivery is the earlier step (the bytes must exist locally before * this member's own checkpoint can even attempt to materialize * against them), so checking byte-fence first surfaces the more * fundamental blocker first in log/error ordering — but either fence * alone is sufficient to refuse a state-changing op, so the final * admit/refuse outcome does not depend on this ordering. Same * allowlist as the byte-fence: replication + read-only/status ops * stay admitted (they let materialization proceed and re-verify); * everything else is refused with a loud non-serving error while the * member STAYS ALIVE and keeps retrying (controller-never-self- * terminate). */ if (upgrade_materialization_fenced(ha->raft_c)) { switch (opcode) { /* Replication ops — let commit/apply advance + re-verify */ case OPCode_heartbeat: case OPCode_replConfig: case OPCode_replConfigPush: case OPCode_upgradeConfigAck: case OPCode_hello: case OPCode_walAck: case OPCode_replEntry: case OPCode_replAck: case OPCode_replCatchUpDone: case OPCode_walCatchupEntry: case OPCode_snapshotTransfer: /* Read-only queries / status — answerable while fenced */ case OPCode_get: case OPCode_probeConfigTag: case OPCode_ctrlstatus: break; /* fall through to the quorum check below */ default: /* State-changing op while this member's own upgrade-snapshot * materialization is incomplete/unverified: refuse. */ return -XCE_ERR_NOT_READY; } } /* State-changing operations require majority so committed entries * are durable across failover. Read-only queries and replication * ops are always allowed. */ if (!ha_has_write_quorum(ha)) { /* Quorum-transition observability backstop (Req 8.1, 8.3): record the * loss edge if a membership-change site missed it. The edge detector * makes this silent on every call after the first — repeated rejected * operations do NOT produce a log storm. */ ha_log_quorum_transition(ha); switch (opcode) { /* Replication ops — always allowed (they build majority) */ case OPCode_heartbeat: case OPCode_replConfig: case OPCode_replConfigPush: case OPCode_upgradeConfigAck: case OPCode_hello: case OPCode_walAck: case OPCode_replEntry: case OPCode_replAck: case OPCode_replCatchUpDone: case OPCode_walCatchupEntry: case OPCode_snapshotTransfer: /* Read-only queries */ case OPCode_get: case OPCode_probeConfigTag: /* Read-only controller status (controller-status 7.1, 7.2): * answers the local identity/health view even without quorum — * precisely when an operator most wants it. */ case OPCode_ctrlstatus: /* Fact-bearing ops — reports of events that already happened * (worker job transitions/rusage, scheduler pend reasons). * Rejecting them loses the fact forever: the reporter never * retries (bug-396). Admit so the RETAIN path can append and * hold them until quorum returns (Req 3.2; umbrella 2.8 — * facts are retained, never dropped). */ case OPCode_notify: case OPCode_update: case OPCode_setpreason: case OPCode_clrpreason: return 0; default: return -XCE_ERR_NOT_READY; } } return 0; case CTRL_INIT: /* Allow election RPCs AND primary→standby replication messages. * During the startup stagger a primary may already connect and push * config — accepting these ensures the node doesn't miss the initial * bootstrap and can transition to CTRL_STANDBY immediately. */ if (opcode == OPCode_requestVote || opcode == OPCode_preVoteRequest || opcode == OPCode_probeConfigTag || opcode == OPCode_heartbeat || opcode == OPCode_hello || opcode == OPCode_nop || opcode == OPCode_replConfigPush || opcode == OPCode_walCatchupEntry || opcode == OPCode_snapshotTransfer || /* Read-only controller status — answerable in every role * (controller-status 7.1, 7.2). */ opcode == OPCode_ctrlstatus) return 0; return -XCE_ERR_NOT_READY; case CTRL_ELECTING: /* Allow election RPCs AND primary→standby replication messages. * A node campaigning for leadership must still accept config, WAL, * and heartbeats from the current primary — if it wins the election * it needs current state; if it loses, it becomes STANDBY and must * resume replication without having missed pushes. Only job/client * ops are rejected (we're not ready to serve). */ if (opcode == OPCode_requestVote || opcode == OPCode_preVoteRequest || opcode == OPCode_probeConfigTag || opcode == OPCode_heartbeat || opcode == OPCode_hello || opcode == OPCode_nop || opcode == OPCode_replConfigPush || opcode == OPCode_walCatchupEntry || opcode == OPCode_snapshotTransfer || /* Read-only controller status — answerable in every role * (controller-status 7.1, 7.2). */ opcode == OPCode_ctrlstatus) return 0; return -XCE_ERR_NOT_READY; case CTRL_STANDBY: { /* Allow ops that standbys legitimately handle */ switch (opcode) { case OPCode_nop: case OPCode_reply: case OPCode_hello: case OPCode_join: case OPCode_shutdown: case OPCode_heartbeat: case OPCode_redirect: case OPCode_steppingDown: case OPCode_replEntry: case OPCode_replAck: case OPCode_replSnapshot: case OPCode_replCatchUpDone: case OPCode_replConfig: case OPCode_replConfigPush: case OPCode_requestVote: case OPCode_preVoteRequest: case OPCode_probeConfigTag: case OPCode_walAck: case OPCode_upgradeConfigAck: case OPCode_snapshotTransfer: case OPCode_walCatchupEntry: /* Read-only controller status — a standby answers with its local * view (it reports the primary it knows) (controller-status 7.1, 7.2). */ case OPCode_ctrlstatus: return 0; default: return -XCE_ERR_STANDBY; } } default: return 0; } } /* ha_config_promotion_fenced (ha-term-safety task 4.3, R7.9): a member is * non-promotable while config serve-fenced — its latest committed config marker * references bytes it does not hold/match, so it must not become authoritative. * Pure atomic read via config_serve_fenced (no I/O) — safe on the election path. */ int ha_config_promotion_fenced(const ha_state_t *ha) { if (!ha) return 0; return config_serve_fenced(ha->raft_c) ? 1 : 0; } /* ── Redirect helpers used by join.c ───────────────────────── */ /* * ha_should_redirect: returns 1 when this standby knows the primary's * address and should redirect incoming join requests there. * Returns 0 for PRIMARY role, or for STANDBY with no known primary yet. */ int ha_should_redirect(const ha_state_t *ha) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) return 0; if (ha->primary_host[0] == '\0' || ha->primary_port == 0) return 0; return 1; } /* * ha_no_primary_known: returns 1 when this node is a STANDBY but does not * yet know the primary's address. Callers should reply E_XCE_NOMASTER so * the client retries rather than treating the error as permanent. */ int ha_no_primary_known(const ha_state_t *ha) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) return 0; return (ha->primary_host[0] == '\0' || ha->primary_port == 0) ? 1 : 0; } /* * ha_register_persist_hooks: register the three persistence hooks on the * cluster's hook system, guarded by the hooks_registered idempotency flag. * * ha-single-leader-role task 2.2: extracted from ha_promote_to_primary so * BOTH roads to PRIMARY (the raft become-leader callback and the non-raft * ha_promote_to_primary path) share one hook-registration sequence and * cannot diverge. Safe on a zero-init cluster_t (zeroed hooksys → * xce_add_hook returns -EINVAL harmlessly; hooks_registered still set to 1). * Requires non-NULL c (dereferences &c->hooksys). */ static void ha_register_persist_hooks(cluster_t *c) { if (c->ha.hooks_registered) return; xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun, &c->persist); xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun, &c->persist); xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun, &c->persist); c->ha.hooks_registered = 1; } /* * ha_primary_log_append (ha-term-safety task 3.1, R1.1/R1.2/R2.1): the SINGLE * primary append entry point. Routes a local primary append (job-state entry or * the own-term promotion no-op) through the raft library's raft_log_append, so * the LIBRARY stamps the live election term (s->term) and assigns the LSN * (s->log_len+1), then drives on_persist_entry — which (primary role) performs * the actual WAL write via persist_wal_async. This makes the library index the * authority for both the per-entry term (R1) and the LSN, retiring the * hand-rolled bypass that assigned the LSN directly through persist_wal_async. * * Co-tracking invariant (closes the task 2.1 deferral): because raft_log_append * is the only primary append path, every primary append advances BOTH the * library index (s->log_len) and the persist write cursor (p->current_lsn) by * exactly one, in lockstep — so p->current_lsn == s->log_len + 1 always holds and * the LSN the library assigns equals the LSN on_persist_entry writes. raft_init * (task 2.5) seeds s->log_len from the recovered durable floor, which equals * p->current_lsn - 1 at boot, establishing the invariant before the first append. * * Persist-before-act: raft_log_append calls on_persist_entry BEFORE advancing * s->log_len and propagates a non-zero persist rc without mutating the log (no * phantom entry) — preserving persist-before-ACK and the bug-850 contiguity * guard beneath the WAL write. * * Legacy fallback (non-raft road): when the raft library is not in RAFT_LEADER * state (raft_log_append returns -EPERM) — the init-time non-raft promotion road * (ha_promote_to_primary) and engine-less unit-test contexts — the library is * not tracking the log, so there is no co-tracking hazard. We append locally * through the legacy persist_wal_async path (which still records the live term * via the persist_set_term seam set on the prior promotion) so those roads keep * working unchanged. Once raft drives leadership in production, the -EPERM * branch is never taken and the library is the sole LSN/term source. * * out_lsn (optional) receives the assigned 1-based LSN on success. * Returns 0 on success; propagates a non-zero persist rc on append failure. */ int ha_primary_log_append(cluster_t *c, uint32_t type, const void *payload, uint32_t len, uint64_t *out_lsn) { if (!c) return -EINVAL; /* Pack the in-flight adapter envelope [raft_entry_blob_t{type,len}][payload] * — the SAME format on_persist_entry (ha_cb_persist_entry_stub) unpacks, so * the encoder and decoder share the single wal_types.h definition. */ size_t blob_len = sizeof(raft_entry_blob_t) + len; uint8_t stackbuf[256]; uint8_t *blob = (blob_len <= sizeof(stackbuf)) ? stackbuf : malloc(blob_len); if (!blob) return -ENOMEM; raft_entry_blob_t *bh = (raft_entry_blob_t *)blob; bh->type = type; bh->payload_len = len; if (payload && len > 0) memcpy(blob + sizeof(*bh), payload, len); uint64_t idx = 0; int rc = raft_log_append(&c->ha.raft, blob, blob_len, &idx); if (rc == -EPERM) { /* Non-raft road: the library is not LEADER and is not tracking the log. * Append locally through the legacy path (no library index to co-track). * Stamp the live election term (ha->term) so the entry still records the * leader's term (R1.1) even off the raft road — matching on_persist_entry's * persist_set_term on the raft path. */ persist_set_term(&c->persist, atomic_load_explicit(&c->ha.term, memory_order_acquire)); rc = persist_wal_async(&c->persist, type, payload, len); if (rc == 0) { pthread_mutex_lock(&c->persist.lock); idx = c->persist.current_lsn - 1; /* the LSN just assigned */ pthread_mutex_unlock(&c->persist.lock); } } if (blob != stackbuf) free(blob); if (rc == 0 && out_lsn) *out_lsn = idx; return rc; } /* ha_repl_ack_abs_peer_idx (ha-term-safety task 3.3 remediation): resolve the * ABSOLUTE library peer index from an ACK endpoint's stored peer_idx, branching * on endpoint type EXACTLY as the standby_endpoints scan at ha.c:6187-6189. * * Two endpoint types reach the walAck branch of repl_ack_decode_and_dispatch, * and their stored peer_idx has DIFFERENT meaning: * - Promoted vote ep (vote_ep_ctx_t, ep_is_vote != 0): peer_idx is a * self-EXCLUDED "votes" index (relative). Map it back to the absolute index * — the inverse of raft_cb_send_vote_request's votes_idx remap (self_idx == * my_priority): abs = votes_idx < self_idx ? votes_idx : votes_idx + 1. * - Heartbeat standby ep (standby_ep_ctx_t, ctx==NULL, ep_is_vote == 0): * peer_idx is ALREADY the absolute candidate index — pass it through. * * The earlier unconditional remap corrupted the heartbeat path: a standby ep's * already-absolute index was remapped a second time, crediting the WRONG peer * (false quorum / stuck commit) or, near the array bound, an OUT-OF-BOUNDS * match_index[] write. This resolver is the single source of truth so the * production call site and the unit test exercise identical logic. */ int ha_repl_ack_abs_peer_idx(ha_state_t *ha, int ep_is_vote, int peer_idx) { int self_idx = ha->my_priority; /* self_idx == my_priority by convention */ if (ep_is_vote) return (peer_idx < self_idx) ? peer_idx : peer_idx + 1; return peer_idx; /* heartbeat standby ep: already absolute */ } /* ha-term-safety task 3.3 (R4.1/R4.5): drive the raft library on a replication * ACK. Route the reply through raft_on_append_entries_reply (updating * match_index[]/next_index[]) then raft_advance_commit, so the library * recomputes the current-term-gated commit (Figure-8) and MAY fire * on_commit_advanced — the task-2.4 callback feeding the synchronous barrier. * * Peer-index contract: `abs_peer_idx` is the ABSOLUTE library peer index * (j != self_idx), the same index space the library uses for * match_index[]/next_index[]. The CALLER resolves it per endpoint type via * ha_repl_ack_abs_peer_idx (promoted-vote eps carry a relative votes index; * heartbeat standby eps carry an already-absolute index) — this seam must NOT * remap again or it would corrupt the heartbeat path. * * Defense-in-depth: a peer index outside [0, raft.n_peers) (and never self) is * rejected loudly rather than passed into raft_on_append_entries_reply, whose * match_index[] write is unchecked. This fences an OOB / wrong-peer write at the * controller boundary (no assert inside src/lib/raft/) and self-heals — the * caller simply skips this ACK's library accounting. * * The library drops a stale-term reply (reply_term != s->term) inside * raft_on_append_entries_reply, so a stale ACK never touches match_index[] and * never advances commit. raft_advance_commit is leader-only (-EPERM otherwise); * on a non-leader the call is a harmless no-op. Neither library call performs * I/O — safe on the iomgr ACK path. */ int ha_raft_on_repl_ack(cluster_t *c, int abs_peer_idx, uint64_t acked_lsn, uint64_t reply_term) { if (!c) return -EINVAL; /* Defense-in-depth bounds/self check: the library's match_index[] write is * unchecked, so reject a bogus index here (fail loud, never corrupt). */ int self_idx = c->ha.my_priority; int n_peers = c->ha.raft.n_peers; if (abs_peer_idx < 0 || abs_peer_idx >= n_peers || abs_peer_idx == self_idx) { ERROR("ha_raft_on_repl_ack: rejecting out-of-range peer index %d " "(n_peers=%d self_idx=%d) — skipping library ACK accounting", abs_peer_idx, n_peers, self_idx); return -EINVAL; } /* A successful WalAck reports the standby's matched index (acked_lsn). The * library applies it only when reply_term == s->term (stale-term drop). */ (void)raft_on_append_entries_reply(&c->ha.raft, abs_peer_idx, /*success*/1, acked_lsn, reply_term); /* Recompute the current-term-gated commit; fires on_commit_advanced exactly * once on a strict advance of a current-term entry (else holds). Leader-only: * -EPERM off the leader leaves commit_index unchanged with no callback. */ (void)raft_advance_commit(&c->ha.raft); return 0; } int ha_raft_on_repl_nack(cluster_t *c, int abs_peer_idx, uint64_t reply_term, uint64_t *corrected_lsn_out) { if (!c) return -EINVAL; /* Same defense-in-depth bounds/self check as ha_raft_on_repl_ack. */ int self_idx = c->ha.my_priority; int n_peers = c->ha.raft.n_peers; if (abs_peer_idx < 0 || abs_peer_idx >= n_peers || abs_peer_idx == self_idx) { ERROR("ha_raft_on_repl_nack: rejecting out-of-range peer index %d " "(n_peers=%d self_idx=%d) — skipping library NACK accounting", abs_peer_idx, n_peers, self_idx); return -EINVAL; } /* A rejected AppendEntries reports a log conflict at the sent * prevLogIndex/prevLogTerm. match_index is unused on the failure branch * (raft.c: "failure: next_index[peer] = Max(next_index[peer]-1, 1). Does * NOT advance commit.") -- pass 0. The library applies the backoff only * when reply_term == s->term (stale-term drop), same as the ack path. */ (void)raft_on_append_entries_reply(&c->ha.raft, abs_peer_idx, /*success*/0, /*match_index unused*/0, reply_term); if (corrected_lsn_out) { uint64_t next = c->ha.raft.next_index[abs_peer_idx]; *corrected_lsn_out = (next > 0) ? next - 1 : 0; } return 0; } /* * ha_arm_own_term_gate: append the own-term WAL_NOOP and arm the commit * engine's writable_lsn gate (ha-single-leader-role task 2.2, Req 1.4/2.1/2.2). * * The shared promotion tail both roads to PRIMARY run. Sequence: * 1. persist_wal_async(WAL_NOOP) — appends a header-only entry and * broadcasts it to standbys via repl_broadcast (one call does both). * 2. derive noop_lsn = current_lsn - 1 under p->lock (the append * post-incremented current_lsn; persist.c:1019-1021 pattern). * 3. commit_submit(noop_lsn, COMMIT_MODE_RETAIN) — lock-free, non-blocking, * registration-free observability bookkeeping (records nothing). * 4. commit_set_writable_lsn(noop_lsn) — arms the own-term gate so the * engine withholds WAIT completion until committed_lsn >= noop_lsn. * * Non-blocking by construction: every call here returns without waiting for * the no-op to commit (Req 1.4 — status honesty does not wait). * * Degraded-but-safe (design §Error Strategy): if the no-op append fails, * no LSN exists, so the gate stays UNSET; promotion STILL COMPLETES. The * universal commit barrier still requires per-entry majority of a new-term * entry, so the first state-changing entry of this term becomes the de-facto * own-term vehicle. We do NOT block and do NOT abort promotion. */ static void ha_arm_own_term_gate(cluster_t *c) { /* ha-term-safety task 3.1: route the own-term promotion no-op through the * library (ha_primary_log_append → raft_log_append) so the WAL_NOOP records * the newly established term (R1.2) and the library assigns its LSN. On the * non-raft promotion road this falls back to the legacy append; either way * the returned LSN is the no-op's WAL LSN. */ uint64_t noop_lsn = 0; int rc = ha_primary_log_append(c, WAL_NOOP, NULL, 0, &noop_lsn); if (rc < 0) { ERROR("HA: own-term WAL_NOOP append failed: %d", rc); WARN("HA: own-term gate left UNSET (degraded-but-safe); first " "new-term entry becomes the own-term commit vehicle"); return; } /* Observability-only bookkeeping (lock-free, non-blocking, records * nothing per entry — commit state derives from registry watermarks). */ (void)commit_submit(&c->commit, noop_lsn, COMMIT_MODE_RETAIN, 0); /* Arm the own-term gate: completion of any WAIT is withheld until * committed_lsn >= noop_lsn. */ commit_set_writable_lsn(&c->commit, noop_lsn); INFO("HA: own-term noop appended lsn=%" PRIu64 ", writable_lsn gate armed", noop_lsn); } /* * ha_promote_to_primary: the NON-RAFT promotion road (cluster.c init-time, * legacy XCE_TESTING evaluation path, tests). This path OWNS its term, so it * calls ha_do_promote (which increments the term + sets role=PRIMARY). It * then runs the SAME shared sequence the raft road runs: register hooks, * start the heartbeat, and arm the own-term gate (noop + writable_lsn). * * Every road to PRIMARY arms the own-term gate. The noop+gate tail is * guarded on c != NULL (callers always pass a real cluster here). */ int ha_promote_to_primary(cluster_t *c) { ha_do_promote(&c->ha); /* non-raft path owns its term increment */ /* Shared promotion sequence — must match the raft become-leader tail: * hooks live BEFORE the no-op append so the persistence seam is active * before any racing client write. */ ha_register_persist_hooks(c); /* Arm periodic heartbeat to standbys — keeps their watchdog alive when * the cluster is idle (no WAL entries or config pushes). */ ha_start_primary_heartbeat(c); /* Own-term no-op + writable_lsn gate (Req 1.4, 2.1, 2.2). */ ha_arm_own_term_gate(c); return 0; } /* * ha_demote_to_standby: disable persistence hooks, then set role=STANDBY * and reset the hooks_registered idempotency guard (via ha_do_demote). */ int ha_demote_to_standby(cluster_t *c) { /* Disable persistence hooks before changing role */ if (c->ha.hooks_registered) { xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun); xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun); xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun); } ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */ return 0; } /* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */ /* * ha_wait_for_heartbeat_or_timeout: called from the startup path after * ha_init() when the node needs to discover whether a primary already exists. * * Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where * deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter * * The heartbeat handler (cluster.c) posts heartbeat_seen when it receives * OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits * OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler. * * Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived); * ha_do_demote() was called (role = CTRL_STANDBY). * Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT); * caller is responsible for driving the election. */ ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha) { /* Seed jitter PRNG on first use */ if (ha->jitter_seed == 0) ha_seed_jitter(&ha->jitter_seed, ha->my_priority); /* Compute timeout: election_timeout + priority_step * priority + random jitter. * Higher-priority (lower index) nodes time out sooner, giving them a head * start in the election before lower-priority nodes begin campaigning. */ long to_ms = (long)ha->config.election_timeout_ms; to_ms += (long)ha->my_priority * (long)ha->config.election_priority_step_ms; if (ha->config.election_jitter_max_ms > 0) { to_ms += (long)(xorshift64_next(&ha->jitter_seed) % ((uint64_t)ha->config.election_jitter_max_ms + 1)); } /* sem_timedwait requires an absolute CLOCK_REALTIME deadline */ struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); deadline.tv_sec += to_ms / 1000; deadline.tv_nsec += (to_ms % 1000) * 1000000L; if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; } int rc = sem_timedwait(&ha->heartbeat_seen, &deadline); if (rc == 0) { /* Heartbeat arrived before the deadline — demote to standby */ ha_do_demote(ha); return CTRL_STANDBY; } /* ETIMEDOUT (or any other error): treat as election timeout */ return CTRL_ELECTING; } /* ── Term Durability (Req 9) ──────────────────────────────────────────────── * * ha_term_file_t: on-disk format for ha-term.bin. * Fields are in host byte order (single-host durability, no cross-endian migration). * CRC32 covers all preceding bytes via zlib crc32(). */ #define HA_TERM_MAGIC 0xDEADBEEFCAFEBABEULL typedef struct { uint64_t magic; uint64_t term; uint64_t voted_for_term; char voted_for_name[XCE_MAXNAME_LEN]; uint32_t crc32; } ha_term_file_t; /* * ha_persist_term: atomically write ha_term_file_t to {workdir}/ha-term.bin. * Uses write-to-tmp + fsync + rename for durability (existing project pattern). * If workdir is NULL, the persist is skipped (test mode or no workdir configured). * Req 9.1: must be called before any vote grant or term increment is reported. */ void ha_persist_term(ha_state_t *ha, const char *workdir) { if (!workdir) return; /* test mode: skip file I/O */ ha_term_file_t f; memset(&f, 0, sizeof(f)); f.magic = HA_TERM_MAGIC; f.term = atomic_load_explicit(&ha->term, memory_order_acquire); f.voted_for_term = ha->voted_for_term; strncpy(f.voted_for_name, ha->voted_for_name, sizeof(f.voted_for_name) - 1); f.voted_for_name[sizeof(f.voted_for_name) - 1] = '\0'; /* CRC32 covers all bytes before the crc32 field itself */ size_t crc_len = offsetof(ha_term_file_t, crc32); f.crc32 = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len); /* Write to tmp then rename — atomic from filesystem perspective */ char path[1024], tmp[1040]; snprintf(path, sizeof(path), "%s/ha-term.bin", workdir); snprintf(tmp, sizeof(tmp), "%s/ha-term.bin.tmp", workdir); int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) return; /* best-effort; caller must not rely on this for correctness */ ssize_t written = write(fd, &f, sizeof(f)); if (written == (ssize_t)sizeof(f)) fsync(fd); close(fd); if (written == (ssize_t)sizeof(f)) rename(tmp, path); } /* * ha_restore_term: read {workdir}/ha-term.bin at startup and restore * ha->term and ha->voted_for_* into ha_state_t. * * On missing file (ENOENT): no-op — term stays 0. Req 9.3. * On CRC/magic mismatch: log WARN, zero ha->term and ha->voted_for_*. Req 9.2. * On valid file: atomically restore ha->term; copy voted_for fields. Req 9.2. * * Returns 0 on success (including gracefully-handled file-not-found and CRC bad), * -errno on unexpected I/O error. * If workdir is NULL, function is a no-op (safe for unit tests without a workdir). */ int ha_restore_term(ha_state_t *ha, const char *workdir) { if (!workdir) return 0; char path[1024]; snprintf(path, sizeof(path), "%s/ha-term.bin", workdir); int fd = open(path, O_RDONLY); if (fd < 0) { if (errno == ENOENT) return 0; /* first boot — no file yet, term stays 0 (Req 9.3) */ return -errno; } ha_term_file_t f; memset(&f, 0, sizeof(f)); ssize_t n = read(fd, &f, sizeof(f)); close(fd); if (n != (ssize_t)sizeof(f)) { WARN("HA: ha-term.bin short read (%zd/%zu) — ignoring, starting at term 0", n, sizeof(f)); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); return 0; } /* Verify magic */ if (f.magic != HA_TERM_MAGIC) { WARN("HA: ha-term.bin bad magic (0x%016" PRIx64 ") — ignoring, starting at term 0", f.magic); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); return 0; } /* Verify CRC32 */ size_t crc_len = offsetof(ha_term_file_t, crc32); uint32_t computed = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len); if (computed != f.crc32) { WARN("HA: ha-term.bin CRC mismatch (expected 0x%08x got 0x%08x) — ignoring, starting at term 0", computed, f.crc32); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); return 0; } /* Valid file — restore state (Req 9.2) */ atomic_store_explicit(&ha->term, f.term, memory_order_release); ha->voted_for_term = f.voted_for_term; strncpy(ha->voted_for_name, f.voted_for_name, sizeof(ha->voted_for_name) - 1); ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0'; return 0; } /* ── ha_heartbeat_is_fresh (Req 6.2) ──────────────────────────────────────── * * Returns 1 if the elapsed time since the last primary message * (ha->last_activity, written by ha_on_heartbeat_received) is strictly less * than ha->config.heartbeat_timeout_ms (i.e. the primary is still alive). * Returns 0 if no primary message has been seen yet (last_activity == 0) or * the timeout has expired. * * Uses CLOCK_MONOTONIC seconds to avoid wall-clock skew; whole-second * granularity is sufficient for the 10 s default timeout. */ int ha_heartbeat_is_fresh(ha_state_t *ha) { time_t last = ha->last_activity; /* time_t load is atomic on LP64 */ if (last == 0) return 0; /* no primary message seen yet */ struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); long ms = (long)(now.tv_sec - last) * 1000L; return ms < (long)ha->config.heartbeat_timeout_ms; } /* ── Voter freshness floor (election safety) ─────────────────────────────── * * Compute the voter's effective freshness floor as the per-field max of its * local state and its learned-committed cluster state. Outdated candidates * (below this floor) are denied so the cluster can never be rolled back below * what it has already committed. See ha.h for full contract. */ void ha_vote_self_freshness(ha_state_t *ha, uint64_t local_config_tag, uint64_t local_applied_lsn, uint64_t *out_tag, uint64_t *out_lsn) { uint64_t learned_tag = atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire); uint64_t learned_lsn = atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire); if (out_tag) *out_tag = (local_config_tag > learned_tag) ? local_config_tag : learned_tag; if (out_lsn) *out_lsn = (local_applied_lsn > learned_lsn) ? local_applied_lsn : learned_lsn; } /* ── Candidate advertised LSN (election safety, ha-commit-durability 4.2) ─── * * A candidate advertises EXACTLY its durable WAL tip on the LSN axis — the * bare `persist_durable_floor(p)`, NOT the folded * max(durable_floor, last_known_committed_lsn) that ha_vote_self_freshness * computes for the VOTER floor. * * Why bare, not folded: ha_vote_self_freshness unconditionally returns * max(local_lsn, last_known_committed_lsn). Feeding it durable_floor would let * a node restarted with a TRUNCATED log (durable_floor low) but a stale * volatile learned_committed (high) advertise the stale-high value and WIN — * exactly the truncated-candidate-wins hole this task closes (design §1a/§5a). * A candidate may only claim a log position it can durably back. * * The candidate's TAG axis is unchanged: it still uses * max(local_config_tag, learned_tag) (the config epoch is orthogonal to log * durability — design §1a). Only the LSN axis bypasses the fold. * * NULL-safe (persist_durable_floor guards NULL → 0) for the test/seam paths. */ uint64_t ha_candidate_advertise_lsn(persist_t *p) { return persist_durable_floor(p); } /* bug-848: see ha.h. After local WAL recovery, advertise the recovered durable * tip for catch-up so the primary streams only the missing suffix instead of * re-streaming + overwriting consistent records. Monotonic: never lowers a * higher applied_lsn (e.g. if catch-up already advanced it). */ void ha_sync_applied_lsn_from_recovery(ha_state_t *ha, persist_t *p) { if (!ha || !p) return; uint64_t recovered = persist_durable_floor(p); uint64_t cur = atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); if (recovered > cur) atomic_store_explicit(&ha->applied_lsn, recovered, memory_order_release); } /* bug-1042: re-sync the raft library's log_len / commit_index / term from the * recovered WAL AFTER persist_recover. * * Init-ordering hazard: ha_init_raft (myapp.c, before the election) sources * raft.initial_log_len from persist_durable_floor — but on a STANDBY-REJOIN boot * persist_recover has not run yet at that point, so the library comes up with * log_len=0. After cluster_init_phase2 runs persist_recover (recovering e.g. * durable_floor=7), the library still thinks its log is EMPTY. Every incoming * AppendEntries then fails logOk (prevLogIndex>=1 against log_len=0 → reject), * so the rejoined member is PERMANENTLY stuck as a non-converging standby and * the primary re-streams [tip+1..committed] every heartbeat forever. * * Fix: after recovery, raise the library's log_len to the recovered WAL tip and * its commit_index to the recovered durable floor (clamped to log_len, honoring * the commit_index<=log_len invariant), and align raft.term with the persisted * term so the per-LSN (term,index) reconcile reads consistent state. Monotonic: * never lowers an already-higher library position (e.g. if catch-up advanced it * before this runs). The library stores no entry bytes — on_get_entry_term reads * the recovered per-LSN term index — so raising log_len is sufficient for logOk. */ void ha_sync_raft_log_from_recovery(ha_state_t *ha, persist_t *p) { if (!ha || ha->raft_c == NULL || !p) return; uint64_t recovered_tip = persist_durable_floor(p); if (recovered_tip > ha->raft.log_len) ha->raft.log_len = recovered_tip; /* commit_index advances to the recovered durable floor but never past the * log tip (library invariant commit_index <= log_len), and never decreases. */ uint64_t recovered_commit = recovered_tip; if (recovered_commit > ha->raft.log_len) recovered_commit = ha->raft.log_len; if (recovered_commit > ha->raft.commit_index) ha->raft.commit_index = recovered_commit; /* Align the library term with the persisted/restored HA term so the * (term,lsn) reconcile and own-term gate read a consistent value after a * rejoin (ha_restore_term already loaded ha->term before ha_init_raft). */ uint64_t ha_term = atomic_load_explicit(&ha->term, memory_order_acquire); if (ha_term > ha->raft.term) ha->raft.term = ha_term; } /* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ──────────────────────── * * handle_request_vote: process an incoming OP_requestVote RPC. * * Decision order (fail-fast): * 0. role gate (Req 6.1, 6.2): * - CTRL_PRIMARY → deny * - CTRL_STANDBY with fresh heartbeat → deny * 1. in_catch_up → deny (Req 2.5) * 2. req term < my term → deny (Req 2.2) * 3. req term > my term → update term, clear voted_for (Req 2.3) * 4. already voted for a DIFFERENT candidate in this term → deny (Req 2.6) * 5. freshness gate: candidate not at least as fresh → deny (Req 2.4) * 6. grant vote, record voted_for, persist (Req 9.1) * * Returns 1 = granted, 0 = denied, <0 = error. * * TEST-ONLY (XCE_TESTING): production vote dispatch goes through * cluster.c → raft_on_vote_request. This function is the legacy * entry point used by test_ha_vote_gate.c and test_ha_election.c * to exercise role-gate logic in isolation without a full raft state. */ #ifdef XCE_TESTING int handle_request_vote(ha_state_t *ha, uint64_t term, const char *candidate_id, uint64_t config_tag, uint64_t last_lsn, uint64_t my_config_tag, uint64_t my_last_lsn, int in_catch_up_flag) { /* Req 2.3: higher term → step down BEFORE any role gate. * * Raft §5.1: "If RPC request or response contains term T > currentTerm: * set currentTerm = T, convert to follower." * * Without this ordering, a CTRL_PRIMARY node at term=N would deny a vote * request at term=N+1 without ever seeing the higher term, leaving two * nodes both believing they lead → split-brain. */ uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire); if (term > my_term) { atomic_store_explicit(&ha->term, term, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); /* Step down from leader: a higher-term election is in progress and we * must not block it with the role gate below. */ ctrl_role_t cur_role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (cur_role == CTRL_PRIMARY) { ha_do_demote(ha); INFO("HA: stepping down (term %" PRIu64 " → %" PRIu64 ") — higher-term election in progress", my_term, term); } my_term = term; } /* Req 2.2: stale term → deny, do not update any state */ if (term < my_term) { return 0; } /* Req 6.1, 6.2: role-based deny gate. * Evaluated AFTER the term catch-up above: if we just stepped down due to * a higher term, role is now CTRL_STANDBY and we fall through to voting. * PRIMARY at the current term still denies votes (protecting the live * primary from same-term challengers). */ ctrl_role_t my_role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (my_role == CTRL_PRIMARY) return 0; /* deny — I am primary; protect against same-term challengers */ if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 && ha_heartbeat_is_fresh(ha)) return 0; /* deny — my primary is alive (active primary connection) */ /* Req 2.5: catching-up standbys deny all votes */ if (in_catch_up_flag) return 0; /* Protect voted_for check+set from concurrent vote requests on different vtasks */ pthread_mutex_lock(&ha->lock); /* Req 2.6: already voted for a DIFFERENT candidate in this term → deny. * Same-candidate re-grant is idempotent (Raft §5.2: grant iff votedFor ∈ * {Nil, candidateId}) — matches the duplicate-vote check in raft.c. */ if (ha->voted_for_term == term && ha->voted_for_name[0] != '\0' && strcmp(ha->voted_for_name, candidate_id) != 0) { pthread_mutex_unlock(&ha->lock); return 0; } /* Req 2.4 + election-safety fix #2: freshness gate against the EFFECTIVE * floor — the max of the voter's local state and its learned-committed * cluster state. A candidate that is below what the cluster has already * committed (config_tag or, at equal tag, LSN) would roll the cluster back * and must be denied even when this voter's local state is colder. * Deny if candidate has a lower config_tag, or equal config_tag with lower * LSN; grant if strictly fresher or equally fresh. */ uint64_t eff_tag = 0, eff_lsn = 0; ha_vote_self_freshness(ha, my_config_tag, my_last_lsn, &eff_tag, &eff_lsn); if (config_tag < eff_tag) { pthread_mutex_unlock(&ha->lock); return 0; } if (config_tag == eff_tag && last_lsn < eff_lsn) { pthread_mutex_unlock(&ha->lock); return 0; } /* Grant the vote: record voted_for and persist before responding (Req 9.1) */ ha->voted_for_term = term; strncpy(ha->voted_for_name, candidate_id, sizeof(ha->voted_for_name) - 1); ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0'; pthread_mutex_unlock(&ha->lock); /* workdir is not available here — persist with NULL (test mode or caller * wraps this function with workdir before sending the network reply). * Production dispatch path calls ha_persist_term(ha, workdir) explicitly * before this function writes its network reply. */ ha_persist_term(ha, NULL); return 1; } #endif /* XCE_TESTING */ /* ── raft-drive-async Task 2.1: reconnect-offload feature flag (ALL builds) ── * * drive_async_reconnect_offload gates the peer reconnect path: * ON (production default) → primary_heartbeat_fn submits the peer reconnect * to the slow lane and returns immediately; the delaymgr thread NEVER * performs the blocking connect inline (ha-async-io.md thread-role * contract — a blocking connect on the heartbeat/election-timeout thread * stalls liveness to healthy peers and triggers spurious elections). * OFF → legacy inline blocking connect on the delaymgr thread; retained only * so the TDD RED test can demonstrate the pre-change stall. * * Compiled in ALL builds (production reads it on every heartbeat tick); the * setter is the test seam to flip RED↔GREEN. */ static _Atomic int g_drive_async_reconnect_offload = 1; void ha_set_reconnect_offload_for_test(int on) { atomic_store_explicit(&g_drive_async_reconnect_offload, on ? 1 : 0, memory_order_release); } int ha_reconnect_offload_enabled(void) { return atomic_load_explicit(&g_drive_async_reconnect_offload, memory_order_acquire); } /* raft-drive-async task 3.2: SnapshotSend feature flag. * * DEFAULT ON (1): the snapshot transfer runs on a slow-lane worker, off the * caller (io_worker/drive) thread. The bug-1197-family borrowed-endpoint * hazard (snapshot_lane_fn's `ep` is shared with the io_worker epoll loop, * not lane-owned) is fixed: ha_snapshot_send_dispatch takes an endpoint_ref() * before submit and snapshot_lane_fn releases it via endpoint_unref() on * completion — the general bug-1298 refcount system (every heap-ep teardown * site already routes through endpoint_unref, see cluster.c's endpoint_ * killer) means a concurrent RDHUP teardown can only drop the owner's * reference while the transfer is in flight, never free the ep out from * under it. Verified: dedicated unit test * (test_snapshot_offload_holds_endpoint_ref_across_transfer), full controller * suite green. * * Coupled to g_drive_async_peerctrl_dispatch (bug-1198 requires both ON * together — see that flag's comment) and flips ON with it. The cold-start * pool-wiring self-deadlock (bug-1376) found flipping both ON together in a * live force-snapshot soak is root-caused and fixed — see * g_drive_async_peerctrl_dispatch_enabled's comment. * * Keyed per standby (Req 1.5) so at most one snapshot per peer is in flight; * at most one persist_checkpoint in flight (existing atomic guard). Compiled * in ALL builds; the setter is the test seam to flip RED↔GREEN. */ static _Atomic int g_drive_async_snapshot_offload = 1; void ha_set_snapshot_offload_for_test(int on) { atomic_store_explicit(&g_drive_async_snapshot_offload, on ? 1 : 0, memory_order_release); } int ha_snapshot_offload_enabled(void) { return atomic_load_explicit(&g_drive_async_snapshot_offload, memory_order_acquire); } /* raft-drive-async task 3.3: ConfigPush feature flag. * * DEFAULT OFF (0): the requester-less config push runs INLINE on the caller * (io_worker/drive) thread — byte-for-byte the legacy behavior, i.e. * push_config_transport_only's fan-out followed by its blocking * wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS). * * Landed DORMANT alongside the Task 3.1/3.2 flags because the offload borrows * standby endpoints it does not own (bug-1197 family — push_config_transport_only * reads c->ha.standby_endpoints): moving the push to a lane thread while the * io_worker may RDHUP-free those endpoints would be a cross-thread UAF. * push_config_transport_only's hazard was DISTINCT from the snapshot (3.2) * lane's borrowed-ep hazard — it grabbed raw endpoint_t* pointers for ALL * standbys under c->ha.lock, released the lock, then spawned one pthread per * standby that read/wrote those endpoints unprotected for up to * CONF_PUSH_TIMEOUT_MS, a race that existed regardless of this flag's value * (the pthread-per-standby fan-out is not the io_worker, so "runs inline on * io_worker" never actually serialized it against a concurrent RDHUP * teardown). Logged as bug-1381, **fixed 2026-07-02**: push_config_transport_only * (config_repl.c) now takes its own endpoint_ref() on each standby ep at the * same point the pointer is snapshotted under c->ha.lock, with a matching * endpoint_unref() from push_thread (or from this function's own cleanup if a * thread never started) — mirroring this same file's snapshot-lane fix. * * **Flip attempted 2026-07-02, REVERTED to OFF**: bug-1381 alone did not clear * the bar. Flag-ON force-snapshot soak went 0/3 pass ("no snapshot install" / * "did not reconverge to 3-of-3" on the freshly-rebuilt xce.build binary — NOT * a stale-binary artifact); a flag-OFF control on the identical build went * 1/2 pass with the SAME failure signature, so the underlying defect is * PRE-EXISTING and orthogonal to this flag (root cause: `ha_send_snapshot_to_standby`, * config_repl.c, never checks `xce_snapshot_chunk_req_raw`'s return in its * chunk loop, so a chunk that fails to enqueue — e.g. ECONNRESET during the * empty-rejoin reconnect window — is silently dropped and the sender still * logs "transfer complete" and returns 0; logged as a new bug, see buglog). * Even so, flag-ON's 0/3 vs flag-OFF's 1/2 is a small enough sample that a * TIMING interaction (configpush lane traffic narrowing the window the * snapshot lane has to complete before reconnect) can't be ruled out either. * Flag stayed OFF until the chunk-loss bug (bug-1386) AND a second, * independently-discovered bug (bug-1389) were fixed and re-validated. * * **bug-1386 FIXED 2026-07-02** (config_repl.c): the send-side chunk loop now * checks xce_snapshot_chunk_req_raw's return and stops+propagates on failure * instead of silently continuing and reporting "complete". * * **bug-1389 FIXED 2026-07-02** (this file + ha.h): re-validating bug-1386's * fix surfaced a SECOND, independent defect — force-snapshot chunks are * dispatched across the vtask pool with no ordering guarantee (a live soak * showed arrival order 0,1,3,4,2,5,... across cpu#0-3 workers), but * ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in * ARRIVAL order instead of at its own chunk_offset — reordered chunks landed * at the wrong file position (same total size, wrong content). A live crc32 * correlation between send and receive proved every chunk's CONTENT survived * the wire intact, ruling out capnp/transport entirely. Fixed via * pwrite()-at-chunk_offset (order-independent) plus a * snapshot_recv_count/snapshot_recv_total completion tracker (guarded by * ha->lock) so finalize fires only once every chunk has actually arrived, * not merely because the current call happens to carry is_last. * * **Flag re-flipped to ON 2026-07-02** after both bugs closed: full controller * suite green, live-validated 3/3 --force-snapshot PASS (genuine snapshot * installs confirmed) + 6-round --chaotic PASS on the flag-ON binary. * * ONLY the requester-less call sites (post-election promotion, heartbeat-driven * HA_REPL_PUSH_CONFIG) route through this flag; the admin-upgrade barrier path * (push_config_to_standbys) is unaffected. Compiled in ALL builds; the setter * is the test seam to flip RED<->GREEN. */ static _Atomic int g_drive_async_configpush_offload = 1; void ha_set_configpush_offload_for_test(int on) { atomic_store_explicit(&g_drive_async_configpush_offload, on ? 1 : 0, memory_order_release); } int ha_configpush_offload_enabled(void) { return atomic_load_explicit(&g_drive_async_configpush_offload, memory_order_acquire); } /* raft-drive-async task 3.1: PeerCtrlDispatch feature flag. * * DEFAULT ON (1): the registered peer-control handler hands the decoded * packet to the vtask pool and returns promptly, instead of running its * routing body inline on the iomgr io_worker thread. * * Gate items closed: the task 3.4 deadlock-freedom proof (LivenessAdmission, * task 1.6), the bug-1197 endpoint-quiescence fix (3.4a), bug-1198 (closed by * the snapshot-offload flag's endpoint_ref/endpoint_unref fix — see that * flag's comment), and bug-1376 (below). * * bug-1376 (self-deadlock, root-caused + fixed): a live --force-snapshot soak * (a genuinely EMPTY node cold-starting and rejoining — the one scenario * chaotic mode never exercises, since it restarts nodes with WAL intact) hung * with both flags ON. Root cause: peer_ctrl_msg_handler held ep->dispatch_ * inflight across the ENTIRE peer_ctrl_dispatch_to_pool call, including its * inline fallback when ha->pool is not yet wired (the common cold-start * shape — a fresh node's own self-election can resolve, via an inbound vote * reply, before cluster_init_phase2 finishes wiring the pool). That inline * body can reach raft_cb_become_follower, which — now that cluster_init_ * phase2 has run synchronously earlier in the SAME call chain and wired the * pool — tears down its own loser vote-eps via the deferred path * (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting * for dispatch_inflight==0 on THIS SAME ep. That count is held by this exact * call, on this exact thread, and cannot reach 0 until the spin itself * returns — a permanent self-deadlock. Fixed: peer_ctrl_dispatch_to_pool now * releases dispatch_inflight at the exact right point for each of its 3 exit * paths — before calling the inline body on every fallback path (pool==NULL, * OOM, enqueue failure), since io_worker is single-threaded and the inline * path never enqueues anything, so there is no pool-worker race left to * protect against once inline execution is committed to; only after the * (non-blocking) enqueue completes on the success path, preserving the * original protection for the genuinely concurrent pool-worker case. Unit * test: test_vep_self_teardown_does_not_deadlock (bounded-timeout join, * RED->GREEN). Live-verified: --force-snapshot soak (the exact repro shape) * now completes without hanging, confirmed a real snapshot install, * committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak * with both flags ON also passed clean. * * Compiled in ALL builds (production reads it on every peer-control message); * the setter is the test seam to flip OFF for a regression comparison. */ static _Atomic int g_drive_async_peerctrl_dispatch_enabled = 1; void ha_set_peerctrl_dispatch_for_test(int on) { atomic_store_explicit(&g_drive_async_peerctrl_dispatch_enabled, on ? 1 : 0, memory_order_release); } int ha_peerctrl_dispatch_enabled(void) { return atomic_load_explicit(&g_drive_async_peerctrl_dispatch_enabled, memory_order_acquire); } /* raft-drive-async task 3.1: test-only observation seam (plain global, NULL in * production — zero cost, same idiom as ha_nop_req_hook). Invoked at the top * of peer_ctrl_msg_handler_inline before any role routing. */ void (*ha_peerctrl_body_probe)(endpoint_t *ep) = NULL; /* ── Test-only hook shims (XCE_TESTING only) ────────────────────────────── * Type definitions (election_timeout_ctx_t), extern declarations * (cluster_init_phase2, push_config_transport_only, * ha_send_ctrlhello_to_all_standbys), and static forward declarations * (ha_heartbeat_is_fresh) have been moved to the forward-declarations block * near the top of this file so that the raft callback implementations * (Task 6.2) can reference them. */ #ifdef XCE_TESTING /* * ha_phase2_hook: test-only seam. * When non-NULL, ha_elect_evaluate_and_signal() calls this INSTEAD OF the real * cluster_init_phase2() so tests can count phase2 calls without a live cluster_t. */ int (*ha_phase2_hook)(void *c) = NULL; /* * ha_send_ctrlhello_hook: test-only seam for ha_send_ctrlhello_to_all_standbys. * When non-NULL, the post-election path calls this INSTEAD OF the real function * (cluster.c is not linked in unit-test binary). Req 7.2. */ int (*ha_send_ctrlhello_hook)(void *c) = NULL; /* * ha_push_config_hook: test-only seam for push_config_transport_only. * When non-NULL, the post-election path calls this INSTEAD OF the real function * (config_repl.c's push_config_transport_only needs a real cluster). Req 7.1. */ int (*ha_push_config_hook)(void *c) = NULL; /* * ha_nop_req_hook: test-only seam for xce_nop_req in primary_heartbeat_fn. * When non-NULL, primary_heartbeat_fn calls this INSTEAD OF xce_nop_req so * tests can count NOP sends without a live endpoint. Req 3.2, task 2.2. */ int (*ha_nop_req_hook)(endpoint_t *ep) = NULL; /* * ha_heartbeat_req_hook: test-only seam for the HA heartbeat send in * primary_heartbeat_fn. When non-NULL, primary_heartbeat_fn calls this INSTEAD * OF xce_ha_heartbeat_req with the SAME (term, committed_lsn, * committed_config_tag) it stamps onto the wire, so tests can assert the * heartbeat carries the primary's own current committed state. Req 1.1, 3.4. */ int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term, uint64_t committed_lsn, uint64_t committed_config_tag) = NULL; /* * ha_reconnect_connect_hook: test-only seam for the BLOCKING half of the * offloaded peer reconnect (raft-drive-async Task 2.1). When non-NULL, the * slow-lane reconnect worker calls this INSTEAD OF the real bounded * endpoint_connect + iomgr_monitor, so a test can drive the reconnect path * deterministically (e.g. park the worker to simulate an unreachable peer). * Returns 0 to simulate success, <0 to simulate failure. Req 2.1, 2.2, 2.5. */ int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx) = NULL; #ifdef XCE_TESTING /* * ha_standby_evicted_count_for_test: counts how many stale same-peer outbound * channels on_standby_connected evicted while resolving the inbound-vs-outbound * connection-ownership race (raft-drive-async Task 2.2, Req 2.6). Test-only * observability — never read on the production path. */ _Atomic int ha_standby_evicted_count_for_test = 0; #endif /* * ha_repl_push_config_for_standby_hook: test-only seam for the per-standby * config push triggered by the heartbeat-reply decision loop (Task 3.3). * When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF * push_config_transport_only() when the decision is HA_REPL_PUSH_CONFIG. * Requirements: 1.4 (ha-repl-hardening) */ int (*ha_repl_push_config_for_standby_hook)(void *c, int peer_idx) = NULL; /* * ha_repl_stream_wal_hook: test-only seam for ha_stream_wal_to_standby. * When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF * ha_stream_wal_to_standby() when the decision is HA_REPL_STREAM_WAL. * Requirements: 5.1–5.5 (ha-repl-hardening) */ int (*ha_repl_stream_wal_hook)(void *ep, const char *wal_path, uint64_t standby_lsn, uint64_t committed_lsn) = NULL; /* * ha_repl_send_snapshot_hook: test-only seam for ha_send_snapshot_to_standby. * When non-NULL, ha_dispatch_repl_action_for_standby() calls this INSTEAD OF * ha_send_snapshot_to_standby() when the decision is HA_REPL_SEND_SNAPSHOT. * Requirements: 4.1–4.5 (ha-repl-hardening) */ int (*ha_repl_send_snapshot_hook)(int peer_idx) = NULL; /* * ha_configpush_probe: test-only observation seam for the requester-less * config-push offload (raft-drive-async Task 3.3). When non-NULL, * ha_config_push_do_send() invokes it right around push_config_transport_only * (on BOTH the inline and the lane path), so a test can record the running * thread and optionally park the lane worker in flight to prove the * cluster-wide single-flight dedup. NULL in production (zero cost). */ void (*ha_configpush_probe)(void) = NULL; #endif /* XCE_TESTING */ /* ── Election Evaluation (Req 2.1, 2.7, 2.8) ─────────────────────────────── * * ha_elect_evaluate_and_signal: evaluate accumulated votes; set ctx->result * and post ctx->done when quorum is reached or all peers replied. * * Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. * ctx->votes_granted starts at 1 (self-vote); ctx->n_peers is n_other_peers. * Quorum = ceil((n_peers + 1) / 2) = (n_peers + 1 + 1) / 2 = (n_peers + 2) / 2. * * Yield gate (Req 2.7): if any responding peer has a higher config_tag, or * equal config_tag with a higher LSN than the candidate → yield (CTRL_STANDBY). * Candidate's own tag/lsn is stored in ctx->votes[0].my_tag / my_lsn. * * Fatal gate (Req 2.8): if candidate has config_tag=0 and any peer has tag>0 * → CTRL_STANDBY (never PRIMARY with no config when peer has config). * * Returns CTRL_PRIMARY or CTRL_STANDBY; also sets ctx->result. */ ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx) { /* Guard: called from handle_request_vote_reply under ctx->lock; called from * unit-test path (peers==NULL) without threads — done_posted prevents double * execution if two threads both hit quorum simultaneously. */ if (ctx->done_posted) return (ctx->result == 1) ? CTRL_PRIMARY : CTRL_STANDBY; ctrl_role_t role_result; /* Fix 2 (CRITICAL): zero-peer guard — single-node shortcircuit. * n_peers==0 means no other nodes; votes[] flexible array has 0 entries, * so accessing votes[0] would be out-of-bounds. Self-vote (votes_granted=1) * always satisfies quorum=1. No fresher-peer or fatal-gate checks apply. */ if (ctx->n_peers == 0) { ctx->result = 1; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_PRIMARY; goto apply_role; } { int total_nodes = ctx->n_peers + 1; /* include self */ int quorum = (total_nodes + 1) / 2; /* ceil(total/2) */ /* Req 2.8: fatal gate — candidate has no config but a peer does */ uint64_t my_tag = ctx->votes[0].my_tag; uint64_t my_lsn = ctx->votes[0].my_lsn; if (my_tag == 0 && ctx->max_resp_tag > 0) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } /* Check quorum */ if (ctx->votes_granted < quorum) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } /* Req 2.7: yield gate — fresher responding peer */ if (ctx->max_resp_tag > my_tag) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } if (ctx->max_resp_tag == my_tag && ctx->max_resp_lsn > my_lsn) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } /* Quorum met and no fresher peer → PRIMARY */ ctx->result = 1; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_PRIMARY; } apply_role: /* ── Task 4.1: apply role and call cluster_init_phase2 ───────────────── * * After determining the election result, update ha->role and (on first boot) * call cluster_init_phase2. This runs once per process lifetime — the * phase2_done flag prevents re-entry on subsequent elections (RDHUP failover). * * The cluster_t pointer is carried in ctx->c (NULL in unit tests). * * Cancel the election timeout watchdog if we won (it would be a no-op now * but the explicit cancel avoids a spurious second evaluate call). */ { cluster_t *c = ctx->c; int is_first_boot = !ha->phase2_done; if (role_result == CTRL_PRIMARY) { /* Cancel election timeout watchdog (quorum already reached) */ if (ctx->timeout_ctx) { election_timeout_ctx_t *etc = (election_timeout_ctx_t *)ctx->timeout_ctx; atomic_store(&etc->cancelled, 1); } /* First boot: call cluster_init_phase2 to load config+data (Req 9.2) */ if (is_first_boot) { int p2rc = 0; #ifdef XCE_TESTING /* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */ if (ha_phase2_hook != NULL) p2rc = ha_phase2_hook(c); #else if (c != NULL) p2rc = cluster_init_phase2(c); #endif if (p2rc == 0) { ha->phase2_done = 1; } else { /* bug-1477: do NOT latch phase2_done on failure. Unlike * raft_cb_become_leader, this legacy evaluation path has * ALREADY signaled the caller's ctx->done semaphore with * result=1 above (before "apply_role:") — refusing * promotion here would leave that signal inconsistent * with never becoming CTRL_PRIMARY. Proceed to * CTRL_PRIMARY as before; ha_schedule_phase2_retry wires * the commit engine shortly after instead of never. */ WARN("HA FSM: cluster_init_phase2 failed on promotion " "(rc=%d) — proceeding to CTRL_PRIMARY, retrying " "phase2 (bug-1477)", p2rc); if (c != NULL) ha_schedule_phase2_retry(c); } } /* ha-single-leader-role task 3.1 (Req 1.1, 1.5): enter CTRL_PRIMARY * DIRECTLY — symmetric with raft_cb_become_leader's rewrite. There * is no transitional COMMITTING state and no committing-timeout * watchdog; new-leader safety comes from the own-term WAL_NOOP + * writable_lsn gate armed below (ha_arm_own_term_gate), not from a * fourth role value. The legacy XCE_TESTING election path OWNS its * term (no raft library in test mode); it historically did NOT * increment the term in this branch, so we preserve that — no * ha_do_promote, no new term store (research.md §4 finding 2). */ atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire), is_first_boot); /* Push config to standbys with lower config_tag (Req 7.1). * Task 3.3: requester-less promotion push → offloadable (dormant * flag OFF = inline, unchanged; flag ON = slow lane). */ #ifdef XCE_TESTING if (ha_push_config_hook != NULL) ha_push_config_hook(c); #else if (c != NULL) ha_config_push_offloadable(c, NULL, 0, NULL, 0); #endif /* Send CtrlHelloRequest to all standbys to trigger WAL catch-up (Req 7.2) */ #ifdef XCE_TESTING if (ha_send_ctrlhello_hook != NULL) ha_send_ctrlhello_hook(c); #else if (c != NULL) ha_send_ctrlhello_to_all_standbys(c); #endif /* ha-single-leader-role task 3.1: route promotion side effects * through the SAME shared sequence raft_cb_become_leader runs so the * test-driven (legacy) and raft-driven promotion paths cannot diverge * (design §Legacy Path Symmetry). Guarded on a real cluster_t — the * helpers dereference &c->hooksys / &c->persist / &c->commit, and the * unit-test evaluation path passes c == NULL. Identical role / hook / * heartbeat / no-watchdog / own-term-gate state results either road. */ if (c) { /* Register the persistence hooks BEFORE the own-term no-op append, * so the persistence seam is live before any racing client write. * Idempotent via the hooks_registered guard. */ ha_register_persist_hooks(c); /* Periodic heartbeat to standbys (idempotency guard prevents * double-arming if already started). */ ha_start_primary_heartbeat(c); /* Own-term WAL_NOOP + writable_lsn gate (Req 1.4, 2.1, 2.2) — * non-blocking; append failure degrades safe (gate unset, * promotion still completes). */ ha_arm_own_term_gate(c); } } else { /* STANDBY result */ /* Cancel election timeout watchdog — it is now a no-op but explicit * cancellation matches spec wording and avoids a spurious lock-acquire * if the timeout fires after STANDBY is decided. */ if (ctx->timeout_ctx) { election_timeout_ctx_t *etc = (election_timeout_ctx_t *)ctx->timeout_ctx; atomic_store(&etc->cancelled, 1); } /* First boot: call cluster_init_phase2 so structures are ready; * config arrives via primary's replication push (Req 9.2, 7.3) */ if (is_first_boot) { int p2rc = 0; #ifdef XCE_TESTING /* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */ if (ha_phase2_hook != NULL) p2rc = ha_phase2_hook(c); #else if (c != NULL) p2rc = cluster_init_phase2(c); #endif if (p2rc == 0) { ha->phase2_done = 1; } else { /* bug-1477: do NOT latch phase2_done on failure — a * standby can safely stay passive while * ha_schedule_phase2_retry retries. */ WARN("HA FSM: cluster_init_phase2 failed on " "become-standby (rc=%d) — member stays alive, will " "retry (bug-1477)", p2rc); if (c != NULL) ha_schedule_phase2_retry(c); } } atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire), is_first_boot); /* Primary connects outbound and pushes config via ctrlhello. * Standby is fully passive here — no fetch_config_from_primary. */ } } return role_result; } /* ── Vote Reply Handler (Req 2.2–2.6) ────────────────────────────────────── * * handle_request_vote_reply: process a reply to an OP_requestVote RPC. * * Called by each per-peer vote-request thread once the peer replies. * Updates ctx->votes[peer_idx] with the reply fields, tracks the freshest * responding peer in ctx->max_resp_tag/max_resp_lsn, increments counters, * and evaluates whether to signal ctx->done. * * Quorum = ceil((n_peers + 1) / 2) = (n_peers + 2) / 2 * (n_peers = number of other peers, not counting self; votes_granted starts at 1) * * Posts ctx->done via ha_elect_evaluate_and_signal when: * - votes_granted reaches quorum (fast path — no need to wait for all replies), OR * - n_replied reaches n_peers (all peers accounted for, win or lose) */ /* * ha_record_vote_reply: record a decoded vote reply into ctx->votes[peer_idx] * WITHOUT signalling the legacy promotion path (bug-828). * * Records the granting flag, peer config_tag, and peer LSN under ctx->lock with * the same idempotency guard handle_request_vote_reply uses, and updates the * vote tally + freshest-responder tracking. It does NOT call * ha_elect_evaluate_and_signal — so it never posts ctx->done and never runs the * legacy apply_role promotion side-effects. * * This is the seam the PRODUCTION vote_reply_dispatch path calls so the granting * voter's advertised LSN reaches ha_compute_safe_committed_lsn (via ctx->votes[]) * while the raft library remains the SOLE production promotion driver * (raft_on_vote_reply → on_become_leader). Routing the record through here (not * handle_request_vote_reply) avoids a second, racing promotion signal. * * Returns non-zero if this reply newly reached the legacy quorum/all-replied * threshold (so handle_request_vote_reply knows to signal); 0 otherwise. * Must be called WITHOUT ctx->lock held (it takes the lock internally). */ static int ha_record_vote_reply_locked(ha_elect_ctx_t *ctx, int peer_idx, int vote_granted, uint64_t peer_config_tag, uint64_t peer_lsn) { /* Guard against out-of-range index */ if (peer_idx < 0 || peer_idx >= ctx->n_peers) return 0; pthread_mutex_lock(&ctx->lock); ha_vote_entry_t *entry = &ctx->votes[peer_idx]; /* Idempotency guard: ignore duplicate replies */ if (entry->replied) { pthread_mutex_unlock(&ctx->lock); return 0; } /* Record reply fields */ entry->replied = 1; entry->vote_granted = vote_granted ? 1 : 0; entry->peer_tag = peer_config_tag; entry->peer_lsn = peer_lsn; /* Update counters */ if (vote_granted) ctx->votes_granted++; ctx->n_replied++; /* Track the freshest responding peer for the yield gate (Req 2.7) */ if (peer_config_tag > ctx->max_resp_tag) { ctx->max_resp_tag = peer_config_tag; ctx->max_resp_lsn = peer_lsn; } else if (peer_config_tag == ctx->max_resp_tag && peer_lsn > ctx->max_resp_lsn) { ctx->max_resp_lsn = peer_lsn; } /* Compute whether the legacy quorum/all-replied threshold is now reached. * Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. */ int total_nodes = ctx->n_peers + 1; int quorum = (total_nodes + 1) / 2; int reached = (ctx->votes_granted >= quorum || ctx->n_replied >= ctx->n_peers); pthread_mutex_unlock(&ctx->lock); return reached; } /* Public record-only seam (bug-828): records the reply but never signals the * legacy promotion path. Used by the production vote_reply_dispatch path. */ void ha_record_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_config_tag, uint64_t peer_lsn) { (void)ha; /* fencing: caller bumps ha->term before calling here */ (void)term; if (!ctx) return; (void)ha_record_vote_reply_locked(ctx, peer_idx, vote_granted, peer_config_tag, peer_lsn); } void handle_request_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_config_tag, uint64_t peer_lsn) { (void)term; /* fencing: callers bump ha->term if peer's term > ours, before calling here */ if (peer_idx < 0 || peer_idx >= ctx->n_peers) return; /* Record the reply (shared with the production record-only seam) and learn * whether the legacy quorum/all-replied threshold is now reached. */ int reached = ha_record_vote_reply_locked(ctx, peer_idx, vote_granted, peer_config_tag, peer_lsn); /* Decide whether to signal the waiting thread. * ha_elect_evaluate_and_signal takes ctx->lock; its done_posted guard * prevents double sem_post if two threads reach quorum simultaneously. * Skip if the index was out of range / a duplicate reply (reached==0 and * nothing recorded) — the duplicate case is already idempotent in * ha_record_vote_reply_locked, which returns 0 without re-counting. */ if (reached) { pthread_mutex_lock(&ctx->lock); ha_elect_evaluate_and_signal(ha, ctx); pthread_mutex_unlock(&ctx->lock); } } /* ── Task 2.2: iomgr-native vote send path ────────────────────────────────── * * vote_ep_ctx_t: per-peer context stored in ep->data for each outgoing vote * endpoint. on_vote_connected fires after the TLS handshake completes (when * endpoint_t.ssl_handshake_done transitions to 1 in io_handler). */ /* vote_ep_ctx_t: typedef moved to Task 6.2 forward-declaration block above */ /* * peer_ctrl_msg_handler / vote_reply_decode_and_dispatch / * repl_ack_decode_and_dispatch: forward declarations. * peer_ctrl_msg_handler is the single iomgr handler set in on_vote_connected * and never reassigned — it routes by ha->role at dispatch time. */ static int peer_ctrl_msg_handler(void *ep_raw); static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt); static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx, io_packet_t *pkt); /* ha-prevote task 5.1: preVoteReply decode/route; defined alongside the * real-vote decoder, feeds the later-defined pre_vote_reply_dispatch. */ static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt); static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c, int peer_idx, int granted); #ifdef XCE_TESTING /* * xce_vote_request_hook: test-only seam. * * When non-NULL, on_vote_connected() calls this INSTEAD OF the real * xce_request() so tests can capture the call without requiring a live * iomgr or SSL endpoint. The hook receives the fully built * xce_request_t and may inspect all fields. * * Pattern mirrors xce_retry_send_hook in request.c. */ int (*xce_vote_request_hook)(endpoint_t *ep, xce_request_t *req) = NULL; #endif /* XCE_TESTING */ /* * on_vote_connected: iomgr callback fired after TLS handshake completes on an * outgoing vote endpoint. * * Steps: * 1. Cast ep->data to vote_ep_ctx_t. * 2. Build request_vote_req_t from ctx fields. * 3. Build xce_request_t{opcode=OPCode_requestVote, body.requestvotemsg=&vrq}. * 4. Call xce_vote_request_hook (test seam, XCE_TESTING only) — then * call xce_request() to queue the packet on the iomgr send path. * 5. Set ep->msg_handler = vote_reply_msg_handler so the next inbound * packet is decoded as a RequestVoteReply. * * Req 2.3: send vote after TLS handshake. * Req 1.2: vote RPC via iomgr path (xce_request enqueues to endpoint_enq). */ static void on_vote_connected(endpoint_t *ep) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; /* Build the RequestVoteRequest payload */ request_vote_req_t vrq; memset(&vrq, 0, sizeof(vrq)); vrq.term = vctx->my_term; vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : ""); vrq.config_tag = vctx->my_tag; vrq.last_log_lsn = vctx->my_lsn; /* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */ vrq.last_log_term = vctx->my_last_log_term; vrq.last_log_index = vctx->my_last_log_index; /* Wrap in xce_request_t */ xce_request_t req; memset(&req, 0, sizeof(req)); req.opcode = OPCode_requestVote; req.body_which = Request_body_requestvotemsg; req.body.requestvotemsg = &vrq; #ifdef XCE_TESTING if (xce_vote_request_hook != NULL) xce_vote_request_hook(ep, &req); else xce_request(ep, &req); #else xce_request(ep, &req); #endif /* Set state-aware dispatcher — routes by ha->role at message-arrival time. * Never reassigned: ELECTING → vote reply; PRIMARY → repl ack. */ ep->msg_handler = peer_ctrl_msg_handler; } /* * on_pre_vote_connected: iomgr callback fired after the TLS handshake completes * on an outgoing PreVote endpoint (ha-prevote task 4.1). * * Mirrors on_vote_connected but emits OPCode_preVoteRequest (reusing the * RequestVoteRequest payload struct + union arm, per the wire schema in task * 1.1). prospective_term (current+1) rides in the term field; nothing is * persisted. The reply-routing handler for preVoteReply is wired in task 5.1; * here we set the same state-aware dispatcher used by the real-vote path. */ static void on_pre_vote_connected(endpoint_t *ep) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; request_vote_req_t vrq; memset(&vrq, 0, sizeof(vrq)); vrq.term = vctx->my_term; /* prospective term = current + 1 */ vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : ""); vrq.config_tag = vctx->my_tag; vrq.last_log_lsn = vctx->my_lsn; /* ha-term-safety task 1.2: candidate's Raft (lastLogTerm, lastLogIndex). */ vrq.last_log_term = vctx->my_last_log_term; vrq.last_log_index = vctx->my_last_log_index; xce_request_t req; memset(&req, 0, sizeof(req)); req.opcode = OPCode_preVoteRequest; req.body_which = Request_body_requestvotemsg; req.body.requestvotemsg = &vrq; #ifdef XCE_TESTING if (xce_vote_request_hook != NULL) xce_vote_request_hook(ep, &req); else xce_request(ep, &req); #else xce_request(ep, &req); #endif ep->msg_handler = peer_ctrl_msg_handler; } /* * xce_vote_reply_hook: test-only seam for vote_reply_msg_handler. * * Forward-declared here (before vote_reply_msg_handler) so the handler body * can reference it. Defined only under XCE_TESTING; guarded below. * * When non-NULL, vote_reply_msg_handler() calls this INSTEAD OF the real * handle_request_vote_reply() so tests can capture decoded fields without * a live iomgr. Receives the decoded scalar fields — no capnp pointers. * * Pattern mirrors xce_vote_request_hook used in task 2.2. */ #ifdef XCE_TESTING int (*xce_vote_reply_hook)(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn) = NULL; #endif /* XCE_TESTING */ /* * vote_reply_dispatch: internal helper — delegates to the raft library or * the test hook (when XCE_TESTING and hook is non-NULL). * * Production path: raft_on_vote_reply owns quorum tracking and fires * on_become_leader/on_become_follower via the raft callbacks. * Test path: xce_vote_reply_hook captures decoded fields without real iomgr. */ static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx, uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn) { #ifdef XCE_TESTING if (xce_vote_reply_hook != NULL) { xce_vote_reply_hook(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn); return; } #endif /* bug-828 (PRIMARY fix): route the decoded reply into the election context * BEFORE driving the raft library. Without this, the granting voter's * advertised peer_lsn was discarded — ctx->votes[idx].vote_granted stayed 0, * so a clean failover's ha_compute_safe_committed_lsn (read in * raft_cb_become_leader / on_become_leader) saw only the self-vote * (n_granted=1) and truncated committed entries down to self_lsn. * * Record-only: ha_record_vote_reply records ctx->votes[idx] under ctx->lock * but does NOT post ctx->done / run the legacy apply_role promotion path. * raft_on_vote_reply remains the SOLE production promotion driver (it owns * the quorum tally and fires on_become_leader) — so there is no double * promotion. `idx` is vctx->peer_idx, the local ctx->votes[] index (set in * raft_cb_send_vote_request via the absolute→local mapping), which is exactly * the index ha_compute_safe_committed_lsn and the test hook use. */ ha_record_vote_reply(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn); /* bug-1417: raft_on_vote_reply's from_idx must be the ABSOLUTE cluster * index (the space vote_replies[]/self_idx live in — raft_start_election * pre-marks vote_replies[self_idx] and sends to every absolute peer != * self_idx, raft.c:113-153), NOT the compressed self-excluded votes_idx * `idx` carries for ctx->votes[] above. Passing the compressed idx * straight through collides with vote_replies[self_idx] whenever a * peer's compressed votes_idx numerically equals self_idx (true for any * peer at or above self_idx in a >2-node cluster, e.g. self_idx=1's * absolute peer 2 compresses to votes_idx=1) — raft_on_vote_reply's * per-peer dedup guard then silently treats the genuine reply as an * already-counted duplicate and drops it, so a middle-priority node can * never reach quorum on that peer's vote alone. ha_repl_ack_abs_peer_idx * (ha.c:3779) already implements the exact inverse of * raft_cb_send_vote_request's votes_idx remap and is unit-tested; reuse * it here (ep_is_vote=1) rather than duplicating the arithmetic. * * bug-1435: raft_on_vote_reply mutates raft_state_t's plain (non-atomic) * votes_granted/replies_received/election_decided fields and can fire * on_become_leader/on_become_follower synchronously. With raft-drive-async * pool dispatch (default ON), a DIFFERENT peer's vote reply can be * processed on a DIFFERENT pool worker concurrently — without raft_lock, * two concurrent calls can race: one reaches quorum and promotes to * PRIMARY, while the other, racing on the pre-increment vote count, * independently satisfies the "all replied, no quorum" branch and * immediately fires on_become_follower for the same term, silently * reverting role to STANDBY right after promotion (verified data race — * every request thereafter is correctly, faithfully rejected as STANDBY * by ha_dispatch_gate, matching the observed symptom exactly). Hold * raft_lock across the whole call so the state-machine evaluation and any * callback it fires are atomic with respect to other raft_state_t * entry points. */ pthread_mutex_lock(&ha->raft_lock); raft_on_vote_reply(&ha->raft, ha_repl_ack_abs_peer_idx(ha, /*ep_is_vote=*/1, idx), vote_granted, term); pthread_mutex_unlock(&ha->raft_lock); } /* * vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch * to vote_reply_dispatch. Factored from vote_reply_msg_handler so that * peer_ctrl_msg_handler can invoke the same logic without code duplication. * Packet ownership (raft-drive-async task 3.1): the caller * (peer_ctrl_msg_handler) has ALREADY taken `pkt` off ep->fsm_in.pkt and * cleared ep->fsm_in.pkt to NULL — `pkt` is passed in explicitly and this * function frees it on every path. Clearing in the caller (not here) is what * makes the off-iomgr dispatch path race-free: the body never touches * ep->fsm_in.pkt, so a concurrent io_worker read on the same endpoint cannot * collide with the in-flight packet. vote_reply_dispatch can still reach * raft_cb_become_follower → endpoint_fini(ep) (loser path) safely because * ep->fsm_in.pkt is already NULL. */ static int vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt) { ha_state_t *ha = vctx->ha; ha_elect_ctx_t *ctx = vctx->ctx; int idx = vctx->peer_idx; int ok = 0; uint64_t vr_term = 0; int vr_granted = 0; uint64_t vr_tag = 0; uint64_t vr_lsn = 0; if (!pkt || !pkt->data || pkt->sz == 0) goto done; { struct capn capn_ctx; struct Request outer_rq; Request_ptr outer_rp; struct RequestReply rep_s; struct RequestVoteReply vr_raw; int capn_ok; capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data, (size_t)pkt->sz, 0); if (capn_ok != 0) goto done; outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1); read_Request(&outer_rq, outer_rp); if (outer_rq.body_which != Request_body_replymsg || !outer_rq.body.replymsg.p.data) { capn_free(&capn_ctx); goto done; } read_RequestReply(&rep_s, outer_rq.body.replymsg); if (rep_s.data_which != RequestReply_data_requestvotereplymsg) { capn_free(&capn_ctx); goto done; } memset(&vr_raw, 0, sizeof(vr_raw)); read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg); capn_free(&capn_ctx); vr_term = (uint64_t)vr_raw.term; vr_granted = vr_raw.voteGranted ? 1 : 0; vr_tag = (uint64_t)vr_raw.peerConfigTag; vr_lsn = (uint64_t)vr_raw.peerLastLogLsn; ok = 1; } done: if (pkt) { free_packet(pkt); free(pkt); } if (ok) { uint64_t cur = atomic_load_explicit(&ha->term, memory_order_acquire); while (vr_term > cur) { if (atomic_compare_exchange_weak_explicit(&ha->term, &cur, vr_term, memory_order_acq_rel, memory_order_acquire)) break; } vote_reply_dispatch(ha, ctx, idx, vr_term, vr_granted, vr_tag, vr_lsn); } else { vote_reply_dispatch(ha, ctx, idx, atomic_load_explicit(&ha->term, memory_order_acquire), 0, 0, 0); } return 0; } /* * pre_vote_reply_decode_and_dispatch: decode a preVoteReply packet on an * outbound PreVote endpoint and feed it to the PreVote tally (ha-prevote task * 5.1). Mirrors vote_reply_decode_and_dispatch exactly — the PreVote reply * reuses the identical RequestVoteReply payload / union arm — but routes to * pre_vote_reply_dispatch (the raft-library PreVote tally) instead of the * real-vote tally, and deliberately performs NO term adoption (a PreVote reply * never changes our term; Req 1.2). vctx->peer_idx is the ABSOLUTE cluster * index that raft_on_pre_vote_reply expects (set in ha_send_pre_vote_request). * Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly * (already taken off ep->fsm_in.pkt by peer_ctrl_msg_handler); this function * frees it on every path and never touches ep->fsm_in.pkt. */ static int pre_vote_reply_decode_and_dispatch(vote_ep_ctx_t *vctx, io_packet_t *pkt) { ha_state_t *ha = vctx->ha; cluster_t *c = vctx->c; int idx = vctx->peer_idx; /* ABSOLUTE cluster index */ int ok = 0; int vr_granted = 0; if (!pkt || !pkt->data || pkt->sz == 0) goto done; { struct capn capn_ctx; struct Request outer_rq; Request_ptr outer_rp; struct RequestReply rep_s; struct RequestVoteReply vr_raw; int capn_ok; capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data, (size_t)pkt->sz, 0); if (capn_ok != 0) goto done; outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1); read_Request(&outer_rq, outer_rp); if (outer_rq.body_which != Request_body_replymsg || !outer_rq.body.replymsg.p.data) { capn_free(&capn_ctx); goto done; } read_RequestReply(&rep_s, outer_rq.body.replymsg); if (rep_s.data_which != RequestReply_data_requestvotereplymsg) { capn_free(&capn_ctx); goto done; } memset(&vr_raw, 0, sizeof(vr_raw)); read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg); capn_free(&capn_ctx); vr_granted = vr_raw.voteGranted ? 1 : 0; ok = 1; } done: if (pkt) { free_packet(pkt); free(pkt); } /* A malformed / wrong-body reply counts as a non-grant (best-effort * delivery, per design Event Contract). No term adoption either way. */ pre_vote_reply_dispatch(ha, c, idx, ok ? vr_granted : 0); return 0; } /* * repl_ack_decode_and_dispatch: decode an inbound packet on a promoted vote * endpoint while role == CTRL_PRIMARY. * * - OPCode_replCatchUpDone → INFO no-op (ack-counting removed in task 4.1) * - OPCode_reply with stateReportMsg → ha_process_standby_state (Task 3.1) * - OPCode_reply with walackreplymsg → repl_on_ack (Task 4.1) * - OPCode_reply with replconfigpushreplymsg → repl_on_config_ack * (barrier-rollout task 5.2 r1 / ha-config-bootstrap task 8.1) * - OPCode_nop or OPCode_reply with any other body → silently ignored * - anything else → WARN + drop */ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx, io_packet_t *pkt) { ha_state_t *ha = vctx->ha; cluster_t *c = vctx->c; /* Packet ownership (raft-drive-async task 3.1): `pkt` is passed in * explicitly (peer_ctrl_msg_handler already cleared ep->fsm_in.pkt); this * function frees it and never touches ep->fsm_in.pkt. * * raft-drive-async tasks 3.2/3.3 (FUTURE): the long ops reachable from the * stateReport branch below — ha_dispatch_repl_action_for_standby's * snapshot send / config push — are where the SlowLane offload lands. For * task 3.1 the body only relocates onto the pool and still runs bounded * decode/route work inline in the vtask entry. */ if (!pkt || !pkt->data || pkt->sz == 0) { if (pkt) { free_packet(pkt); free(pkt); } return 0; } struct capn capn_ctx; struct Request outer_rq; Request_ptr outer_rp; int capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data, (size_t)pkt->sz, 0); free_packet(pkt); free(pkt); if (capn_ok != 0) return 0; outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1); read_Request(&outer_rq, outer_rp); int opcode = (int)outer_rq.opcode; if (opcode == OPCode_replCatchUpDone) { capn_free(&capn_ctx); /* ha-single-leader-role task 4.1: the force-promotion ack-counting * machinery has been deleted — promotion is driven solely by the * heartbeat-driven decision loop and per-entry majority ack. Standbys * still send OPCode_replCatchUpDone and the dispatch gate still admits * it; it is now an INFO-logged no-op. */ (void)ha; (void)c; INFO("HA: replCatchUpDone received (no-op; ack-counting machinery removed in task 4.1)"); } else if (opcode == OPCode_reply && outer_rq.body_which == Request_body_replymsg && outer_rq.body.replymsg.p.data) { /* Heartbeat NOP reply — decode inner RequestReply for stateReport */ struct RequestReply rep_s; memset(&rep_s, 0, sizeof(rep_s)); read_RequestReply(&rep_s, outer_rq.body.replymsg); if (rep_s.data_which == RequestReply_data_stateReportMsg) { /* Decode stateReport and call convergence tracker (Task 3.1). * read_StateReport reads scalar fields directly from the capnp * pointer without heap allocation — safe to call before capn_free. */ struct StateReport sr_raw; memset(&sr_raw, 0, sizeof(sr_raw)); read_StateReport(&sr_raw, rep_s.data.stateReportMsg); state_report_t sr; sr.applied_lsn = (uint64_t)sr_raw.appliedLsn; sr.config_tag = (uint64_t)sr_raw.configTag; sr.term = (uint64_t)sr_raw.term; capn_free(&capn_ctx); /* bug-1421: a primary never learns of a follower's higher term via * the AE-response path (an AE reject is dropped before it reaches * raft_on_append_entries_reply — see ha_raft_on_repl_ack). The * heartbeat-reply state report is the fallback term-discovery * vector: it fires on every convergence cycle regardless of * replication outcome, so a stale-term primary steps down here * instead of retrying AppendEntries against a follower that will * reject every entry forever. raft_on_higher_term_observed only * touches (term, voted_for, role) — never peer-indexed * bookkeeping — so it is safe to call before peer_idx is resolved. * bug-1435: serialize against concurrent vote-reply/vote-request * processing (see ha.h raft_lock doc) — this call can also fire * on_become_follower, racing the same fields. */ pthread_mutex_lock(&ha->raft_lock); (void)raft_on_higher_term_observed(&ha->raft, sr.term); pthread_mutex_unlock(&ha->raft_lock); /* peer_idx from standby_ep_ctx_t (layout-compatible with vote_ep_ctx_t) */ int peer_idx = vctx->peer_idx; uint64_t committed_lsn = 0; uint64_t committed_tag = 0; uint64_t last_written = 0; /* latest WAL LSN — may be > committed */ #ifndef XCE_TESTING if (c) { committed_lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); committed_tag = atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire); /* current_lsn is the next LSN to write; last written = current - 1. * Read without lock — monotonic, approximate value is sufficient * for the replication decision. */ last_written = (c->persist.current_lsn > 0) ? c->persist.current_lsn - 1 : 0; } #else /* In test mode, c may be NULL; caller sets ha fields directly */ committed_lsn = atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire); committed_tag = atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire); (void)c; #endif ha_process_standby_state(ha, peer_idx, sr.config_tag, sr.applied_lsn, committed_tag, committed_lsn); /* Derive oldest_wal_lsn from the WAL directory. * The oldest retained segment's start LSN is the boundary — * standbys below it need a snapshot (Task 3.5b). */ uint64_t oldest_wal_lsn = 0; #ifndef XCE_TESTING if (c && c->persist.wal_path[0]) { extern uint64_t ha_compute_oldest_wal_lsn(const char *wal_path); oldest_wal_lsn = ha_compute_oldest_wal_lsn(c->persist.wal_path); } #endif /* Replication target (bug-1478): committed_lsn, with the * last_written bootstrap fallback ONLY when committed_lsn is * genuinely stuck at 0 — see ha_compute_repl_target_lsn's own * doc comment for why an unconditional max() permanently wedges * catch-up once a local-only checkpoint trails a real, * already-advancing committed_lsn. */ uint64_t target_lsn = ha_compute_repl_target_lsn(committed_lsn, last_written); ha_dispatch_repl_action_for_standby(ha, c, ep, peer_idx, committed_tag, target_lsn, oldest_wal_lsn); } else if (rep_s.data_which == RequestReply_data_walackreplymsg) { /* Task 4.1: WAL ACK reply from standby — extract lsn + success, * call repl_on_ack to update acked_lsn and signal majority waiter. */ struct WalAckReply war_raw; memset(&war_raw, 0, sizeof(war_raw)); read_WalAckReply(&war_raw, rep_s.data.walackreplymsg); uint64_t acked_lsn = (uint64_t)war_raw.lsn; uint64_t reply_term = (uint64_t)war_raw.term; int success = war_raw.success ? 1 : 0; capn_free(&capn_ctx); if (success && c) { repl_on_ack(&c->persist.repl, ep, acked_lsn); TRACE("repl_ack_decode_and_dispatch: WAL ACK from standby" " lsn=%lu success=1", (unsigned long)acked_lsn); /* ha-term-safety task 3.3 (R4.1/R4.5): route the ACK through the * raft library so it updates match_index[]/next_index[] and * recomputes the current-term-gated commit (raft_advance_commit). * This fires on_commit_advanced (task 2.4) ONLY when a quorum * agrees on a CURRENT-TERM entry — making the library the live * commit source alongside the durable barrier; a stale-term reply * is dropped inside the library. Driven on the same iomgr ACK * path; neither library call performs I/O. * * Resolve the ABSOLUTE library peer index per endpoint type FIRST * (remediation): this branch fires for BOTH promoted-vote eps * (vctx->ctx != NULL, peer_idx is a relative votes index) AND * heartbeat standby eps (vctx->ctx == NULL, peer_idx is already * absolute) — see the authoritative scan at ha.c:6187-6189. A * blind remap would double-map the heartbeat path onto the wrong * (or out-of-range) match_index[] slot. */ int ack_abs_idx = ha_repl_ack_abs_peer_idx( &c->ha, /*ep_is_vote*/ vctx->ctx != NULL, vctx->peer_idx); ha_raft_on_repl_ack(c, ack_abs_idx, acked_lsn, reply_term); /* ha-convergence-recovery task 4.2 (D6, Req 5.1/5.2/7.1): * ACK-driven catch-up — continue streaming to a standby still * behind target, or re-include it at parity. Shares the exact * decision the unit test drives via ha_walack_continuation_for_test. * Scheduling is O(1) on iomgr; all WAL file I/O runs in the * deferred continuation task. */ ha_walack_on_ack(c, ep, vctx->peer_idx, acked_lsn); } else if (!success) { /* bug-1421: a WAL NACK can also carry a higher term (the * standby rejected this entry as stale) — route it through * the same term-discovery path as the heartbeat state report * so the primary steps down instead of retrying forever. * bug-1435: serialize against concurrent raft_state_t access * (see ha.h raft_lock doc). */ pthread_mutex_lock(&ha->raft_lock); (void)raft_on_higher_term_observed(&ha->raft, reply_term); pthread_mutex_unlock(&ha->raft_lock); /* bug-1479: a same-or-lower-term NACK means an ordinary Raft * log conflict (ae_reject_logmismatch), NOT a stale-term * primary — the case above doesn't cover it. Before this fix * the reject was silently dropped: the library's own, * already-correct nextIndex backoff (raft_on_append_entries_ * reply) was never invoked, so the primary kept re- * broadcasting NEW entries assuming agreement at an index * the standby had already rejected, forever — the standby * would eventually get silently evicted via * MAX_PENDING_ACKS, but nothing would ever stream it back * into sync (repl_start_catchup's own driver, the ACK-driven * continuation, needs a real ACK to fire, and a log- * conflicted standby can never send one while stuck). * Immediately (not after 16 silent failures) back off via * the library and mark catch-up at the corrected position — * ha_dispatch_repl_action_for_standby's heartbeat-driven * decision (below) is what actually streams to it. */ if (c) { int ack_abs_idx = ha_repl_ack_abs_peer_idx( &c->ha, /*ep_is_vote*/ vctx->ctx != NULL, vctx->peer_idx); uint64_t corrected_lsn = 0; if (ha_raft_on_repl_nack(c, ack_abs_idx, reply_term, &corrected_lsn) == 0) { repl_start_catchup(&c->persist.repl, ep, corrected_lsn); } } WARN("repl_ack_decode_and_dispatch: WAL NACK from standby" " lsn=%lu success=0 — marking catch-up", (unsigned long)acked_lsn); } } else if (rep_s.data_which == RequestReply_data_replconfigpushreplymsg) { /* Config-push ACK from standby (ha-commit-barrier-rollout task * 5.2 remediation r1; completes the unlanded half of * ha-config-bootstrap task 8.1): the standby replies to * OP_replConfigPush with replconfigpushreplymsg carrying * {accepted, versionTag} (config_repl.c:744-752). Decode the * acked tag and apply it to the replicator so the configAck * barrier (repl_wait_majority_ack) can observe the covering * ACK — without this branch every multi-member config upgrade * would time out. read_ReplConfigPushReply reads scalars only * — safe to call before capn_free. */ struct ReplConfigPushReply rcp_raw; memset(&rcp_raw, 0, sizeof(rcp_raw)); read_ReplConfigPushReply(&rcp_raw, rep_s.data.replconfigpushreplymsg); uint64_t acked_tag = (uint64_t)rcp_raw.versionTag; int accepted = rcp_raw.accepted ? 1 : 0; capn_free(&capn_ctx); if (accepted && c) { repl_on_config_ack(&c->persist.repl, ep, acked_tag); TRACE("repl_ack_decode_and_dispatch: config ACK from standby" " tag=%016" PRIx64 " accepted=1", acked_tag); } else if (!accepted) { WARN("repl_ack_decode_and_dispatch: config push rejected by" " standby (tag=%016" PRIx64 ") — ignoring", acked_tag); } } else { capn_free(&capn_ctx); } } else if (opcode == OPCode_nop) { /* Plain NOP with no body — silently ignore */ capn_free(&capn_ctx); } else { capn_free(&capn_ctx); WARN("repl_ack_decode_and_dispatch: unexpected opcode %d on promoted ep", opcode); } return 0; } /* * peer_ctrl_msg_handler_inline: the state-aware peer-control routing body. * Routes by ha->role at dispatch time — no explicit handler reassignment needed. * CTRL_ELECTING → vote_reply_decode_and_dispatch * CTRL_PRIMARY → repl_ack_decode_and_dispatch * CTRL_STANDBY + PreVote → pre_vote_reply_decode_and_dispatch (task 5.1) * any other role → drop packet (WARN) * * Packet ownership (raft-drive-async task 3.1): `pkt` is passed in explicitly — * the caller (peer_ctrl_msg_handler) has already taken it off ep->fsm_in.pkt and * cleared that slot to NULL. This body and the routers it calls free `pkt` on * every path and NEVER touch ep->fsm_in.pkt, which is what lets this body run * off the iomgr thread (on the vtask pool) without colliding with a concurrent * io_worker read on the same endpoint. * * This body is run either INLINE on the io_worker thread (dispatch flag OFF — * the production default; identical to the pre-task behavior) or on a vtask * pool worker (flag ON; via peer_ctrl_vtask_entry). * * PreVote note (ha-prevote task 5.1): a preVoteReply arrives on the candidate's * OUTBOUND PreVote endpoint while the candidate's role is still CTRL_STANDBY * (a PreVote round performs no role change — Req 1.1). Without this branch the * reply would hit the default case and be dropped, stranding the round until it * times out. We gate on raft.pre_vote_in_progress so a STANDBY node with no * active round still drops unexpected traffic. */ static int peer_ctrl_msg_handler_inline(endpoint_t *ep, io_packet_t *pkt) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; ha_state_t *ha = vctx->ha; /* raft-drive-async task 3.1: test-only observation seam — fires before any * routing so a test can count/timestamp/park the body. NULL in production. */ if (ha_peerctrl_body_probe) ha_peerctrl_body_probe(ep); ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); /* PreVote replies land while CTRL_STANDBY with a round in progress. */ if (role == CTRL_STANDBY && ha->raft.pre_vote_in_progress) return pre_vote_reply_decode_and_dispatch(vctx, pkt); switch (role) { case CTRL_ELECTING: return vote_reply_decode_and_dispatch(vctx, pkt); case CTRL_PRIMARY: return repl_ack_decode_and_dispatch(ep, vctx, pkt); default: if (pkt) { free_packet(pkt); free(pkt); } WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role); return 0; } } /* * peer_ctrl_vtask_entry (raft-drive-async task 3.1): vtask pool entry that runs * the peer-control routing body off the iomgr thread. Extracts (ep, pkt) from * the VTASK_IO task data — the SAME io_task_data_t shape the lib vtask path * uses — takes ownership of pkt out of the task data (so vtask_fini, which only * frees the data struct, cannot reach it) and hands it to the body, which frees * it. No double-free: pkt is freed exactly once inside the body; run_task then * frees the task + data struct. endpoint_unblock_tasks (per-endpoint * serialization, Req 4.1/4.2) is driven by run_task after this returns. */ static int peer_ctrl_vtask_entry(vtask_t *task) { io_task_data_t *iod = &task->data->u.iodata; endpoint_t *ep = iod->ep; io_packet_t *pkt = iod->pkt; iod->pkt = NULL; /* body owns pkt now; nothing else may reference it */ if (ep == NULL) { if (pkt) { free_packet(pkt); free(pkt); } return -EINVAL; } return peer_ctrl_msg_handler_inline(ep, pkt); } /* * peer_ctrl_classify (raft-drive-async task 3.1): assign the LivenessAdmission * task class for an inbound peer-control message. This handler decodes vote * replies, PreVote replies and WAL/config ACKs — all RECEIPT decode + commit * eval, which is bounded liveness work. Per the design's deadlock-freedom * argument (R4.5), ack/commit-advance decode MUST be VTASK_LIVENESS so it is * NEVER admission-gated: a quorum of acks can always find a reserved worker to * advance committed_lsn even when the pool is saturated by *_BLOCK tasks. * * (The vote-GRANT term fsync — VTASK_LIVE_BLOCK, R4.6 — is handled on the * INBOUND RequestVote handler, not on these reply/ack endpoints; and the long * config-push/snapshot ops are offloaded to the SlowLane in tasks 3.2/3.3. So * everything this handler dispatches is liveness work today.) * * ┌─ FLAG-ON HAZARD CLOSED (raft-drive-async finding-2; bug-1198) ───────────┐ * │ LIVENESS is correct only once the snapshot send runs OFF the pool. The │ * │ CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT → │ * │ persist_checkpoint (a BLOCKING fdatasync) in │ * │ repl_ack_decode_and_dispatch → ha_dispatch_repl_action_for_standby. │ * │ Running that blocking op while tagged VTASK_LIVENESS (never gated) would │ * │ break the deadlock-freedom reservation — a burst of snapshot sends could │ * │ occupy every liveness worker and starve the acks that advance │ * │ committed_lsn. │ * │ │ * │ CLOSED: the borrowed-endpoint lifetime hazard (bug-1197) is fixed by an │ * │ endpoint_ref/endpoint_unref bracket around the lane dispatch (see │ * │ ha_snapshot_send_dispatch / snapshot_lane_fn), so WHENEVER both flags are │ * │ ON together the repl-ack path's snapshot send is ALWAYS offloaded before │ * │ peer_ctrl_classify ever tags it VTASK_LIVENESS on the pool — bug-1198 │ * │ cannot recur. Both flags now default ON (bug-1376, the cold-start │ * │ pool-wiring self-deadlock found flipping both ON, is root-caused + fixed │ * │ — see g_drive_async_peerctrl_dispatch_enabled's comment). │ * └──────────────────────────────────────────────────────────────────────────┘ */ static vtask_class_t peer_ctrl_classify(ha_state_t *ha, io_packet_t *pkt) { (void)ha; (void)pkt; return VTASK_LIVENESS; } /* ── bug-1197: deferred vote-ep teardown ────────────────────────────────────── * * When g_drive_async_peerctrl_dispatch_enabled is ON, raft_cb_become_follower * runs on a pool worker and must endpoint_fini() every loser vote-ep. But * another pool worker can be mid-vtask (or have a queued vtask) on one of those * same eps reading ep->data (a vote_ep_ctx_t) — freeing ep->data underneath it * is a UAF. Instead of freeing inline we enqueue a teardown vtask BOUND to the * ep (VTASK_IO): the pool's per-ep busy serialization guarantees it runs after * any in-flight or earlier-queued same-ep vtask completes. We also mark the ep * tearing_down + iomgr_unmonitor it BEFORE enqueue so the io_worker drops any * late same-batch event instead of dispatching a new vtask onto a doomed ep. */ #ifdef XCE_TESTING /* test-only ordering probe: fired (with the real ep) at the top of the teardown * vtask body, before endpoint_fini, so a test can assert teardown ran strictly * after the last live same-ep use. NULL in production. */ void (*ha_vep_teardown_probe)(endpoint_t *) = NULL; /* test-only drain probe: fired in peer_ctrl_msg_handler AFTER the tearing_down * check passes but BEFORE the pool enqueue (inside the dispatch_inflight * bracket), so a test can park a dispatch in the drained window and prove * endpoint_mark_tearing_down blocks until the enqueue completes. NULL in * production. */ void (*ha_peerctrl_predispatch_probe)(endpoint_t *) = NULL; #endif /* Runs on a pool worker, bound VTASK_IO to ep, strictly AFTER any in-flight or * queued same-ep peer-control vtask (per-ep busy FIFO). On the loser path * ectx->votes[i].ep is the sole reference to ep, so this owns the free. */ static int vote_ep_teardown_vtask_entry(vtask_t *task) { io_task_data_t *iod = &task->data->u.iodata; endpoint_t *ep = iod->ep; if (ep != NULL) { #ifdef XCE_TESTING if (ha_vep_teardown_probe) ha_vep_teardown_probe(ep); #endif /* bug-1298: drop the creator/owner reference. We DON'T fini/free here * and DON'T NULL iod->ep: this vtask still holds the per-vtask reference * (taken in vtask_enq), so the ep stays alive through run_task's post-body * endpoint_unblock_tasks; run_task then drops the per-vtask ref, and that * last endpoint_unref performs the fini+free. No UAF, no NULL hack. */ endpoint_unref(ep); } return 0; } /* Build + enqueue the teardown vtask. Caller has ALREADY * endpoint_mark_tearing_down(ep) + iomgr_unmonitor(ep). Controllers never * self-terminate: on any alloc/enqueue failure we fail loud (WARN) and tear the * ep down inline — with the pool unwired or the task rejected there is no other * worker that could be racing this ep, so an inline endpoint_fini+free is safe. */ static void vote_ep_teardown_enqueue(ha_state_t *ha, endpoint_t *ep) { vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool; vtask_data_t *data = calloc(1, sizeof(*data)); vtask_t *task = calloc(1, sizeof(*task)); if (pool == NULL || data == NULL || task == NULL) { WARN("bug-1197: vote-ep teardown enqueue unavailable (pool=%p data=%p " "task=%p) — tearing down inline (stay alive)", (void *)pool, (void *)data, (void *)task); free(data); free(task); endpoint_unref(ep); /* bug-1298: no vtask enqueued — drop owner ref → free */ return; } data->vtype = VTASK_IO; data->u.iodata.ep = ep; /* drives per-ep serialization in bind_vtask */ data->u.iodata.pkt = NULL; vtask_init(task, (vtask_func_t)vote_ep_teardown_vtask_entry, data); /* VTASK_LIVENESS is never admission-gated, so the teardown can always be * enqueued even when the pool is saturated by blocking tasks. */ if (vtask_enq_classified(pool, task, VTASK_LIVENESS) < 0) { WARN("bug-1197: vote-ep teardown enqueue failed — tearing down inline " "(stay alive)"); free(data); free(task); /* bug-1298: vtask_enq already dropped the per-vtask ref it took on the * failed enqueue, so refcnt is back to the owner ref — drop it → free. */ endpoint_unref(ep); } } /* The flag-ON loser-path teardown for one vote-ep: mark tearing_down, stop epoll * wakeups, then defer the free onto the ep's per-ep FIFO. ORDERING (bug-1197): * - endpoint_mark_tearing_down sets the flag under ep->lock; the io_worker's * late-event gate (peer_ctrl_msg_handler, ON branch) reads it under ep->lock * via endpoint_is_tearing_down, so any dispatch DECISION taken after the * mark observes it and drops — nothing new is enqueued for this ep. * - iomgr_unmonitor stops FUTURE epoll events (an event already in the current * batch is handled by the tearing_down gate above). * - the teardown vtask is itself VTASK_IO bound to ep, so the pool's per-ep * busy serialization runs it strictly after any in-flight / earlier-queued * same-ep vtask completes. * NOTE: we cannot hold ep->lock ACROSS the dispatch enqueue (vtask_enq_classified * → dataq_enq takes the taskq lock, while the pool dispatcher takes ep->lock * UNDER the taskq lock in vtask_eligible → endpoint_is_task_blocked — an AB/BA * inversion). The check is therefore done under ep->lock and the enqueue * outside it; see the io_worker gate for the residual-window note. */ static void vote_ep_teardown_async(ha_state_t *ha, endpoint_t *ep) { endpoint_mark_tearing_down(ep); if (ep->iomgr != NULL) iomgr_unmonitor(ep->iomgr, ep); vote_ep_teardown_enqueue(ha, ep); } #ifdef XCE_TESTING /* ha_vote_ep_teardown_for_test: drive the flag-ON per-ep async teardown branch * for a single ep without staging a full election/become_follower. */ void ha_vote_ep_teardown_for_test(ha_state_t *ha, endpoint_t *ep) { vote_ep_teardown_async(ha, ep); } #endif /* * peer_ctrl_dispatch_to_pool (raft-drive-async task 3.1): hand the decoded * peer-control packet to the vtask pool and return promptly. Per-endpoint * arrival-order serialization is automatic — bind_vtask calls * endpoint_block_tasks(ep) and the dispatcher won't bind a second task for the * same endpoint until this one completes (Req 4.1/4.2). * * Controllers NEVER self-terminate (CLAUDE.md): on any failure to enqueue * (pool unwired, OOM, or admission reject) we fail loud and run the body INLINE * rather than drop a consensus message — staying alive and correct. * * ┌─ FLAG-ON HAZARD CLOSED (raft-drive-async finding-1; bug-1197, task 3.4a) ─┐ * │ With this dispatch enabled, raft_cb_become_follower (pool worker) tearing │ * │ down peer vote-eps could race the single io_worker thread reading the │ * │ same ep's data for a same-batch queued event — a freed-vctx UAF. Fixed by │ * │ deferred teardown-as-vtask: become_follower marks each loser vote-ep │ * │ tearing_down + iomgr_unmonitor's it BEFORE enqueuing a per-ep-bound │ * │ teardown vtask (vote_ep_teardown_vtask_entry), so the pool's per-ep busy │ * │ serialization guarantees it runs strictly after any in-flight/queued │ * │ same-ep peer-control vtask. An RCU-like dispatch_inflight drain closes the │ * │ TOCTOU window between the flag-check and enqueue. See the bug-1197 │ * │ deferred vote-ep teardown block below and vote_ep_teardown_async. │ * └──────────────────────────────────────────────────────────────────────────┘ */ /* * bug-1376 (self-deadlock, found investigating the raft-drive-async flag * flip): the caller (peer_ctrl_msg_handler) holds ep->dispatch_inflight * incremented across this whole call. That counter exists to stop a POOL * WORKER thread from touching a queued vtask's ep concurrently with a * teardown — it protects the brief window between the tearing_down check * and a (non-blocking) enqueue. It must NOT still be held while running the * INLINE fallback body: peer_ctrl_msg_handler_inline can recursively reach * raft_cb_become_follower -> vote_ep_teardown_async -> endpoint_mark_ * tearing_down on THIS SAME ep (a node's own vote-request ep receiving the * very reply that makes it step down), which spins waiting for * dispatch_inflight==0 — a count only this call itself holds, on the same * thread, unable to ever reach 0 until this call returns. Permanent hang * (io_worker is single-threaded, so nothing else could ever be racing this * ep while we run inline — dispatch_inflight has nothing left to protect * once we've committed to the inline path). * * Fix: release dispatch_inflight at the exact right point for whichever of * the 3 exit paths fires below, instead of the caller holding it across the * whole call. The ENQUEUE-success path still releases only after the * (non-blocking) enqueue completes, preserving the original protection for * the genuinely concurrent pool-worker case; every INLINE-fallback path * releases BEFORE calling the inline body, since no pool worker can ever be * racing an ep whose message we are about to process synchronously * ourselves. */ static int peer_ctrl_dispatch_to_pool(endpoint_t *ep, io_packet_t *pkt) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; ha_state_t *ha = vctx->ha; vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool; if (pool == NULL) { WARN("peer_ctrl_dispatch_to_pool: vtask pool not wired — running " "inline (stay alive, never drop a peer-control message)"); atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return peer_ctrl_msg_handler_inline(ep, pkt); } vtask_data_t *data = (vtask_data_t *)calloc(1, sizeof(*data)); vtask_t *task = (vtask_t *)calloc(1, sizeof(*task)); if (data == NULL || task == NULL) { WARN("peer_ctrl_dispatch_to_pool: OOM allocating vtask — running " "inline (stay alive)"); free(data); free(task); atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return peer_ctrl_msg_handler_inline(ep, pkt); } data->vtype = VTASK_IO; data->u.iodata.ep = ep; /* drives per-endpoint serialization in bind_vtask */ data->u.iodata.pkt = pkt; /* ownership transferred into the task */ vtask_init(task, (vtask_func_t)peer_ctrl_vtask_entry, data); /* raft-drive-async task 3.1: the task is the SOLE owner of pkt (the shim * already cleared ep->fsm_in.pkt). Mark it so taskq_item_killer frees pkt * if this task is destroyed while still queued (never executed) at * vtaskmgr_fini — without this the pkt would leak on shutdown. The * executed path frees pkt inside peer_ctrl_vtask_entry (and NULLs * iod->pkt), so there is no double-free. */ task->owns_io_pkt = 1; vtask_class_t cls = peer_ctrl_classify(ha, pkt); int rc = vtask_enq_classified(pool, task, cls); if (rc < 0) { /* Task NOT enqueued — we still own task/data/pkt. LIVENESS is never * gated so this should not happen; if it does, fail loud and run inline * (never drop). Free the wrapper structs only — pkt is freed by the * inline body below. */ WARN("peer_ctrl_dispatch_to_pool: vtask_enq_classified failed (rc=%d) " "— running inline (stay alive)", rc); free(data); free(task); atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return peer_ctrl_msg_handler_inline(ep, pkt); } /* Enqueued: release now that the (non-blocking) enqueue itself has * completed. The queued vtask is what a pool worker will process; a * concurrent teardown drain now correctly waits on the per-ep busy/FIFO * gate (bind_vtask) for that vtask, not on this call. */ atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return 0; /* returns promptly; the body runs on a pool worker */ } /* * peer_ctrl_msg_handler: the registered iomgr msg_handler for peer-control * endpoints (set in on_vote_connected / on_pre_vote_connected / on_standby_ * connected). Runs on the io_worker thread. * * raft-drive-async task 3.1: takes ownership of the inbound packet off * ep->fsm_in.pkt and clears that slot IMMEDIATELY (so a subsequent io_worker * read on this endpoint allocates a fresh packet and cannot collide with the * one we are about to process), then either runs the routing body inline * (dispatch flag OFF — production default, byte-for-byte the prior behavior) or * hands it to the vtask pool and returns promptly (flag ON). */ static int peer_ctrl_msg_handler(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; io_packet_t *pkt = ep->fsm_in.pkt; ep->fsm_in.pkt = NULL; /* take ownership before any routing/dispatch */ if (!ha_peerctrl_dispatch_enabled()) return peer_ctrl_msg_handler_inline(ep, pkt); /* bug-1197 late-event gate (ON path only): if this ep is being torn down, * a same-batch event may still be delivered by the io_worker after the * teardown thread called iomgr_unmonitor. Dropping it here keeps the doomed * ep off the vtask pool. * * The window between the tearing_down check and the pool enqueue is closed * by ep->dispatch_inflight: we increment it (seq_cst) BEFORE reading the * flag and decrement it AFTER the enqueue completes. endpoint_mark_tearing_ * down sets the flag (under ep->lock) and then DRAINS this counter to 0 * before returning. So once the teardown caller's mark returns, every * dispatch that read the flag as false has already finished its enqueue; * because the teardown vtask is enqueued only AFTER mark returns, that * peer-control vtask is strictly FIFO-before the teardown vtask (and gates * on the same ep busy bit). Any dispatch starting after the flag is set * reads it true here and drops. No same-ep vtask is ever enqueued after the * teardown vtask — the window is fully closed. We must NOT hold ep->lock * across the enqueue (it would invert with the taskq lock, see * vote_ep_teardown_async); the counter is the lock-free handshake instead. */ atomic_fetch_add_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); if (endpoint_is_tearing_down(ep)) { if (pkt) { free_packet(pkt); free(pkt); } atomic_fetch_sub_explicit(&ep->dispatch_inflight, 1, memory_order_seq_cst); return 0; } #ifdef XCE_TESTING /* test-only seam: fires AFTER the flag-check passed but BEFORE the enqueue, * so a test can park a dispatch in exactly the drained window and prove * endpoint_mark_tearing_down blocks until the enqueue completes. */ if (ha_peerctrl_predispatch_probe) ha_peerctrl_predispatch_probe(ep); #endif /* bug-1376: peer_ctrl_dispatch_to_pool owns the matching fetch_sub now * (released at the exact right point for whichever of its 3 exit paths * fires — see its own comment) instead of a single unconditional * decrement here after it returns. See that function for why. */ return peer_ctrl_dispatch_to_pool(ep, pkt); } #ifdef XCE_TESTING /* * vote_reply_build_pkt_for_test: test helper that encodes a RequestVoteReply * into a newly allocated io_packet_t that vote_reply_msg_handler_for_test can decode. * * Wire format mirrors the real peer reply: * Request{opcode=reply, body.replymsg=RequestReply{ * data_which=requestvotereplymsg, * data.requestvotereplymsg=RequestVoteReply{term, voteGranted, ...} * }} * * Returns a freshly allocated io_packet_t* on success, NULL on failure. * Handler takes ownership: it calls free_packet()+free() when done. * Tests set ep->fsm_in.pkt to the returned pointer before calling the handler. */ io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn) { struct capn cs; struct capn_segment *seg; struct RequestVoteReply vr_capn; RequestVoteReply_ptr vr_ptr; struct RequestReply rr_capn; RequestReply_ptr rr_ptr; struct Request rq_capn; Request_ptr rq_ptr; io_packet_t *pkt; int rc; capn_init_malloc(&cs); seg = capn_root(&cs).seg; /* RequestVoteReply — all scalar fields */ memset(&vr_capn, 0, sizeof(vr_capn)); vr_capn.term = term; vr_capn.voteGranted = vote_granted ? 1 : 0; vr_capn.peerConfigTag = peer_tag; vr_capn.peerLastLogLsn = peer_lsn; vr_ptr = new_RequestVoteReply(seg); write_RequestVoteReply(&vr_capn, vr_ptr); /* RequestReply envelope */ memset(&rr_capn, 0, sizeof(rr_capn)); rr_capn.seq = 1; rr_capn.opcode = OPCode_requestVote; rr_capn.status = 0; rr_capn.data_which = RequestReply_data_requestvotereplymsg; rr_capn.data.requestvotereplymsg = vr_ptr; rr_ptr = new_RequestReply(seg); write_RequestReply(&rr_capn, rr_ptr); /* Outer Request{opcode=reply} */ memset(&rq_capn, 0, sizeof(rq_capn)); rq_capn.opcode = OPCode_reply; rq_capn.seq = 1; rq_capn.body_which = Request_body_replymsg; rq_capn.body.replymsg = rr_ptr; rq_ptr = new_Request(seg); write_Request(&rq_capn, rq_ptr); capn_setp(capn_root(&cs), 0, rq_ptr.p); pkt = calloc(1, sizeof(*pkt)); if (!pkt) { capn_free(&cs); return NULL; } pkt->sz = capn_size(&cs); rc = alloc_packet(pkt); if (rc < 0) { capn_free(&cs); free(pkt); return NULL; } capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0); capn_free(&cs); return pkt; } /* * vote_reply_msg_handler_for_test: test entry point for vote-reply decode path. * Temporarily forces role to CTRL_ELECTING so peer_ctrl_msg_handler routes to * vote_reply_decode_and_dispatch, then restores the original role. * Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred. */ int vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; ha_state_t *ha = vctx ? vctx->ha : NULL; int32_t saved_role = ha ? atomic_load_explicit(&ha->role, memory_order_acquire) : 0; if (ha) atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release); ep->fsm_in.pkt = pkt; int rc = peer_ctrl_msg_handler(ep); if (ha) atomic_store_explicit(&ha->role, saved_role, memory_order_release); return rc; } /* * vote_ep_ctx_build_for_test: test helper that allocates a vote_ep_ctx_t, * fills it from the supplied parameters, and wires up ep->data and * ep->on_connected so the test can drive on_vote_connected indirectly. * * Returns ep (same pointer passed in) for call-chain convenience. * Caller is responsible for free(ep->data) after use. */ endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t my_term, uint64_t my_tag, uint64_t my_lsn, const char *my_name) { vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx)); if (!vctx) return ep; /* test will fail when on_connected is NULL */ vctx->ha = ha; vctx->c = NULL; vctx->ctx = ctx; vctx->peer_idx = peer_idx; vctx->my_term = my_term; vctx->my_tag = my_tag; vctx->my_lsn = my_lsn; vctx->my_name = my_name; ep->data = vctx; ep->on_connected = on_vote_connected; return ep; } #ifdef XCE_TESTING /* * vote_ep_wire_on_fini_for_test (ha-elect-ctx-lifecycle task 3.3): wire the * real vote_ep_on_fini hook onto ep->on_fini, exactly as raft_cb_send_vote_request * / raft_cb_send_pre_vote_request do in production. vote_ep_on_fini is static * to this TU, so tests that build an endpoint via vote_ep_ctx_build_for_test * and need to drive a REAL endpoint_fini() (to prove the completion hook * fires and releases the carrier reference) call this to wire it first. */ void vote_ep_wire_on_fini_for_test(endpoint_t *ep) { endpoint_set_on_fini(ep, vote_ep_on_fini); } #endif /* * election_timeout_fn_for_test: test helper that exercises the election timeout * callback logic with an explicit cancelled_val instead of an atomic field. * Allows testing the cancelled-gate without setting up a real delaymgr. * * Returns 0 in all cases (mirrors the real callback's return value). */ int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx, int cancelled_val) { if (cancelled_val) return 0; pthread_mutex_lock(&ctx->lock); ha_elect_evaluate_and_signal(ha, ctx); pthread_mutex_unlock(&ctx->lock); return 0; } /* * repl_catchup_done_pkt_for_test: build a minimal io_packet_t encoding * Request{opcode=replCatchUpDone, body_which=nodata}. * Used by peer_ctrl_msg_handler tests to verify the repl-ack dispatch path. */ io_packet_t *repl_catchup_done_pkt_for_test(void) { struct capn cs; struct capn_segment *seg; struct Request rq_capn; Request_ptr rq_ptr; io_packet_t *pkt; int rc; capn_init_malloc(&cs); seg = capn_root(&cs).seg; memset(&rq_capn, 0, sizeof(rq_capn)); rq_capn.opcode = OPCode_replCatchUpDone; rq_capn.seq = 1; rq_capn.body_which = Request_body_nomsg; rq_ptr = new_Request(seg); write_Request(&rq_capn, rq_ptr); capn_setp(capn_root(&cs), 0, rq_ptr.p); pkt = calloc(1, sizeof(*pkt)); if (!pkt) { capn_free(&cs); return NULL; } pkt->sz = capn_size(&cs); rc = alloc_packet(pkt); if (rc < 0) { capn_free(&cs); free(pkt); return NULL; } capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0); capn_free(&cs); return pkt; } /* * vote_ep_err_handler_for_test: call the static vote_ep_err_handler on ep. * Allows tests to invoke the err_handler without going through iomgr/endpoint machinery. */ int vote_ep_err_handler_for_test(endpoint_t *ep) { return vote_ep_err_handler((void *)ep); } /* * peer_ctrl_msg_handler_for_test: expose peer_ctrl_msg_handler for tests. * Sets ep->fsm_in.pkt then invokes the handler. Packet ownership transferred. */ int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt) { ep->fsm_in.pkt = pkt; return peer_ctrl_msg_handler(ep); } /* * pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL * decode/route path (peer_ctrl_msg_handler) while the candidate is CTRL_STANDBY * with a PreVote round in progress — the exact production condition under which * a PreVote reply arrives. Unlike vote_reply_msg_handler_for_test it does NOT * force the role: it proves the STANDBY+pre_vote_in_progress branch added in * task 5.1 actually routes the reply to the PreVote tally (instead of dropping * it). Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred. */ int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt) { ep->fsm_in.pkt = pkt; return peer_ctrl_msg_handler(ep); } /* * vote_ep_ctx_set_cluster_for_test: set the cluster_t pointer inside the * vote_ep_ctx_t stored at ep->data. Tests that need the promoted-endpoint * dispatch path to reach ha_promote_to_primary(c) with a live cluster pointer * must call this after vote_ep_ctx_build_for_test (which sets vctx->c = NULL * by default). * Requirements: task 7.3 (ha-peer-connection-reuse). */ void vote_ep_ctx_set_cluster_for_test(endpoint_t *ep, cluster_t *c) { if (!ep || !ep->data) return; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; vctx->c = c; } /* * raft_cb_become_leader_for_test: drive the leader promotion path without * a live cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress * production-only calls (phase2, push_config, ctrlhello). */ void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term) { ha->raft_c = NULL; raft_cb_become_leader((void *)ha, term); } #ifdef XCE_TESTING /* ha_arm_promotion_fence_timer_for_test (ha-commit-durability task 4.4): * expose the file-static bounded-window arm path so the durability suite can * drive the exact deferral wired into raft_cb_become_leader without staging a * full election. Production never references this symbol (XCE_TESTING only). */ void ha_arm_promotion_fence_timer_for_test(cluster_t *c, bool truncated, uint64_t durable_floor, uint64_t target_hwm) { ha_arm_promotion_fence_timer(&c->ha, c, truncated, durable_floor, target_hwm); } #endif /* * raft_cb_become_follower_for_test: drive the follower path without a live * cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production * calls (phase2, fetch_config); ha_arm_heartbeat_timeout_vtask is a no-op * when c=NULL so no watchdog thread is spawned. */ void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term) { ha->raft_c = NULL; raft_cb_become_follower((void *)ha, term, -1); } /* * raft_cb_become_follower_with_cluster_for_test (ha-convergence-recovery * task 2.1): drive the follower path with a LIVE cluster pointer so the * cluster-dependent demotion-convergence branch (task 2.2) is exercised * against a real persist/hooksys/delaymgr fixture. Sets ha->raft_c = c so * the become-follower callback sees a real cluster (unlike the NULL-cluster * shim above). raft_cb_become_follower is file-static, so this seam must * live here in ha.c. */ void raft_cb_become_follower_with_cluster_for_test(cluster_t *c, uint64_t term) { c->ha.raft_c = c; raft_cb_become_follower((void *)&c->ha, term, -1); } /* * raft_cb_become_leader_with_cluster_for_test (fix-promotion-standby-registration): * drive the leader promotion path with a LIVE cluster pointer so the * cluster-dependent standby-registration branch (repl_add_standby for each * surviving vote endpoint whose votes[i].peer is set) is exercised against a * real persist/repl/commit/delaymgr fixture. Sets ha->raft_c = c so the * become-leader callback sees a real cluster (unlike the NULL-cluster shim * raft_cb_become_leader_for_test above, which suppresses the c-dependent * registration block entirely). raft_cb_become_leader is file-static, so this * seam must live here in ha.c. Symmetric with * raft_cb_become_follower_with_cluster_for_test. */ void raft_cb_become_leader_with_cluster_for_test(cluster_t *c, uint64_t term) { c->ha.raft_c = c; raft_cb_become_leader((void *)&c->ha, term); } #endif /* XCE_TESTING */ /* ── Task 2.4 / raft-adapter-conformance task 3.1: vote endpoint err handlers ── * * vote_ep_handle_promoted_disconnect: shared CTRL_PRIMARY-role cleanup for a * dropped promoted connection (standby-endpoint deregistration, WAL * replicator deregistration, quorum-transition logging). Extracted * byte-for-byte from the formerly-single vote_ep_err_handler so both the * real-vote and PreVote on_err handlers apply the identical cleanup — this * branch's behavior is unchanged by the split. */ static void vote_ep_handle_promoted_disconnect(ha_state_t *ha, endpoint_t *ep, vote_ep_ctx_t *vctx) { /* Promoted connection dropped — remove from standby_endpoints */ if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int i = 0; i < ha->standby_endpoints->count; i++) { if (zarray_get(ha->standby_endpoints, i) == ep) { zarray_del(ha->standby_endpoints, i); break; } } pthread_mutex_unlock(&ha->lock); } /* Task 2.2: deregister from WAL replicator. * Req 1.2: when a standby disconnects, the HA layer shall deregister * the endpoint from the replicator via repl_remove_standby. * vctx->c may be NULL in test mode — guard accordingly. */ if (vctx->c) repl_remove_standby(&vctx->c->persist.repl, ep); /* Quorum-transition observability (Req 8.1): a standby just dropped — the * write-quorum count changed. Record the loss edge once (silent if * quorum is still held after the removal). */ ha_log_quorum_transition(ha); } /* * vote_ep_err_handler: on_err callback for outgoing REAL-VOTE endpoints * (registered in raft_cb_send_vote_request). Called by iomgr when the * endpoint encounters an error (connect failure, SSL error, remote close * before reply). * * Election-phase branch counts a post-connect failure through * vote_reply_dispatch — the SAME production tally path a connect-time * failure (raft_cb_send_vote_request's own rc<0 branches, bug-1423) or a * delivered denial (vote_reply_msg_handler) already uses — instead of the * legacy handle_request_vote_reply, which bypasses raft_on_vote_reply * (raft-adapter-conformance Requirement 4.2). */ static int vote_ep_err_handler(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; if (!vctx) return 0; ha_state_t *ha = vctx->ha; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) { vote_ep_handle_promoted_disconnect(ha, ep, vctx); } else { /* Election phase: count as a denied vote through the production * tally, same path a connect-time failure or a delivered denial * uses (raft-adapter-conformance Requirement 4.2). */ vote_reply_dispatch(ha, vctx->ctx, vctx->peer_idx, vctx->my_term, /*vote_granted=*/0, 0, 0); } /* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data. * vctx is freed in ha_election_cleanup() when all per-peer contexts release. */ return 0; } /* * pre_vote_ep_err_handler: on_err callback for outgoing PREVOTE endpoints * (registered in raft_cb_send_pre_vote_request). Mirrors vote_ep_err_handler * above for the PreVote endpoint kind — the two kinds share vote_ep_ctx_t but * are already registered at distinct call sites, so the split (rather than a * new discriminant field on vote_ep_ctx_t) mirrors an existing seam. * * Election-phase branch counts a post-connect failure through * pre_vote_reply_dispatch — the SAME production tally path a connect-time * failure (raft_cb_send_pre_vote_request's own rc<0 branches, bug-1423 PreVote * twin) already uses — instead of the legacy handle_request_vote_reply * (raft-adapter-conformance Requirement 4.3). */ static int pre_vote_ep_err_handler(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; if (!vctx) return 0; ha_state_t *ha = vctx->ha; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) { vote_ep_handle_promoted_disconnect(ha, ep, vctx); } else { /* Election phase: count as a not-granted PreVote reply through the * production tally, same path a connect-time failure uses * (raft-adapter-conformance Requirement 4.3). */ pre_vote_reply_dispatch(ha, vctx->c, vctx->peer_idx, /*granted=*/0); } /* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data. * vctx is freed in ha_election_cleanup() when all per-peer contexts release. */ return 0; } #ifdef XCE_TESTING /* * pre_vote_ep_err_handler_for_test: call the static pre_vote_ep_err_handler on * ep. Allows tests to invoke the PreVote err_handler without going through * iomgr/endpoint machinery — the PreVote twin of vote_ep_err_handler_for_test * (raft-adapter-conformance task 3.2). */ int pre_vote_ep_err_handler_for_test(endpoint_t *ep) { return pre_vote_ep_err_handler((void *)ep); } #endif /* XCE_TESTING */ /* ── ha_init_raft: one-time raft state machine initialization ──────────────── * * Called once at startup after ha_restore_term() to wire the raft state * machine with the correct cluster size and callbacks. Subsequent elections * use raft_start_election() / raft_on_leader_lost() on this same state. * * n_peers: total candidates_count from cluster config (includes self). */ void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers) { /* Map persisted voted_for_name back to a peer index for initial_voted_for. * This lets raft_check_and_start_election (RDHUP path) see the prior vote * without re-reading the file. */ int restored_voted_for = -1; if (ha->voted_for_name[0] != '\0' && c && c->myconf) { const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; for (int i = 0; i < pcfg->candidates_count; i++) { if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) { restored_voted_for = i; break; } } } /* ha-term-safety task 2.5 (R1.5, R2.1): restore the durable log/commit floor * from the recovered WAL so the library boots at the recovered tip — not an * empty log at LSN 0. The library never reads storage; the caller restores * initial_log_len / initial_commit_index here (same discipline as * initial_term / initial_voted_for above). * * initial_log_len = recovered durable last-log index * (persist_durable_floor: re-derived from the WAL tip * by persist_recover; 0 on empty first boot). * initial_commit_index = recovered committed LSN (committed_lsn). Recovery * re-derives the durable tip but the post-election * safe-commit point governs the committed position; * we pass whatever committed prefix is established at * init (0 on a fresh boot). Clamp to log_len to honor * the library invariant commit_index <= log_len. * * The recovered last-log TERM is surfaced on demand via on_get_entry_term( * log_len) (the library's logOk freshness gate reads it) — there is no * separate initial last-log-term field. */ uint64_t initial_log_len = c ? persist_durable_floor(&c->persist) : 0; uint64_t initial_commit_index = c ? atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire) : 0; if (initial_commit_index > initial_log_len) initial_commit_index = initial_log_len; raft_config_t rcfg; memset(&rcfg, 0, sizeof(rcfg)); rcfg.n_peers = n_peers; rcfg.self_idx = ha->my_priority; rcfg.election_timeout_ms = ha->config.election_timeout_ms; rcfg.my_priority = ha->my_priority; rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire); rcfg.initial_voted_for = restored_voted_for; rcfg.initial_log_len = initial_log_len; rcfg.initial_commit_index = initial_commit_index; rcfg.cbs = &s_real_cbs; rcfg.ctx = ha; raft_init(&ha->raft, &rcfg); ha->n_peers = n_peers; ha->raft_c = c; INFO("HA: raft initialized (n_peers=%d, self=%d, term=%" PRIu64 ", log_len=%" PRIu64 ", commit_index=%" PRIu64 ")", n_peers, ha->my_priority, atomic_load_explicit(&ha->term, memory_order_acquire), initial_log_len, initial_commit_index); } /* ── ha-prevote task 4.1: candidate-side PreVote orchestration ───────────── * * The PreVote round is a side-effect-free majority probe interposed BEFORE the * destructive truncate + CTRL_ELECTING + real-election sequence. During the * round the node stays CTRL_STANDBY and mutates no persistent state (term / * voted_for / role). Only on a WIN does it enter the unchanged real-election * path; on a LOSS / round-timeout it stays STANDBY, re-arms the liveness * watchdog, and logs campaign abandonment. * * This task implements the round machinery + the WON/LOST branches. The * failover triggers (ha_on_primary_ctrl_fd_rdhup / liveness_poll_fn / * ha_on_heartbeat_received) are rewired to call this in task 4.2 — until then * the round is INERT in production. */ /* * prevote_round_ctx_t: delaymgr context for the PreVote round timeout. * cancelled is set when the round is decided (WON or LOST) before the timeout * fires, so a late timeout task is a no-op. */ typedef struct { _Atomic int cancelled; ha_state_t *ha; cluster_t *c; } prevote_round_ctx_t; /* Active round-timeout context. _Atomic so arm (vtask) vs timeout-fire * (delaymgr) vs cancel-on-decision is well-defined. */ static prevote_round_ctx_t * _Atomic s_prevote_round_ctx = NULL; #ifdef XCE_TESTING /* * ha_prevote_rearm_liveness_hook: test-only seam. When non-NULL, the LOST / * timeout branch calls this INSTEAD OF ha_arm_heartbeat_timeout_vtask so a unit * test can count re-arms without a live delaymgr (Req 5.4). */ int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c) = NULL; #endif /* ha_prevote_rearm_liveness: re-arm the standby liveness watchdog after a lost * round so a later genuine failure still triggers a new PreVote round (the * watchdog does not re-arm itself once it has fired). */ static void ha_prevote_rearm_liveness(ha_state_t *ha, cluster_t *c) { #ifdef XCE_TESTING if (ha_prevote_rearm_liveness_hook != NULL) { ha_prevote_rearm_liveness_hook(ha, c); return; } #endif ha_arm_heartbeat_timeout_vtask(ha, c); } /* ha_prevote_cancel_round_timeout: mark the active round-timeout context * cancelled (the delaymgr task frees it on fire) and clear the static pointer * so a later decision/timeout sees no active round. */ static void ha_prevote_cancel_round_timeout(void) { prevote_round_ctx_t *old = atomic_load_explicit(&s_prevote_round_ctx, memory_order_acquire); if (old) { atomic_store_explicit(&old->cancelled, 1, memory_order_release); atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release); } } /* * ha_enter_real_election: the WON-branch sequence — truncate uncommitted WAL * above the committed boundary, then enter the UNCHANGED real-election path * (ha_start_election_async transitions to CTRL_ELECTING and delegates the term * bump + self-vote to raft_start_election). Factored here so both the PreVote * WON branch (task 4.1) and the rewired trigger (task 4.2) can call it; this * task does NOT change ha_on_primary_ctrl_fd_rdhup's own behaviour. * * c == NULL (unit-test mode): skip truncate + the live election; just transition * the role to CTRL_ELECTING so the WON transition is observable. */ static void ha_enter_real_election(ha_state_t *ha, cluster_t *c) { if (!c) { /* Test mode: the real election needs iomgr/delaymgr; just mark the * destructive transition so the WON branch is observable. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release); return; } /* bug-828 final slice (Leader Completeness): do NOT pre-truncate the WAL * here. The earlier code truncated above ha->last_known_committed_lsn * BEFORE the vote — but that watermark is the LOCALLY-LEARNED commit point, * which LAGS the true cluster commit point (the primary commits an entry on * a durable majority, then dies before a heartbeat carries that committed * LSN to this standby). persist_truncate_wal_above clamps durable_lsn down * to the cutoff, so a standby that had DURABLY persisted committed entries * [committed+1 .. tip] would discard them and then advertise the stale-low * cutoff as its durable floor. Every voter doing the same made * safe-committed-from-votes collapse to the stale watermark, and the * AUTHORITATIVE post-election truncate (raft_cb_become_leader, * persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale * prefix — losing committed jobs on a clean failover (bug-828/795). * * Raft model: a candidate does NOT truncate its log before campaigning. It * keeps its full durable log and advertises its real durable tip * (ha_candidate_advertise_lsn == persist_durable_floor), so voters holding * more committed entries deny a colder candidate and the winner has every * committed entry (Leader Completeness). Divergent UNCOMMITTED tails are * removed ONLY by the authoritative post-election safe-commit truncate * (above the majority-held point) and by the library's per-entry (term,lsn) * reconcile (on_truncate_from) during subsequent AppendEntries — never by a * stale-watermark pre-truncate that can erase committed state. */ /* Candidate freshness for the RequestVote payload (ha-commit-durability * task 4.2). The TAG axis uses the same config-epoch fold the voter side * computes (ha_vote_self_freshness), but the LSN axis advertises the BARE * durable WAL tip — a candidate may only claim a log position it can * durably back, so a truncated-restart node cannot over-advertise a stale * volatile learned-committed value and win (design §1a/§5a). */ uint64_t cand_tag = 0, cand_lsn = 0; if (c->myconf) { uint64_t local_tag = c->myconf->config_version_tag; uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL); cand_lsn = ha_candidate_advertise_lsn(&c->persist); } int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1; const char *my_name = (c->myconf) ? c->myconf->name : ""; ha->raft_c = c; #ifdef XCE_TESTING /* Test seam: the WON-branch real election is the same step the pre-rewire * rdhup body fired, so the existing rdhup-integration tests intercept it * here (capturing my_tag/my_lsn / counting election starts) — they now * drive it through the PreVote→WON path (ha-prevote task 4.2). * * Mark the role CTRL_ELECTING before invoking the hook: the real * ha_start_election_async transitions to CTRL_ELECTING as its first step, * so a test that intercepts the election must still observe the destructive * role transition the WON branch performs (the spy hook does not start a * real election that would set it). */ if (ha_rdhup_election_hook != NULL) { atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release); if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) { const xce_cluster_config_t *cfg = &c->myconf->peers_cfg; int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0; xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES]; int np = 0; for (int i = 0; i < cfg->candidates_count && np < n; i++) { if (strcmp(cfg->candidates[i].name, c->myhost ? c->myhost->name : "") != 0) peer_ptrs[np++] = &cfg->candidates[i]; } ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn, my_name); } return; } #endif /* ha_start_election_async transitions to CTRL_ELECTING and drives the * UNCHANGED raft_start_election (term++ + self-vote). Peer connections are * driven by raft_cb_send_vote_request; the peers[] arg is unused. */ ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name); } /* * ha_start_prevote_async: start a PreVote round (ha-prevote task 4.1). * * Snapshots self tag/lsn into the round metadata carrier (ha->elect_ctx, read * by raft_cb_send_pre_vote_request), calls raft_begin_pre_vote (self-grant + * per-peer send via the callback), and arms a round-timeout bounded by * election_timeout_ms (Req 5.4 — no unbounded delay). The node stays * CTRL_STANDBY; NOTHING persistent is mutated (no term/voted_for/role change). * * If raft_begin_pre_vote returns RAFT_PRE_VOTE_WON immediately (single-node * cluster: self-grant is already a majority), the WON branch runs at once. * * c == NULL (unit-test mode): the send callback returns early and the round * timeout is a no-op; the round state lives entirely in raft_state_t so the * reply-driven branches are still fully exercisable via the test seam. */ void ha_start_prevote_async(ha_state_t *ha, cluster_t *c) { if (!ha) return; ha->raft_c = c; /* Snapshot self freshness for the PreVote request payload (same floor as a * real vote). Stored in the metadata carrier ha->elect_ctx so * raft_cb_send_pre_vote_request can read my_tag/my_lsn. */ uint64_t cand_tag = 0, cand_lsn = 0; int n_peers = ha->n_peers > 0 ? ha->n_peers : ha->raft.n_peers; if (c && c->myconf) { uint64_t local_tag = c->myconf->config_version_tag; uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); /* Task 4.2: TAG via the config-epoch fold, LSN as the bare durable WAL * tip (PreVote must not over-advertise either — design §1a). */ ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, NULL); cand_lsn = ha_candidate_advertise_lsn(&c->persist); n_peers = c->myconf->peers_cfg.candidates_count; int n_other = (n_peers > 1) ? n_peers - 1 : 0; ha_elect_ctx_t *mc = calloc(1, sizeof(*mc) + (size_t)n_other * sizeof(mc->votes[0])); if (mc) { mc->n_peers = n_other; mc->result = -1; mc->c = c; mc->self_tag = cand_tag; mc->self_lsn = cand_lsn; pthread_mutex_init(&mc->lock, NULL); sem_init(&mc->done, 0, 0); atomic_init(&mc->refcount, 1); /* ha-elect-ctx-lifecycle task 2.4: bias reference */ if (n_other > 0) { mc->votes[0].my_tag = cand_tag; mc->votes[0].my_lsn = cand_lsn; } /* ha-elect-ctx-lifecycle task 2.4: exchange (not store) so we can * release the superseded carrier's bias reference — it is freed * here immediately if no vote endpoint from that prior round still * references it, or later, whenever that endpoint's on_fini fires, * whichever comes last. Replaces the old deliberate leak (see * design.md's Architecture §Mechanism). */ ha_elect_ctx_t *old_ctx = (ha_elect_ctx_t *)atomic_exchange_explicit( &ha->elect_ctx, mc, memory_order_acq_rel); ha_elect_ctx_release(old_ctx); } ha->n_peers = n_peers; } INFO("HA: starting PreVote round (prospective term=%" PRIu64 ", n_peers=%d) " "— staying CTRL_STANDBY, no persist", ha->raft.term + 1, n_peers); /* raft_begin_pre_vote: self-grant + per-peer send; mutates only the * pre_vote_* tally in raft_state_t (no term/voted_for/role change). * bug-1435: serialize against concurrent raft_state_t access (see ha.h * raft_lock doc) — pre_votes_granted/pre_vote_replies[] are just as * unprotected as the real-vote tally. */ pthread_mutex_lock(&ha->raft_lock); int rc = raft_begin_pre_vote(&ha->raft); pthread_mutex_unlock(&ha->raft_lock); /* Arm the round timeout (bounded by election_timeout_ms). Production only; * the round state is in raft_state_t so tests exercise the branches via the * reply seam without the timer. */ ha_prevote_cancel_round_timeout(); if (c && c->delaymgr.running) { prevote_round_ctx_t *tctx = calloc(1, sizeof(*tctx)); if (tctx) { atomic_init(&tctx->cancelled, 0); tctx->ha = ha; tctx->c = c; atomic_store_explicit(&s_prevote_round_ctx, tctx, memory_order_release); extern int ha_prevote_round_timeout_fn(void *arg); int trc = delay_task_ms(&c->delaymgr, (int)ha->config.election_timeout_ms, 1, (delay_task_entry_fn_t)ha_prevote_round_timeout_fn, tctx); if (trc < 0) { WARN("HA: ha_start_prevote_async: round-timeout schedule failed: %d", trc); atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release); free(tctx); } } } /* Single-node: the self-grant is already a majority — win immediately. */ if (rc == RAFT_PRE_VOTE_WON) { ha_prevote_cancel_round_timeout(); INFO("HA: PreVote won immediately (single-node) — entering real election"); ha_enter_real_election(ha, c); } } /* * pre_vote_reply_dispatch: feed a PreVote reply into the raft tally and branch * on its outcome (ha-prevote task 4.1). peer_idx is the ABSOLUTE cluster index * of the replying peer (the same index raft_begin_pre_vote sent to). * * RAFT_PRE_VOTE_WON → cancel round timeout, run the real-election sequence * (truncate + CTRL_ELECTING + ha_start_election_async). * RAFT_PRE_VOTE_LOST → stay CTRL_STANDBY, re-arm liveness, log abandonment. * RAFT_PRE_VOTE_PENDING → no-op (await more replies). * * Mirrors vote_reply_dispatch → raft_on_vote_reply, but the raft library makes * NO role/term change here — the controller acts on the returned status. * * Production caller (task 5.1): pre_vote_reply_decode_and_dispatch, reached from * peer_ctrl_msg_handler when a preVoteReply arrives on the outbound PreVote * endpoint. Also driven by the XCE_TESTING reply seam. */ static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c, int peer_idx, int granted) { /* bug-1435: serialize against concurrent raft_state_t access (see ha.h * raft_lock doc). raft_lock is RECURSIVE: the WON branch below calls * ha_enter_real_election -> ... -> raft_start_election synchronously on * this same thread, which re-acquires raft_lock at its own call site — * safe by design (same-thread nesting), never blocks a legitimate chain. */ pthread_mutex_lock(&ha->raft_lock); int outcome = raft_on_pre_vote_reply(&ha->raft, peer_idx, granted); pthread_mutex_unlock(&ha->raft_lock); if (outcome == RAFT_PRE_VOTE_WON) { ha_prevote_cancel_round_timeout(); INFO("HA: PreVote round WON — entering real election (term will be %" PRIu64 ")", ha->raft.term + 1); ha_enter_real_election(ha, c); return; } if (outcome == RAFT_PRE_VOTE_LOST) { ha_prevote_cancel_round_timeout(); /* Req 7.3: log campaign abandonment distinctly from a genuine failover * so operators can tell a suppressed disruptive election apart. */ INFO("HA: PreVote round LOST — abandoning campaign, staying CTRL_STANDBY " "(no term increment); re-arming liveness watchdog"); /* Stay CTRL_STANDBY — no persist, no role change (Req 1.4). */ ha_prevote_rearm_liveness(ha, c); return; } /* RAFT_PRE_VOTE_PENDING — await more replies, do nothing. */ } /* * ha_prevote_round_timeout_fn: delaymgr task body for the PreVote round timeout. * A timeout that fires before a decision is treated as a LOSS (Req 5.4). */ int ha_prevote_round_timeout_fn(void *arg) { prevote_round_ctx_t *ctx = (prevote_round_ctx_t *)arg; /* bug (2026-07-10 chaotic-soak ASAN finding): clear s_prevote_round_ctx * for THIS ctx BEFORE freeing it, unconditionally -- regardless of which * branch below runs. The prior code only cleared the static pointer * inside the "round still in progress" branch; when pre_vote_in_progress * was ALREADY false here (the round was decided via a reply arriving on * a different path, one that doesn't itself clear this pointer) this * function fell straight through to free(ctx) below with * s_prevote_round_ctx still pointing at the about-to-be-freed memory. * The NEXT PreVote round's ha_start_prevote_async then calls * ha_prevote_cancel_round_timeout() defensively before arming its own * timer, reads that dangling pointer, and writes through it -- * heap-use-after-free (confirmed via ASAN on a live chaotic HA soak, * both crashing nodes wedged with an identical ha.c:6712 UAF). CAS so a * newer round's context (if one has already replaced this one by the * time this delaymgr task runs) is never incorrectly cleared. */ { prevote_round_ctx_t *expected = ctx; atomic_compare_exchange_strong_explicit( &s_prevote_round_ctx, &expected, NULL, memory_order_acq_rel, memory_order_relaxed); } if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) { #ifndef XCE_TESTING free(ctx); #endif return 0; } ha_state_t *ha = ctx->ha; cluster_t *c = ctx->c; /* Only act if the round is still in progress (not already decided). */ if (ha->raft.pre_vote_in_progress) { ha->raft.pre_vote_in_progress = 0; WARN("HA: PreVote round timed out before a decision — treating as LOSS, " "staying CTRL_STANDBY; re-arming liveness watchdog (Req 5.4)"); ha_prevote_rearm_liveness(ha, c); } #ifndef XCE_TESTING free(ctx); #endif return 0; } #ifdef XCE_TESTING /* * pre_vote_reply_dispatch_for_test: drive the real pre_vote_reply_dispatch with * decoded (peer_idx, granted) fields, bypassing iomgr (ha-prevote task 4.1). */ void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, int peer_idx, int granted) { pre_vote_reply_dispatch(ha, c, peer_idx, granted); } /* * ha_prevote_round_timeout_fire_for_test: simulate the round timeout firing * before a decision, driving the LOST branch (Req 5.4). */ void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c) { prevote_round_ctx_t ctx; memset(&ctx, 0, sizeof(ctx)); ctx.ha = ha; ctx.c = c; atomic_init(&ctx.cancelled, 0); ha_prevote_round_timeout_fn(&ctx); } #endif /* XCE_TESTING */ /* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ──── * * Non-blocking election initiator. Sets up the raft state machine and * delegates all election mechanics to raft_start_election(). * * raft_start_election() handles: * - term increment and self-vote (via on_persist callback) * - single-node shortcircuit (fires on_become_leader immediately) * - multi-node RPC fan-out (via on_send_vote_request callback per peer) * - election timeout scheduling (via on_schedule_timeout callback) * * ha_elect_ctx_t is still allocated here because raft_cb_send_vote_request * reads ha->elect_ctx to retrieve my_tag/my_lsn for the capnp VoteRequest * payload. Vote counting and quorum decisions are now owned by the raft * library (raft_on_vote_reply fires on_become_leader/follower callbacks). * * Requirements: 1.3, 2.1 (shortcircuit), 2.4 (async vote send), 9.2 */ void ha_start_election_async(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) { (void)peers; /* peer connections driven by raft_cb_send_vote_request */ (void)my_name; /* self-vote name stored in on_persist via raft_start_election */ /* Step 0: raft is initialized once at startup by ha_init_raft(); nothing * to do here. ha->raft_c and ha->n_peers are already set. */ /* Step 1: transition to CTRL_ELECTING before firing RPCs. * raft_start_election fires on_become_leader / on_send_vote_request below; * the role must already be ELECTING so ha_dispatch_gate() admits vote RPCs. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release); INFO("HA FSM: → CTRL_ELECTING (async, n_peers=%d)", n_peers); /* Step 2: pre-allocate ha_elect_ctx_t so raft_cb_send_vote_request can * read my_tag/my_lsn from it when building the capnp VoteRequest payload. * Vote counting is now owned by the raft library; this ctx is only a * metadata carrier for tag/lsn. For multi-node elections it is kept alive * until all per-peer vote_ep_ctx_t endpoints finish (iomgr lifetime). * For single-node, raft_start_election fires on_become_leader immediately * so no ctx reads occur after return. * * n_peers = total cluster size (including self), matching raft convention. * standby_converged[] is indexed by votes_idx (0 .. n_peers-2), so * convergence loops iterate i < n_peers - 1 (skip self). */ int n_other = (n_peers > 1) ? n_peers - 1 : 0; /* Store total cluster size (including self) — matches raft_config_t.n_peers */ ha->n_peers = n_peers; ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + (size_t)n_other * sizeof(ctx->votes[0])); if (!ctx) { WARN("ha_start_election_async: calloc ctx failed"); return; } ctx->n_peers = n_other; ctx->votes_granted = 1; /* self-vote placeholder (raft tracks actual count) */ ctx->n_replied = 0; ctx->max_resp_tag = 0; ctx->max_resp_lsn = 0; ctx->result = -1; ctx->done_posted = 0; ctx->c = c; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); atomic_init(&ctx->refcount, 1); /* ha-elect-ctx-lifecycle task 2.4: bias reference */ /* Store candidate's own tag/lsn unconditionally in self_lsn/self_tag (task 6.3): * ha_compute_safe_committed_lsn uses these for the majority computation, * including the single-node case where votes[] is empty. */ ctx->self_lsn = my_lsn; ctx->self_tag = my_config_tag; /* Also store in votes[0] for raft_cb_send_vote_request (multi-node only) */ if (n_other > 0) { ctx->votes[0].my_tag = my_config_tag; ctx->votes[0].my_lsn = my_lsn; } /* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx. * ha-elect-ctx-lifecycle task 2.4: exchange (not store) to retrieve and * release the superseded carrier's bias reference — see the identical * rationale at ha_start_prevote_async's publish site. */ ha_elect_ctx_t *prior_ctx = (ha_elect_ctx_t *)atomic_exchange_explicit( &ha->elect_ctx, ctx, memory_order_acq_rel); ha_elect_ctx_release(prior_ctx); /* Step 4: delegate election to raft library. * * A node with no valid local config (my_config_tag == 0) must not start an * election — any peer with a real config tag will beat it in the freshness * gate and the node would waste a term increment. It stays in CTRL_ELECTING, * accepts incoming vote requests (granting to fresher peers), and will * become STANDBY after the winner sends it a heartbeat/ctrlhello. * n_peers and raft state are already initialised above so RDHUP retries work. * * raft_start_election when allowed: * - increments ha->raft.term and records self-vote (fires on_persist) * - single-node: fires on_become_leader → raft_cb_become_leader * - multi-node: fires on_send_vote_request per peer → raft_cb_send_vote_request * then fires on_schedule_timeout → raft_cb_schedule_timeout * * No direct ha->term++ or ha->voted_for_* writes happen in this function * after this call; the raft library owns those via on_persist. */ /* Req 4.3 / ha-config-bootstrap Req 2.8: * A node with no local config (my_config_tag == 0) must not start an * election if any peer has a valid config — it cannot win (freshness gate * denies it) and it would waste a term increment. It stays in CTRL_ELECTING, * accepts incoming vote requests (granting to fresher peers), and becomes * STANDBY after the winner sends a heartbeat. * * Exception: if ALL peers also have config_tag=0 (cluster cold-start with no * configured node), someone must win — allow election normally. * * my_config_tag comes from myconf->config_version_tag set by load_primary_config; * n_peers and the peer list from --peers are always available via the parameters. * We check if any peer returned a non-zero tag via the xce_probe_config_tag path * already called in myapp_run before ha_start_election_async. */ if (my_config_tag == 0 && n_peers > 1) { INFO("ha_start_election_async: no local config (tag=0, n_peers=%d) — " "participating as voter only, not starting own election", n_peers); /* Transition to CTRL_STANDBY so myapp_run schedules fetch_config_from_primary. * The primary will push config via ctrlhello once it becomes CTRL_PRIMARY. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); /* ha-elect-ctx-lifecycle task 2.4: no vote sends will happen from this * node this round — ctx was never captured by any vote endpoint, so * this release (refcount 1->0) frees it immediately, same outcome as * the old direct destroy+free, now routed through the single release * helper for consistency. */ atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release); ha_elect_ctx_release(ctx); return; } /* bug-1435: serialize against concurrent raft_state_t access (see ha.h * raft_lock doc). This call can fire on_become_leader synchronously * (single-node case) or on_send_vote_request per peer + on_schedule_ * timeout (multi-node) — none of which re-enter a raft_* call, so the * lock is held only across this one call. */ pthread_mutex_lock(&ha->raft_lock); raft_start_election(&ha->raft); pthread_mutex_unlock(&ha->raft_lock); /* For single-node, raft_start_election fires on_become_leader synchronously. * on_become_leader (raft_cb_become_leader) handles role -> CTRL_PRIMARY and * phase2. Release the ctx here since no raft callbacks * will reference it after raft_start_election returns for single-node * (no vote endpoint is ever created — votes[] is empty). */ if (n_peers == 1) { atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release); ha_elect_ctx_release(ctx); return; } INFO("ha_start_election_async: returning (n_peers=%d, raft election started)", n_peers); /* Returns immediately — election result delivered via raft on_become_leader/follower */ } #ifdef XCE_TESTING /* * ha_rdhup_election_hook: test-only seam for ha_start_election_async in the * RDHUP failover path. When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls * this instead of the real ha_start_election_async(). Req 6.3, 6.5, 9.3. */ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) = NULL; #endif /* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ─────────────────── * * Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback), * 6.4 (no heartbeat within timeout → elect), 6.8 (role flip) * 10.5 (INFO logs on FSM transitions) */ /* * ha_begin_failover: the single failover entry point (ha-prevote task 4.2). * * Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and * the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route * here AFTER their connection-teardown bookkeeping and the config-less * voter-only guard. It starts a side-effect-free PreVote round and returns; * it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those * destructive steps are deferred to PreVote success (ha_enter_real_election, * task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL * and term untouched (Req 1.1, 4.3, 4.4). */ /* ha_election_stagger_ms: see ha.h. Same formula as the startup stagger * (myapp.c) — priority baseline plus fresh random jitter, re-randomized each * call so repeated split-vote collisions converge (Raft §5.2; bug-606). */ uint32_t ha_election_stagger_ms(ha_state_t *ha) { if (!ha) return 0; if (ha->jitter_seed == 0) ha_seed_jitter(&ha->jitter_seed, ha->my_priority); uint32_t step = ha->config.election_priority_step_ms; uint32_t jmax = ha->config.election_jitter_max_ms; uint32_t delay = (uint32_t)ha->my_priority * step; if (jmax > 0) delay += (uint32_t)(xorshift64_next(&ha->jitter_seed) % ((uint64_t)jmax + 1)); return delay; } /* ha_jittered_election_delay: see ha.h. Canonical Raft randomized election * timeout — [base, 2*base], re-drawn each campaign — applied to the per-campaign * watchdog so a constant cold-connection grant RTT cannot resonate just past a * fixed deadline forever (task 5.7). Independent of election_jitter_max_ms and * never below the configured floor. */ uint32_t ha_jittered_election_delay(ha_state_t *ha, uint32_t base) { if (!ha || base == 0) return base; if (ha->jitter_seed == 0) ha_seed_jitter(&ha->jitter_seed, ha->my_priority); uint32_t extra = (uint32_t)(xorshift64_next(&ha->jitter_seed) % ((uint64_t)base + 1)); return base + extra; } /* failover_stagger_ctx_t: carries ha/c into the delaymgr-scheduled PreVote start. * Freed by ha_failover_stagger_fn (one-shot). */ typedef struct { ha_state_t *ha; cluster_t *c; } failover_stagger_ctx_t; /* ha_failover_stagger_fn: delaymgr body that starts the PreVote round after the * per-node stagger (bug-606). Guard: if a primary connected during the stagger * (we are a STANDBY with a fresh heartbeat), the cluster already has a leader — * skip the PreVote entirely instead of needlessly probing a live primary. */ static int ha_failover_stagger_fn(void *arg) { failover_stagger_ctx_t *ctx = (failover_stagger_ctx_t *)arg; ha_state_t *ha = ctx->ha; cluster_t *c = ctx->c; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) { INFO("HA: failover stagger elapsed but a primary is alive — skipping " "PreVote (bug-606)"); free(ctx); return 0; } ha_start_prevote_async(ha, c); free(ctx); return 0; } /* CLOCK_MONOTONIC milliseconds — for the RDHUP failover debounce (5.10b). */ static uint64_t ha_mono_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L); } void ha_begin_failover(ha_state_t *ha, cluster_t *c) { if (!ha) return; /* Stamp the failover-trigger time so a rapid RDHUP cadence from a dead * primary's fd cannot re-trigger a fresh failover on every event (5.10b * debounce in ha_on_primary_ctrl_fd_rdhup). Any trigger source (RDHUP or the * deferred heartbeat-timeout stagger) records here. */ atomic_store_explicit(&ha->last_failover_ms, ha_mono_ms(), memory_order_release); INFO("HA: beginning failover via PreVote (no truncate / no term bump until " "PreVote wins) — staying CTRL_STANDBY"); /* Stagger the PreVote start per node so two survivors that lost the primary * simultaneously do not campaign in lock-step and split the vote forever * (Raft §5.2; bug-606). Production only: in unit-test mode (c == NULL) or * with no running delaymgr, start synchronously so the existing failover / * PreVote tests observe the unchanged immediate behaviour. */ uint32_t stagger = ha_election_stagger_ms(ha); if (c && c->delaymgr.running && stagger > 0) { failover_stagger_ctx_t *sctx = calloc(1, sizeof(*sctx)); if (sctx) { sctx->ha = ha; sctx->c = c; INFO("HA: staggering PreVote start by %ums (priority=%d) to avoid " "split vote (bug-606)", stagger, ha->my_priority); int rc = delay_task_ms(&c->delaymgr, (int)stagger, 1, ha_failover_stagger_fn, sctx); if (rc >= 0) return; /* Scheduling failed — fall through to a synchronous start. */ WARN("HA: ha_begin_failover: stagger schedule failed (%d) — " "starting PreVote immediately", rc); free(sctx); } } ha_start_prevote_async(ha, c); } void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) { if (!ha) return; /* raft-ha-core-safety task 5.6: do NOT restart failover while an election is * already in flight. A won PreVote enters the real election (CTRL_ELECTING), * or a PreVote round may be mid-decision (pre_vote_in_progress). The dead * primary's control fd keeps RDHUP'ing on a deterministic ~15s cadence — if * each RDHUP re-triggers a fresh ha_begin_failover it PREEMPTS the in-progress * real election before it can complete, bumping the term every cycle and * keeping two surviving candidates dueling in lockstep (split vote, unbounded * term escalation; research.md §"Task 5.5 fix VERIFIED"). Drop the RDHUP: * close the dead control fd and let the in-progress round finish or time out — * its election timeout is jittered, which desynchronizes the duel so one * survivor wins. A node NOT currently electing falls through to the normal * 5.2 suppress / genuine-loss failover path below. */ { ctrl_role_t cur = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (cur == CTRL_ELECTING || ha->raft.pre_vote_in_progress) { if (ha->primary_ctrl_fd >= 0) { close(ha->primary_ctrl_fd); ha->primary_ctrl_fd = -1; } INFO("HA: primary_ctrl_fd RDHUP but an election is already in progress " "(role=%d, prevote=%d) — dropping the RDHUP, not restarting " "failover (task 5.6: don't preempt the in-progress election / " "churn the term)", (int)cur, ha->raft.pre_vote_in_progress); return; } } /* raft-ha-core-safety 5.10b: RDHUP failover DEBOUNCE. The 5.6 guard above * only catches an RDHUP while the role is INSTANTANEOUSLY CTRL_ELECTING / * pre_vote_in_progress. But a dead primary's fd RDHUPs faster than an * election settles, and each RDHUP that lands in the gap AFTER a real * election loses (role briefly back at CTRL_STANDDBY before the next attempt) * re-triggers a fresh failover → a term bump per RDHUP (observed: 48 * begin_failovers, terms 6→24 in one round; research.md §"5.10b DEEPER"). * Suppress an RDHUP-triggered failover for up to 2× the election timeout * after the last trigger — long enough to cover the jittered [T,2T] election * watchdog (5.7) so the election's OWN retry runs without the external RDHUP * cadence piling on extra term bumps. A genuinely-dead primary is still * failed over — at most once per window, not once per RDHUP. First trigger * (last_failover_ms == 0) is never debounced. */ { uint64_t now_ms = ha_mono_ms(); uint64_t last = atomic_load_explicit(&ha->last_failover_ms, memory_order_acquire); uint64_t cooldown_ms = 2u * (uint64_t)ha->config.election_timeout_ms; if (last != 0 && now_ms >= last && (now_ms - last) < cooldown_ms) { if (ha->primary_ctrl_fd >= 0) { close(ha->primary_ctrl_fd); ha->primary_ctrl_fd = -1; } INFO("HA: primary_ctrl_fd RDHUP within failover cooldown (%lums since " "last trigger, cooldown=%lums) — debouncing, not restarting " "failover (5.10b: a sustained RDHUP cadence must not ladder the " "term; the in-flight election's own jittered retry handles it)", (unsigned long)(now_ms - last), (unsigned long)cooldown_ms); return; } } /* Cancel the liveness poll task first (Req 5.4): prevents a concurrent * poll fn from triggering a second failover call after EPOLLRDHUP fires. */ ha_disarm_heartbeat_timeout_vtask(ha); /* Close and null the primary control fd. Connection-teardown bookkeeping * stays at the trigger regardless of the PreVote outcome (ha-prevote task * 4.2). */ if (ha->primary_ctrl_fd >= 0) { close(ha->primary_ctrl_fd); ha->primary_ctrl_fd = -1; } /* raft-ha-core-safety task 5.2: SPURIOUS-FAILOVER SUPPRESSION. * * A TRANSIENT control-channel RDHUP while the current-term primary is still * alive (a heartbeat arrived within heartbeat_timeout_ms) must NOT launch an * unwinnable PreVote failover. The live stall (research.md §"Phase 4/5"): a * recovered standby whose channel to the live primary flaps treats each * RDHUP as "primary died" → PreVote → peers see the primary alive and deny → * round times out → reconnect → repeat ~15s, so the node prevote-spins * forever instead of rejoining. Mirror the guard ha_failover_stagger_fn * already applies on the DEFERRED path (a flap = the primary re-established * the channel, so ha_heartbeat_is_fresh is the right liveness signal — * the same one handle_request_vote uses to deny same-term challengers). * * Stay CTRL_STANDBY and await the primary's reconnect (handle_ctrlhello); * re-arm the liveness watchdog so a GENUINE subsequent loss is still caught * (the watchdog resets last_activity to now, granting a fresh grace window — * if the primary truly never heartbeats again it fires a real failover after * heartbeat_timeout_ms). GENUINE loss (stale heartbeat) falls through to the * failover path below unchanged; Election Safety is unaffected (no term bump, * no role change here). */ ctrl_role_t rdhup_role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (rdhup_role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) { INFO("HA: primary_ctrl_fd RDHUP but the primary is alive (fresh " "heartbeat) — suppressing PreVote failover, staying CTRL_STANDBY " "and awaiting reconnect (spurious-failover suppression, " "raft-ha-core-safety 5.2)"); if (c) ha_arm_heartbeat_timeout_vtask(ha, c); return; } /* ha-prevote task 4.2: the failover trigger NO LONGER transitions to * CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and * STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is * deferred to PreVote success (ha_enter_real_election). The node only ever * inflates its term / unseats a healthy primary if a PreVote majority is * first obtained (Req 1.1, 4.3, 4.4). */ INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying " "CTRL_STANDBY until PreVote wins) (Req 1.1)"); if (!c) { /* Unit-test mode: no persist / no live transport. Still begin the * round so the deferred-destructive behaviour is observable (the round * state lives in raft_state_t; ha_enter_real_election with c==NULL only * flips the role on a WIN). */ ha_begin_failover(ha, c); return; } /* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening * Req 10.5): a config-less node (config_version_tag == 0) must NEVER start * an RDHUP re-election while a configured peer exists. Two such nodes would * grant each other (the freshness gate only denies a LOWER tag; tag0==tag0 * with equal lsn is granted), forming a majority and electing a config-less * primary over the node that actually holds the config — which then pushes * an empty config and breaks replication. * * This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a * config-less node can never win a PreVote (the freshness gate denies it), * so there is no point starting a round — and starting one would needlessly * probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote * still GRANTS incoming vote requests to the fresher configured peer * regardless of role, and this node transitions/fetches config when the * configured winner connects and pushes config via ctrlhello. * * Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false, * so a lone node still self-elects (its PreVote trivially wins). */ uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0; int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1; if (my_config_tag == 0 && n_peers_guard > 1) { INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — " "voter only, not starting PreVote/RDHUP re-election; awaiting " "configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard); atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); return; } /* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async * (it snapshots self freshness into ha->elect_ctx so the PreVote request * payload — and, on a WIN, the deferred real election — advertise the * effective freshness floor, NOT the stale persist.committed_lsn: see the * bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */ ha_begin_failover(ha, c); } /* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */ /* * liveness_poll_ctx_t: context for the delaymgr-based standby liveness task. * * Requirements: 1.2, 1.4, 1.5, 1.6, 5.4 * * Replaces hb_vtask_t + hb_timeout_thread. No pthread, no semaphore. * The poll fn reads ha->last_activity (written by ha_on_heartbeat_received) * and fires ha_on_primary_ctrl_fd_rdhup if elapsed >= heartbeat_timeout_ms. * * cancelled is _Atomic so that writes from ha_disarm_heartbeat_timeout_vtask * (main/vtask context) are visible to liveness_poll_fn (delaymgr worker thread) * without a data race. */ typedef struct { ha_state_t *ha; cluster_t *c; _Atomic int cancelled; } liveness_poll_ctx_t; /* Static pointer to the active context. _Atomic so concurrent arm/disarm * vs poll-fn execution is well-defined. Access pattern: * arm — allocates and stores (single writer, vtask context) * disarm — sets cancelled, frees (single writer, vtask context) * poll_fn — reads via passed arg only; never dereferences s_liveness_ctx */ static liveness_poll_ctx_t * _Atomic s_liveness_ctx = NULL; /* rdhup_failover_vtask_entry / ha_dispatch_rdhup_failover: offload * ha_on_primary_ctrl_fd_rdhup (which chains into ha_start_election_async -> * raft_start_election -> on_persist -> ha_persist_term's synchronous fsync) * off the delaymgr thread onto a VTASK_LIVE_BLOCK vtask, mirroring * ha_dispatch_election_start. This is the real, common production trigger for * a standby's failover election (heartbeat-timeout watchdog) -- delaymgr must * only detect the timeout and hand off, never run the election itself. */ typedef struct { ha_state_t *ha; cluster_t *c; } rdhup_failover_dispatch_t; static int rdhup_failover_vtask_entry(vtask_t *task) { rdhup_failover_dispatch_t *d = (rdhup_failover_dispatch_t *)task->data->u.generic; ha_on_primary_ctrl_fd_rdhup(d->ha, d->c); free(d); return 0; } static void ha_dispatch_rdhup_failover(ha_state_t *ha, cluster_t *c) { vtaskmgr_t *pool = (vtaskmgr_t *)ha->pool; vtask_data_t *data = calloc(1, sizeof(*data)); vtask_t *task = vtask_alloc(); rdhup_failover_dispatch_t *d = calloc(1, sizeof(*d)); if (pool == NULL || data == NULL || task == NULL || d == NULL) { WARN("ha_dispatch_rdhup_failover: vtask unavailable (pool=%p data=%p " "task=%p d=%p) — failing over inline (stay alive)", (void *)pool, (void *)data, (void *)task, (void *)d); free(data); free(task); free(d); ha_on_primary_ctrl_fd_rdhup(ha, c); return; } d->ha = ha; d->c = c; data->vtype = VTASK_GENERIC; data->u.generic = d; vtask_init(task, (vtask_func_t)rdhup_failover_vtask_entry, data); if (vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) < 0) { WARN("ha_dispatch_rdhup_failover: vtask enqueue failed — failing over " "inline (stay alive)"); free(data); free(task); free(d); ha_on_primary_ctrl_fd_rdhup(ha, c); } } /* * liveness_poll_fn: delaymgr task replacing hb_timeout_thread. * * Checks whether the primary has been silent for >= heartbeat_timeout_ms. * If so, calls ha_on_primary_ctrl_fd_rdhup and does not re-arm. * If not, re-arms itself for the next poll interval. * If cancelled, returns immediately without any action. * * Also skips if role != CTRL_STANDBY (Req 5.4): prevents duplicate failover * if EPOLLRDHUP already transitioned the role to CTRL_ELECTING. */ static int liveness_poll_fn(void *arg) { liveness_poll_ctx_t *ctx = (liveness_poll_ctx_t *)arg; /* Check cancelled flag first — set by ha_arm_heartbeat_timeout_vtask when * re-arming replaces this ctx. When cancelled, free the ctx here (the * re-armer cannot safely free it because this task may already be running * on a delaymgr thread). Do NOT re-arm. * * In XCE_TESTING builds callers pass stack-allocated ctxs, so we must * not free in that path. Production builds always heap-allocate via * ha_arm_heartbeat_timeout_vtask. */ if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) { #ifndef XCE_TESTING free(ctx); #endif return 0; } ha_state_t *ha = ctx->ha; cluster_t *c = ctx->c; /* Skip if not standby — EPOLLRDHUP may have already changed the role */ if (atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY) return 0; /* Compute elapsed time since last primary message */ struct timespec now_ts; clock_gettime(CLOCK_MONOTONIC, &now_ts); time_t now = now_ts.tv_sec; time_t last = ha->last_activity; /* time_t write is atomic on LP64 */ time_t elapsed = (last > 0) ? (now - last) : 0; uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000; if (last > 0 && (uint32_t)elapsed >= timeout_s) { /* Silence exceeded timeout — trigger failover */ WARN("HA: liveness poll: no primary activity for %lus (timeout=%us) → failover", (unsigned long)elapsed, timeout_s); ha_dispatch_rdhup_failover(ha, c); /* Do NOT re-arm — failover handles re-election */ return 0; } /* Still fresh — re-arm for next poll interval */ if (c && c->delaymgr.running) { uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2; if (poll_ms < 100) poll_ms = 100; delay_task_ms(&c->delaymgr, (int)poll_ms, 1, (delay_task_entry_fn_t)liveness_poll_fn, ctx); } return 0; } #ifdef XCE_TESTING /* * liveness_poll_fn_test_entry: test-only entry point that calls liveness_poll_fn * directly with a pre-constructed ctx, bypassing delaymgr. * Only compiled when XCE_TESTING is defined. */ void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx) { liveness_poll_fn(ctx); } /* * primary_heartbeat_fn_for_test: call primary_heartbeat_fn directly with c. * Bypasses delaymgr scheduling so tests can drive the idle-NOP logic without * a live timer. Only compiled when XCE_TESTING is defined. (Task 2.2) */ int primary_heartbeat_fn_for_test(cluster_t *c) { return primary_heartbeat_fn(c); } #endif void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c) { if (!ha) return; /* Disarm any existing poll ctx by setting cancelled=1. * Do NOT free it here — the delaymgr task may already be running on * another thread and will free the ctx itself when it sees cancelled=1. * Clearing s_liveness_ctx prevents a second re-arm attempt from operating * on the stale pointer. */ liveness_poll_ctx_t *old = atomic_load_explicit(&s_liveness_ctx, memory_order_acquire); if (old) { atomic_store_explicit(&old->cancelled, 1, memory_order_release); atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release); } liveness_poll_ctx_t *ctx = calloc(1, sizeof(*ctx)); if (!ctx) return; ctx->ha = ha; ctx->c = c; atomic_init(&ctx->cancelled, 0); /* Initialise last_activity to now so the first poll computes elapsed from * the arm time rather than from zero. */ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); ha->last_activity = ts.tv_sec; atomic_store_explicit(&s_liveness_ctx, ctx, memory_order_release); uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2; if (poll_ms < 100) poll_ms = 100; INFO("HA: standby liveness poll armed (timeout=%ums, poll=%ums)", ha->config.heartbeat_timeout_ms, poll_ms); if (c && c->delaymgr.running) { int rc = delay_task_ms(&c->delaymgr, (int)poll_ms, 1, (delay_task_entry_fn_t)liveness_poll_fn, ctx); if (rc < 0) WARN("HA: ha_arm_heartbeat_timeout_vtask: delay_task_ms failed: %d", rc); } } /* ha_discover_primary: if CTRL_ELECTING, step down to STANDBY on receiving * a valid primary message. A campaigning node that sees a live primary must * defer — continuing the election risks split-brain (bug-353). */ void ha_discover_primary(ha_state_t *ha, cluster_t *c) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role != CTRL_ELECTING) return; uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire); INFO("HA: discovered live primary while ELECTING (term=%" PRIu64 ") — stepping down", term); raft_cb_become_follower(ha, term, -1); } bool ha_msg_term_is_stale(ha_state_t *ha, uint64_t msg_term) { if (!ha || msg_term == 0) return false; uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire); return msg_term < term; } void ha_on_heartbeat_received(ha_state_t *ha) { if (!ha) return; struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); ha->last_activity = ts.tv_sec; tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec); /* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader * is alive — a PreVote round started by a transient blip must be ABANDONED. * Cancel the in-progress round (clear the raft round state + the controller * round-timeout handle) and re-attach as a standby. No persistent term / * voted_for / role change happens here: the node never left CTRL_STANDBY * during the round, so re-attaching is a no-op on role/term — exactly the * non-disruptive rejoin the spec requires. */ if (ha->raft.pre_vote_in_progress) { ha->raft.pre_vote_in_progress = 0; ha->raft.pre_votes_granted = 0; ha_prevote_cancel_round_timeout(); INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round " "and re-attaching as CTRL_STANDBY (leader alive, no term change) " "(Req 4.2)"); } } /* * ha_observe_heartbeat_committed: reconcile-free monotonic learn of the * primary's committed position from a heartbeat (ha-heartbeat-committed- * propagation, Req 2.1, 2.2, 2.3, 3.1, 3.3). * * Single-receiver-thread path (controller request handling), so a relaxed * compare-then-store is sufficient. The config tag is written BEFORE the LSN * (both release-ordered) so a concurrent acquire-load reader that observes the * advanced LSN never pairs it with a stale tag. * * Advisory only: NO ha_rejoin_reconcile / NO WAL truncation / NO change to any * produced/persisted committed, role, term, or applied state. This is the * explicit distinction from cluster_apply_ctrlhello_reply (which truncates). */ void ha_observe_heartbeat_committed(cluster_t *c, uint64_t committed_lsn, uint64_t committed_config_tag) { if (!c) return; ha_state_t *ha = &c->ha; uint64_t cur = atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire); /* Monotonic no-op: a zero (absent/liveness-only) or non-advancing LSN must * leave the learned position — and its config tag — untouched (Req 2.2). */ if (committed_lsn == 0 || committed_lsn <= cur) return; /* Tag moves with the LSN, never independently regressed (Req 2.3): publish * the tag first so the LSN's release store carries the matching tag. */ atomic_store_explicit(&ha->last_known_committed_config_tag, committed_config_tag, memory_order_release); atomic_store_explicit(&ha->last_known_committed_lsn, committed_lsn, memory_order_release); /* Reconcile-free: nothing else is touched (Req 3.1, 3.3). */ tlog(TLOG_DEBUG3, "HA: learned committed_lsn=%" PRIu64 " config_tag=%" PRIu64 " from heartbeat (was %" PRIu64 ")", committed_lsn, committed_config_tag, cur); } /* * ha_standby_advance_committed_and_apply (2026-07-10, controller-invariant * audit finding — commit/apply decoupling extended to the standby): drive * this node's OWN p->committed_lsn from a learned primary committed position, * clamped to what this node has actually durably received, then kick the * SAME apply pump (persist_apply_pump_kick -> apply_committed_entry()) the * primary's own online-apply and restart replay use. * * This is the missing half of the standby ingest path's commit-then-apply * story: ha_apply_single_entry / ha_apply_term_entry_via_library persist + * ACK a replicated entry unconditionally (required for durability and the * primary's commit majority) but no longer apply it — entity materialization * now waits for THIS function to observe that the entry is actually * committed, never merely durable-on-receipt. Safe against a lagging * standby: the clamp to local_tip means committed_lsn here can never run * ahead of what this node has itself received, so persist_apply_pump_run's * apply_pump_read_entry can never be asked to read an LSN this node doesn't * have on disk yet. * * Race-safe against a CONCURRENT writer (2026-07-10, controller-invariant- * audit finding): this function is gated to non-primary roles by its * callers, but raft_cb_become_leader's own persist_promotion_advance_committed * call happens BEFORE role flips to CTRL_PRIMARY (ha.c, the role store sits * ~170 lines after the advance call) -- so a still-in-flight WAL catch-up * entry (this function is also called from ha_apply_wal_catchup_entry's exit * points) can legitimately run on another thread during that window, racing * this function against the promotion path's own advance. Both now go * through persist_committed_lsn_advance_atomic's CAS loop, which is safe * against that race by construction (never loses an update, never * regresses) -- no shared lock needed across the two call sites. */ static void ha_standby_advance_committed_and_apply(cluster_t *c, uint64_t learned_committed_lsn) { if (!c || learned_committed_lsn == 0) return; persist_t *p = &c->persist; pthread_mutex_lock(&p->lock); uint64_t local_tip = p->current_lsn > 0 ? p->current_lsn - 1 : 0; pthread_mutex_unlock(&p->lock); uint64_t target = learned_committed_lsn < local_tip ? learned_committed_lsn : local_tip; persist_committed_lsn_advance_atomic(p, target); persist_apply_pump_kick(p, c->ha.pool); } /* * ha_consume_heartbeat_committed: standby OP_nop consume branch — the * non-primary gate (Req 3.4) + the stale-term guard (Req 3.2), then the * monotonic observe (Req 1.2). Reconcile-free; updates nothing on a NULL body, * a failed gate, or a zero committed value (Req 4.2). Pure additive helper — * the caller runs the unchanged liveness/reply path first (Req 4.1). */ void ha_consume_heartbeat_committed(cluster_t *c, const heartbeat_msg_t *hb) { if (!c || !hb) return; /* (3.4) A primary uses its own authoritative committed — never an inbound * heartbeat value. Only non-primary nodes consume. */ ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&c->ha.role, memory_order_acquire); if (role == CTRL_PRIMARY) return; /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat through the raft * library as a 0-entry AppendEntries so the LIBRARY performs UpdateTerm — a * heartbeat at a strictly-higher term makes this node adopt the term and step * down. This REPLACES the prior hand-rolled term-guard that only DROPPED a * stale heartbeat (hb->term < myterm) and never adopted a higher term. * * The heartbeat carries no entries and no prev tuple, so we present the * node's own log tip as (prevLogIndex,prevLogTerm) — logOk holds (already- * have at the tip) for an equal/higher-term heartbeat, and UpdateTerm runs * first regardless. raft_init must have wired the raft state (production * boot); guard for the pre-init placeholder where raft_c is unset. */ if (c->ha.raft_c == c) { uint64_t my_log_len = c->ha.raft.log_len; uint64_t my_tip_term = (my_log_len > 0) ? persist_entry_term(&c->persist, my_log_len) : 0; wal_catchup_entry_t hbae; memset(&hbae, 0, sizeof(hbae)); hbae.n_entries = 0; /* heartbeat: zero entries */ hbae.term = hb->term; /* leader's live term (UpdateTerm) */ hbae.prev_log_index = my_log_len; /* already-have at our tip */ hbae.prev_log_term = my_tip_term; hbae.leader_commit = hb->committed_lsn; (void)ha_consume_append_entries(c, &hbae, /*leader_idx*/-1); } /* (3.2) Stale-term guard for the committed LEARN below: after UpdateTerm a * heartbeat from a primary whose term is behind ours is stale/partitioned — * do not let its committed inflate the vote-freshness floor. Re-read our term * (it may have advanced via the library UpdateTerm above). */ uint64_t myterm = atomic_load_explicit(&c->ha.term, memory_order_acquire); if (hb->term < myterm) return; /* (1.2) Monotonic, reconcile-free learn of the committed position. Zero/ * non-advancing LSN no-ops inside the observe (Req 4.2). */ ha_observe_heartbeat_committed(c, hb->committed_lsn, hb->committed_config_tag); /* 2026-07-10: drive this node's own committed_lsn + apply pump from the * position just learned (see ha_standby_advance_committed_and_apply) — * the commit-then-apply half of the standby ingest path. */ ha_standby_advance_committed_and_apply(c, hb->committed_lsn); } void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha) { (void)ha; liveness_poll_ctx_t *ctx = atomic_load_explicit(&s_liveness_ctx, memory_order_acquire); if (!ctx) return; atomic_store_explicit(&ctx->cancelled, 1, memory_order_release); atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release); } /* ── Primary heartbeat sender + late-joiner outbound connect ─────────────── */ /* * standby_ep_ctx_t: per-peer context for outbound standby connections opened * by the primary during the heartbeat task. * * Layout deliberately matches vote_ep_ctx_t for the first 4 fields so that * peer_ctrl_msg_handler (which casts ep->data to vote_ep_ctx_t*) can safely * read ha, c, and peer_idx. ctx is always NULL here (no active election). */ typedef struct { ha_state_t *ha; cluster_t *c; ha_elect_ctx_t *ctx; /* always NULL — not a vote endpoint */ int peer_idx; /* index into c->myconf->peers_cfg.candidates[] */ /* Task 1.1 (ha-heartbeat-liveness): last time a push/nop was sent to this * standby (CLOCK_MONOTONIC seconds). Initialized to 0 at allocation; * written by on_standby_connected and primary_heartbeat_fn (task 2.2). */ time_t last_push_time; } standby_ep_ctx_t; #ifdef XCE_TESTING /* * standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data, * and return ep. last_push_time is initialised to 0 (calloc). * Caller must free(ep->data) after the test. (Task 2.2) */ endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha, cluster_t *c, int peer_idx) { standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx)); if (!sctx) return NULL; sctx->ha = ha; sctx->c = c; sctx->ctx = NULL; sctx->peer_idx = peer_idx; /* last_push_time = 0 from calloc — tests set it explicitly */ ep->data = sctx; return ep; } /* * on_standby_connected_for_test: call on_standby_connected directly. * ep->data must be a valid standby_ep_ctx_t (use standby_ep_ctx_build_for_test). * xce_request will fail on a null fd — tests only check last_push_time. (Task 2.2) */ int on_standby_connected_for_test(endpoint_t *ep) { on_standby_connected(ep); return 0; } /* * standby_ep_err_handler_primary_for_test: expose the outbound-standby error * handler so a test can tear down a heap endpoint the production way (remove * from standby_endpoints + replicator, free ep->data) before reclaiming the * endpoint itself. (raft-drive-async Task 2.2) */ int standby_ep_err_handler_primary_for_test(endpoint_t *ep) { return standby_ep_err_handler_primary(ep); } #endif /* XCE_TESTING */ /* * on_standby_connected: iomgr on_connected callback for outbound standby * endpoints created by the heartbeat task. * * Sends OPCode_hello (CtrlHelloRequest) so the receiving standby runs * handle_ctrlhello and triggers WAL catch-up + config push. Registers the * endpoint in ha->standby_endpoints so future heartbeats reach it. */ static void on_standby_connected(endpoint_t *ep) { standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data; ha_state_t *ha = sctx->ha; cluster_t *c = sctx->c; /* * Register in standby_endpoints, resolving the inbound-vs-outbound * connection-ownership race (raft-drive-async Task 2.2, Req 2.6/4.4; * ha-async-io.md connection ownership). * * The hazard: BOTH raft_cb_become_leader (promotes surviving vote eps) and * this path (late-joiner / reconnect) zarray_store into standby_endpoints * keyed only by ep-POINTER identity. So while an outbound reconnect to * peer X is in flight, a DIFFERENT ep for the same peer X (a just-promoted * vote-ep, or a stale prior channel whose RDHUP has not yet been processed) * may already be registered. Leaving both would create two live outbound * state-push channels to one peer — heartbeats/WAL fan out to a dead/stale * duplicate (lost/reordered messages, Req 2.6) and the older endpoint is * orphaned in the replicator. * * Resolution (deterministic, per-peer): the freshly-connected channel wins. * Under ha->lock collect any DIFFERENT ep with the same peer_idx and drop * it from the array; then register `ep`. The actual teardown of the stale * ep runs AFTER releasing ha->lock because standby_ep_err_handler_primary * re-takes ha->lock (and touches the replicator) — avoiding self-deadlock * and holding the lock across no I/O (R4.4: iomgr/registration state mutated * only via the lock-guarded interface, never racing the event loop). */ endpoint_t *stale_same_peer = NULL; if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); int already = 0; for (int i = 0; i < ha->standby_endpoints->count; i++) { endpoint_t *cur = (endpoint_t *)zarray_get(ha->standby_endpoints, i); if (cur == ep) { already = 1; continue; } /* Different ep, same peer → stale duplicate to evict. */ standby_ep_ctx_t *cur_sctx = cur ? (standby_ep_ctx_t *)cur->data : NULL; if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx) { stale_same_peer = cur; zarray_del(ha->standby_endpoints, i); /* count shrank by one; the surviving scan continues but we * only ever expect a single same-peer duplicate. */ break; } } if (!already) zarray_store(ha->standby_endpoints, ep); pthread_mutex_unlock(&ha->lock); } /* Tear down the evicted stale channel OUTSIDE ha->lock. Mirror the library * disconnect path (handle_error → iomgr_unmonitor → err_handler) so we * never free an endpoint whose fd is still in the epoll set (R4.4: mutate * iomgr/registration state ONLY via the lock-guarded interface, never * racing the event loop). err-handler removes it from standby_endpoints + * the replicator and frees ep->data; we then reclaim the endpoint itself. * Never self-terminate — this is routine ownership reconciliation. */ if (stale_same_peer) { WARN("HA: peer[%d] reconnected — evicting stale duplicate outbound " "channel to keep exactly one live connection (Req 2.6)", sctx->peer_idx); if (stale_same_peer->iomgr) iomgr_unmonitor((iomgr_t *)stale_same_peer->iomgr, stale_same_peer); (void)standby_ep_err_handler_primary(stale_same_peer); /* err-handler set ep->data = NULL and freed sctx; bug-1298: drop the * owner ref — endpoint_unref reclaims outq/ssl/packets and frees the * calloc'd struct when the last reference goes (deferred if a pool vtask * still references this just-unmonitored ep). */ endpoint_unref(stale_same_peer); #ifdef XCE_TESTING atomic_fetch_add(&ha_standby_evicted_count_for_test, 1); #endif } /* Reset failure counter so the next disconnect restarts logging from attempt 1. */ ha->peer_connect_fail_count[sctx->peer_idx] = 0; /* Record last_push_time before any send so the first heartbeat tick after * connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */ { struct timespec _ts; clock_gettime(CLOCK_MONOTONIC, &_ts); sctx->last_push_time = _ts.tv_sec; } /* Send ctrlhello — triggers handle_ctrlhello on the standby side */ uint64_t lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); uint64_t tag = c->myconf ? c->myconf->config_version_tag : 0; char *name = c->myconf ? c->myconf->name : ""; ctrl_hello_req_t hello = { .name = name, .key = NULL, .applied_lsn = lsn, .config_tag = tag }; xce_request_t req; memset(&req, 0, sizeof(req)); req.opcode = OPCode_hello; req.body_which = Request_body_ctrlhellomsg; req.body.ctrlhellomsg = &hello; int rc = xce_request(ep, &req); if (rc < 0) WARN("HA: ctrlhello to peer[%d] failed: %d", sctx->peer_idx, rc); else INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent", sctx->peer_idx); /* Task 2.1: register late-joiner with WAL replicator. * Req 1.1: primary must register standby with repl_add_standby when * establishing the standby control channel (late-joiner outbound connect). * Host/port from candidates[peer_idx] — the peer we connected to. */ if (c && c->myconf && sctx->peer_idx >= 0 && sctx->peer_idx < c->myconf->peers_cfg.candidates_count) { const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[sctx->peer_idx]; int rc = repl_add_standby(&c->persist.repl, ep, peer->hostname, (uint32_t)peer->port); if (rc >= 0) INFO("HA: repl registered late-joiner standby peer[%d] (slot %d, %s:%u)", sctx->peer_idx, rc, peer->hostname, (uint32_t)peer->port); else WARN("HA: repl_add_standby late-joiner peer[%d] failed: %d", sctx->peer_idx, rc); } /* Quorum-transition observability (Req 8.2): a late-joiner standby connected — * the write-quorum count changed. Record the restore edge once (silent if * quorum was already held). */ ha_log_quorum_transition(ha); /* Set state-aware dispatcher for subsequent messages on this endpoint */ ep->msg_handler = peer_ctrl_msg_handler; } /* * standby_ep_err_handler_primary: error handler for outbound standby * endpoints created by the heartbeat task. Removes from standby_endpoints * and frees the context — the next heartbeat tick will attempt reconnect. */ static int standby_ep_err_handler_primary(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data; if (!sctx) return 0; ha_state_t *ha = sctx->ha; cluster_t *c = sctx->c; if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int i = 0; i < ha->standby_endpoints->count; i++) { if (zarray_get(ha->standby_endpoints, i) == ep) { zarray_del(ha->standby_endpoints, i); break; } } pthread_mutex_unlock(&ha->lock); } /* Task 2.2: deregister late-joiner standby from WAL replicator before * freeing the context. Req 1.2: RDHUP or connection error triggers * repl_remove_standby so the replicator stops sending WAL entries to it. */ if (c) repl_remove_standby(&c->persist.repl, ep); free(sctx); ep->data = NULL; return 0; } /* * reconnect_ctx_t: owns the prepared endpoint across the in-flight reconnect * window (raft-drive-async Task 2.1). Built on the delaymgr tick by the * non-blocking prep (ha_connect_prep_peer), consumed once on a slow-lane * worker by reconnect_lane_fn, then freed. The ctx keeps `ep`/`sctx` valid * for the whole window so the lane worker never touches freed memory while the * delaymgr thread races ahead to the next tick (Req 2.3 lifetime). */ typedef struct { cluster_t *c; int peer_idx; /* ABSOLUTE candidate index */ endpoint_t *ep; /* prepared, not yet connected */ standby_ep_ctx_t *sctx; /* == ep->data */ int should_log; } reconnect_ctx_t; /* * ha_connect_prep_peer: the NON-BLOCKING half of the outbound peer connect. * Allocates the endpoint context, initialises the endpoint, sets the SSL * client state and the on-connected / err handlers — all of which are * non-blocking and safe to run on the delaymgr tick. Does NOT perform the * blocking connect (that is ha_connect_finish_peer, run on the slow lane). * * Returns a heap reconnect_ctx_t* (caller owns) on success, or NULL on an * allocation failure (nothing leaked). */ static reconnect_ctx_t *ha_connect_prep_peer(cluster_t *c, int peer_idx) { ha_state_t *ha = &c->ha; standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx)); if (!sctx) return NULL; sctx->ha = ha; sctx->c = c; sctx->peer_idx = peer_idx; endpoint_t *ep = endpoint_get(); if (!ep) { free(sctx); return NULL; } endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0); endpoint_set_peer(ep, c->myhost); ep->data = sctx; ep->iomgr = &c->iomgr; if (ha->client_ssl_ctx) { SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx; ep->ssl = SSL_new(client_ctx); if (ep->ssl) SSL_set_connect_state(ep->ssl); } endpoint_set_on_connected(ep, on_standby_connected); endpoint_set_err_handler(ep, standby_ep_err_handler_primary); int n = ++ha->peer_connect_fail_count[peer_idx]; int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0); reconnect_ctx_t *rc_ctx = calloc(1, sizeof(*rc_ctx)); if (!rc_ctx) { ep->data = NULL; endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (not monitored) */ free(sctx); return NULL; } rc_ctx->c = c; rc_ctx->peer_idx = peer_idx; rc_ctx->ep = ep; rc_ctx->sctx = sctx; rc_ctx->should_log = should_log; return rc_ctx; } /* * ha_connect_finish_peer: the BLOCKING half of the outbound peer connect — * the bounded connect + lock-guarded iomgr registration. MUST run on a * slow-lane worker, NEVER on the delaymgr tick (Req 2.1, 2.2; ha-async-io.md * thread-role contract). * * On success the endpoint is connecting and registered with iomgr; on_connected * (on_standby_connected) fires later from the iomgr loop. On failure the * endpoint is torn down here and the controller simply retries on the next * heartbeat tick (controllers never self-terminate). * * Returns 0 on a successful connect+register, <0 otherwise. * * NOTE (Task 2.2 seam): this implements the happy-path connect+register + * teardown-on-failure. The inbound-vs-outbound connection-ownership race * resolution, the no-lost/no-reordered-message guarantee on success, and the * richer failure/retry hardening are owned by Task 2.2 — they layer on top of * this happy path without changing this function's contract (return 0/<0, * caller clears the per-peer in-flight flag). */ static int ha_connect_finish_peer(reconnect_ctx_t *rc_ctx) { cluster_t *c = rc_ctx->c; endpoint_t *ep = rc_ctx->ep; standby_ep_ctx_t *sctx = rc_ctx->sctx; int peer_idx = rc_ctx->peer_idx; const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[peer_idx]; #ifdef XCE_TESTING if (ha_reconnect_connect_hook) { /* Deterministic test path: the hook stands in for the real bounded * connect + register so a test can park the worker (unreachable peer) * or return a synthetic success/failure without a live socket. On * the test path we do NOT touch the real endpoint fd. */ int hrc = ha_reconnect_connect_hook(c, peer_idx); /* The hook stands in for the real connect+iomgr_monitor: it NEVER * registers the synthetic endpoint with iomgr, so the prepped ep/sctx * are unowned on BOTH success and failure here — reclaim them in either * case (test-only path; production reclaims via iomgr/err-handler). * Reverting this to free only on failure re-introduces the 2.1 * GREEN-test endpoint leak masked by detect_leaks=0. */ ep->data = NULL; endpoint_unref(ep); /* bug-1298: refcnt 1->0 → fini+free (test path) */ free(sctx); return hrc; } #endif if (rc_ctx->should_log) INFO("HA: peer[%d] (%s) not connected — initiating outbound connect " "(attempt %d)", peer_idx, peer->hostname, c->ha.peer_connect_fail_count[peer_idx]); int rc = endpoint_connect(ep, peer->hostname, peer->port, DEFAULT_CONNECT_TIMEOUT_MS); if (rc < 0) { if (rc_ctx->should_log) WARN("HA: connect to peer[%d] (%s:%u) failed: %d", peer_idx, peer->hostname, peer->port, rc); /* bug-1298: ep->data cleared (sctx freed separately below) then drop the * owner ref — endpoint_unref does fini+free of the calloc'd struct. */ ep->data = NULL; endpoint_unref(ep); free(sctx); return rc; } rc = iomgr_monitor(ep); /* lock-guarded epoll registration (Req 4.4) */ if (rc < 0) { ep->data = NULL; endpoint_unref(ep); /* bug-1298: fini+free (deferred if a vtask refs it) */ free(sctx); return rc; } INFO("HA: connecting outbound to peer[%d] (%s:%u)", peer_idx, peer->hostname, peer->port); return 0; } /* * reconnect_lane_fn: the slow-lane worker entry (raft-drive-async Task 2.1). * Runs the bounded blocking connect off the delaymgr thread, then ALWAYS * clears the per-peer in-flight flag (success or failure) so the dedup never * wedges and the next heartbeat tick can retry an unreachable peer (Req 2.5, * 2.4 — never self-terminates). Frees the reconnect ctx. */ static void reconnect_lane_fn(void *arg) { reconnect_ctx_t *rc_ctx = (reconnect_ctx_t *)arg; if (!rc_ctx) return; cluster_t *c = rc_ctx->c; int peer_idx = rc_ctx->peer_idx; (void)ha_connect_finish_peer(rc_ctx); /* Clear in-flight LAST, after the endpoint is fully connected/registered * or torn down — so a concurrent heartbeat tick cannot enqueue a second * reconnect that races this one's endpoint setup (Req 2.5). */ atomic_store_explicit(&c->ha.reconnect_inflight[peer_idx], 0, memory_order_release); free(rc_ctx); } /* * ha_connect_to_peer: open a new outbound TLS connection to a peer candidate * and register it as a standby channel. Called from primary_heartbeat_fn for * peers that are absent from standby_endpoints. * * Offload flag ON (default, Task 2.1): prepares the endpoint (non-blocking) * and submits the bounded connect to the slow lane with a per-peer dedup key, * then returns immediately — the delaymgr thread NEVER blocks on connect. * * Offload flag OFF (legacy / TDD RED only): runs the blocking connect inline * on the calling (delaymgr) thread. Retained solely so the RED test can show * the pre-change stall; production runs with the flag ON. */ static void ha_connect_to_peer(cluster_t *c, int peer_idx) { ha_state_t *ha = &c->ha; if (!ha_reconnect_offload_enabled()) { /* ── Legacy inline path (flag OFF) — blocks the caller. ── */ reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx); if (!rc_ctx) return; (void)ha_connect_finish_peer(rc_ctx); free(rc_ctx); return; } /* ── Offload path (flag ON) — non-blocking on the delaymgr thread. ── * * Per-peer single-flight: CAS the in-flight flag 0->1. If it was already * 1, a reconnect to this peer is already in flight on the lane — enqueue * nothing (Req 2.5). This guard is the authoritative dedup; the lane's * own per-key dedup is a redundant backstop. */ int expected = 0; if (!atomic_compare_exchange_strong_explicit( &ha->reconnect_inflight[peer_idx], &expected, 1, memory_order_acq_rel, memory_order_acquire)) { return; /* already reconnecting this peer */ } reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx); if (!rc_ctx) { /* prep failed (alloc) — release the in-flight guard so the next tick * retries; controller stays alive (Req 2.4). */ atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0, memory_order_release); return; } /* Submit the blocking connect to the slow lane. Dedup key is the * per-peer absolute index (+1 so it is non-zero; key 0 disables dedup). */ uint64_t key = (uint64_t)(peer_idx + 1); int srv = slowlane_submit(&c->slowlane, key, reconnect_lane_fn, rc_ctx); if (srv == -EEXIST) { /* Lane still holds this peer's dedup key from a PRIOR reconnect task. * This is the narrow window where that task already cleared our * reconnect_inflight flag (its last action) but the lane has not yet * removed its key. Our CAS therefore succeeded (flag was 0) even * though the lane refuses the submit. We MUST release the flag we just * set — otherwise no task owns its clear and dedup wedges permanently. * Tear down our duplicate prep; the next tick retries cleanly. */ rc_ctx->ep->data = NULL; endpoint_unref(rc_ctx->ep); /* bug-1298: fini+free the calloc'd struct */ free(rc_ctx->sctx); free(rc_ctx); atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0, memory_order_release); return; } if (srv < 0 && srv != -EAGAIN) { /* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a * failure: the task is still queued and will run, so we keep the flag * set and let reconnect_lane_fn clear it. Only on a true failure do * we tear down and release the guard so the next tick retries. */ WARN("HA: slowlane_submit reconnect peer[%d] failed: %d — retry next tick", peer_idx, srv); rc_ctx->ep->data = NULL; endpoint_unref(rc_ctx->ep); /* bug-1298: fini+free the calloc'd struct */ free(rc_ctx->sctx); free(rc_ctx); atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0, memory_order_release); return; } /* 0 or -EAGAIN: the task is enqueued and will run exactly once on a lane * worker; reconnect_lane_fn clears the in-flight flag on completion. */ } /* * primary_heartbeat_fn: self-perpetuating one-shot delaymgr task. * * On each tick: * 1. Sends OPCode_nop to all connected standbys (keeps watchdog alive). * 2. Scans the peers list; for any peer NOT in standby_endpoints, opens a * fresh outbound TLS connection so late joiners are bootstrapped without * the standby ever initiating an outbound connect. * * Stops naturally on demotion: the role check prevents re-arming. */ static int primary_heartbeat_fn(void *arg) { cluster_t *c = (cluster_t *)arg; ha_state_t *ha = &c->ha; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role != CTRL_PRIMARY) return 0; /* demoted — stop sending NOPs */ /* 1. Heartbeat all connected standbys — idle-triggered NOP (Req 3.1-3.4). * * For each standby ep, compute elapsed time since the last outbound push. * Send NOP only when elapsed >= heartbeat_interval_ms so that WAL entries, * config pushes, and ctrlhello suppress redundant NOPs (Req 3.4). */ { struct timespec _hb_ts; clock_gettime(CLOCK_MONOTONIC, &_hb_ts); time_t now = _hb_ts.tv_sec; int interval_s = (int)(ha->config.heartbeat_interval_ms / 1000); if (interval_s <= 0) interval_s = HA_DEFAULT_HEARTBEAT_INTERVAL_MS / 1000; if (ha->standby_endpoints) { /* Liveness eviction threshold (wall clock, matches ep->ts_last_in). */ time_t now_wall = time(NULL); uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000; if (timeout_s == 0) timeout_s = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS / 1000; /* Stamp the primary's OWN current committed position + term onto * every heartbeat this tick (Req 1.1, 3.4). Read atomically here, * once per tick, so every standby sees a consistent snapshot. Use * the primary's authoritative state — NOT the standby-side * last_known_* fields. */ uint64_t hb_term = atomic_load_explicit(&ha->term, memory_order_acquire); uint64_t hb_committed_lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); uint64_t hb_committed_config_tag = atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire); /* Endpoints evicted this tick; their repl deregistration runs AFTER * ha->lock is released (repl lock-ordering — mirrors the err handlers). */ endpoint_t *evicted[MAX_HA_CANDIDATES]; int n_evicted = 0; pthread_mutex_lock(&ha->lock); int n = ha->standby_endpoints->count; for (int i = 0; i < n; i++) { endpoint_t *ep = (endpoint_t *)zarray_get(ha->standby_endpoints, i); if (!ep) continue; standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data; if (!sctx) continue; /* Liveness eviction (symmetric with the standby's liveness poll): * a standby that was responsive at least once (ts_last_in > 0) but * has sent nothing for longer than heartbeat_timeout is dead or was * restarted. A wedged/dead endpoint left in the array makes the * reconnect scan below treat the peer as still-connected, so the * restarted node never receives a fresh ctrlhello and loops * elections forever (terms climbing unbounded). Evict it here; the * reconnect scan re-establishes a fresh channel on this/next tick. * Removal from standby_endpoints is the load-bearing action — the * endpoint object itself is reclaimed by its err handler when the * dead connection finally surfaces an epoll event (bounded). */ if (ep->ts_last_in > 0 && (now_wall - ep->ts_last_in) > (time_t)timeout_s) { WARN("HA: standby[%d] silent for %lds (timeout=%us) — evicting " "dead endpoint, will reconnect", i, (long)(now_wall - ep->ts_last_in), timeout_s); zarray_del(ha->standby_endpoints, i); if (n_evicted < MAX_HA_CANDIDATES) evicted[n_evicted++] = ep; i--; n--; continue; } time_t elapsed = now - sctx->last_push_time; if (elapsed >= (time_t)interval_s) { /* Heartbeat carries the primary's own committed + term * (Req 1.1, 3.4). */ #ifdef XCE_TESTING int rc; if (ha_heartbeat_req_hook) rc = ha_heartbeat_req_hook(ep, hb_term, hb_committed_lsn, hb_committed_config_tag); else if (ha_nop_req_hook) rc = ha_nop_req_hook(ep); else rc = xce_ha_heartbeat_req(ep, hb_term, hb_committed_lsn, hb_committed_config_tag); #else int rc = xce_ha_heartbeat_req(ep, hb_term, hb_committed_lsn, hb_committed_config_tag); #endif if (rc < 0) WARN("HA: heartbeat nop to standby[%d] failed: %d", i, rc); else sctx->last_push_time = now; } } pthread_mutex_unlock(&ha->lock); for (int k = 0; k < n_evicted; k++) repl_remove_standby(&c->persist.repl, evicted[k]); } } /* 2. Reconnect to any peer missing from standby_endpoints */ if (c->myconf) { const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; for (int i = 0; i < pcfg->candidates_count; i++) { if (i == ha->my_priority) continue; /* skip self */ /* Check if this peer already has a standby endpoint. * * Two endpoint types can be in standby_endpoints: * - Promoted vote eps: ep->data = vote_ep_ctx_t, peer_idx is * votes_idx (relative, self excluded). Convert absolute i to * votes_idx using the same formula as raft_cb_send_vote_request. * - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL), * peer_idx is absolute candidate index. Compare directly with i. */ int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1; int found = 0; if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int j = 0; j < ha->standby_endpoints->count; j++) { endpoint_t *sep = (endpoint_t *)zarray_get( ha->standby_endpoints, j); if (!sep || !sep->data) continue; standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)sep->data; int match_idx = (sctx->ctx != NULL) ? votes_idx_for_i /* promoted vote ep: relative */ : i; /* standby ep: absolute */ if (sctx->peer_idx == match_idx) { found = 1; break; } } pthread_mutex_unlock(&ha->lock); } if (!found) ha_connect_to_peer(c, i); } } /* runcount=-1 (infinite repeat): the delaymgr re-enqueues this task * automatically. The role check at entry stops work on demotion. */ return 0; } void ha_start_primary_heartbeat(cluster_t *c) { if (!c) return; ha_state_t *ha = &c->ha; int interval_ms = (int)ha->config.heartbeat_interval_ms; if (interval_ms <= 0) interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS; /* Idempotency: only arm once per process lifetime. With runcount=-1 the * delaymgr re-enqueues the task automatically; duplicate tasks would stack * on every re-election. */ { static _Atomic int s_armed = 0; int expected = 0; if (!atomic_compare_exchange_strong(&s_armed, &expected, 1)) return; /* already armed — task is still running */ } /* Guard: skip if delaymgr is not running (e.g. unit tests with zeroed cluster_t). * In production the delaymgr is always started before any promotion. */ if (!c->delaymgr.running) { INFO("HA: heartbeat skipped (delaymgr not running)"); return; } /* 2026-07-10 (chaotic-soak finding): fire the FIRST heartbeat synchronously, * right now, instead of only arming the periodic delay_task_ms below. * delay_task_ms's first invocation fires only AFTER interval_ms elapses * (it is a periodic timer, not "run now then repeat") -- so despite this * function's own long-standing comment/doc ("start primary heartbeat * immediately so standbys receive NOP"), a freshly-promoted leader was * actually waiting a full heartbeat_interval_ms (default 3000ms) before a * reconnected/catching-up standby ever got a heartbeat to reply to. That * reply (stateReport) is what lets the primary learn the standby's true * applied_lsn (ha_process_standby_state) and start catch-up streaming -- * which gates the per-entry majority-ack that advances committed_lsn past * this election's conservative safe_committed_lsn floor. A live chaotic * soak showed jobs above safe_lsn (legitimately majority-committed before * the old primary died, per Leader Completeness) invisible via * expect-job/joblist for the full 3s wait even after the * persist_promotion_advance_committed fix above -- because nothing * corroborated them was ever ASKED for that long. primary_heartbeat_fn * only performs non-blocking async sends (xce_ha_heartbeat_req), so * calling it inline here carries the same non-blocking-I/O contract as * the rest of promotion and needs no vtask/thread-role change. */ primary_heartbeat_fn(c); /* runcount=-1 (infinite): delaymgr re-enqueues automatically. * The role check in primary_heartbeat_fn no-ops on demotion. */ int rc = delay_task_ms(&c->delaymgr, interval_ms, -1, (delay_task_entry_fn_t)primary_heartbeat_fn, c); if (rc < 0) WARN("ha_start_primary_heartbeat: delay_task_ms failed: %d", rc); else INFO("HA: primary heartbeat armed (interval=%dms, rc=%d)", interval_ms, rc); }