#define _GNU_SOURCE #include "ha.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include /* Circular-include safety: cluster_def.h includes ha.h (include-guarded), * and ha.h only forward-declares cluster_t — no cycle. */ #include "xce/cluster_def.h" #include "xce/errnox.h" #include "controller.capnp.h" #include "persist.h" #include "request.h" #include "utils.h" /* ── HA dispatch gate error codes (defined here; declared extern in errnox.h) ── */ int XCE_ERR_NOT_READY = E_XCE_NOT_READY; int XCE_ERR_STANDBY = E_XCE_STANDBY; /* No-op stubs removed: s_real_cbs (below) is used exclusively since task 6.2. */ /* ── Forward declarations needed by real raft callbacks (Task 6.2) ──────── * These static functions and types are defined later in this file; declared * here so the callback implementations can reference them. */ /* vote_ep_ctx_t: per-peer context for outgoing vote endpoints (full def below) */ typedef struct { ha_state_t *ha; cluster_t *c; ha_elect_ctx_t *ctx; int peer_idx; uint64_t my_term; uint64_t my_tag; uint64_t my_lsn; const char *my_name; } vote_ep_ctx_t; /* election_timeout_ctx_t: delaymgr callback context for raft election timeout */ typedef struct { _Atomic int cancelled; ha_state_t *ha; cluster_t *c; ha_elect_ctx_t *ctx; } election_timeout_ctx_t; /* committing_timeout_ctx_t: owns the heap allocation; freed by the callback. */ typedef struct { _Atomic int cancelled; ha_state_t *ha; cluster_t *c; } committing_timeout_ctx_t; /* cluster_init_phase2 / push_config_to_standbys / ha_send_ctrlhello_to_all_standbys: * defined in cluster.c / config_repl.c; only referenced in production builds. * (XCE_TESTING uses hook shims and does not link cluster.c.) */ #ifndef XCE_TESTING extern int cluster_init_phase2(cluster_t *c); extern void ha_send_ctrlhello_to_all_standbys(cluster_t *c); extern int push_config_to_standbys(cluster_t *c, const uint8_t *bytes, size_t blen, const uint8_t sha256[32], uint64_t new_tag); #endif static void on_vote_connected(endpoint_t *ep); static int vote_ep_err_handler(void *ep_raw); static int committing_timeout_fn(void *arg); static int ha_heartbeat_is_fresh(ha_state_t *ha); static int primary_heartbeat_fn(void *arg); static void on_standby_connected(endpoint_t *ep); /* ── Real raft callback implementations (Task 6.2) ───────────── * * These are registered as s_real_cbs in ha_init, replacing the no-op stubs. * They are wired into the active election path in task 6.3. * * All callbacks receive ctx = ha_state_t*. * ha->raft_c is set by ha_start_election_async before any raft_* calls. * When ha->raft_c is NULL (unit test mode), callbacks that require cluster_t * are no-ops. */ /* * raft_cb_send_vote_request: encode and send a capnp VoteRequest to peer. * * peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[]. * Uses the existing iomgr vote-send path (on_vote_connected / vote_ep_ctx_t). * Reads the current election context from ha->elect_ctx (set before raft_start_election). * Returns 0 on success; non-zero if no cluster context or alloc failure. */ static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c || !c->myconf) return 1; /* test mode */ const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; if (peer_idx < 0 || peer_idx >= pcfg->candidates_count) return 1; ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (!ectx) return 1; /* no election in progress */ xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx]; /* Compute the local index of peer_idx within ectx->votes[]: * votes[] has n_peers - 1 = n_other entries for peers excluding self. * Map: votes_idx = peer_idx if peer_idx < self_idx, else peer_idx - 1. */ int self_idx = ha->my_priority; /* self_idx == my_priority by convention */ int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1; if (votes_idx < 0 || votes_idx >= ectx->n_peers) return 1; /* Allocate per-peer context */ vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx)); if (!vctx) { handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); return 1; } vctx->ha = ha; vctx->c = c; vctx->ctx = ectx; vctx->peer_idx = votes_idx; vctx->my_term = term; /* tag and lsn come from votes[0] which holds our own values */ vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0; vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0; vctx->my_name = c->myconf ? c->myconf->name : ""; /* Get a fresh endpoint */ endpoint_t *ep = endpoint_get(); if (!ep) { free(vctx); handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); return 1; } endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0); if (c) endpoint_set_peer(ep, c->myhost); ep->data = vctx; ep->iomgr = &c->iomgr; if (ha->client_ssl_ctx) { SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx; ep->ssl = SSL_new(client_ctx); if (ep->ssl) SSL_set_connect_state(ep->ssl); } endpoint_set_on_connected(ep, on_vote_connected); endpoint_set_err_handler(ep, vote_ep_err_handler); int rc = endpoint_connect(ep, peer->hostname, peer->port); if (rc < 0) { endpoint_fini(ep); if (!c) handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); return 1; } rc = iomgr_monitor(ep); if (rc < 0) { endpoint_fini(ep); /* frees ep->data (vctx) via FREEUP */ handle_request_vote_reply(ha, ectx, votes_idx, term, 0, 0, 0); return 1; } /* Track endpoint for promotion/close after election result */ ectx->votes[votes_idx].ep = ep; return 0; } /* * raft_cb_vote_eligible: freshness gate delegated from raft library. * * from_idx is the absolute candidate index of the requester. * Applies the same checks as handle_request_vote: * - role gate: PRIMARY or COMMITTING → deny * - fresh heartbeat on STANDBY → deny * - in_catch_up → deny * - freshness: candidate config_tag/lsn must be ≥ ours * Returns 1 to grant, 0 to deny. */ static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term) { (void)from_idx; (void)term; ha_state_t *ha = (ha_state_t *)ctx; /* Role gate */ ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY || role == CTRL_PRIMARY_COMMITTING) return 0; /* Fresh heartbeat on STANDBY → primary is alive, deny */ if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) return 0; /* In catch-up → deny */ if (atomic_load_explicit(&ha->in_catch_up, memory_order_acquire)) return 0; /* Freshness gate requires knowing the candidate's config_tag/lsn. * The raft library does not pass these; they are carried in the capnp * RequestVoteRequest fields. For the adapter, freshness is checked in * handle_request_vote (called from the iomgr decode path in task 6.3). * Here we return 1 (eligible) so the raft library proceeds; the full * freshness gate is enforced by the caller before invoking raft_on_vote_request. * This is safe because the caller (task 6.3 dispatcher) already applies * the freshness check from handle_request_vote before delegating here. */ return 1; } /* raft_election_timeout_fn: delaymgr callback for raft election timeout. * Fires raft_on_election_timeout after delay_ms if not cancelled. * Owns the election_timeout_ctx_t — frees it on exit. */ static int raft_election_timeout_fn(void *arg) { election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg; ha_state_t *ha = t->ha; /* Clear the stored pointer before checking cancelled so on_cancel_timeout * is a safe no-op if it races with this callback. */ ha->raft_election_timeout_ctx = NULL; if (atomic_load(&t->cancelled)) { free(t); return 0; } raft_on_election_timeout(&ha->raft); free(t); return 0; } /* * raft_cb_schedule_timeout: schedule election timeout via delaymgr. * * Creates an election_timeout_ctx_t and calls delay_task_ms. * The pointer is stored in ha->raft_election_timeout_ctx for cancellation. */ static void raft_cb_schedule_timeout(void *ctx, uint32_t delay_ms) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c) return; /* test mode */ election_timeout_ctx_t *t = calloc(1, sizeof(*t)); if (!t) return; atomic_init(&t->cancelled, 0); t->ha = ha; t->c = c; t->ctx = NULL; /* raft path does not use ha_elect_ctx_t */ int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1, raft_election_timeout_fn, t); if (rc < 0) { WARN("raft_cb_schedule_timeout: delay_task_ms failed: %d", rc); free(t); return; } ha->raft_election_timeout_ctx = t; } /* * raft_cb_cancel_timeout: cancel the pending election timeout. * Sets cancelled=1 on the stored election_timeout_ctx_t. */ static void raft_cb_cancel_timeout(void *ctx) { ha_state_t *ha = (ha_state_t *)ctx; election_timeout_ctx_t *t = (election_timeout_ctx_t *)ha->raft_election_timeout_ctx; if (t) atomic_store(&t->cancelled, 1); ha->raft_election_timeout_ctx = NULL; } /* raft_rdhup_delay_fn: delaymgr callback for RDHUP-triggered delayed election. * Owns its ctx (election_timeout_ctx_t) — frees it on exit. * Calls raft_check_and_start_election to start election only if not already voted. */ static int raft_rdhup_delay_fn(void *arg) { election_timeout_ctx_t *t = (election_timeout_ctx_t *)arg; ha_state_t *ha = t->ha; if (!atomic_load(&t->cancelled)) raft_check_and_start_election(&ha->raft); free(t); return 0; } /* * raft_cb_start_election_after_delay: schedule RDHUP retry timer. * After delay_ms, calls raft_check_and_start_election to start election * only if this node hasn't already voted for another candidate. */ static void raft_cb_start_election_after_delay(void *ctx, uint32_t delay_ms) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; if (!c) return; /* test mode */ /* Priority 0 → delay_ms == 0: highest-priority node starts immediately. */ if (delay_ms == 0) { raft_check_and_start_election(&ha->raft); return; } election_timeout_ctx_t *t = calloc(1, sizeof(*t)); if (!t) return; atomic_init(&t->cancelled, 0); t->ha = ha; t->c = c; t->ctx = NULL; int rc = delay_task_ms(&c->delaymgr, (int)delay_ms, 1, raft_rdhup_delay_fn, t); if (rc < 0) { WARN("raft_cb_start_election_after_delay: delay_task_ms failed: %d", rc); free(t); } } /* * raft_cb_become_leader: node won the election — enter COMMITTING state. * * Equivalent of ha_elect_evaluate_and_signal CTRL_PRIMARY path: * - enter CTRL_PRIMARY_COMMITTING * - call cluster_init_phase2 on first boot * - reset repl_ack_count, push config, send ctrlhello * - arm committing timeout watchdog * * ha->raft_c must be non-NULL (production path only; test mode uses noop stubs). */ static void raft_cb_become_leader(void *ctx, uint64_t term) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; /* Cancel any pending election timeout */ raft_cb_cancel_timeout(ctx); /* Sync ha->term from raft term (raft owns term increments) */ atomic_store_explicit(&ha->term, term, memory_order_release); int is_first_boot = !ha->phase2_done; /* First boot: call cluster_init_phase2 */ if (is_first_boot) { #ifdef XCE_TESTING if (ha_phase2_hook != NULL) ha_phase2_hook(c); #else if (c != NULL) cluster_init_phase2(c); #endif ha->phase2_done = 1; } /* Enter COMMITTING: block job traffic until majority ack */ atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); INFO("HA FSM (raft): → CTRL_PRIMARY_COMMITTING (term=%" PRIu64 ", first_boot=%d)", term, is_first_boot); /* Reset ack counter for this committing round */ atomic_store(&ha->repl_ack_count, 0); /* Promote surviving vote endpoints to standby replication channels. * Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to * ha->standby_endpoints so push_config and ctrlhello reach them. */ { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (ectx && ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int i = 0; i < ectx->n_peers; i++) { if (ectx->votes[i].ep != NULL) { zarray_store(ha->standby_endpoints, ectx->votes[i].ep); INFO("HA: promoted vote ep[%d] → standby channel", i); } } pthread_mutex_unlock(&ha->lock); } } if (c) { /* Push config to standbys — read raw bytes from disk so standbys * can save the file. NULL bytes are passed only in test mode. */ #ifdef XCE_TESTING if (ha_push_config_hook != NULL) ha_push_config_hook(c); #else { uint8_t *cfg_bytes = NULL; size_t cfg_len = 0; extern int conf_read_raw(const char *, uint8_t **, size_t *); if (c->myconf && c->myconf->conffile && conf_read_raw(c->myconf->conffile, &cfg_bytes, &cfg_len) == 0) { push_config_to_standbys(c, cfg_bytes, cfg_len, c->myconf->config_sha256, c->myconf->config_version_tag); free(cfg_bytes); } else { push_config_to_standbys(c, NULL, 0, NULL, c->myconf ? c->myconf->config_version_tag : 0); } } #endif /* Send CtrlHelloRequest to all standbys to trigger WAL catch-up */ #ifdef XCE_TESTING if (ha_send_ctrlhello_hook != NULL) ha_send_ctrlhello_hook(c); #else ha_send_ctrlhello_to_all_standbys(c); #endif /* Start primary heartbeat immediately in COMMITTING so standbys receive * NOP during the committing window and their liveness poll does not fire. * ha_start_primary_heartbeat is also called in ha_promote_to_primary; * the idempotency guard (role check at fn top) prevents double-arming. */ ha_start_primary_heartbeat(c); /* Arm committing timeout watchdog */ { /* Cancel any prior committing timeout (e.g. from a previous term) */ committing_timeout_ctx_t *prev = (committing_timeout_ctx_t *) atomic_exchange(&ha->committing_timeout_ctx, NULL); if (prev) { atomic_store(&prev->cancelled, 1); } } committing_timeout_ctx_t *ct = calloc(1, sizeof(*ct)); if (ct) { atomic_init(&ct->cancelled, 0); ct->ha = ha; ct->c = c; atomic_store(&ha->committing_timeout_ctx, ct); int rc = delay_task_ms(&c->delaymgr, (int)ha->config.committing_timeout_ms, 1, committing_timeout_fn, ct); if (rc < 0) { WARN("raft_cb_become_leader: delay_task_ms for committing timeout failed: %d", rc); atomic_store(&ha->committing_timeout_ctx, NULL); free(ct); } } } } /* * raft_cb_become_follower: node lost election or saw higher term — enter STANDBY. * * Syncs ha->term from the raft-reported term, sets role to CTRL_STANDBY, * calls cluster_init_phase2 on first boot (standbys also need it to * receive config replication push from the new primary). */ static void raft_cb_become_follower(void *ctx, uint64_t term, int leader_idx) { (void)leader_idx; ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; /* Cancel any pending election timeout */ raft_cb_cancel_timeout(ctx); /* Cancel any pending committing timeout — we are no longer leader. * Without this, a node that stepped down due to a higher-term vote * request would still promote to PRIMARY when the old committing * timeout fires → split-brain. */ { committing_timeout_ctx_t *ct = (committing_timeout_ctx_t *) atomic_exchange(&ha->committing_timeout_ctx, NULL); if (ct) { atomic_store(&ct->cancelled, 1); } } /* Sync ha->term */ atomic_store_explicit(&ha->term, term, memory_order_release); int is_first_boot = !ha->phase2_done; /* First boot: call cluster_init_phase2 so structures are ready */ if (is_first_boot) { #ifdef XCE_TESTING if (ha_phase2_hook != NULL) ha_phase2_hook(c); #else if (c != NULL) cluster_init_phase2(c); #endif ha->phase2_done = 1; } /* Close all outgoing vote endpoints (loser path). * The primary will push config and ctrlhello via promoted connections; * the standby waits passively on its listener for the inbound connection. */ { ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( &ha->elect_ctx, memory_order_acquire); if (ectx) { for (int i = 0; i < ectx->n_peers; i++) { if (ectx->votes[i].ep != NULL) { INFO("HA: closing vote ep[%d] (loser path)", i); endpoint_fini(ectx->votes[i].ep); ectx->votes[i].ep = NULL; } } } } atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); INFO("HA FSM (raft): → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)", term, is_first_boot); /* Arm heartbeat watchdog — fires ha_on_primary_ctrl_fd_rdhup if primary * sends no ctrlhello within election_timeout_ms. */ ha_arm_heartbeat_timeout_vtask(ha, c); } /* * raft_cb_persist: persist term + voted_for to disk before granting vote. * * Maps raft integer voted_for index to a candidate name using cluster config. * Updates ha->term, ha->voted_for_term, ha->voted_for_name then calls * ha_persist_term(ha, workdir). * * voted_for == -1 means "no vote"; voted_for >= 0 is the candidate index. * Returns 0 on success, non-zero on error (causes raft library to deny vote). */ static int raft_cb_persist(void *ctx, uint64_t term, int voted_for) { ha_state_t *ha = (ha_state_t *)ctx; cluster_t *c = ha->raft_c; /* Store ha-term.bin under {workdir}/{myname}/data/ alongside WAL/snapshot. */ char data_dir[1024] = {0}; if (c && c->myconf && c->myconf->workdir && c->myconf->name) snprintf(data_dir, sizeof(data_dir), "%s/%s/data", c->myconf->workdir, c->myconf->name); const char *workdir = data_dir[0] ? data_dir : NULL; /* Update ha->term */ atomic_store_explicit(&ha->term, term, memory_order_release); /* Map voted_for index to name */ pthread_mutex_lock(&ha->lock); if (voted_for == -1) { ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); } else { ha->voted_for_term = term; /* Look up candidate name by index */ if (c && c->myconf && voted_for < c->myconf->peers_cfg.candidates_count) { const char *name = c->myconf->peers_cfg.candidates[voted_for].name; strncpy(ha->voted_for_name, name, sizeof(ha->voted_for_name) - 1); ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0'; } else { /* Test mode or out-of-range: use "self" placeholder */ if (voted_for == ha->my_priority) { strncpy(ha->voted_for_name, "self", sizeof(ha->voted_for_name) - 1); } else { memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); } } } pthread_mutex_unlock(&ha->lock); ha_persist_term(ha, workdir); return 0; } /* raft_cb_trace: emit raft state-machine events at DEBUG3 level. * Active when controller is started with -vvv; suppressed otherwise. */ static void raft_cb_trace(void *ctx, const char *event, const char *detail) { ha_state_t *ha = (ha_state_t *)ctx; tlog(TLOG_DEBUG3, "RAFT[%s] term=%" PRIu64 " event=%s%s%s", ha->my_priority == 0 ? "p0" : ha->my_priority == 1 ? "p1" : "p2", atomic_load_explicit(&ha->term, memory_order_relaxed), event, detail ? " " : "", detail ? detail : ""); } static const raft_callbacks_t s_real_cbs = { .on_send_vote_request = raft_cb_send_vote_request, .on_vote_eligible = raft_cb_vote_eligible, .on_become_leader = raft_cb_become_leader, .on_become_follower = raft_cb_become_follower, .on_persist = raft_cb_persist, .on_schedule_timeout = raft_cb_schedule_timeout, .on_cancel_timeout = raft_cb_cancel_timeout, .on_start_election_after_delay = raft_cb_start_election_after_delay, .on_trace = raft_cb_trace, }; /* ── PRNG seeding ───────────────────────────────────────────── */ void ha_seed_jitter(uint64_t *seed, int my_priority) { uint8_t buf[8] = {0}; int fd = open("/dev/urandom", O_RDONLY); if (fd >= 0) { ssize_t n = 0; while (n < 8) { ssize_t r = read(fd, buf + n, (size_t)(8 - n)); if (r <= 0) break; n += r; } close(fd); if (n == 8) { memcpy(seed, buf, 8); if (*seed != 0) return; } } struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); *seed = (uint64_t)ts.tv_nsec ^ ((uint64_t)my_priority << 56) ^ ((uint64_t)getpid() << 16); } int ha_init(ha_state_t *ha, const ha_config_t *cfg, int my_priority) { if (!ha || !cfg) return -EINVAL; /* Zero first so ha_fini() is safe even if validation fails below */ memset(ha, 0, sizeof(*ha)); if (cfg->heartbeat_interval_ms < 2000) return -EINVAL; if (cfg->heartbeat_timeout_ms < cfg->heartbeat_interval_ms) return -EINVAL; if (cfg->election_timeout_ms < 500) return -EINVAL; ha->config = *cfg; ha->my_priority = my_priority; atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); atomic_store_explicit(&ha->applied_lsn, 0ULL, memory_order_release); atomic_store_explicit(&ha->promotion_failed, 0, memory_order_relaxed); atomic_store_explicit(&ha->promotion_retry_count, 0, memory_order_relaxed); atomic_store_explicit(&ha->in_catch_up, 0, memory_order_relaxed); /* failover_requested / watchdog_stop deleted — watchdog replaced by ha_arm_heartbeat_timeout_vtask */ pthread_mutex_init(&ha->lock, NULL); ha->ssl_ctx = NULL; ha->jitter_seed = 0; ha->standby_endpoints = NULL; ha->primary_ctrl_fd = -1; /* election-iomgr-refactor Task 2.1: explicit zero-init for documentation clarity * (memset above already zeroes these, but being explicit is better documentation) */ ha->client_ssl_ctx = NULL; ha->phase2_done = 0; ha->n_peers = 0; atomic_store_explicit(&ha->repl_ack_count, 0, memory_order_relaxed); /* ha-heartbeat-liveness Task 1.1: zero-init liveness timestamp. * Written by ha_on_heartbeat_received (task 2.1); read by liveness_poll_fn. */ ha->last_activity = 0; /* ha-repl-hardening Task 1.4: per-standby convergence tracking arrays. * memset(ha, 0, sizeof(*ha)) above already zeroes these arrays; the explicit * note documents the invariant: on entry to CTRL_PRIMARY_COMMITTING every * standby starts as not-converged (tag=0, lsn=0, converged=false). */ /* heartbeat_seen starts at 0: no heartbeat seen yet. * ha_wait_for_heartbeat_or_timeout() calls sem_timedwait on this. */ sem_init(&ha->heartbeat_seen, 0, 0); /* ── Task 6.1/6.2: Initialize embedded raft_state_t ───────────────────── * * Wire real callbacks (s_real_cbs) so the state machine is ready when * ha_start_election_async re-initialises raft with the correct n_peers / * self_idx / initial_term after ha_restore_term(). * * n_peers = 1 and self_idx = 0 are safe placeholders here; they are * overwritten by ha_raft_reinit() (called from ha_start_election_async) * which runs after ha_restore_term() and knows the real cluster size. * * initial_term = 0 / initial_voted_for = -1 are first-boot defaults; * ha_raft_reinit() restores the persisted values before any election fires. * * ha->raft_c is NULL until ha_start_election_async sets it. */ ha->raft_c = NULL; ha->raft_election_timeout_ctx = NULL; raft_config_t raft_cfg; memset(&raft_cfg, 0, sizeof(raft_cfg)); raft_cfg.n_peers = 1; /* placeholder; overwritten by ha_raft_reinit */ raft_cfg.self_idx = my_priority; /* self_idx == priority by convention */ raft_cfg.election_timeout_ms = cfg->election_timeout_ms; raft_cfg.my_priority = my_priority; raft_cfg.initial_term = 0; /* ha_restore_term() updates ha->term after ha_init */ raft_cfg.initial_voted_for = -1; /* no vote on fresh init */ raft_cfg.cbs = &s_real_cbs; raft_cfg.ctx = ha; raft_init(&ha->raft, &raft_cfg); return 0; } /* SIGUSR1 handler / ha_install_sigusr1() deleted — failover_requested replaced by * ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */ void ha_do_promote(ha_state_t *ha) { /* Monotonically increasing fencing token — standbys reject entries from lower terms */ atomic_fetch_add_explicit(&ha->term, 1, memory_order_acq_rel); atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY, memory_order_release); /* COH-5: new primary starts a fresh CRC chain; standbys reset their chain on catch-up */ ha->last_received_crc = WAL_CRC_NONE; /* Req 10.5: log FSM transition to PRIMARY */ INFO("HA FSM: → CTRL_PRIMARY (term=%" PRIu64 ")", atomic_load_explicit(&ha->term, memory_order_acquire)); } void ha_do_demote(ha_state_t *ha) { atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); /* Reset so the next ha_promote_to_primary() re-registers hooks idempotently */ ha->hooks_registered = 0; /* Hook disable requires hooksys_t — done by ha_demote_to_standby(cluster_t*) */ /* Req 10.5: log FSM transition to STANDBY */ INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ")", atomic_load_explicit(&ha->term, memory_order_acquire)); } /* probe_thread() / wait_any_probe() deleted — only called by am_i_primary() (election-iomgr-refactor) */ /* am_i_primary() deleted — replaced by ha_start_election_async (election-iomgr-refactor) */ /* * ha_is_primary: lock-free role check. */ int ha_is_primary(const ha_state_t *ha) { return (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire) == CTRL_PRIMARY; } /* ha_check_priority_stepdown() deleted — replaced by quorum election (Req 2.1) */ /* start_heartbeat_listener() / stop_heartbeat_listener() deleted — never called (election-iomgr-refactor) */ /* ── P3-6: Test-only failover shim ────────────────────────────── */ #ifdef XCE_TESTING uint64_t ha_trigger_manual_failover_for_test(ha_state_t *ha) { ha_do_demote(ha); ha_do_promote(ha); return atomic_load_explicit(&ha->term, memory_order_acquire); } #endif /* ha_check_heartbeat_nocluster() / ha_watchdog_thread() / ha_start_watchdog() / * ha_stop_watchdog() deleted — watchdog pthread replaced by * ha_arm_heartbeat_timeout_vtask (delaymgr) in election-iomgr-refactor */ void ha_fini(ha_state_t *ha) { if (!ha) return; pthread_mutex_destroy(&ha->lock); sem_destroy(&ha->heartbeat_seen); /* Fix ASAN leak: free the SSL context if one was allocated */ if (ha->ssl_ctx) { SSL_CTX_free((SSL_CTX *)ha->ssl_ctx); ha->ssl_ctx = NULL; } } /* ── Task 2.1/2.2: WAL catch-up entry application (standby side) ───────────── * * ha_apply_wal_catchup_entry: apply WAL catch-up entries received from the * primary to local persistent storage and advance applied_lsn on success. * * Two modes (Req 5.3, 5.5): * * Batch mode (entry->n_entries > 0): * Iterate entry->entries[0..n_entries-1] in order. Apply each sub-entry * individually via apply_wal_entry(). NULL sub-entries are treated as * corrupt and skipped (Req 3.3). After ALL sub-entries are processed, * advance applied_lsn to last-successfully-applied lsn+1 so the next * heartbeat reply reflects post-batch state (Req 5.3). * * Single-entry mode (entry->n_entries == 0): * Backward-compatible path from task 2.1. Apply entry->lsn/entry_type * directly and advance applied_lsn to lsn+1 on success. * * In both modes: * - applied_lsn is advanced atomically and monotonically. * - On error for any individual entry: log ERROR, skip, continue (Req 3.3). * - applied_lsn is in-memory only; not persisted to disk (design §Durability). * - Standby stays in CTRL_STANDBY throughout (Req 3.4). * * Returns 0 on success (even if some batch entries were skipped due to errors). * Returns -EINVAL if ha, p, or entry is NULL. */ /* ha_advance_applied_lsn: atomically advance ha->applied_lsn to new_lsn * if new_lsn > current value (monotonic). */ static void ha_advance_applied_lsn(ha_state_t *ha, uint64_t new_lsn) { uint64_t old_lsn = (uint64_t)atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); while (new_lsn > old_lsn) { if (atomic_compare_exchange_weak_explicit( &ha->applied_lsn, &old_lsn, new_lsn, memory_order_acq_rel, memory_order_acquire)) break; } } /* ha_apply_single_entry: apply one decoded wal_catchup_entry_t sub-entry. * Returns 0 on success, non-zero on error (caller logs and skips). */ static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e) { wal_entry_hdr_t hdr; memset(&hdr, 0, sizeof(hdr)); hdr.lsn = e->lsn; hdr.type = (uint32_t)e->entry_type; hdr.payload_len = 0; hdr.crc = 0; hdr.prev_crc = 0; return apply_wal_entry(p, &hdr, NULL); } int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p, wal_catchup_entry_t *entry) { if (!ha || !p || !entry) return -EINVAL; /* ── Batch mode: entry->n_entries > 0 ─────────────────────────────────── */ if (entry->n_entries > 0) { /* Apply each sub-entry in LSN order. Track the last successfully * applied LSN so we can advance applied_lsn after the full batch. * Req 5.5: applied_lsn advances only after the full batch is written. * Req 3.3: NULL or failed entries are logged and skipped; no crash. */ uint64_t last_good_lsn = 0; int had_good = 0; for (int i = 0; i < entry->n_entries; i++) { wal_catchup_entry_t *sub = entry->entries[i]; if (!sub) { ERROR("ha_apply_wal_catchup_entry: NULL sub-entry at " "batch index %d — skipping", i); continue; } int rc = ha_apply_single_entry(p, sub); if (rc != 0) { ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed " "for batch index %d lsn=%" PRIu64 " type=%u rc=%d " "— skipping entry", i, sub->lsn, (unsigned)sub->entry_type, rc); continue; } last_good_lsn = sub->lsn; had_good = 1; } /* Advance applied_lsn to last_good_lsn+1 after the full batch. * Only advance if at least one entry was applied successfully. */ if (had_good) ha_advance_applied_lsn(ha, last_good_lsn + 1); return 0; } /* ── Single-entry mode (task 2.1 backward-compatible path) ────────────── */ int rc = ha_apply_single_entry(p, entry); if (rc != 0) { ERROR("ha_apply_wal_catchup_entry: apply_wal_entry failed for " "lsn=%" PRIu64 " type=%u rc=%d — skipping entry", entry->lsn, (unsigned)entry->entry_type, rc); return rc; } /* Advance applied_lsn atomically (lsn+1 mirrors persist->current_lsn * after a successful apply). */ ha_advance_applied_lsn(ha, entry->lsn + 1); return 0; } /* ha_nop_reply_state_report — populate *sr from ha state and wire into reply. * Reads ha->applied_lsn (atomic) and myconf->config_version_tag, stores them * in *sr, then sets reply_out->data_which = stateReportMsg and * reply_out->data.state_report = sr. * Caller keeps *sr alive until xce_reply() serialises the message. * Requirements: 1.1, 1.3, 1.4 (ha-repl-hardening spec) */ void ha_nop_reply_state_report(ha_state_t *ha, myconf_t *myconf, state_report_t *sr, xce_reply_t *reply_out) { sr->applied_lsn = atomic_load_explicit(&ha->applied_lsn, memory_order_acquire); sr->config_tag = myconf ? myconf->config_version_tag : 0; reply_out->data_which = RequestReply_data_stateReportMsg; reply_out->data.state_report = sr; } /* * ha_process_standby_state — update per-standby convergence tracking. * * Called on each heartbeat reply from a standby (CTRL_PRIMARY_COMMITTING or * CTRL_PRIMARY). Stores (reported_tag, reported_lsn) into the per-standby * arrays, evaluates convergence, and logs INFO on the first * not-converged → converged transition (Req 7.2). Regression resets the flag * (Req 2.5). * * Convergence: reported_tag >= committed_tag AND reported_lsn >= committed_lsn. * * Requirements: 1.2, 2.2, 2.5 (ha-repl-hardening spec) */ void ha_process_standby_state(ha_state_t *ha, int peer_idx, uint64_t reported_tag, uint64_t reported_lsn, uint64_t committed_tag, uint64_t committed_lsn) { if (peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES) return; /* Store reported values */ ha->standby_reported_tag[peer_idx] = reported_tag; ha->standby_reported_lsn[peer_idx] = reported_lsn; /* Evaluate convergence */ bool was_converged = ha->standby_converged[peer_idx]; bool now_converged = (reported_tag >= committed_tag) && (reported_lsn >= committed_lsn); ha->standby_converged[peer_idx] = now_converged; /* Log INFO on first transition to converged (Req 7.2) */ if (now_converged && !was_converged) { INFO("HA: standby[%d] converged: config_tag=%" PRIu64 " applied_lsn=%" PRIu64 " (committed_tag=%" PRIu64 " committed_lsn=%" PRIu64 ")", peer_idx, reported_tag, reported_lsn, committed_tag, committed_lsn); } } /* ── Dispatch gate (req 1.3, 1.4) ────────────────────────────────────────── * * Called from myreq_handler() in cluster.c before the opcode switch. * Returns 0 when the operation is permitted for the current role. * * Primary-only ops are any op that is not one of the always-allowed set: * nop, reply, hello, join, shutdown, heartbeat, redirect, * steppingDown, replEntry, replAck, replSnapshot, replCatchUpDone, * replConfig, replConfigPush, requestVote, probeConfigTag, * walAck, upgradeConfigAck, snapshotTransfer, walCatchupEntry. * * For CTRL_STANDBY the check is: op is in the primary-only set → deny. * For CTRL_INIT / CTRL_ELECTING the allowlist is tighter: only * OP_requestVote and OP_probeConfigTag pass through. */ int ha_dispatch_gate(ha_state_t *ha, int opcode) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); switch (role) { case CTRL_PRIMARY: return 0; case CTRL_INIT: /* Allow election probes, config probes, AND heartbeats so the primary's * periodic heartbeat can reach the handler and post heartbeat_seen, * triggering the CTRL_INIT → CTRL_STANDBY transition (Req 1.1). */ if (opcode == OPCode_requestVote || opcode == OPCode_probeConfigTag || opcode == OPCode_heartbeat) return 0; return -XCE_ERR_NOT_READY; case CTRL_ELECTING: if (opcode == OPCode_requestVote || opcode == OPCode_probeConfigTag) return 0; return -XCE_ERR_NOT_READY; case CTRL_PRIMARY_COMMITTING: /* Req 5.1, 5.2, 8.1: Primary won election but has not yet received * majority replication ack. Allow only replication-related and * administrative opcodes; deny job dispatch and queue management * with XCE_ERR_NOT_READY. */ switch (opcode) { case OPCode_heartbeat: case OPCode_replConfig: case OPCode_replConfigPush: case OPCode_upgradeConfigAck: case OPCode_hello: case OPCode_walAck: case OPCode_replEntry: case OPCode_replAck: case OPCode_replCatchUpDone: return 0; default: return -XCE_ERR_NOT_READY; } case CTRL_STANDBY: { /* Allow ops that standbys legitimately handle */ switch (opcode) { case OPCode_nop: case OPCode_reply: case OPCode_hello: case OPCode_join: case OPCode_shutdown: case OPCode_heartbeat: case OPCode_redirect: case OPCode_steppingDown: case OPCode_replEntry: case OPCode_replAck: case OPCode_replSnapshot: case OPCode_replCatchUpDone: case OPCode_replConfig: case OPCode_replConfigPush: case OPCode_requestVote: case OPCode_probeConfigTag: case OPCode_walAck: case OPCode_upgradeConfigAck: case OPCode_snapshotTransfer: case OPCode_walCatchupEntry: return 0; default: return -XCE_ERR_STANDBY; } } default: return 0; } } /* ── Redirect helpers used by join.c ───────────────────────── */ /* * ha_should_redirect: returns 1 when this standby knows the primary's * address and should redirect incoming join requests there. * Returns 0 for PRIMARY role, or for STANDBY with no known primary yet. */ int ha_should_redirect(const ha_state_t *ha) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) return 0; if (ha->primary_host[0] == '\0' || ha->primary_port == 0) return 0; return 1; } /* * ha_no_primary_known: returns 1 when this node is a STANDBY but does not * yet know the primary's address. Callers should reply E_XCE_NOMASTER so * the client retries rather than treating the error as permanent. */ int ha_no_primary_known(const ha_state_t *ha) { ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY) return 0; return (ha->primary_host[0] == '\0' || ha->primary_port == 0) ? 1 : 0; } /* * ha_promote_to_primary: increment term, set role=PRIMARY, then register * persistence hooks (idempotency guard via hooks_registered). * Safe to call on a zero-init cluster_t (zeroed hooksys → xce_add_hook * returns -EINVAL harmlessly; hooks_registered is still set to 1). */ int ha_promote_to_primary(cluster_t *c) { ha_do_promote(&c->ha); /* Register persistence hooks — idempotency guard prevents double-add */ if (c->ha.hooks_registered == 0) { xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun, &c->persist); xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun, &c->persist); xce_add_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun, &c->persist); c->ha.hooks_registered = 1; } /* Arm periodic heartbeat to standbys — keeps their watchdog alive when * the cluster is idle (no WAL entries or config pushes). */ ha_start_primary_heartbeat(c); return 0; } /* * ha_demote_to_standby: disable persistence hooks, then set role=STANDBY * and reset the hooks_registered idempotency guard (via ha_do_demote). */ int ha_demote_to_standby(cluster_t *c) { /* Disable persistence hooks before changing role */ if (c->ha.hooks_registered) { xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_JOB, (hook_fn_t)persist_job_hook_fun); xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_QUEUE, (hook_fn_t)persist_queue_hook_fun); xce_disable_hook(&c->hooksys, XCE_CONTROLLER_HOOK_WORKER, (hook_fn_t)persist_worker_hook_fun); } ha_do_demote(&c->ha); /* sets role=STANDBY, resets hooks_registered=0 */ return 0; } /* ── HA Config Bootstrap: startup heartbeat wait (Req 1.1, 1.2) ─────────── */ /* * ha_wait_for_heartbeat_or_timeout: called from the startup path after * ha_init() when the node needs to discover whether a primary already exists. * * Blocks on sem_timedwait(&ha->heartbeat_seen, &deadline) where * deadline = now(CLOCK_REALTIME) + election_timeout_ms + jitter * * The heartbeat handler (cluster.c) posts heartbeat_seen when it receives * OPCode_heartbeat and ha->role == CTRL_INIT. ha_dispatch_gate() permits * OPCode_heartbeat in CTRL_INIT state so the packet reaches the handler. * * Returns CTRL_STANDBY — semaphore was posted (heartbeat arrived); * ha_do_demote() was called (role = CTRL_STANDBY). * Returns CTRL_ELECTING — sem_timedwait timed out (ETIMEDOUT); * caller is responsible for driving the election. */ ctrl_role_t ha_wait_for_heartbeat_or_timeout(ha_state_t *ha) { /* Seed jitter PRNG on first use */ if (ha->jitter_seed == 0) ha_seed_jitter(&ha->jitter_seed, ha->my_priority); /* Compute timeout: election_timeout + priority_step * priority + random jitter. * Higher-priority (lower index) nodes time out sooner, giving them a head * start in the election before lower-priority nodes begin campaigning. */ long to_ms = (long)ha->config.election_timeout_ms; to_ms += (long)ha->my_priority * (long)ha->config.election_priority_step_ms; if (ha->config.election_jitter_max_ms > 0) { to_ms += (long)(xorshift64_next(&ha->jitter_seed) % ((uint64_t)ha->config.election_jitter_max_ms + 1)); } /* sem_timedwait requires an absolute CLOCK_REALTIME deadline */ struct timespec deadline; clock_gettime(CLOCK_REALTIME, &deadline); deadline.tv_sec += to_ms / 1000; deadline.tv_nsec += (to_ms % 1000) * 1000000L; if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; } int rc = sem_timedwait(&ha->heartbeat_seen, &deadline); if (rc == 0) { /* Heartbeat arrived before the deadline — demote to standby */ ha_do_demote(ha); return CTRL_STANDBY; } /* ETIMEDOUT (or any other error): treat as election timeout */ return CTRL_ELECTING; } /* ── Term Durability (Req 9) ──────────────────────────────────────────────── * * ha_term_file_t: on-disk format for ha-term.bin. * Fields are in host byte order (single-host durability, no cross-endian migration). * CRC32 covers all preceding bytes via zlib crc32(). */ #define HA_TERM_MAGIC 0xDEADBEEFCAFEBABEULL typedef struct { uint64_t magic; uint64_t term; uint64_t voted_for_term; char voted_for_name[XCE_MAXNAME_LEN]; uint32_t crc32; } ha_term_file_t; /* * ha_persist_term: atomically write ha_term_file_t to {workdir}/ha-term.bin. * Uses write-to-tmp + fsync + rename for durability (existing project pattern). * If workdir is NULL, the persist is skipped (test mode or no workdir configured). * Req 9.1: must be called before any vote grant or term increment is reported. */ void ha_persist_term(ha_state_t *ha, const char *workdir) { if (!workdir) return; /* test mode: skip file I/O */ ha_term_file_t f; memset(&f, 0, sizeof(f)); f.magic = HA_TERM_MAGIC; f.term = atomic_load_explicit(&ha->term, memory_order_acquire); f.voted_for_term = ha->voted_for_term; strncpy(f.voted_for_name, ha->voted_for_name, sizeof(f.voted_for_name) - 1); f.voted_for_name[sizeof(f.voted_for_name) - 1] = '\0'; /* CRC32 covers all bytes before the crc32 field itself */ size_t crc_len = offsetof(ha_term_file_t, crc32); f.crc32 = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len); /* Write to tmp then rename — atomic from filesystem perspective */ char path[1024], tmp[1040]; snprintf(path, sizeof(path), "%s/ha-term.bin", workdir); snprintf(tmp, sizeof(tmp), "%s/ha-term.bin.tmp", workdir); int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (fd < 0) return; /* best-effort; caller must not rely on this for correctness */ ssize_t written = write(fd, &f, sizeof(f)); if (written == (ssize_t)sizeof(f)) fsync(fd); close(fd); if (written == (ssize_t)sizeof(f)) rename(tmp, path); } /* * ha_restore_term: read {workdir}/ha-term.bin at startup and restore * ha->term and ha->voted_for_* into ha_state_t. * * On missing file (ENOENT): no-op — term stays 0. Req 9.3. * On CRC/magic mismatch: log WARN, zero ha->term and ha->voted_for_*. Req 9.2. * On valid file: atomically restore ha->term; copy voted_for fields. Req 9.2. * * Returns 0 on success (including gracefully-handled file-not-found and CRC bad), * -errno on unexpected I/O error. * If workdir is NULL, function is a no-op (safe for unit tests without a workdir). */ int ha_restore_term(ha_state_t *ha, const char *workdir) { if (!workdir) return 0; char path[1024]; snprintf(path, sizeof(path), "%s/ha-term.bin", workdir); int fd = open(path, O_RDONLY); if (fd < 0) { if (errno == ENOENT) return 0; /* first boot — no file yet, term stays 0 (Req 9.3) */ return -errno; } ha_term_file_t f; memset(&f, 0, sizeof(f)); ssize_t n = read(fd, &f, sizeof(f)); close(fd); if (n != (ssize_t)sizeof(f)) { WARN("HA: ha-term.bin short read (%zd/%zu) — ignoring, starting at term 0", n, sizeof(f)); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); return 0; } /* Verify magic */ if (f.magic != HA_TERM_MAGIC) { WARN("HA: ha-term.bin bad magic (0x%016" PRIx64 ") — ignoring, starting at term 0", f.magic); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); return 0; } /* Verify CRC32 */ size_t crc_len = offsetof(ha_term_file_t, crc32); uint32_t computed = (uint32_t)crc32(0L, (const Bytef *)&f, (uInt)crc_len); if (computed != f.crc32) { WARN("HA: ha-term.bin CRC mismatch (expected 0x%08x got 0x%08x) — ignoring, starting at term 0", computed, f.crc32); atomic_store_explicit(&ha->term, 0ULL, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); return 0; } /* Valid file — restore state (Req 9.2) */ atomic_store_explicit(&ha->term, f.term, memory_order_release); ha->voted_for_term = f.voted_for_term; strncpy(ha->voted_for_name, f.voted_for_name, sizeof(ha->voted_for_name) - 1); ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0'; return 0; } /* ── ha_heartbeat_is_fresh (Req 6.2) ──────────────────────────────────────── * * Returns 1 if the elapsed time since ha->last_heartbeat is strictly less than * ha->config.heartbeat_timeout_ms (i.e. the primary is still alive). * Returns 0 if the heartbeat is stale (timeout has expired). * * Uses CLOCK_MONOTONIC to avoid wall-clock skew. */ static int ha_heartbeat_is_fresh(ha_state_t *ha) { struct timespec now, last; clock_gettime(CLOCK_MONOTONIC, &now); pthread_mutex_lock(&ha->lock); last = ha->last_heartbeat; pthread_mutex_unlock(&ha->lock); long ms = (now.tv_sec - last.tv_sec) * 1000L + (now.tv_nsec - last.tv_nsec) / 1000000L; return ms < (long)ha->config.heartbeat_timeout_ms; } /* ── Vote Handler (Req 2.2–2.6, 6.1, 6.2, 6.4, 9.1) ──────────────────────── * * handle_request_vote: process an incoming OP_requestVote RPC. * * Decision order (fail-fast): * 0. role gate (Req 6.1, 6.2): * - CTRL_PRIMARY or CTRL_PRIMARY_COMMITTING → deny * - CTRL_STANDBY with fresh heartbeat → deny * 1. in_catch_up → deny (Req 2.5) * 2. req term < my term → deny (Req 2.2) * 3. req term > my term → update term, clear voted_for (Req 2.3) * 4. already voted for a DIFFERENT candidate in this term → deny (Req 2.6) * 5. freshness gate: candidate not at least as fresh → deny (Req 2.4) * 6. grant vote, record voted_for, persist (Req 9.1) * * Returns 1 = granted, 0 = denied, <0 = error. */ int handle_request_vote(ha_state_t *ha, uint64_t term, const char *candidate_id, uint64_t config_tag, uint64_t last_lsn, uint64_t my_config_tag, uint64_t my_last_lsn, int in_catch_up_flag) { /* Req 2.3: higher term → step down BEFORE any role gate. * * Raft §5.1: "If RPC request or response contains term T > currentTerm: * set currentTerm = T, convert to follower." * * Without this ordering, a node in CTRL_PRIMARY_COMMITTING at term=N * would deny a vote request at term=N+1 without ever seeing the higher * term. Both nodes then independently promote to PRIMARY via their * committing timeout → split-brain. */ uint64_t my_term = atomic_load_explicit(&ha->term, memory_order_acquire); if (term > my_term) { atomic_store_explicit(&ha->term, term, memory_order_release); ha->voted_for_term = 0; memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); /* Step down from leader/committing: a higher-term election is in * progress and we must not block it with the role gate below. */ ctrl_role_t cur_role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (cur_role == CTRL_PRIMARY || cur_role == CTRL_PRIMARY_COMMITTING) { ha_do_demote(ha); INFO("HA: stepping down (term %" PRIu64 " → %" PRIu64 ") — higher-term election in progress", my_term, term); } my_term = term; } /* Req 2.2: stale term → deny, do not update any state */ if (term < my_term) { return 0; } /* Req 6.1, 6.2: role-based deny gate. * Evaluated AFTER the term catch-up above: if we just stepped down due to * a higher term, role is now CTRL_STANDBY and we fall through to voting. * PRIMARY and PRIMARY_COMMITTING at the current term still deny votes * (protecting the live primary from same-term challengers). */ ctrl_role_t my_role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (my_role == CTRL_PRIMARY || my_role == CTRL_PRIMARY_COMMITTING) return 0; /* deny — I am primary (or committing); protect against challengers */ if (my_role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) return 0; /* deny — my primary is alive */ /* Req 2.5: catching-up standbys deny all votes */ if (in_catch_up_flag) return 0; /* Protect voted_for check+set from concurrent vote requests on different vtasks */ pthread_mutex_lock(&ha->lock); /* Req 2.6: already voted for a different candidate in this term → deny */ if (ha->voted_for_term == term && ha->voted_for_name[0] != '\0' && strcmp(ha->voted_for_name, candidate_id) != 0) { pthread_mutex_unlock(&ha->lock); return 0; } /* Req 2.4: freshness gate * Deny if candidate has a lower config_tag, or equal config_tag with lower LSN. * Grant if candidate is strictly fresher or equally fresh. */ if (config_tag < my_config_tag) { pthread_mutex_unlock(&ha->lock); return 0; } if (config_tag == my_config_tag && last_lsn < my_last_lsn) { pthread_mutex_unlock(&ha->lock); return 0; } /* Grant the vote: record voted_for and persist before responding (Req 9.1) */ ha->voted_for_term = term; strncpy(ha->voted_for_name, candidate_id, sizeof(ha->voted_for_name) - 1); ha->voted_for_name[sizeof(ha->voted_for_name) - 1] = '\0'; pthread_mutex_unlock(&ha->lock); /* workdir is not available here — persist with NULL (test mode or caller * wraps this function with workdir before sending the network reply). * Production dispatch path calls ha_persist_term(ha, workdir) explicitly * before this function writes its network reply. */ ha_persist_term(ha, NULL); return 1; } /* ── Test-only hook shims (XCE_TESTING only) ────────────────────────────── * Type definitions (election_timeout_ctx_t, committing_timeout_ctx_t), * extern declarations (cluster_init_phase2, push_config_to_standbys, * ha_send_ctrlhello_to_all_standbys), and static forward declarations * (committing_timeout_fn, ha_heartbeat_is_fresh) have been moved to the * forward-declarations block near the top of this file so that the raft * callback implementations (Task 6.2) can reference them. */ #ifdef XCE_TESTING /* * ha_phase2_hook: test-only seam. * When non-NULL, ha_elect_evaluate_and_signal() calls this INSTEAD OF the real * cluster_init_phase2() so tests can count phase2 calls without a live cluster_t. */ int (*ha_phase2_hook)(void *c) = NULL; /* * ha_send_ctrlhello_hook: test-only seam for ha_send_ctrlhello_to_all_standbys. * When non-NULL, the COMMITTING path calls this INSTEAD OF the real function * (cluster.c is not linked in unit-test binary). Req 7.2. */ int (*ha_send_ctrlhello_hook)(void *c) = NULL; /* * ha_push_config_hook: test-only seam for push_config_to_standbys. * When non-NULL, the COMMITTING path calls this INSTEAD OF the real function * (config_repl.c's push_config_to_standbys needs a real cluster). Req 7.1. */ int (*ha_push_config_hook)(void *c) = NULL; /* * ha_nop_req_hook: test-only seam for xce_nop_req in primary_heartbeat_fn. * When non-NULL, primary_heartbeat_fn calls this INSTEAD OF xce_nop_req so * tests can count NOP sends without a live endpoint. Req 3.2, task 2.2. */ int (*ha_nop_req_hook)(endpoint_t *ep) = NULL; #endif /* XCE_TESTING */ /* ── Election Evaluation (Req 2.1, 2.7, 2.8) ─────────────────────────────── * * ha_elect_evaluate_and_signal: evaluate accumulated votes; set ctx->result * and post ctx->done when quorum is reached or all peers replied. * * Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. * ctx->votes_granted starts at 1 (self-vote); ctx->n_peers is n_other_peers. * Quorum = ceil((n_peers + 1) / 2) = (n_peers + 1 + 1) / 2 = (n_peers + 2) / 2. * * Yield gate (Req 2.7): if any responding peer has a higher config_tag, or * equal config_tag with a higher LSN than the candidate → yield (CTRL_STANDBY). * Candidate's own tag/lsn is stored in ctx->votes[0].my_tag / my_lsn. * * Fatal gate (Req 2.8): if candidate has config_tag=0 and any peer has tag>0 * → CTRL_STANDBY (never PRIMARY with no config when peer has config). * * Returns CTRL_PRIMARY or CTRL_STANDBY; also sets ctx->result. */ ctrl_role_t ha_elect_evaluate_and_signal(ha_state_t *ha, ha_elect_ctx_t *ctx) { /* Guard: called from handle_request_vote_reply under ctx->lock; called from * unit-test path (peers==NULL) without threads — done_posted prevents double * execution if two threads both hit quorum simultaneously. */ if (ctx->done_posted) return (ctx->result == 1) ? CTRL_PRIMARY : CTRL_STANDBY; ctrl_role_t role_result; /* Fix 2 (CRITICAL): zero-peer guard — single-node shortcircuit. * n_peers==0 means no other nodes; votes[] flexible array has 0 entries, * so accessing votes[0] would be out-of-bounds. Self-vote (votes_granted=1) * always satisfies quorum=1. No fresher-peer or fatal-gate checks apply. */ if (ctx->n_peers == 0) { ctx->result = 1; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_PRIMARY; goto apply_role; } { int total_nodes = ctx->n_peers + 1; /* include self */ int quorum = (total_nodes + 1) / 2; /* ceil(total/2) */ /* Req 2.8: fatal gate — candidate has no config but a peer does */ uint64_t my_tag = ctx->votes[0].my_tag; uint64_t my_lsn = ctx->votes[0].my_lsn; if (my_tag == 0 && ctx->max_resp_tag > 0) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } /* Check quorum */ if (ctx->votes_granted < quorum) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } /* Req 2.7: yield gate — fresher responding peer */ if (ctx->max_resp_tag > my_tag) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } if (ctx->max_resp_tag == my_tag && ctx->max_resp_lsn > my_lsn) { ctx->result = 0; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_STANDBY; goto apply_role; } /* Quorum met and no fresher peer → PRIMARY */ ctx->result = 1; ctx->done_posted = 1; sem_post(&ctx->done); role_result = CTRL_PRIMARY; } apply_role: /* ── Task 4.1: apply role and call cluster_init_phase2 ───────────────── * * After determining the election result, update ha->role and (on first boot) * call cluster_init_phase2. This runs once per process lifetime — the * phase2_done flag prevents re-entry on subsequent elections (RDHUP failover). * * The cluster_t pointer is carried in ctx->c (NULL in unit tests). * * Cancel the election timeout watchdog if we won (it would be a no-op now * but the explicit cancel avoids a spurious second evaluate call). */ { cluster_t *c = ctx->c; int is_first_boot = !ha->phase2_done; if (role_result == CTRL_PRIMARY) { /* Cancel election timeout watchdog (quorum already reached) */ if (ctx->timeout_ctx) { election_timeout_ctx_t *etc = (election_timeout_ctx_t *)ctx->timeout_ctx; atomic_store(&etc->cancelled, 1); } /* First boot: call cluster_init_phase2 to load config+data (Req 9.2) */ if (is_first_boot) { #ifdef XCE_TESTING /* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */ if (ha_phase2_hook != NULL) ha_phase2_hook(c); #else if (c != NULL) cluster_init_phase2(c); #endif ha->phase2_done = 1; } /* Enter COMMITTING: block job traffic until majority ack (Req 4.1, 8.1) */ atomic_store_explicit(&ha->role, (int32_t)CTRL_PRIMARY_COMMITTING, memory_order_release); INFO("HA FSM: → CTRL_PRIMARY_COMMITTING (term=%" PRIu64 ", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire), is_first_boot); /* Reset ack counter for this committing round (Req 5.2) */ atomic_store(&ha->repl_ack_count, 0); /* Push config to standbys with lower config_tag (Req 7.1) */ #ifdef XCE_TESTING if (ha_push_config_hook != NULL) ha_push_config_hook(c); #else if (c != NULL) push_config_to_standbys(c, NULL, 0, NULL, 0); #endif /* Send CtrlHelloRequest to all standbys to trigger WAL catch-up (Req 7.2) */ #ifdef XCE_TESTING if (ha_send_ctrlhello_hook != NULL) ha_send_ctrlhello_hook(c); #else if (c != NULL) ha_send_ctrlhello_to_all_standbys(c); #endif /* Arm committing timeout watchdog (degraded-mode fallback, Req 8.4) */ if (c) { /* Cancel any prior committing timeout */ committing_timeout_ctx_t *prev = (committing_timeout_ctx_t *) atomic_exchange(&ha->committing_timeout_ctx, NULL); if (prev) { atomic_store(&prev->cancelled, 1); } committing_timeout_ctx_t *ct = calloc(1, sizeof(*ct)); if (ct) { atomic_init(&ct->cancelled, 0); ct->ha = ha; ct->c = c; atomic_store(&ha->committing_timeout_ctx, ct); int rc = delay_task_ms(&c->delaymgr, (int)ha->config.committing_timeout_ms, 1, committing_timeout_fn, ct); if (rc < 0) { WARN("ha_elect_evaluate_and_signal: delay_task_ms " "for committing timeout failed: %d", rc); atomic_store(&ha->committing_timeout_ctx, NULL); free(ct); } } } } else { /* STANDBY result */ /* Cancel election timeout watchdog — it is now a no-op but explicit * cancellation matches spec wording and avoids a spurious lock-acquire * if the timeout fires after STANDBY is decided. */ if (ctx->timeout_ctx) { election_timeout_ctx_t *etc = (election_timeout_ctx_t *)ctx->timeout_ctx; atomic_store(&etc->cancelled, 1); } /* First boot: call cluster_init_phase2 so structures are ready; * config arrives via primary's replication push (Req 9.2, 7.3) */ if (is_first_boot) { #ifdef XCE_TESTING /* In test mode, use hook if set; skip real phase2 (cluster.c not linked) */ if (ha_phase2_hook != NULL) ha_phase2_hook(c); #else if (c != NULL) cluster_init_phase2(c); #endif ha->phase2_done = 1; } atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); INFO("HA FSM: → CTRL_STANDBY (term=%" PRIu64 ", first_boot=%d)", atomic_load_explicit(&ha->term, memory_order_acquire), is_first_boot); /* Primary connects outbound and pushes config via ctrlhello. * Standby is fully passive here — no fetch_config_from_primary. */ } } return role_result; } /* ── Vote Reply Handler (Req 2.2–2.6) ────────────────────────────────────── * * handle_request_vote_reply: process a reply to an OP_requestVote RPC. * * Called by each per-peer vote-request thread once the peer replies. * Updates ctx->votes[peer_idx] with the reply fields, tracks the freshest * responding peer in ctx->max_resp_tag/max_resp_lsn, increments counters, * and evaluates whether to signal ctx->done. * * Quorum = ceil((n_peers + 1) / 2) = (n_peers + 2) / 2 * (n_peers = number of other peers, not counting self; votes_granted starts at 1) * * Posts ctx->done via ha_elect_evaluate_and_signal when: * - votes_granted reaches quorum (fast path — no need to wait for all replies), OR * - n_replied reaches n_peers (all peers accounted for, win or lose) */ void handle_request_vote_reply(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_config_tag, uint64_t peer_lsn) { (void)term; /* fencing: callers bump ha->term if peer's term > ours, before calling here */ /* Guard against out-of-range index */ if (peer_idx < 0 || peer_idx >= ctx->n_peers) return; pthread_mutex_lock(&ctx->lock); ha_vote_entry_t *entry = &ctx->votes[peer_idx]; /* Idempotency guard: ignore duplicate replies */ if (entry->replied) { pthread_mutex_unlock(&ctx->lock); return; } /* Record reply fields */ entry->replied = 1; entry->vote_granted = vote_granted ? 1 : 0; entry->peer_tag = peer_config_tag; entry->peer_lsn = peer_lsn; /* Update counters */ if (vote_granted) ctx->votes_granted++; ctx->n_replied++; /* Track the freshest responding peer for the yield gate (Req 2.7) */ if (peer_config_tag > ctx->max_resp_tag) { ctx->max_resp_tag = peer_config_tag; ctx->max_resp_lsn = peer_lsn; } else if (peer_config_tag == ctx->max_resp_tag && peer_lsn > ctx->max_resp_lsn) { ctx->max_resp_lsn = peer_lsn; } /* Decide whether to signal the waiting thread. * Quorum = ceil((n_peers + 1) / 2) where n_peers+1 is total cluster size. * n_peers is the peer count excluding self; self-vote is already in votes_granted. * ha_elect_evaluate_and_signal is called under ctx->lock; its done_posted guard * prevents double sem_post if two threads reach quorum simultaneously. */ int total_nodes = ctx->n_peers + 1; int quorum = (total_nodes + 1) / 2; if (ctx->votes_granted >= quorum || ctx->n_replied >= ctx->n_peers) ha_elect_evaluate_and_signal(ha, ctx); pthread_mutex_unlock(&ctx->lock); } /* ── Task 2.2: iomgr-native vote send path ────────────────────────────────── * * vote_ep_ctx_t: per-peer context stored in ep->data for each outgoing vote * endpoint. on_vote_connected fires after the TLS handshake completes (when * endpoint_t.ssl_handshake_done transitions to 1 in io_handler). */ /* vote_ep_ctx_t: typedef moved to Task 6.2 forward-declaration block above */ /* * peer_ctrl_msg_handler / vote_reply_decode_and_dispatch / * repl_ack_decode_and_dispatch: forward declarations. * peer_ctrl_msg_handler is the single iomgr handler set in on_vote_connected * and never reassigned — it routes by ha->role at dispatch time. */ static int peer_ctrl_msg_handler(void *ep_raw); static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx); static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx); #ifdef XCE_TESTING /* * xce_vote_request_hook: test-only seam. * * When non-NULL, on_vote_connected() calls this INSTEAD OF the real * xce_request() so tests can capture the call without requiring a live * iomgr or SSL endpoint. The hook receives the fully built * xce_request_t and may inspect all fields. * * Pattern mirrors xce_retry_send_hook in request.c. */ int (*xce_vote_request_hook)(endpoint_t *ep, xce_request_t *req) = NULL; #endif /* XCE_TESTING */ /* * on_vote_connected: iomgr callback fired after TLS handshake completes on an * outgoing vote endpoint. * * Steps: * 1. Cast ep->data to vote_ep_ctx_t. * 2. Build request_vote_req_t from ctx fields. * 3. Build xce_request_t{opcode=OPCode_requestVote, body.requestvotemsg=&vrq}. * 4. Call xce_vote_request_hook (test seam, XCE_TESTING only) — then * call xce_request() to queue the packet on the iomgr send path. * 5. Set ep->msg_handler = vote_reply_msg_handler so the next inbound * packet is decoded as a RequestVoteReply. * * Req 2.3: send vote after TLS handshake. * Req 1.2: vote RPC via iomgr path (xce_request enqueues to endpoint_enq). */ static void on_vote_connected(endpoint_t *ep) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; /* Build the RequestVoteRequest payload */ request_vote_req_t vrq; memset(&vrq, 0, sizeof(vrq)); vrq.term = vctx->my_term; vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : ""); vrq.config_tag = vctx->my_tag; vrq.last_log_lsn = vctx->my_lsn; /* Wrap in xce_request_t */ xce_request_t req; memset(&req, 0, sizeof(req)); req.opcode = OPCode_requestVote; req.body_which = Request_body_requestvotemsg; req.body.requestvotemsg = &vrq; #ifdef XCE_TESTING if (xce_vote_request_hook != NULL) xce_vote_request_hook(ep, &req); else xce_request(ep, &req); #else xce_request(ep, &req); #endif /* Set state-aware dispatcher — routes by ha->role at message-arrival time. * Never reassigned: ELECTING → vote reply; COMMITTING/PRIMARY → repl ack. */ ep->msg_handler = peer_ctrl_msg_handler; } /* * xce_vote_reply_hook: test-only seam for vote_reply_msg_handler. * * Forward-declared here (before vote_reply_msg_handler) so the handler body * can reference it. Defined only under XCE_TESTING; guarded below. * * When non-NULL, vote_reply_msg_handler() calls this INSTEAD OF the real * handle_request_vote_reply() so tests can capture decoded fields without * a live iomgr. Receives the decoded scalar fields — no capnp pointers. * * Pattern mirrors xce_vote_request_hook used in task 2.2. */ #ifdef XCE_TESTING int (*xce_vote_reply_hook)(ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn) = NULL; #endif /* XCE_TESTING */ /* * vote_reply_dispatch: internal helper — delegates to the raft library or * the test hook (when XCE_TESTING and hook is non-NULL). * * Production path: raft_on_vote_reply owns quorum tracking and fires * on_become_leader/on_become_follower via the raft callbacks. * Test path: xce_vote_reply_hook captures decoded fields without real iomgr. */ static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx, uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn) { #ifdef XCE_TESTING if (xce_vote_reply_hook != NULL) { xce_vote_reply_hook(ha, ctx, idx, term, vote_granted, peer_tag, peer_lsn); return; } #endif (void)ctx; (void)peer_tag; (void)peer_lsn; /* consumed by raft library via callbacks */ raft_on_vote_reply(&ha->raft, idx, vote_granted, term); } /* * vote_reply_decode_and_dispatch: decode a RequestVoteReply packet and dispatch * to vote_reply_dispatch. Factored from vote_reply_msg_handler so that * peer_ctrl_msg_handler can invoke the same logic without code duplication. * Packet ownership: caller passes ep->fsm_in.pkt; this function frees it. */ static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) { ha_state_t *ha = vctx->ha; ha_elect_ctx_t *ctx = vctx->ctx; int idx = vctx->peer_idx; io_packet_t *pkt = ep->fsm_in.pkt; int ok = 0; uint64_t vr_term = 0; int vr_granted = 0; uint64_t vr_tag = 0; uint64_t vr_lsn = 0; if (!pkt || !pkt->data || pkt->sz == 0) goto done; { struct capn capn_ctx; struct Request outer_rq; Request_ptr outer_rp; struct RequestReply rep_s; struct RequestVoteReply vr_raw; int capn_ok; capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data, (size_t)pkt->sz, 0); if (capn_ok != 0) goto done; outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1); read_Request(&outer_rq, outer_rp); if (outer_rq.body_which != Request_body_replymsg || !outer_rq.body.replymsg.p.data) { capn_free(&capn_ctx); goto done; } read_RequestReply(&rep_s, outer_rq.body.replymsg); if (rep_s.data_which != RequestReply_data_requestvotereplymsg) { capn_free(&capn_ctx); goto done; } memset(&vr_raw, 0, sizeof(vr_raw)); read_RequestVoteReply(&vr_raw, rep_s.data.requestvotereplymsg); capn_free(&capn_ctx); vr_term = (uint64_t)vr_raw.term; vr_granted = vr_raw.voteGranted ? 1 : 0; vr_tag = (uint64_t)vr_raw.peerConfigTag; vr_lsn = (uint64_t)vr_raw.peerLastLogLsn; ok = 1; } done: if (pkt) { free_packet(pkt); free(pkt); } if (ok) { uint64_t cur = atomic_load_explicit(&ha->term, memory_order_acquire); while (vr_term > cur) { if (atomic_compare_exchange_weak_explicit(&ha->term, &cur, vr_term, memory_order_acq_rel, memory_order_acquire)) break; } vote_reply_dispatch(ha, ctx, idx, vr_term, vr_granted, vr_tag, vr_lsn); } else { vote_reply_dispatch(ha, ctx, idx, atomic_load_explicit(&ha->term, memory_order_acquire), 0, 0, 0); } return 0; } /* * repl_ack_decode_and_dispatch: decode an inbound packet on a promoted vote * endpoint while role == CTRL_PRIMARY_COMMITTING or CTRL_PRIMARY. * * - OPCode_replCatchUpDone → ha_repl_ack_received * - OPCode_reply with stateReportMsg → ha_process_standby_state (Task 3.1) * - OPCode_nop or OPCode_reply without stateReport → silently ignored * - anything else → WARN + drop */ static int repl_ack_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx) { ha_state_t *ha = vctx->ha; cluster_t *c = vctx->c; io_packet_t *pkt = ep->fsm_in.pkt; if (!pkt || !pkt->data || pkt->sz == 0) { if (pkt) { free_packet(pkt); free(pkt); } return 0; } struct capn capn_ctx; struct Request outer_rq; Request_ptr outer_rp; int capn_ok = capn_init_mem(&capn_ctx, (const uint8_t *)pkt->data, (size_t)pkt->sz, 0); free_packet(pkt); free(pkt); ep->fsm_in.pkt = NULL; if (capn_ok != 0) return 0; outer_rp.p = capn_getp(capn_root(&capn_ctx), 0, 1); read_Request(&outer_rq, outer_rp); int opcode = (int)outer_rq.opcode; if (opcode == OPCode_replCatchUpDone) { capn_free(&capn_ctx); ha_repl_ack_received(ha, c); } else if (opcode == OPCode_reply && outer_rq.body_which == Request_body_replymsg && outer_rq.body.replymsg.p.data) { /* Heartbeat NOP reply — decode inner RequestReply for stateReport */ struct RequestReply rep_s; memset(&rep_s, 0, sizeof(rep_s)); read_RequestReply(&rep_s, outer_rq.body.replymsg); if (rep_s.data_which == RequestReply_data_stateReportMsg) { /* Decode stateReport and call convergence tracker (Task 3.1). * read_StateReport reads scalar fields directly from the capnp * pointer without heap allocation — safe to call before capn_free. */ struct StateReport sr_raw; memset(&sr_raw, 0, sizeof(sr_raw)); read_StateReport(&sr_raw, rep_s.data.stateReportMsg); state_report_t sr; sr.applied_lsn = (uint64_t)sr_raw.appliedLsn; sr.config_tag = (uint64_t)sr_raw.configTag; capn_free(&capn_ctx); /* peer_idx from standby_ep_ctx_t (layout-compatible with vote_ep_ctx_t) */ int peer_idx = vctx->peer_idx; uint64_t committed_lsn = 0; uint64_t committed_tag = 0; #ifndef XCE_TESTING if (c) { committed_lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); committed_tag = atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire); } #else /* In test mode, c may be NULL; caller sets ha fields directly */ committed_lsn = atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire); committed_tag = atomic_load_explicit(&ha->committed_config_tag, memory_order_acquire); (void)c; #endif ha_process_standby_state(ha, peer_idx, sr.config_tag, sr.applied_lsn, committed_tag, committed_lsn); } else { capn_free(&capn_ctx); } } else if (opcode == OPCode_nop) { /* Plain NOP with no body — silently ignore */ capn_free(&capn_ctx); } else { capn_free(&capn_ctx); WARN("repl_ack_decode_and_dispatch: unexpected opcode %d on promoted ep", opcode); } return 0; } /* * peer_ctrl_msg_handler: state-aware iomgr handler set once in on_vote_connected. * Routes by ha->role at dispatch time — no explicit handler reassignment needed. * CTRL_ELECTING → vote_reply_decode_and_dispatch * CTRL_PRIMARY_COMMITTING / * CTRL_PRIMARY → repl_ack_decode_and_dispatch * any other role → drop packet (WARN) */ static int peer_ctrl_msg_handler(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; ha_state_t *ha = vctx->ha; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); switch (role) { case CTRL_ELECTING: return vote_reply_decode_and_dispatch(ep, vctx); case CTRL_PRIMARY_COMMITTING: case CTRL_PRIMARY: return repl_ack_decode_and_dispatch(ep, vctx); default: { io_packet_t *pkt = ep->fsm_in.pkt; if (pkt) { free_packet(pkt); free(pkt); ep->fsm_in.pkt = NULL; } WARN("peer_ctrl_msg_handler: unexpected role %d, packet dropped", (int)role); return 0; } } } #ifdef XCE_TESTING /* * vote_reply_build_pkt_for_test: test helper that encodes a RequestVoteReply * into a newly allocated io_packet_t that vote_reply_msg_handler_for_test can decode. * * Wire format mirrors the real peer reply: * Request{opcode=reply, body.replymsg=RequestReply{ * data_which=requestvotereplymsg, * data.requestvotereplymsg=RequestVoteReply{term, voteGranted, ...} * }} * * Returns a freshly allocated io_packet_t* on success, NULL on failure. * Handler takes ownership: it calls free_packet()+free() when done. * Tests set ep->fsm_in.pkt to the returned pointer before calling the handler. */ io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn) { struct capn cs; struct capn_segment *seg; struct RequestVoteReply vr_capn; RequestVoteReply_ptr vr_ptr; struct RequestReply rr_capn; RequestReply_ptr rr_ptr; struct Request rq_capn; Request_ptr rq_ptr; io_packet_t *pkt; int rc; capn_init_malloc(&cs); seg = capn_root(&cs).seg; /* RequestVoteReply — all scalar fields */ memset(&vr_capn, 0, sizeof(vr_capn)); vr_capn.term = term; vr_capn.voteGranted = vote_granted ? 1 : 0; vr_capn.peerConfigTag = peer_tag; vr_capn.peerLastLogLsn = peer_lsn; vr_ptr = new_RequestVoteReply(seg); write_RequestVoteReply(&vr_capn, vr_ptr); /* RequestReply envelope */ memset(&rr_capn, 0, sizeof(rr_capn)); rr_capn.seq = 1; rr_capn.opcode = OPCode_requestVote; rr_capn.status = 0; rr_capn.data_which = RequestReply_data_requestvotereplymsg; rr_capn.data.requestvotereplymsg = vr_ptr; rr_ptr = new_RequestReply(seg); write_RequestReply(&rr_capn, rr_ptr); /* Outer Request{opcode=reply} */ memset(&rq_capn, 0, sizeof(rq_capn)); rq_capn.opcode = OPCode_reply; rq_capn.seq = 1; rq_capn.body_which = Request_body_replymsg; rq_capn.body.replymsg = rr_ptr; rq_ptr = new_Request(seg); write_Request(&rq_capn, rq_ptr); capn_setp(capn_root(&cs), 0, rq_ptr.p); pkt = calloc(1, sizeof(*pkt)); if (!pkt) { capn_free(&cs); return NULL; } pkt->sz = capn_size(&cs); rc = alloc_packet(pkt); if (rc < 0) { capn_free(&cs); free(pkt); return NULL; } capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0); capn_free(&cs); return pkt; } /* * vote_reply_msg_handler_for_test: test entry point for vote-reply decode path. * Temporarily forces role to CTRL_ELECTING so peer_ctrl_msg_handler routes to * vote_reply_decode_and_dispatch, then restores the original role. * Caller owns ep->data (vote_ep_ctx_t); pkt ownership is transferred. */ int vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt) { vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; ha_state_t *ha = vctx ? vctx->ha : NULL; int32_t saved_role = ha ? atomic_load_explicit(&ha->role, memory_order_acquire) : 0; if (ha) atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release); ep->fsm_in.pkt = pkt; int rc = peer_ctrl_msg_handler(ep); if (ha) atomic_store_explicit(&ha->role, saved_role, memory_order_release); return rc; } /* * vote_ep_ctx_build_for_test: test helper that allocates a vote_ep_ctx_t, * fills it from the supplied parameters, and wires up ep->data and * ep->on_connected so the test can drive on_vote_connected indirectly. * * Returns ep (same pointer passed in) for call-chain convenience. * Caller is responsible for free(ep->data) after use. */ endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t my_term, uint64_t my_tag, uint64_t my_lsn, const char *my_name) { vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx)); if (!vctx) return ep; /* test will fail when on_connected is NULL */ vctx->ha = ha; vctx->c = NULL; vctx->ctx = ctx; vctx->peer_idx = peer_idx; vctx->my_term = my_term; vctx->my_tag = my_tag; vctx->my_lsn = my_lsn; vctx->my_name = my_name; ep->data = vctx; ep->on_connected = on_vote_connected; return ep; } /* * election_timeout_fn_for_test: test helper that exercises the election timeout * callback logic with an explicit cancelled_val instead of an atomic field. * Allows testing the cancelled-gate without setting up a real delaymgr. * * Returns 0 in all cases (mirrors the real callback's return value). */ int election_timeout_fn_for_test(ha_state_t *ha, ha_elect_ctx_t *ctx, int cancelled_val) { if (cancelled_val) return 0; pthread_mutex_lock(&ctx->lock); ha_elect_evaluate_and_signal(ha, ctx); pthread_mutex_unlock(&ctx->lock); return 0; } /* * committing_timeout_fn_for_test: test helper that exercises the committing * timeout callback logic with an explicit cancelled_val. The cluster pointer * may be NULL; the function checks the role on `ha` and calls * ha_promote_to_primary(c) only when role == CTRL_PRIMARY_COMMITTING && c != NULL. * * Unlike the production callback, this shim does NOT free the ctx structure — * it is stack/test-owned and must not be heap-freed by the shim. * * Returns 0 in all cases (mirrors the real callback). */ int committing_timeout_fn_for_test(ha_state_t *ha, cluster_t *c, int cancelled_val) { if (cancelled_val) return 0; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY_COMMITTING && c != NULL) ha_promote_to_primary(c); return 0; } /* * repl_catchup_done_pkt_for_test: build a minimal io_packet_t encoding * Request{opcode=replCatchUpDone, body_which=nodata}. * Used by peer_ctrl_msg_handler tests to verify the COMMITTING dispatch path. */ io_packet_t *repl_catchup_done_pkt_for_test(void) { struct capn cs; struct capn_segment *seg; struct Request rq_capn; Request_ptr rq_ptr; io_packet_t *pkt; int rc; capn_init_malloc(&cs); seg = capn_root(&cs).seg; memset(&rq_capn, 0, sizeof(rq_capn)); rq_capn.opcode = OPCode_replCatchUpDone; rq_capn.seq = 1; rq_capn.body_which = Request_body_nomsg; rq_ptr = new_Request(seg); write_Request(&rq_capn, rq_ptr); capn_setp(capn_root(&cs), 0, rq_ptr.p); pkt = calloc(1, sizeof(*pkt)); if (!pkt) { capn_free(&cs); return NULL; } pkt->sz = capn_size(&cs); rc = alloc_packet(pkt); if (rc < 0) { capn_free(&cs); free(pkt); return NULL; } capn_write_mem(&cs, (uint8_t *)pkt->data, (size_t)pkt->sz, 0); capn_free(&cs); return pkt; } /* * vote_ep_err_handler_for_test: call the static vote_ep_err_handler on ep. * Allows tests to invoke the err_handler without going through iomgr/endpoint machinery. */ int vote_ep_err_handler_for_test(endpoint_t *ep) { return vote_ep_err_handler((void *)ep); } /* * peer_ctrl_msg_handler_for_test: expose peer_ctrl_msg_handler for tests. * Sets ep->fsm_in.pkt then invokes the handler. Packet ownership transferred. */ int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt) { ep->fsm_in.pkt = pkt; return peer_ctrl_msg_handler(ep); } /* * vote_ep_ctx_set_cluster_for_test: set the cluster_t pointer inside the * vote_ep_ctx_t stored at ep->data. Tests that need ha_repl_ack_received * to call ha_promote_to_primary(c) when quorum is reached must call this * after vote_ep_ctx_build_for_test (which sets vctx->c = NULL by default). * Requirements: task 7.3 (ha-peer-connection-reuse). */ void vote_ep_ctx_set_cluster_for_test(endpoint_t *ep, cluster_t *c) { if (!ep || !ep->data) return; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; vctx->c = c; } /* * raft_cb_become_leader_for_test: drive the leader promotion path without * a live cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress * production-only calls (phase2, push_config, ctrlhello, committing timer). */ void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term) { ha->raft_c = NULL; raft_cb_become_leader((void *)ha, term); } /* * raft_cb_become_follower_for_test: drive the follower path without a live * cluster. Passes ha as ctx; sets ha->raft_c = NULL to suppress production * calls (phase2, fetch_config); ha_arm_heartbeat_timeout_vtask is a no-op * when c=NULL so no watchdog thread is spawned. */ void raft_cb_become_follower_for_test(ha_state_t *ha, uint64_t term) { ha->raft_c = NULL; raft_cb_become_follower((void *)ha, term, -1); } #endif /* XCE_TESTING */ /* ── Task 4.1: committing_timeout_fn — delaymgr callback ─────────────────── * * Fires ha_promote_to_primary (degraded mode) if the majority ack threshold * has not been reached within committing_timeout_ms. Req 8.4. * Owns the committing_timeout_ctx_t — frees it on exit. */ static int committing_timeout_fn(void *arg) { committing_timeout_ctx_t *t = (committing_timeout_ctx_t *)arg; if (atomic_load(&t->cancelled)) { free(t); return 0; } ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&t->ha->role, memory_order_acquire); if (role == CTRL_PRIMARY_COMMITTING) { WARN("HA: committing timeout — majority ack not received within timeout; " "promoting to PRIMARY (degraded mode, Req 8.4)"); if (t->c) ha_promote_to_primary(t->c); } free(t); return 0; } /* ── Task 4.1: ha_repl_ack_received ──────────────────────────────────────── * * Called when a standby ACKs config+WAL catch-up (OPCode_replCatchUpDone). * Increments repl_ack_count; when majority threshold (quorum-1 peers, since * primary counts itself) is reached, disarms the committing timeout and calls * ha_promote_to_primary. Req 8.2. */ void ha_repl_ack_received(ha_state_t *ha, cluster_t *c) { if (!ha) return; int acks = atomic_fetch_add(&ha->repl_ack_count, 1) + 1; int quorum = (ha->n_peers + 2) / 2; /* majority of total cluster */ if (acks >= quorum - 1) { /* -1: primary counts itself */ INFO("HA: majority ack received (acks=%d, quorum=%d) → CTRL_PRIMARY", acks, quorum); if (c) ha_promote_to_primary(c); } } /* ── Task 2.4: vote_ep_err_handler ───────────────────────────────────────── * * Called by iomgr when the outgoing vote endpoint encounters an error * (connect failure, SSL error, remote close before reply). * Counts the peer as denied: grant=0, tag=0, lsn=0. */ static int vote_ep_err_handler(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; if (!vctx) return 0; ha_state_t *ha = vctx->ha; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role == CTRL_PRIMARY_COMMITTING || role == CTRL_PRIMARY) { /* Promoted connection dropped — remove from standby_endpoints */ if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int i = 0; i < ha->standby_endpoints->count; i++) { if (zarray_get(ha->standby_endpoints, i) == ep) { zarray_del(ha->standby_endpoints, i); break; } } pthread_mutex_unlock(&ha->lock); } } else { /* Election phase: count as denied vote */ handle_request_vote_reply(ha, vctx->ctx, vctx->peer_idx, vctx->my_term, 0, /* vote_granted = false */ 0, 0); } /* Do NOT free vctx here — peer_ctrl_msg_handler may still reference ep->data. * vctx is freed in ha_election_cleanup() when all per-peer contexts release. */ return 0; } /* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ──── * * Non-blocking election initiator. Sets up the raft state machine and * delegates all election mechanics to raft_start_election(). * * raft_start_election() handles: * - term increment and self-vote (via on_persist callback) * - single-node shortcircuit (fires on_become_leader immediately) * - multi-node RPC fan-out (via on_send_vote_request callback per peer) * - election timeout scheduling (via on_schedule_timeout callback) * * ha_elect_ctx_t is still allocated here because raft_cb_send_vote_request * reads ha->elect_ctx to retrieve my_tag/my_lsn for the capnp VoteRequest * payload. Vote counting and quorum decisions are now owned by the raft * library (raft_on_vote_reply fires on_become_leader/follower callbacks). * * Requirements: 1.3, 2.1 (shortcircuit), 2.4 (async vote send), 9.2 */ void ha_start_election_async(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) { (void)peers; /* peer connections driven by raft_cb_send_vote_request */ (void)my_name; /* self-vote name stored in on_persist via raft_start_election */ /* Step 0: wire cluster pointer for raft callbacks. * Must be set before any raft_* call so callbacks can reach cluster_t. */ ha->raft_c = c; /* Step 0b: re-initialise raft_state_t with the real cluster size and * the term/voted_for restored from disk by ha_restore_term(). * * ha->raft.term may be 0 from ha_init while ha->term was restored to a * higher value by ha_restore_term() — reconcile here before the election. * * self_idx = ha->my_priority (index of this node in candidates[]). * n_peers = total cluster size (candidates_count); passed directly. * * voted_for: map ha->voted_for_name back to a peer index (-1 if none). */ { int restored_voted_for = -1; if (ha->voted_for_name[0] != '\0' && c && c->myconf) { const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; for (int i = 0; i < pcfg->candidates_count; i++) { if (strcmp(pcfg->candidates[i].name, ha->voted_for_name) == 0) { restored_voted_for = i; break; } } } raft_config_t rcfg; memset(&rcfg, 0, sizeof(rcfg)); rcfg.n_peers = n_peers; /* total candidates_count */ rcfg.self_idx = ha->my_priority; rcfg.election_timeout_ms = ha->config.election_timeout_ms; rcfg.my_priority = ha->my_priority; rcfg.initial_term = atomic_load_explicit(&ha->term, memory_order_acquire); rcfg.initial_voted_for = restored_voted_for; rcfg.cbs = &s_real_cbs; rcfg.ctx = ha; raft_init(&ha->raft, &rcfg); } /* Step 1: store peer count for ha_repl_ack_received quorum calculation */ ha->n_peers = n_peers; /* Step 2: transition to CTRL_ELECTING before firing RPCs. * raft_start_election fires on_become_leader / on_send_vote_request below; * the role must already be ELECTING so ha_dispatch_gate() admits vote RPCs. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, memory_order_release); INFO("HA FSM: → CTRL_ELECTING (async, n_peers=%d)", n_peers); /* Step 3: pre-allocate ha_elect_ctx_t so raft_cb_send_vote_request can * read my_tag/my_lsn from it when building the capnp VoteRequest payload. * Vote counting is now owned by the raft library; this ctx is only a * metadata carrier for tag/lsn. For multi-node elections it is kept alive * until all per-peer vote_ep_ctx_t endpoints finish (iomgr lifetime). * For single-node, raft_start_election fires on_become_leader immediately * so no ctx reads occur after return. */ int n_other = (n_peers > 1) ? n_peers - 1 : 0; ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + (size_t)n_other * sizeof(ctx->votes[0])); if (!ctx) { WARN("ha_start_election_async: calloc ctx failed"); return; } ctx->n_peers = n_other; ctx->votes_granted = 1; /* self-vote placeholder (raft tracks actual count) */ ctx->n_replied = 0; ctx->max_resp_tag = 0; ctx->max_resp_lsn = 0; ctx->result = -1; ctx->done_posted = 0; ctx->c = c; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); /* Store candidate's own tag/lsn; votes[0] is read by raft_cb_send_vote_request */ if (n_other > 0) { ctx->votes[0].my_tag = my_config_tag; ctx->votes[0].my_lsn = my_lsn; } /* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx */ atomic_store_explicit(&ha->elect_ctx, ctx, memory_order_release); /* Step 4: delegate election to raft library. * * A node with no valid local config (my_config_tag == 0) must not start an * election — any peer with a real config tag will beat it in the freshness * gate and the node would waste a term increment. It stays in CTRL_ELECTING, * accepts incoming vote requests (granting to fresher peers), and will * become STANDBY after the winner sends it a heartbeat/ctrlhello. * n_peers and raft state are already initialised above so RDHUP retries work. * * raft_start_election when allowed: * - increments ha->raft.term and records self-vote (fires on_persist) * - single-node: fires on_become_leader → raft_cb_become_leader * - multi-node: fires on_send_vote_request per peer → raft_cb_send_vote_request * then fires on_schedule_timeout → raft_cb_schedule_timeout * * No direct ha->term++ or ha->voted_for_* writes happen in this function * after this call; the raft library owns those via on_persist. */ /* Req 4.3 / ha-config-bootstrap Req 2.8: * A node with no local config (my_config_tag == 0) must not start an * election if any peer has a valid config — it cannot win (freshness gate * denies it) and it would waste a term increment. It stays in CTRL_ELECTING, * accepts incoming vote requests (granting to fresher peers), and becomes * STANDBY after the winner sends a heartbeat. * * Exception: if ALL peers also have config_tag=0 (cluster cold-start with no * configured node), someone must win — allow election normally. * * my_config_tag comes from myconf->config_version_tag set by load_primary_config; * n_peers and the peer list from --peers are always available via the parameters. * We check if any peer returned a non-zero tag via the xce_probe_config_tag path * already called in myapp_run before ha_start_election_async. */ if (my_config_tag == 0 && n_peers > 1) { INFO("ha_start_election_async: no local config (tag=0, n_peers=%d) — " "participating as voter only, not starting own election", n_peers); /* Transition to CTRL_STANDBY so myapp_run schedules fetch_config_from_primary. * The primary will push config via ctrlhello once it enters COMMITTING. */ atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); /* Free ctx — no vote sends will happen from this node this round. */ atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release); pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); return; } raft_start_election(&ha->raft); /* For single-node, raft_start_election fires on_become_leader synchronously. * on_become_leader (raft_cb_become_leader) handles role → COMMITTING, * phase2, and committing timeout. Free the ctx here since no raft callbacks * will reference it after raft_start_election returns for single-node. */ if (n_peers == 1) { atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release); pthread_mutex_destroy(&ctx->lock); sem_destroy(&ctx->done); free(ctx); return; } INFO("ha_start_election_async: returning (n_peers=%d, raft election started)", n_peers); /* Returns immediately — election result delivered via raft on_become_leader/follower */ } #ifdef XCE_TESTING /* * ha_rdhup_election_hook: test-only seam for ha_start_election_async in the * RDHUP failover path. When non-NULL, ha_on_primary_ctrl_fd_rdhup() calls * this instead of the real ha_start_election_async(). Req 6.3, 6.5, 9.3. */ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name) = NULL; #endif /* ── Task 9.4: EPOLLRDHUP Failover and Heartbeat Timeout ─────────────────── * * Requirements: 6.2 (RDHUP fast path), 6.3 (heartbeat timeout fallback), * 6.4 (no heartbeat within timeout → elect), 6.8 (role flip) * 10.5 (INFO logs on FSM transitions) */ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) { if (!ha) return; /* Cancel the liveness poll task first (Req 5.4): prevents a concurrent * poll fn from triggering a second failover call after EPOLLRDHUP fires. */ ha_disarm_heartbeat_timeout_vtask(ha); /* Close and null the primary control fd */ if (ha->primary_ctrl_fd >= 0) { close(ha->primary_ctrl_fd); ha->primary_ctrl_fd = -1; } /* Transition to CTRL_ELECTING (Req 6.2) */ atomic_store_explicit(&ha->role, CTRL_ELECTING, memory_order_release); INFO("HA: primary_ctrl_fd RDHUP → CTRL_ELECTING (Req 10.5)"); if (!c) return; /* unit test mode: skip persist + election */ /* Truncate uncommitted WAL entries above committed_lsn (Req 6.2) */ uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn, memory_order_acquire); if (committed > 0) persist_truncate_wal_above(&c->persist, committed); /* Task 6.3: delegate RDHUP election trigger to raft library. * * raft_on_leader_lost: * - clears ha->raft.voted_for = -1 (no direct ha->voted_for_* writes here) * - sets ha->raft.role = RAFT_FOLLOWER * - fires on_start_election_after_delay(ctx, priority * election_timeout_ms) * → raft_cb_start_election_after_delay schedules raft_rdhup_delay_fn * which calls raft_check_and_start_election after the priority jitter. * * The manual nanosleep jitter + voted_for clear + skip-if-already-voted * logic is now fully owned by the raft state machine. * * raft->raft_c must be set before raft_on_leader_lost fires callbacks. */ ha->raft_c = c; #ifdef XCE_TESTING if (ha_rdhup_election_hook != NULL) { /* Test seam: build peer args for hook (same as before) so existing * tests that capture ha_start_election_async arguments still work. */ if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) { const xce_cluster_config_t *cfg = &c->myconf->peers_cfg; int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0; xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES]; int np = 0; for (int i = 0; i < cfg->candidates_count && np < n; i++) { if (strcmp(cfg->candidates[i].name, c->myhost ? c->myhost->name : "") != 0) peer_ptrs[np++] = &cfg->candidates[i]; } uint64_t my_tag = c->myconf->config_version_tag; uint64_t my_lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); const char *my_name = c->myconf ? c->myconf->name : ""; ha_rdhup_election_hook(ha, c, np, peer_ptrs, my_tag, my_lsn, my_name); } return; } #endif raft_on_leader_lost(&ha->raft); } /* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */ /* * liveness_poll_ctx_t: context for the delaymgr-based standby liveness task. * * Requirements: 1.2, 1.4, 1.5, 1.6, 5.4 * * Replaces hb_vtask_t + hb_timeout_thread. No pthread, no semaphore. * The poll fn reads ha->last_activity (written by ha_on_heartbeat_received) * and fires ha_on_primary_ctrl_fd_rdhup if elapsed >= heartbeat_timeout_ms. * * cancelled is _Atomic so that writes from ha_disarm_heartbeat_timeout_vtask * (main/vtask context) are visible to liveness_poll_fn (delaymgr worker thread) * without a data race. */ typedef struct { ha_state_t *ha; cluster_t *c; _Atomic int cancelled; } liveness_poll_ctx_t; /* Static pointer to the active context. _Atomic so concurrent arm/disarm * vs poll-fn execution is well-defined. Access pattern: * arm — allocates and stores (single writer, vtask context) * disarm — sets cancelled, frees (single writer, vtask context) * poll_fn — reads via passed arg only; never dereferences s_liveness_ctx */ static liveness_poll_ctx_t * _Atomic s_liveness_ctx = NULL; /* * liveness_poll_fn: delaymgr task replacing hb_timeout_thread. * * Checks whether the primary has been silent for >= heartbeat_timeout_ms. * If so, calls ha_on_primary_ctrl_fd_rdhup and does not re-arm. * If not, re-arms itself for the next poll interval. * If cancelled, returns immediately without any action. * * Also skips if role != CTRL_STANDBY (Req 5.4): prevents duplicate failover * if EPOLLRDHUP already transitioned the role to CTRL_ELECTING. */ static int liveness_poll_fn(void *arg) { liveness_poll_ctx_t *ctx = (liveness_poll_ctx_t *)arg; /* Check cancelled flag first — set by ha_disarm_heartbeat_timeout_vtask */ if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) return 0; ha_state_t *ha = ctx->ha; cluster_t *c = ctx->c; /* Skip if not standby — EPOLLRDHUP may have already changed the role */ if (atomic_load_explicit(&ha->role, memory_order_acquire) != CTRL_STANDBY) return 0; /* Compute elapsed time since last primary message */ struct timespec now_ts; clock_gettime(CLOCK_MONOTONIC, &now_ts); time_t now = now_ts.tv_sec; time_t last = ha->last_activity; /* time_t write is atomic on LP64 */ time_t elapsed = (last > 0) ? (now - last) : 0; uint32_t timeout_s = ha->config.heartbeat_timeout_ms / 1000; if (last > 0 && (uint32_t)elapsed >= timeout_s) { /* Silence exceeded timeout — trigger failover */ WARN("HA: liveness poll: no primary activity for %lus (timeout=%us) → failover", (unsigned long)elapsed, timeout_s); ha_on_primary_ctrl_fd_rdhup(ha, c); /* Do NOT re-arm — failover handles re-election */ return 0; } /* Still fresh — re-arm for next poll interval */ if (c && c->delaymgr.running) { uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2; if (poll_ms < 100) poll_ms = 100; delay_task_ms(&c->delaymgr, (int)poll_ms, 1, (delay_task_entry_fn_t)liveness_poll_fn, ctx); } return 0; } #ifdef XCE_TESTING /* * liveness_poll_fn_test_entry: test-only entry point that calls liveness_poll_fn * directly with a pre-constructed ctx, bypassing delaymgr. * Only compiled when XCE_TESTING is defined. */ void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx) { liveness_poll_fn(ctx); } /* * primary_heartbeat_fn_for_test: call primary_heartbeat_fn directly with c. * Bypasses delaymgr scheduling so tests can drive the idle-NOP logic without * a live timer. Only compiled when XCE_TESTING is defined. (Task 2.2) */ int primary_heartbeat_fn_for_test(cluster_t *c) { return primary_heartbeat_fn(c); } #endif void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c) { if (!ha) return; /* Disarm any existing poll ctx */ liveness_poll_ctx_t *old = atomic_load_explicit(&s_liveness_ctx, memory_order_acquire); if (old) { atomic_store_explicit(&old->cancelled, 1, memory_order_release); free(old); atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release); } liveness_poll_ctx_t *ctx = calloc(1, sizeof(*ctx)); if (!ctx) return; ctx->ha = ha; ctx->c = c; atomic_init(&ctx->cancelled, 0); /* Initialise last_activity to now so the first poll computes elapsed from * the arm time rather than from zero. */ struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); ha->last_activity = ts.tv_sec; atomic_store_explicit(&s_liveness_ctx, ctx, memory_order_release); uint32_t poll_ms = ha->config.heartbeat_interval_ms / 2; if (poll_ms < 100) poll_ms = 100; INFO("HA: standby liveness poll armed (timeout=%ums, poll=%ums)", ha->config.heartbeat_timeout_ms, poll_ms); if (c && c->delaymgr.running) { int rc = delay_task_ms(&c->delaymgr, (int)poll_ms, 1, (delay_task_entry_fn_t)liveness_poll_fn, ctx); if (rc < 0) WARN("HA: ha_arm_heartbeat_timeout_vtask: delay_task_ms failed: %d", rc); } } void ha_on_heartbeat_received(ha_state_t *ha) { if (!ha) return; struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); ha->last_activity = ts.tv_sec; tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec); } void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha) { (void)ha; liveness_poll_ctx_t *ctx = atomic_load_explicit(&s_liveness_ctx, memory_order_acquire); if (!ctx) return; atomic_store_explicit(&ctx->cancelled, 1, memory_order_release); free(ctx); atomic_store_explicit(&s_liveness_ctx, NULL, memory_order_release); } /* ── Primary heartbeat sender + late-joiner outbound connect ─────────────── */ /* * standby_ep_ctx_t: per-peer context for outbound standby connections opened * by the primary during the heartbeat task. * * Layout deliberately matches vote_ep_ctx_t for the first 4 fields so that * peer_ctrl_msg_handler (which casts ep->data to vote_ep_ctx_t*) can safely * read ha, c, and peer_idx. ctx is always NULL here (no active election). */ typedef struct { ha_state_t *ha; cluster_t *c; ha_elect_ctx_t *ctx; /* always NULL — not a vote endpoint */ int peer_idx; /* index into c->myconf->peers_cfg.candidates[] */ /* Task 1.1 (ha-heartbeat-liveness): last time a push/nop was sent to this * standby (CLOCK_MONOTONIC seconds). Initialized to 0 at allocation; * written by on_standby_connected and primary_heartbeat_fn (task 2.2). */ time_t last_push_time; } standby_ep_ctx_t; #ifdef XCE_TESTING /* * standby_ep_ctx_build_for_test: allocate a standby_ep_ctx_t, wire ep->data, * and return ep. last_push_time is initialised to 0 (calloc). * Caller must free(ep->data) after the test. (Task 2.2) */ endpoint_t *standby_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha, cluster_t *c, int peer_idx) { standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx)); if (!sctx) return NULL; sctx->ha = ha; sctx->c = c; sctx->ctx = NULL; sctx->peer_idx = peer_idx; /* last_push_time = 0 from calloc — tests set it explicitly */ ep->data = sctx; return ep; } /* * on_standby_connected_for_test: call on_standby_connected directly. * ep->data must be a valid standby_ep_ctx_t (use standby_ep_ctx_build_for_test). * xce_request will fail on a null fd — tests only check last_push_time. (Task 2.2) */ int on_standby_connected_for_test(endpoint_t *ep) { on_standby_connected(ep); return 0; } #endif /* XCE_TESTING */ /* * on_standby_connected: iomgr on_connected callback for outbound standby * endpoints created by the heartbeat task. * * Sends OPCode_hello (CtrlHelloRequest) so the receiving standby runs * handle_ctrlhello and triggers WAL catch-up + config push. Registers the * endpoint in ha->standby_endpoints so future heartbeats reach it. */ static void on_standby_connected(endpoint_t *ep) { standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data; ha_state_t *ha = sctx->ha; cluster_t *c = sctx->c; /* Register in standby_endpoints under lock (idempotency guard) */ if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); int already = 0; for (int i = 0; i < ha->standby_endpoints->count; i++) { if (zarray_get(ha->standby_endpoints, i) == ep) { already = 1; break; } } if (!already) zarray_store(ha->standby_endpoints, ep); pthread_mutex_unlock(&ha->lock); } /* Record last_push_time before any send so the first heartbeat tick after * connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */ { struct timespec _ts; clock_gettime(CLOCK_MONOTONIC, &_ts); sctx->last_push_time = _ts.tv_sec; } /* Send ctrlhello — triggers handle_ctrlhello on the standby side */ uint64_t lsn = atomic_load_explicit(&c->persist.committed_lsn, memory_order_acquire); uint64_t tag = c->myconf ? c->myconf->config_version_tag : 0; char *name = c->myconf ? c->myconf->name : ""; ctrl_hello_req_t hello = { .name = name, .key = NULL, .applied_lsn = lsn, .config_tag = tag }; xce_request_t req; memset(&req, 0, sizeof(req)); req.opcode = OPCode_hello; req.body_which = Request_body_ctrlhellomsg; req.body.ctrlhellomsg = &hello; int rc = xce_request(ep, &req); if (rc < 0) WARN("HA: ctrlhello to peer[%d] failed: %d", sctx->peer_idx, rc); else INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent", sctx->peer_idx); /* Set state-aware dispatcher for subsequent messages on this endpoint */ ep->msg_handler = peer_ctrl_msg_handler; } /* * standby_ep_err_handler_primary: error handler for outbound standby * endpoints created by the heartbeat task. Removes from standby_endpoints * and frees the context — the next heartbeat tick will attempt reconnect. */ static int standby_ep_err_handler_primary(void *ep_raw) { endpoint_t *ep = (endpoint_t *)ep_raw; standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data; if (!sctx) return 0; ha_state_t *ha = sctx->ha; if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int i = 0; i < ha->standby_endpoints->count; i++) { if (zarray_get(ha->standby_endpoints, i) == ep) { zarray_del(ha->standby_endpoints, i); break; } } pthread_mutex_unlock(&ha->lock); } free(sctx); ep->data = NULL; return 0; } /* * ha_connect_to_peer: open a new outbound TLS connection to a peer candidate * and register it as a standby channel. Called from primary_heartbeat_fn * for peers that are absent from standby_endpoints. */ static void ha_connect_to_peer(cluster_t *c, int peer_idx) { ha_state_t *ha = &c->ha; const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[peer_idx]; standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx)); if (!sctx) return; sctx->ha = ha; sctx->c = c; sctx->peer_idx = peer_idx; endpoint_t *ep = endpoint_get(); if (!ep) { free(sctx); return; } endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0); endpoint_set_peer(ep, c->myhost); ep->data = sctx; ep->iomgr = &c->iomgr; if (ha->client_ssl_ctx) { SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx; ep->ssl = SSL_new(client_ctx); if (ep->ssl) SSL_set_connect_state(ep->ssl); } endpoint_set_on_connected(ep, on_standby_connected); endpoint_set_err_handler(ep, standby_ep_err_handler_primary); int rc = endpoint_connect(ep, peer->hostname, peer->port); if (rc < 0) { WARN("HA: connect to peer[%d] (%s:%u) failed: %d", peer_idx, peer->hostname, peer->port, rc); endpoint_fini(ep); free(sctx); return; } rc = iomgr_monitor(ep); if (rc < 0) { endpoint_fini(ep); free(sctx); return; } INFO("HA: connecting outbound to peer[%d] (%s:%u)", peer_idx, peer->hostname, peer->port); } /* * primary_heartbeat_fn: self-perpetuating one-shot delaymgr task. * * On each tick: * 1. Sends OPCode_nop to all connected standbys (keeps watchdog alive). * 2. Scans the peers list; for any peer NOT in standby_endpoints, opens a * fresh outbound TLS connection so late joiners are bootstrapped without * the standby ever initiating an outbound connect. * * Stops naturally on demotion: the role check prevents re-arming. */ static int primary_heartbeat_fn(void *arg) { cluster_t *c = (cluster_t *)arg; ha_state_t *ha = &c->ha; ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role, memory_order_acquire); if (role != CTRL_PRIMARY && role != CTRL_PRIMARY_COMMITTING) return 0; /* demoted — stop the heartbeat loop */ /* 1. Heartbeat all connected standbys — idle-triggered NOP (Req 3.1-3.4). * * For each standby ep, compute elapsed time since the last outbound push. * Send NOP only when elapsed >= heartbeat_interval_ms so that WAL entries, * config pushes, and ctrlhello suppress redundant NOPs (Req 3.4). */ { struct timespec _hb_ts; clock_gettime(CLOCK_MONOTONIC, &_hb_ts); time_t now = _hb_ts.tv_sec; int interval_s = (int)(ha->config.heartbeat_interval_ms / 1000); if (interval_s <= 0) interval_s = HA_DEFAULT_HEARTBEAT_INTERVAL_MS / 1000; if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); int n = ha->standby_endpoints->count; for (int i = 0; i < n; i++) { endpoint_t *ep = (endpoint_t *)zarray_get(ha->standby_endpoints, i); if (!ep) continue; standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)ep->data; if (!sctx) continue; time_t elapsed = now - sctx->last_push_time; if (elapsed >= (time_t)interval_s) { #ifdef XCE_TESTING int rc = ha_nop_req_hook ? ha_nop_req_hook(ep) : xce_nop_req(ep); #else int rc = xce_nop_req(ep); #endif if (rc < 0) WARN("HA: heartbeat nop to standby[%d] failed: %d", i, rc); else sctx->last_push_time = now; } } pthread_mutex_unlock(&ha->lock); } } /* 2. Reconnect to any peer missing from standby_endpoints */ if (c->myconf) { const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; for (int i = 0; i < pcfg->candidates_count; i++) { if (i == ha->my_priority) continue; /* skip self */ /* Check if this peer already has a standby endpoint. * * Two endpoint types can be in standby_endpoints: * - Promoted vote eps: ep->data = vote_ep_ctx_t, peer_idx is * votes_idx (relative, self excluded). Convert absolute i to * votes_idx using the same formula as raft_cb_send_vote_request. * - Heartbeat standby eps: ep->data = standby_ep_ctx_t (ctx==NULL), * peer_idx is absolute candidate index. Compare directly with i. */ int votes_idx_for_i = (i < ha->my_priority) ? i : i - 1; int found = 0; if (ha->standby_endpoints) { pthread_mutex_lock(&ha->lock); for (int j = 0; j < ha->standby_endpoints->count; j++) { endpoint_t *sep = (endpoint_t *)zarray_get( ha->standby_endpoints, j); if (!sep || !sep->data) continue; standby_ep_ctx_t *sctx = (standby_ep_ctx_t *)sep->data; int match_idx = (sctx->ctx != NULL) ? votes_idx_for_i /* promoted vote ep: relative */ : i; /* standby ep: absolute */ if (sctx->peer_idx == match_idx) { found = 1; break; } } pthread_mutex_unlock(&ha->lock); } if (!found) { INFO("HA: peer[%d] (%s) not connected — initiating outbound connect", i, pcfg->candidates[i].name ? pcfg->candidates[i].name : "?"); ha_connect_to_peer(c, i); } } } /* Re-arm for next interval — only reached when still PRIMARY */ int interval_ms = (int)ha->config.heartbeat_interval_ms; if (interval_ms <= 0) interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS; delay_task_ms(&c->delaymgr, interval_ms, 1, (delay_task_entry_fn_t)primary_heartbeat_fn, c); return 0; } void ha_start_primary_heartbeat(cluster_t *c) { if (!c) return; ha_state_t *ha = &c->ha; int interval_ms = (int)ha->config.heartbeat_interval_ms; if (interval_ms <= 0) interval_ms = HA_DEFAULT_HEARTBEAT_INTERVAL_MS; /* Guard: skip if delaymgr is not running (e.g. unit tests with zeroed cluster_t). * In production the delaymgr is always started before any promotion. */ if (!c->delaymgr.running) return; /* One-shot: the task re-arms itself each time while role == PRIMARY */ int rc = delay_task_ms(&c->delaymgr, interval_ms, 1, (delay_task_entry_fn_t)primary_heartbeat_fn, c); if (rc < 0) WARN("ha_start_primary_heartbeat: delay_task_ms failed: %d", rc); else INFO("HA: primary heartbeat armed (interval=%dms)", interval_ms); }