From 7e19bf6110a858a5e8d00bbd6c5bc5e723aeea0b Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 16 Jun 2026 05:45:06 +0000 Subject: [PATCH] feat(ha-prevote): controller candidate-side PreVote orchestration (task 4.1) ha.c: raft_cb_send_pre_vote_request (emits preVoteRequest), ha_start_prevote_async (side-effect-free round + bounded round timeout), pre_vote_reply_dispatch (raft_on_pre_vote_reply + branch), ha_enter_real_election helper. WON -> truncate + CTRL_ELECTING + ha_start_election_async (unchanged real-election path); LOST/ timeout -> stay CTRL_STANDBY + re-arm liveness + log abandonment; PENDING -> no-op. Round inert until task 4.2 wires triggers; absolute peer-index keyed tally. 6 new test_ha_prevote tests via reply hook; daemon + suite link clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .kiro/specs/ha-prevote/tasks.md | 3 +- src/controller/ha.c | 439 +++++++++++++++++++++ src/controller/ha.h | 19 + src/controller/meson.build | 1 + src/controller/testsuite/controller_test.c | 2 + src/controller/testsuite/test_ha_prevote.c | 338 ++++++++++++++++ 6 files changed, 801 insertions(+), 1 deletion(-) create mode 100644 src/controller/testsuite/test_ha_prevote.c diff --git a/.kiro/specs/ha-prevote/tasks.md b/.kiro/specs/ha-prevote/tasks.md index 5d1f1ee2..5b4b80fc 100644 --- a/.kiro/specs/ha-prevote/tasks.md +++ b/.kiro/specs/ha-prevote/tasks.md @@ -48,7 +48,7 @@ - _Depends: 1.1, 2.1_ - [ ] 4. Core: controller candidate-side orchestration and trigger restructuring (test-first) -- [ ] 4.1 (P) Run the PreVote round and branch on its outcome +- [x] 4.1 (P) Run the PreVote round and branch on its outcome - RED first: add controller tests driving the reply handler through the existing vote-reply hook (no real transport): majority grant → real election started; no majority → node stays standby with term unchanged and the liveness watchdog re-armed; round timeout treated as a loss. - Owns the candidate orchestration in the controller: start the round (no persist, no role change), arm a round timeout bounded by the existing election-probe deadline, and the PreVote-reply handler that feeds the raft tally and branches — on a win perform the existing truncate + electing-transition + start-real-election sequence (unchanged real-election path); on loss log campaign abandonment. The send path emits the PreVote request opcode from task 1.1; transport-side reply routing is wired separately in task 5.1. - Observable: on no-majority the cluster term is unchanged, role stays standby, and a subsequent genuine failure still triggers a new round. @@ -96,3 +96,4 @@ - (1.2) Raft PreVote scaffolding in `raft.h`: outcome codes `RAFT_PRE_VOTE_WON/LOST/PENDING` = 7/8/9; tally fields `pre_vote_in_progress`, `pre_votes_granted`, `pre_vote_replies[RAFT_MAX_PEERS]` appended to `raft_state_t` (after `my_priority`); optional `on_send_pre_vote_request` callback (NULL-checkable). New `#define RAFT_MAX_PEERS 8` bounds the per-peer dedup array — safe: production cap is `MAX_HA_CANDIDATES=4` (src/include/config.h:78), and it matches `RAFT_SIM_MAX_NODES=8`. Tasks 2.1/2.2/2.3 supply the function bodies and MUST keep them pure (no `term`/`voted_for`/`role` mutation, no `on_persist`). Build/test the raft layer with `cd src && nix develop --command bash tools/xce-build.sh lib`. - (3.1) IMPORTANT for controller tasks: `cluster.c` is NOT linked into the unit-test binary (`controller_unit_test_src` in `src/controller/testsuite/meson.build`) — it's daemon-only (`controller_src`). So dispatch cases in cluster.c can't be unit-tested directly; test their composed primitives + contract asserts, and defer end-to-end to scenario tasks 5.2/5.3. `ha.c` IS unit-test-linked (the `test_ha_*` suites exercise it). Controller build/test: `cd src && nix develop --command bash tools/xce-build.sh controller`; the registered meson test name is `"controller test"` (single Unity binary). Known pre-existing `test_commit_barrier_reply` 300s TIMEOUT makes the full suite exit non-zero — validate via the targeted run `nix develop --command meson test -C xce.build --no-rebuild --print-errorlogs "controller test"` and check for zero `:FAIL` (Timeout:1 is the known one). The new prevote opcodes' generated constants are `OPCode_preVoteRequest=38` / `OPCode_preVoteReply=39`. - (3.1, LOW nit) cluster.c PreVote denial LOG reason uses elimination order (stale_term→primary→catch_up→fresh_heartbeat) which differs from `raft_cb_vote_eligible`'s gate order (primary→fresh_heartbeat→catch_up); in the rare STANDBY catch-up+fresh-heartbeat overlap the log string may say `catch_up` while the gate denied on `fresh_heartbeat`. Decision is identical (deny); log-string only. Optional future polish: reorder to match. +- (4.1) ha.c now has `ha_start_prevote_async`, `pre_vote_reply_dispatch`, `ha_enter_real_election` (WON branch = the factored truncate+CTRL_ELECTING+ha_start_election_async helper), a `raft_cb_send_pre_vote_request` callback (registered in `s_real_cbs`), and a round-timeout task. KEY for task 5.1: `pre_vote_reply_dispatch(ha, peer_idx, granted)` takes the ABSOLUTE cluster peer index (matches `raft_begin_pre_vote`'s send loop), NOT the real-vote local `votes[]` index. `pre_vote_reply_dispatch` is `__attribute__((used))` with NO production caller yet — task 5.1's cluster.c `OPCode_preVoteReply` case must call it (replacing the `xce_*` test seam in production). Triggers (rdhup/liveness/heartbeat) are still UNMODIFIED — task 4.2 rewires them to call `ha_start_prevote_async`. diff --git a/src/controller/ha.c b/src/controller/ha.c index 7c5b8978..5635afcf 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -81,6 +81,7 @@ extern int push_config_transport_only(cluster_t *c, const uint8_t *bytes, #endif static void on_vote_connected(endpoint_t *ep); +static void on_pre_vote_connected(endpoint_t *ep); static int vote_ep_err_handler(void *ep_raw); /* ha-single-leader-role task 2.2: shared promotion seam — every road to * PRIMARY registers the persist hooks and arms the own-term gate. */ @@ -203,6 +204,87 @@ static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term) return 0; } +/* + * raft_cb_send_pre_vote_request: encode and send a capnp PreVote request to a + * peer (ha-prevote task 4.1). + * + * Mirrors raft_cb_send_vote_request exactly — same iomgr connect path, same + * vote_ep_ctx_t carrier, same per-peer endpoint tracking — but the connected + * handler (on_pre_vote_connected) emits OPCode_preVoteRequest instead of + * OPCode_requestVote. prospective_term is the candidate's CURRENT+1 term; it + * is NEVER persisted (PreVote is a side-effect-free straw poll). + * + * peer_idx is the absolute candidate index in c->myconf->peers_cfg.candidates[]. + * Returns 0 on success; non-zero in test mode (no cluster) or on alloc failure. + * + * Transport-side reply routing for preVoteReply is wired separately in task 5.1; + * this callback only emits the request. + */ +static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx, + uint64_t prospective_term) +{ + ha_state_t *ha = (ha_state_t *)ctx; + cluster_t *c = ha->raft_c; + if (!c || !c->myconf) return 1; /* test mode — caller owns transport */ + + const xce_cluster_config_t *pcfg = &c->myconf->peers_cfg; + if (peer_idx < 0 || peer_idx >= pcfg->candidates_count) + return 1; + + ha_elect_ctx_t *ectx = (ha_elect_ctx_t *)atomic_load_explicit( + &ha->elect_ctx, memory_order_acquire); + if (!ectx) return 1; /* no round metadata carrier */ + + xce_candidate_t *peer = (xce_candidate_t *)&pcfg->candidates[peer_idx]; + + /* Map absolute peer_idx → votes[] local index (same convention as the + * real-vote send callback). */ + int self_idx = ha->my_priority; + int votes_idx = (peer_idx < self_idx) ? peer_idx : peer_idx - 1; + if (votes_idx < 0 || votes_idx >= ectx->n_peers) + return 1; + + vote_ep_ctx_t *vctx = calloc(1, sizeof(*vctx)); + if (!vctx) return 1; + vctx->ha = ha; + vctx->c = c; + vctx->ctx = ectx; + /* Carry the ABSOLUTE peer index: the PreVote tally in raft_state_t is keyed + * by absolute cluster index (raft_begin_pre_vote loops over i in + * [0, n_peers) and raft_on_pre_vote_reply dedups by that same index). */ + vctx->peer_idx = peer_idx; + vctx->my_term = prospective_term; + vctx->my_tag = (ectx->n_peers > 0) ? ectx->votes[0].my_tag : 0; + vctx->my_lsn = (ectx->n_peers > 0) ? ectx->votes[0].my_lsn : 0; + vctx->my_name = c->myconf ? c->myconf->name : ""; + + endpoint_t *ep = endpoint_get(); + if (!ep) { free(vctx); return 1; } + + endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0); + if (c) endpoint_set_peer(ep, c->myhost); + ep->data = vctx; + ep->iomgr = &c->iomgr; + + if (ha->client_ssl_ctx) { + SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx; + ep->ssl = SSL_new(client_ctx); + if (ep->ssl) + SSL_set_connect_state(ep->ssl); + } + + endpoint_set_on_connected(ep, on_pre_vote_connected); + endpoint_set_err_handler(ep, vote_ep_err_handler); + + int rc = endpoint_connect(ep, peer->hostname, peer->port); + if (rc < 0) { endpoint_fini(ep); return 1; } + + rc = iomgr_monitor(ep); + if (rc < 0) { endpoint_fini(ep); return 1; } + + return 0; +} + /* * raft_cb_vote_eligible: freshness gate delegated from raft library. * @@ -800,6 +882,7 @@ static void raft_cb_trace(void *ctx, const char *event, const char *detail) static const raft_callbacks_t s_real_cbs = { .on_send_vote_request = raft_cb_send_vote_request, + .on_send_pre_vote_request = raft_cb_send_pre_vote_request, .on_vote_eligible = raft_cb_vote_eligible, .on_become_leader = raft_cb_become_leader, .on_become_follower = raft_cb_become_follower, @@ -2609,6 +2692,45 @@ static void on_vote_connected(endpoint_t *ep) ep->msg_handler = peer_ctrl_msg_handler; } +/* + * on_pre_vote_connected: iomgr callback fired after the TLS handshake completes + * on an outgoing PreVote endpoint (ha-prevote task 4.1). + * + * Mirrors on_vote_connected but emits OPCode_preVoteRequest (reusing the + * RequestVoteRequest payload struct + union arm, per the wire schema in task + * 1.1). prospective_term (current+1) rides in the term field; nothing is + * persisted. The reply-routing handler for preVoteReply is wired in task 5.1; + * here we set the same state-aware dispatcher used by the real-vote path. + */ +static void on_pre_vote_connected(endpoint_t *ep) +{ + vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data; + + request_vote_req_t vrq; + memset(&vrq, 0, sizeof(vrq)); + vrq.term = vctx->my_term; /* prospective term = current + 1 */ + vrq.candidate_id = (char *)(vctx->my_name ? vctx->my_name : ""); + vrq.config_tag = vctx->my_tag; + vrq.last_log_lsn = vctx->my_lsn; + + xce_request_t req; + memset(&req, 0, sizeof(req)); + req.opcode = OPCode_preVoteRequest; + req.body_which = Request_body_requestvotemsg; + req.body.requestvotemsg = &vrq; + +#ifdef XCE_TESTING + if (xce_vote_request_hook != NULL) + xce_vote_request_hook(ep, &req); + else + xce_request(ep, &req); +#else + xce_request(ep, &req); +#endif + + ep->msg_handler = peer_ctrl_msg_handler; +} + /* * xce_vote_reply_hook: test-only seam for vote_reply_msg_handler. * @@ -3320,6 +3442,323 @@ void ha_init_raft(ha_state_t *ha, cluster_t *c, int n_peers) atomic_load_explicit(&ha->term, memory_order_acquire)); } +/* ── ha-prevote task 4.1: candidate-side PreVote orchestration ───────────── + * + * The PreVote round is a side-effect-free majority probe interposed BEFORE the + * destructive truncate + CTRL_ELECTING + real-election sequence. During the + * round the node stays CTRL_STANDBY and mutates no persistent state (term / + * voted_for / role). Only on a WIN does it enter the unchanged real-election + * path; on a LOSS / round-timeout it stays STANDBY, re-arms the liveness + * watchdog, and logs campaign abandonment. + * + * This task implements the round machinery + the WON/LOST branches. The + * failover triggers (ha_on_primary_ctrl_fd_rdhup / liveness_poll_fn / + * ha_on_heartbeat_received) are rewired to call this in task 4.2 — until then + * the round is INERT in production. + */ + +/* + * prevote_round_ctx_t: delaymgr context for the PreVote round timeout. + * cancelled is set when the round is decided (WON or LOST) before the timeout + * fires, so a late timeout task is a no-op. + */ +typedef struct { + _Atomic int cancelled; + ha_state_t *ha; + cluster_t *c; +} prevote_round_ctx_t; + +/* Active round-timeout context. _Atomic so arm (vtask) vs timeout-fire + * (delaymgr) vs cancel-on-decision is well-defined. */ +static prevote_round_ctx_t * _Atomic s_prevote_round_ctx = NULL; + +#ifdef XCE_TESTING +/* + * ha_prevote_rearm_liveness_hook: test-only seam. When non-NULL, the LOST / + * timeout branch calls this INSTEAD OF ha_arm_heartbeat_timeout_vtask so a unit + * test can count re-arms without a live delaymgr (Req 5.4). + */ +int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c) = NULL; +#endif + +/* ha_prevote_rearm_liveness: re-arm the standby liveness watchdog after a lost + * round so a later genuine failure still triggers a new PreVote round (the + * watchdog does not re-arm itself once it has fired). */ +static void ha_prevote_rearm_liveness(ha_state_t *ha, cluster_t *c) +{ +#ifdef XCE_TESTING + if (ha_prevote_rearm_liveness_hook != NULL) { + ha_prevote_rearm_liveness_hook(ha, c); + return; + } +#endif + ha_arm_heartbeat_timeout_vtask(ha, c); +} + +/* ha_prevote_cancel_round_timeout: mark the active round-timeout context + * cancelled (the delaymgr task frees it on fire) and clear the static pointer + * so a later decision/timeout sees no active round. */ +static void ha_prevote_cancel_round_timeout(void) +{ + prevote_round_ctx_t *old = atomic_load_explicit(&s_prevote_round_ctx, + memory_order_acquire); + if (old) { + atomic_store_explicit(&old->cancelled, 1, memory_order_release); + atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release); + } +} + +/* + * ha_enter_real_election: the WON-branch sequence — truncate uncommitted WAL + * above the committed boundary, then enter the UNCHANGED real-election path + * (ha_start_election_async transitions to CTRL_ELECTING and delegates the term + * bump + self-vote to raft_start_election). Factored here so both the PreVote + * WON branch (task 4.1) and the rewired trigger (task 4.2) can call it; this + * task does NOT change ha_on_primary_ctrl_fd_rdhup's own behaviour. + * + * c == NULL (unit-test mode): skip truncate + the live election; just transition + * the role to CTRL_ELECTING so the WON transition is observable. + */ +static void ha_enter_real_election(ha_state_t *ha, cluster_t *c) +{ + if (!c) { + /* Test mode: the real election needs iomgr/delaymgr; just mark the + * destructive transition so the WON branch is observable. */ + atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, + memory_order_release); + return; + } + + /* Truncate uncommitted WAL entries above committed_lsn (mirrors the + * existing failover sequence in ha_on_primary_ctrl_fd_rdhup). */ + uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn, + memory_order_acquire); + if (committed > 0) + persist_truncate_wal_above(&c->persist, committed); + + /* Candidate freshness for the RequestVote payload — same floor the voter + * side computes (ha_vote_self_freshness), matching the rdhup path. */ + uint64_t cand_tag = 0, cand_lsn = 0; + if (c->myconf) { + uint64_t local_tag = c->myconf->config_version_tag; + uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn, + memory_order_acquire); + ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, &cand_lsn); + } + + int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1; + const char *my_name = (c->myconf) ? c->myconf->name : ""; + + /* ha_start_election_async transitions to CTRL_ELECTING and drives the + * UNCHANGED raft_start_election (term++ + self-vote). Peer connections are + * driven by raft_cb_send_vote_request; the peers[] arg is unused. */ + ha->raft_c = c; + ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name); +} + +/* + * ha_start_prevote_async: start a PreVote round (ha-prevote task 4.1). + * + * Snapshots self tag/lsn into the round metadata carrier (ha->elect_ctx, read + * by raft_cb_send_pre_vote_request), calls raft_begin_pre_vote (self-grant + + * per-peer send via the callback), and arms a round-timeout bounded by + * election_timeout_ms (Req 5.4 — no unbounded delay). The node stays + * CTRL_STANDBY; NOTHING persistent is mutated (no term/voted_for/role change). + * + * If raft_begin_pre_vote returns RAFT_PRE_VOTE_WON immediately (single-node + * cluster: self-grant is already a majority), the WON branch runs at once. + * + * c == NULL (unit-test mode): the send callback returns early and the round + * timeout is a no-op; the round state lives entirely in raft_state_t so the + * reply-driven branches are still fully exercisable via the test seam. + */ +void ha_start_prevote_async(ha_state_t *ha, cluster_t *c) +{ + if (!ha) return; + + ha->raft_c = c; + + /* Snapshot self freshness for the PreVote request payload (same floor as a + * real vote). Stored in the metadata carrier ha->elect_ctx so + * raft_cb_send_pre_vote_request can read my_tag/my_lsn. */ + uint64_t cand_tag = 0, cand_lsn = 0; + int n_peers = ha->n_peers > 0 ? ha->n_peers : ha->raft.n_peers; + if (c && c->myconf) { + uint64_t local_tag = c->myconf->config_version_tag; + uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn, + memory_order_acquire); + ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, &cand_lsn); + n_peers = c->myconf->peers_cfg.candidates_count; + + int n_other = (n_peers > 1) ? n_peers - 1 : 0; + ha_elect_ctx_t *mc = calloc(1, sizeof(*mc) + + (size_t)n_other * sizeof(mc->votes[0])); + if (mc) { + mc->n_peers = n_other; + mc->result = -1; + mc->c = c; + mc->self_tag = cand_tag; + mc->self_lsn = cand_lsn; + pthread_mutex_init(&mc->lock, NULL); + sem_init(&mc->done, 0, 0); + if (n_other > 0) { + mc->votes[0].my_tag = cand_tag; + mc->votes[0].my_lsn = cand_lsn; + } + /* Do NOT free a prior ctx (live vote endpoints may hold a bare + * pointer to it — same UAF caution as the rdhup path). */ + atomic_store_explicit(&ha->elect_ctx, mc, memory_order_release); + } + ha->n_peers = n_peers; + } + + INFO("HA: starting PreVote round (prospective term=%" PRIu64 ", n_peers=%d) " + "— staying CTRL_STANDBY, no persist", + ha->raft.term + 1, n_peers); + + /* raft_begin_pre_vote: self-grant + per-peer send; mutates only the + * pre_vote_* tally in raft_state_t (no term/voted_for/role change). */ + int rc = raft_begin_pre_vote(&ha->raft); + + /* Arm the round timeout (bounded by election_timeout_ms). Production only; + * the round state is in raft_state_t so tests exercise the branches via the + * reply seam without the timer. */ + ha_prevote_cancel_round_timeout(); + if (c && c->delaymgr.running) { + prevote_round_ctx_t *tctx = calloc(1, sizeof(*tctx)); + if (tctx) { + atomic_init(&tctx->cancelled, 0); + tctx->ha = ha; + tctx->c = c; + atomic_store_explicit(&s_prevote_round_ctx, tctx, + memory_order_release); + extern int ha_prevote_round_timeout_fn(void *arg); + int trc = delay_task_ms(&c->delaymgr, + (int)ha->config.election_timeout_ms, 1, + (delay_task_entry_fn_t)ha_prevote_round_timeout_fn, + tctx); + if (trc < 0) { + WARN("HA: ha_start_prevote_async: round-timeout schedule failed: %d", + trc); + atomic_store_explicit(&s_prevote_round_ctx, NULL, + memory_order_release); + free(tctx); + } + } + } + + /* Single-node: the self-grant is already a majority — win immediately. */ + if (rc == RAFT_PRE_VOTE_WON) { + ha_prevote_cancel_round_timeout(); + INFO("HA: PreVote won immediately (single-node) — entering real election"); + ha_enter_real_election(ha, c); + } +} + +/* + * pre_vote_reply_dispatch: feed a PreVote reply into the raft tally and branch + * on its outcome (ha-prevote task 4.1). peer_idx is the ABSOLUTE cluster index + * of the replying peer (the same index raft_begin_pre_vote sent to). + * + * RAFT_PRE_VOTE_WON → cancel round timeout, run the real-election sequence + * (truncate + CTRL_ELECTING + ha_start_election_async). + * RAFT_PRE_VOTE_LOST → stay CTRL_STANDBY, re-arm liveness, log abandonment. + * RAFT_PRE_VOTE_PENDING → no-op (await more replies). + * + * Mirrors vote_reply_dispatch → raft_on_vote_reply, but the raft library makes + * NO role/term change here — the controller acts on the returned status. + * + * In 4.1 the only caller is the XCE_TESTING reply seam; the production caller + * is the preVoteReply transport dispatch wired in task 5.1. __attribute__((used)) + * keeps the production build clean (no -Wunused-function) until then. + */ +__attribute__((used)) +static void pre_vote_reply_dispatch(ha_state_t *ha, cluster_t *c, + int peer_idx, int granted) +{ + int outcome = raft_on_pre_vote_reply(&ha->raft, peer_idx, granted); + + if (outcome == RAFT_PRE_VOTE_WON) { + ha_prevote_cancel_round_timeout(); + INFO("HA: PreVote round WON — entering real election (term will be %" + PRIu64 ")", ha->raft.term + 1); + ha_enter_real_election(ha, c); + return; + } + + if (outcome == RAFT_PRE_VOTE_LOST) { + ha_prevote_cancel_round_timeout(); + /* Req 7.3: log campaign abandonment distinctly from a genuine failover + * so operators can tell a suppressed disruptive election apart. */ + INFO("HA: PreVote round LOST — abandoning campaign, staying CTRL_STANDBY " + "(no term increment); re-arming liveness watchdog"); + /* Stay CTRL_STANDBY — no persist, no role change (Req 1.4). */ + ha_prevote_rearm_liveness(ha, c); + return; + } + + /* RAFT_PRE_VOTE_PENDING — await more replies, do nothing. */ +} + +/* + * ha_prevote_round_timeout_fn: delaymgr task body for the PreVote round timeout. + * A timeout that fires before a decision is treated as a LOSS (Req 5.4). + */ +int ha_prevote_round_timeout_fn(void *arg) +{ + prevote_round_ctx_t *ctx = (prevote_round_ctx_t *)arg; + + if (atomic_load_explicit(&ctx->cancelled, memory_order_acquire)) { +#ifndef XCE_TESTING + free(ctx); +#endif + return 0; + } + + ha_state_t *ha = ctx->ha; + cluster_t *c = ctx->c; + + /* Only act if the round is still in progress (not already decided). */ + if (ha->raft.pre_vote_in_progress) { + ha->raft.pre_vote_in_progress = 0; + atomic_store_explicit(&s_prevote_round_ctx, NULL, memory_order_release); + WARN("HA: PreVote round timed out before a decision — treating as LOSS, " + "staying CTRL_STANDBY; re-arming liveness watchdog (Req 5.4)"); + ha_prevote_rearm_liveness(ha, c); + } + +#ifndef XCE_TESTING + free(ctx); +#endif + return 0; +} + +#ifdef XCE_TESTING +/* + * pre_vote_reply_dispatch_for_test: drive the real pre_vote_reply_dispatch with + * decoded (peer_idx, granted) fields, bypassing iomgr (ha-prevote task 4.1). + */ +void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, + int peer_idx, int granted) +{ + pre_vote_reply_dispatch(ha, c, peer_idx, granted); +} + +/* + * ha_prevote_round_timeout_fire_for_test: simulate the round timeout firing + * before a decision, driving the LOST branch (Req 5.4). + */ +void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c) +{ + prevote_round_ctx_t ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.ha = ha; + ctx.c = c; + atomic_init(&ctx.cancelled, 0); + ha_prevote_round_timeout_fn(&ctx); +} +#endif /* XCE_TESTING */ + /* ── Task 6.3: ha_start_election_async — delegate to raft_start_election ──── * * Non-blocking election initiator. Sets up the raft state machine and diff --git a/src/controller/ha.h b/src/controller/ha.h index 4b621ad6..82306540 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -801,6 +801,25 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name); +/* + * ha_start_prevote_async: start a side-effect-free PreVote round before any + * real election (ha-prevote task 4.1). + * + * Snapshots self tag/lsn, calls raft_begin_pre_vote (self-grant + per-peer + * preVoteRequest send), and arms a round timeout bounded by election_timeout_ms + * (Req 5.4). The node stays CTRL_STANDBY and mutates NO persistent state + * (no term / voted_for / role change) for the duration of the round. Replies + * are fed to the reply handler (pre_vote_reply_dispatch); on a majority WIN the + * unchanged real-election sequence runs (truncate + CTRL_ELECTING + + * ha_start_election_async), on a LOSS / round-timeout the node stays + * CTRL_STANDBY and re-arms the liveness watchdog. + * + * The failover triggers are rewired to call this in task 4.2; until then the + * round is inert in production. c may be NULL in unit-test mode. + * Requirements: 1.1, 1.4, 1.5, 5.4, 7.3. + */ +void ha_start_prevote_async(ha_state_t *ha, cluster_t *c); + /* * ha_dispatch_repl_action_for_standby: evaluate the replication decision for * one standby and execute the resulting action. diff --git a/src/controller/meson.build b/src/controller/meson.build index a04185b5..5b458520 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -121,6 +121,7 @@ controller_unit_test_src = [ 'testsuite/test_ha_connection_reuse.c', 'testsuite/test_ha_rdhup_integration.c', 'testsuite/test_ha_liveness.c', + 'testsuite/test_ha_prevote.c', 'testsuite/test_commit_engine.c', 'testsuite/test_commit_barrier_rollout.c', 'testsuite/test_commit_barrier_reply.c', diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index 1dd03322..9560a67c 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -49,6 +49,7 @@ extern void test_ha_repl_push(void); extern void test_ha_connection_reuse(void); extern void test_ha_rdhup_integration(void); extern void test_ha_liveness(void); +extern void test_ha_prevote(void); extern void test_commit_engine(void); extern void test_commit_barrier_rollout(void); extern void test_commit_barrier_reply(void); @@ -174,6 +175,7 @@ int main(int argc, char* argv[]) TEST(test_ha_connection_reuse); TEST(test_ha_rdhup_integration); TEST(test_ha_liveness); + TEST(test_ha_prevote); TEST(test_commit_engine); TEST(test_commit_barrier_rollout); TEST(test_commit_barrier_reply); diff --git a/src/controller/testsuite/test_ha_prevote.c b/src/controller/testsuite/test_ha_prevote.c new file mode 100644 index 00000000..007b7fa2 --- /dev/null +++ b/src/controller/testsuite/test_ha_prevote.c @@ -0,0 +1,338 @@ +/* + * test_ha_prevote.c — Unit tests for ha-prevote task 4.1: + * controller candidate-side PreVote orchestration in ha.c. + * + * Scope (task 4.1 only): + * - ha_start_prevote_async: starts a side-effect-free round (no persist, no + * role change, stays CTRL_STANDBY), arms a round timeout. + * - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches — + * WON → real-election sequence (truncate + CTRL_ELECTING + start election) + * LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment + * PENDING → no-op + * - round-timeout firing = LOST branch. + * + * These tests drive the reply path through the XCE_TESTING test seam + * (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the + * existing vote-reply hook pattern. ha.c is unit-test-linked, so the real + * orchestration logic is exercised directly. + * + * Requirements: 1.1, 1.4, 1.5, 5.4, 7.3 + * Boundary: PreVote orchestration (ha.c) + * + * TDD: written first (RED) to specify the exact behaviour before implementation. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include + +#include "ha.h" +#include "netio.h" +#include "raft/raft.h" +#include "xce/cluster_def.h" + +/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */ + +/* + * ha_start_prevote_async: start a PreVote round. No persist, no role change. + * c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op, + * raft_begin_pre_vote's send callback returns early in test mode). + */ +extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c); + +/* + * pre_vote_reply_dispatch_for_test: test entry point that calls the real + * pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without + * a live iomgr. Mirrors the production reply path. + */ +extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, + int peer_idx, int granted); + +/* + * ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task + * firing before a decision (drives the LOST branch). + */ +extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c); + +/* Liveness arm-count hook so tests can prove the watchdog was re-armed on a + * LOST outcome without a live delaymgr. */ +extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c); + +/* ── Test config ─────────────────────────────────────────────────────────── */ + +static ha_config_t g_cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 6000, + .election_timeout_ms = 500, + .probe_connect_timeout_ms = 50, + .max_promotion_retries = 3, + .election_jitter_max_ms = 0, + .election_priority_step_ms = 500, +}; + +/* Re-arm tracking */ +static int g_rearm_calls = 0; +static int stub_rearm(ha_state_t *ha, cluster_t *c) +{ + (void)ha; (void)c; + g_rearm_calls++; + return 0; +} + +/* + * Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer + * grant to win (quorum = 2; self-grant = 1). raft_init in ha_init only sets a + * 1-node placeholder, so we widen the topology directly on the public struct. + */ +static void make_three_node(ha_state_t *ha) +{ + ha->raft.n_peers = 3; + ha->raft.self_idx = ha->my_priority; + ha->raft.role = RAFT_FOLLOWER; + ha->n_peers = 3; +} + +/* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */ + +/* + * ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment + * the persistent term, and must not flip the raft role to CANDIDATE. It sets + * the in-progress flag and self-grant in raft_state_t only. + */ +static void test_prevote_start_no_mutation(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); /* role = CTRL_STANDBY */ + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + uint64_t raft_term_before = ha.raft.term; + int raft_voted_before = ha.raft.voted_for; + + ha_start_prevote_async(&ha, NULL); + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 1.1: role must stay CTRL_STANDBY during the PreVote round"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 1.2: persistent term must not change during the round"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, + "Req 1.2: raft term must not change during the round"); + TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, + "Req 1.2: raft voted_for must not change during the round"); + TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role, + "Req 1.2: raft role must stay FOLLOWER during the round"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "round must be marked in-progress"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, + "self-grant must be recorded (1)"); + + ha_fini(&ha); + printf("test_prevote_start_no_mutation: PASS\n"); +} + +/* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */ + +/* + * 3-node: after starting the round, feeding one peer GRANT reaches quorum (2). + * The WON branch must run the real-election sequence: role becomes + * CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election). + */ +static void test_prevote_majority_grant_starts_real_election(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + ha_start_prevote_async(&ha, NULL); + + /* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); + + int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); + /* WON branch enters the real-election sequence: with c==NULL the election + * cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY — + * the candidate transitioned to CTRL_ELECTING (real-election path). */ + TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, + "Req 1.5: a PreVote WIN must enter the real-election sequence " + "(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "WON must clear the in-progress flag (round over)"); + + ha_fini(&ha); + printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role); +} + +/* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */ + +/* + * 3-node: both peers DENY → no quorum → LOST. The node must stay CTRL_STANDBY, + * leave the persistent term unchanged (Req 1.4), and re-arm the liveness + * watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still + * trigger a new round). + */ +static void test_prevote_no_majority_stays_standby_and_rearms(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + g_rearm_calls = 0; + ha_prevote_rearm_liveness_hook = stub_rearm; + + ha_start_prevote_async(&ha, NULL); + + /* Both non-self peers deny → all replied, no quorum → LOST. */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/); + pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/); + + ha_prevote_rearm_liveness_hook = NULL; + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 1.4: no PreVote majority must keep the node CTRL_STANDBY"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 1.4: no PreVote majority must leave the persistent term unchanged"); + TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, + "Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a " + "later genuine failure still triggers a new round"); + + ha_fini(&ha); + printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n"); +} + +/* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */ + +/* + * If the round times out before a decision, it is treated as a loss: stay + * CTRL_STANDBY, term unchanged, re-arm liveness. + */ +static void test_prevote_round_timeout_is_loss(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + g_rearm_calls = 0; + ha_prevote_rearm_liveness_hook = stub_rearm; + + ha_start_prevote_async(&ha, NULL); + + /* No replies arrive; the round timeout fires. */ + ha_prevote_round_timeout_fire_for_test(&ha, NULL); + + ha_prevote_rearm_liveness_hook = NULL; + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 5.4: a round timeout must leave the persistent term unchanged"); + TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, + "Req 5.4: a timed-out round must re-arm the liveness watchdog"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "round timeout must clear the in-progress flag"); + + ha_fini(&ha); + printf("test_prevote_round_timeout_is_loss: PASS\n"); +} + +/* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */ + +/* + * Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back + * in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round + * (self-grant + in-progress), which a peer grant can again win — proving the + * candidate machinery is reusable, not one-shot. + */ +static void test_prevote_reusable_after_loss(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + ha_prevote_rearm_liveness_hook = stub_rearm; + + /* Round 1: lost. */ + ha_start_prevote_async(&ha, NULL); + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0); + pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); + TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire)); + + /* Round 2: a fresh round arms cleanly (self-grant + in-progress). */ + ha_start_prevote_async(&ha, NULL); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "a new round after a loss must re-arm (in-progress)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, + "a new round must reset the tally to the self-grant"); + + /* Round 2 wins with a peer grant. */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1); + int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); + TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, + "a later genuine failure (new round) must still be able to win and " + "start a real election"); + + ha_prevote_rearm_liveness_hook = NULL; + ha_fini(&ha); + printf("test_prevote_reusable_after_loss: PASS\n"); +} + +/* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */ + +static void test_prevote_pending_is_noop(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + /* 5-node so a single grant is below quorum (3) → PENDING. */ + ha.raft.n_peers = 5; + ha.raft.self_idx = 0; + ha.raft.role = RAFT_FOLLOWER; + ha.n_peers = 5; + + ha_start_prevote_async(&ha, NULL); + + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/); + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "a PENDING (sub-quorum) reply must not change the role"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "a PENDING reply must keep the round in progress"); + + ha_fini(&ha); + printf("test_prevote_pending_is_noop: PASS\n"); +} + +/* ── Suite entry point ──────────────────────────────────────────────────── */ + +void test_ha_prevote(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_prevote_start_no_mutation); + RUN_TEST(test_prevote_majority_grant_starts_real_election); + RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms); + RUN_TEST(test_prevote_round_timeout_is_loss); + RUN_TEST(test_prevote_reusable_after_loss); + RUN_TEST(test_prevote_pending_is_noop); + UNITY_END(); +}