diff --git a/.kiro/specs/ha-prevote/tasks.md b/.kiro/specs/ha-prevote/tasks.md index 5b4b80fc..f157f52d 100644 --- a/.kiro/specs/ha-prevote/tasks.md +++ b/.kiro/specs/ha-prevote/tasks.md @@ -56,7 +56,7 @@ - _Boundary: PreVote orchestration_ - _Depends: 1.1, 2.2, 2.3_ (parallel-safe with 3.1: orchestration lives in a different file than the voter-side dispatch) -- [ ] 4.2 Route failover triggers through PreVote and defer the destructive step +- [x] 4.2 Route failover triggers through PreVote and defer the destructive step - RED first: add tests asserting the failover trigger no longer truncates the WAL or increments the term until PreVote wins, and that an incoming primary heartbeat mid-round cancels the round and re-attaches as standby. - Make both failover entry points (connection RDHUP and heartbeat-timeout poll) begin the PreVote round instead of starting an election directly; move truncate + electing-transition to PreVote success; cancel an in-progress round on heartbeat receipt. - Observable: a frozen-then-resumed standby that loses PreVote leaves the WAL and term untouched and re-attaches when the next heartbeat arrives. @@ -97,3 +97,4 @@ - (3.1) IMPORTANT for controller tasks: `cluster.c` is NOT linked into the unit-test binary (`controller_unit_test_src` in `src/controller/testsuite/meson.build`) — it's daemon-only (`controller_src`). So dispatch cases in cluster.c can't be unit-tested directly; test their composed primitives + contract asserts, and defer end-to-end to scenario tasks 5.2/5.3. `ha.c` IS unit-test-linked (the `test_ha_*` suites exercise it). Controller build/test: `cd src && nix develop --command bash tools/xce-build.sh controller`; the registered meson test name is `"controller test"` (single Unity binary). Known pre-existing `test_commit_barrier_reply` 300s TIMEOUT makes the full suite exit non-zero — validate via the targeted run `nix develop --command meson test -C xce.build --no-rebuild --print-errorlogs "controller test"` and check for zero `:FAIL` (Timeout:1 is the known one). The new prevote opcodes' generated constants are `OPCode_preVoteRequest=38` / `OPCode_preVoteReply=39`. - (3.1, LOW nit) cluster.c PreVote denial LOG reason uses elimination order (stale_term→primary→catch_up→fresh_heartbeat) which differs from `raft_cb_vote_eligible`'s gate order (primary→fresh_heartbeat→catch_up); in the rare STANDBY catch-up+fresh-heartbeat overlap the log string may say `catch_up` while the gate denied on `fresh_heartbeat`. Decision is identical (deny); log-string only. Optional future polish: reorder to match. - (4.1) ha.c now has `ha_start_prevote_async`, `pre_vote_reply_dispatch`, `ha_enter_real_election` (WON branch = the factored truncate+CTRL_ELECTING+ha_start_election_async helper), a `raft_cb_send_pre_vote_request` callback (registered in `s_real_cbs`), and a round-timeout task. KEY for task 5.1: `pre_vote_reply_dispatch(ha, peer_idx, granted)` takes the ABSOLUTE cluster peer index (matches `raft_begin_pre_vote`'s send loop), NOT the real-vote local `votes[]` index. `pre_vote_reply_dispatch` is `__attribute__((used))` with NO production caller yet — task 5.1's cluster.c `OPCode_preVoteReply` case must call it (replacing the `xce_*` test seam in production). Triggers (rdhup/liveness/heartbeat) are still UNMODIFIED — task 4.2 rewires them to call `ha_start_prevote_async`. +- (4.2) Triggers now rewired: `ha_on_primary_ctrl_fd_rdhup` + `liveness_poll_fn` → `ha_begin_failover` → `ha_start_prevote_async` (PreVote first; truncate+CTRL_ELECTING deferred to `ha_enter_real_election` on WON). `ha_on_heartbeat_received` cancels an in-progress round (only when `pre_vote_in_progress`) + re-attaches CTRL_STANDBY, no term change. LOAD-BEARING: `ha_start_prevote_async` checks `raft_begin_pre_vote`'s return and on immediate `RAFT_PRE_VOTE_WON` (single-node, n_peers==1) synchronously calls `ha_enter_real_election` — without this a single-node/immediate-majority node never elects. Full controller suite 563 PASS/0 FAIL with PreVote live. (NOTE: pre-existing deliberate `elect_ctx` ASAN leak ~one-per-round carried into PreVote; out of scope, future cleanup.) diff --git a/src/controller/ha.c b/src/controller/ha.c index 5635afcf..f2251b0e 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -3549,10 +3549,42 @@ static void ha_enter_real_election(ha_state_t *ha, cluster_t *c) int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1; const char *my_name = (c->myconf) ? c->myconf->name : ""; + ha->raft_c = c; + +#ifdef XCE_TESTING + /* Test seam: the WON-branch real election is the same step the pre-rewire + * rdhup body fired, so the existing rdhup-integration tests intercept it + * here (capturing my_tag/my_lsn / counting election starts) — they now + * drive it through the PreVote→WON path (ha-prevote task 4.2). + * + * Mark the role CTRL_ELECTING before invoking the hook: the real + * ha_start_election_async transitions to CTRL_ELECTING as its first step, + * so a test that intercepts the election must still observe the destructive + * role transition the WON branch performs (the spy hook does not start a + * real election that would set it). */ + if (ha_rdhup_election_hook != NULL) { + atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING, + memory_order_release); + if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) { + const xce_cluster_config_t *cfg = &c->myconf->peers_cfg; + int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0; + xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES]; + int np = 0; + for (int i = 0; i < cfg->candidates_count && np < n; i++) { + if (strcmp(cfg->candidates[i].name, + c->myhost ? c->myhost->name : "") != 0) + peer_ptrs[np++] = &cfg->candidates[i]; + } + ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn, + my_name); + } + return; + } +#endif + /* ha_start_election_async transitions to CTRL_ELECTING and drives the * UNCHANGED raft_start_election (term++ + self-vote). Peer connections are * driven by raft_cb_send_vote_request; the peers[] arg is unused. */ - ha->raft_c = c; ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name); } @@ -3922,6 +3954,26 @@ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, * 10.5 (INFO logs on FSM transitions) */ +/* + * ha_begin_failover: the single failover entry point (ha-prevote task 4.2). + * + * Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and + * the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route + * here AFTER their connection-teardown bookkeeping and the config-less + * voter-only guard. It starts a side-effect-free PreVote round and returns; + * it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those + * destructive steps are deferred to PreVote success (ha_enter_real_election, + * task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL + * and term untouched (Req 1.1, 4.3, 4.4). + */ +void ha_begin_failover(ha_state_t *ha, cluster_t *c) +{ + if (!ha) return; + INFO("HA: beginning failover via PreVote (no truncate / no term bump until " + "PreVote wins) — staying CTRL_STANDBY"); + ha_start_prevote_async(ha, c); +} + void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) { if (!ha) return; @@ -3930,23 +3982,31 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) * poll fn from triggering a second failover call after EPOLLRDHUP fires. */ ha_disarm_heartbeat_timeout_vtask(ha); - /* Close and null the primary control fd */ + /* Close and null the primary control fd. Connection-teardown bookkeeping + * stays at the trigger regardless of the PreVote outcome (ha-prevote task + * 4.2). */ if (ha->primary_ctrl_fd >= 0) { close(ha->primary_ctrl_fd); ha->primary_ctrl_fd = -1; } - /* Transition to CTRL_ELECTING (Req 6.2) */ - atomic_store_explicit(&ha->role, CTRL_ELECTING, memory_order_release); - INFO("HA: primary_ctrl_fd RDHUP → CTRL_ELECTING (Req 10.5)"); + /* ha-prevote task 4.2: the failover trigger NO LONGER transitions to + * CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and + * STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is + * deferred to PreVote success (ha_enter_real_election). The node only ever + * inflates its term / unseats a healthy primary if a PreVote majority is + * first obtained (Req 1.1, 4.3, 4.4). */ + INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying " + "CTRL_STANDBY until PreVote wins) (Req 1.1)"); - if (!c) return; /* unit test mode: skip persist + election */ - - /* Truncate uncommitted WAL entries above committed_lsn (Req 6.2) */ - uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn, - memory_order_acquire); - if (committed > 0) - persist_truncate_wal_above(&c->persist, committed); + if (!c) { + /* Unit-test mode: no persist / no live transport. Still begin the + * round so the deferred-destructive behaviour is observable (the round + * state lives in raft_state_t; ha_enter_real_election with c==NULL only + * flips the role on a WIN). */ + ha_begin_failover(ha, c); + return; + } /* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening * Req 10.5): a config-less node (config_version_tag == 0) must NEVER start @@ -3956,128 +4016,33 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) * primary over the node that actually holds the config — which then pushes * an empty config and breaks replication. * - * Mirror the proven ha_start_election_async voter-only guard: skip the - * elect_ctx alloc + the election trigger (raft_on_leader_lost) entirely and - * stay a passive voter (CTRL_STANDBY). handle_request_vote still GRANTS - * incoming vote requests to the fresher configured peer regardless of role, - * and this node transitions/fetches config when the configured winner - * connects and pushes config via ctrlhello. + * This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a + * config-less node can never win a PreVote (the freshness gate denies it), + * so there is no point starting a round — and starting one would needlessly + * probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote + * still GRANTS incoming vote requests to the fresher configured peer + * regardless of role, and this node transitions/fetches config when the + * configured winner connects and pushes config via ctrlhello. * * Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false, - * so a lone node still self-elects regardless of tag. */ + * so a lone node still self-elects (its PreVote trivially wins). */ uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0; int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1; if (my_config_tag == 0 && n_peers_guard > 1) { INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — " - "voter only, not starting RDHUP re-election; awaiting configured " - "primary (ha-config-bootstrap Req 2.8)", n_peers_guard); + "voter only, not starting PreVote/RDHUP re-election; awaiting " + "configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard); atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release); return; } - /* Task 6.3 / 6.7 fix: delegate RDHUP election trigger to raft library. - * - * raft_on_leader_lost: - * - clears ha->raft.voted_for = -1 (no direct ha->voted_for_* writes here) - * - sets ha->raft.role = RAFT_FOLLOWER - * - fires on_start_election_after_delay(ctx, priority * election_timeout_ms) - * → raft_cb_start_election_after_delay schedules raft_rdhup_delay_fn - * which calls raft_check_and_start_election after the priority jitter. - * - * The manual nanosleep jitter + voted_for clear + skip-if-already-voted - * logic is now fully owned by the raft state machine. - * - * raft->raft_c must be set before raft_on_leader_lost fires callbacks. - * - * CRITICAL (Task 6.7): raft_cb_send_vote_request reads ha->elect_ctx to - * get my_tag/my_lsn for the RequestVote payload. On a node that entered - * the voter-only path at first boot (my_config_tag==0), ha->elect_ctx was - * set to NULL and never refreshed. Allocate a fresh ha_elect_ctx_t here - * so that the subsequent raft_start_election finds a valid context. */ - ha->raft_c = c; - - /* Candidate freshness for the RequestVote payload. This MUST mirror the - * voter-side freshness floor (cluster.c OPCode_requestVote): the effective - * floor is max(applied_lsn, last_known_committed_lsn) at the EFFECTIVE - * config tag. A standby never advances persist.committed_lsn (only the - * primary's commit engine writes it) — it advances applied_lsn (WAL - * catch-up) and last_known_committed_lsn (heartbeat replies). Advertising - * persist.committed_lsn here makes every surviving standby advertise a - * stale-low LSN while each peer computes a HIGH floor from applied_lsn; - * the freshness gate (cand_lsn < my_lsn) then trips on both sides, each - * denies the other's vote, no candidate reaches quorum, and the cluster - * spins in an infinite RDHUP→ELECTING→STANDBY loop after the primary dies. - * Using ha_vote_self_freshness makes advertised == voter-floor, so the - * freshest node is always granted by every peer. (Matches the boot-path - * election in myapp_run, which advertises applied_lsn.) */ - uint64_t cand_tag = 0, cand_lsn = 0; - if (c->myconf) { - uint64_t local_tag = c->myconf->config_version_tag; - uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn, - memory_order_acquire); - ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, &cand_lsn); - } - - /* Allocate fresh elect_ctx so raft_cb_send_vote_request can read my_tag/my_lsn */ - if (c && c->myconf) { - int n_peers = c->myconf->peers_cfg.candidates_count; - int n_other = (n_peers > 1) ? n_peers - 1 : 0; - ha_elect_ctx_t *new_ctx = calloc(1, sizeof(*new_ctx) + - (size_t)n_other * sizeof(new_ctx->votes[0])); - if (new_ctx) { - uint64_t my_tag = cand_tag; - uint64_t my_lsn = cand_lsn; - new_ctx->n_peers = n_other; - new_ctx->votes_granted = 1; /* self-vote placeholder */ - new_ctx->result = -1; - new_ctx->c = c; - new_ctx->self_lsn = my_lsn; - new_ctx->self_tag = my_tag; - pthread_mutex_init(&new_ctx->lock, NULL); - sem_init(&new_ctx->done, 0, 0); - if (n_other > 0) { - new_ctx->votes[0].my_tag = my_tag; - new_ctx->votes[0].my_lsn = my_lsn; - } - /* Replace ha->elect_ctx with the fresh ctx. Do NOT free the old - * ctx here: any live vote_ep_ctx_t objects created in a prior - * election round hold a bare pointer to it (vctx->ctx) and will - * dereference it in vote_ep_err_handler / vote_reply_decode_and_dispatch - * when their connection eventually closes. Freeing here would cause - * a UAF. The old ctx is a small allocation; the leak is bounded to - * one per RDHUP cycle and tolerable. */ - atomic_store_explicit(&ha->elect_ctx, new_ctx, memory_order_release); - /* Update n_peers tracking — total cluster size (including self) */ - ha->n_peers = n_peers; - } - - } - -#ifdef XCE_TESTING - if (ha_rdhup_election_hook != NULL) { - /* Test seam: build peer args for hook (same as before) so existing - * tests that capture ha_start_election_async arguments still work. */ - if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) { - const xce_cluster_config_t *cfg = &c->myconf->peers_cfg; - int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0; - xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES]; - int np = 0; - for (int i = 0; i < cfg->candidates_count && np < n; i++) { - if (strcmp(cfg->candidates[i].name, - c->myhost ? c->myhost->name : "") != 0) - peer_ptrs[np++] = &cfg->candidates[i]; - } - uint64_t my_tag = cand_tag; - uint64_t my_lsn = cand_lsn; - const char *my_name = c->myconf ? c->myconf->name : ""; - ha_rdhup_election_hook(ha, c, np, peer_ptrs, my_tag, my_lsn, my_name); - } - return; - } -#endif - - raft_on_leader_lost(&ha->raft); + /* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async + * (it snapshots self freshness into ha->elect_ctx so the PreVote request + * payload — and, on a WIN, the deferred real election — advertise the + * effective freshness floor, NOT the stale persist.committed_lsn: see the + * bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */ + ha_begin_failover(ha, c); } /* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */ @@ -4263,6 +4228,22 @@ void ha_on_heartbeat_received(ha_state_t *ha) clock_gettime(CLOCK_MONOTONIC, &ts); ha->last_activity = ts.tv_sec; tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec); + + /* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader + * is alive — a PreVote round started by a transient blip must be ABANDONED. + * Cancel the in-progress round (clear the raft round state + the controller + * round-timeout handle) and re-attach as a standby. No persistent term / + * voted_for / role change happens here: the node never left CTRL_STANDBY + * during the round, so re-attaching is a no-op on role/term — exactly the + * non-disruptive rejoin the spec requires. */ + if (ha->raft.pre_vote_in_progress) { + ha->raft.pre_vote_in_progress = 0; + ha->raft.pre_votes_granted = 0; + ha_prevote_cancel_round_timeout(); + INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round " + "and re-attaching as CTRL_STANDBY (leader alive, no term change) " + "(Req 4.2)"); + } } void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha) diff --git a/src/controller/ha.h b/src/controller/ha.h index 82306540..e751527c 100644 --- a/src/controller/ha.h +++ b/src/controller/ha.h @@ -820,6 +820,18 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c, */ void ha_start_prevote_async(ha_state_t *ha, cluster_t *c); +/* + * ha_begin_failover: single failover entry point (ha-prevote task 4.2). + * + * Called by both failover triggers (ha_on_primary_ctrl_fd_rdhup and, through + * it, the liveness-timeout poll) AFTER connection-teardown bookkeeping and the + * config-less voter-only guard. Starts a side-effect-free PreVote round + * (ha_start_prevote_async); does NOT truncate the WAL or transition to + * CTRL_ELECTING — those destructive steps are deferred to PreVote success. + * c may be NULL in unit-test mode. Requirements: 1.1, 4.3, 4.4. + */ +void ha_begin_failover(ha_state_t *ha, cluster_t *c); + /* * ha_dispatch_repl_action_for_standby: evaluate the replication decision for * one standby and execute the resulting action. diff --git a/src/controller/meson.build b/src/controller/meson.build index 5b458520..c2502e6e 100644 --- a/src/controller/meson.build +++ b/src/controller/meson.build @@ -122,6 +122,7 @@ controller_unit_test_src = [ 'testsuite/test_ha_rdhup_integration.c', 'testsuite/test_ha_liveness.c', 'testsuite/test_ha_prevote.c', + 'testsuite/test_ha_failover_prevote.c', 'testsuite/test_commit_engine.c', 'testsuite/test_commit_barrier_rollout.c', 'testsuite/test_commit_barrier_reply.c', diff --git a/src/controller/testsuite/controller_test.c b/src/controller/testsuite/controller_test.c index 9560a67c..89cb7813 100644 --- a/src/controller/testsuite/controller_test.c +++ b/src/controller/testsuite/controller_test.c @@ -50,6 +50,7 @@ extern void test_ha_connection_reuse(void); extern void test_ha_rdhup_integration(void); extern void test_ha_liveness(void); extern void test_ha_prevote(void); +extern void test_ha_failover_prevote(void); extern void test_commit_engine(void); extern void test_commit_barrier_rollout(void); extern void test_commit_barrier_reply(void); @@ -176,6 +177,7 @@ int main(int argc, char* argv[]) TEST(test_ha_rdhup_integration); TEST(test_ha_liveness); TEST(test_ha_prevote); + TEST(test_ha_failover_prevote); TEST(test_commit_engine); TEST(test_commit_barrier_rollout); TEST(test_commit_barrier_reply); diff --git a/src/controller/testsuite/test_ha_failover_prevote.c b/src/controller/testsuite/test_ha_failover_prevote.c new file mode 100644 index 00000000..4a66ff72 --- /dev/null +++ b/src/controller/testsuite/test_ha_failover_prevote.c @@ -0,0 +1,371 @@ +/* + * test_ha_failover_prevote.c — ha-prevote task 4.2: + * route the failover triggers (RDHUP + liveness timeout) through PreVote and + * defer the destructive step (truncate + CTRL_ELECTING) until PreVote WINS; + * cancel an in-progress round when a primary heartbeat arrives mid-round. + * + * This is the rewire that activates PreVote on the LIVE failover path. The + * NEW behaviour these tests pin down (which FAILS before the rewire, when the + * triggers elect directly): + * + * (a) Firing the RDHUP failover trigger does NOT transition to CTRL_ELECTING + * or increment the term — it starts a side-effect-free PreVote round and + * the node stays CTRL_STANDBY with pre_vote_in_progress = 1. + * (b) Firing the liveness-timeout trigger goes through the SAME PreVote-first + * path (it inherits the rewired ha_on_primary_ctrl_fd_rdhup). + * (c) A primary heartbeat arriving mid-round CANCELS the round and re-attaches + * as standby with the persistent term unchanged (Req 4.2). + * (d) When the PreVote round then WINS (a peer grants), the trigger path still + * reaches the real election (CTRL_ELECTING) — failover is not blocked. + * (e) The config-less voter-only guard (tag==0, peers>1) still trips and the + * node does NOT even start a PreVote round (it can never win one). + * + * Boundary: PreVote orchestration (trigger path) — ha.c only. + * Requirements: 1.1, 4.2, 4.3, 4.4. + * + * TDD: written FIRST (RED) before the trigger rewire. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +#include "ha.h" +#include "myconf.h" +#include "raft/raft.h" +#include "xce/cluster_def.h" + +/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */ + +extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c); +extern void ha_on_heartbeat_received(ha_state_t *ha); +extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c); + +/* + * liveness_poll_ctx_t: defined privately in ha.c; the layout is mirrored here + * (exactly as test_ha_liveness.c does) so the test can build one for the + * test entry-point without a live delaymgr. + */ +typedef struct { + ha_state_t *ha; + cluster_t *c; + _Atomic int cancelled; +} liveness_poll_ctx_t; + +extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx); + +/* ── Test config ─────────────────────────────────────────────────────────── */ + +static ha_config_t g_cfg = { + .heartbeat_interval_ms = 3000, + .heartbeat_timeout_ms = 6000, + .election_timeout_ms = 500, + .probe_connect_timeout_ms = 50, + .max_promotion_retries = 3, + .election_jitter_max_ms = 0, + .election_priority_step_ms = 500, +}; + +static int g_rearm_calls = 0; +static int stub_rearm(ha_state_t *ha, cluster_t *c) +{ + (void)ha; (void)c; + g_rearm_calls++; + return 0; +} + +/* + * Widen the raft topology to a 3-node cluster (self=0) so a PreVote round + * does NOT trivially self-win (quorum = 2; self-grant = 1). This lets the + * round STAY in progress so we can observe the deferred / cancel behaviour. + */ +static void make_three_node(ha_state_t *ha) +{ + ha->raft.n_peers = 3; + ha->raft.self_idx = 0; + ha->raft.role = RAFT_FOLLOWER; + ha->n_peers = 3; +} + +/* ── Test 1: RDHUP trigger defers the destructive step (Req 1.1, 4.4) ─────── */ + +/* + * Firing ha_on_primary_ctrl_fd_rdhup must NO LONGER transition straight to + * CTRL_ELECTING / truncate / increment the term. Instead it begins a PreVote + * round and stays CTRL_STANDBY with the round in progress. (Unit-test mode, + * c == NULL: ha_enter_real_election — which contains truncate + CTRL_ELECTING — + * must NOT have run, so the role stays CTRL_STANDBY.) + */ +static void test_rdhup_trigger_starts_prevote_not_election(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); /* role = CTRL_STANDBY */ + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + /* Give the handler an fd to close (teardown bookkeeping still happens). */ + int sv[2]; + TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); + ha.primary_ctrl_fd = sv[0]; + + ha_on_primary_ctrl_fd_rdhup(&ha, NULL); /* unit-test mode */ + close(sv[1]); + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 1.1/4.4: RDHUP trigger must start a PreVote round and STAY " + "CTRL_STANDBY, not transition to CTRL_ELECTING"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 4.4: RDHUP trigger must not change the persistent term before " + "PreVote wins"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "Req 1.1: RDHUP trigger must begin a PreVote round (in-progress)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, + "PreVote round must record the self-grant (1)"); + /* The primary fd must still be closed/cleared — teardown bookkeeping stays + * at the trigger regardless of PreVote outcome. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(-1, ha.primary_ctrl_fd, + "RDHUP teardown must still close + clear primary_ctrl_fd"); + + ha_fini(&ha); + printf("test_rdhup_trigger_starts_prevote_not_election: PASS\n"); +} + +/* ── Test 2: liveness-timeout trigger goes through the same PreVote path ──── */ + +/* + * The liveness poll fn, on timeout, must reach the SAME PreVote-first path + * (it calls ha_on_primary_ctrl_fd_rdhup, now rewired). Observable: after the + * timeout, the node is in a PreVote round and STAYS CTRL_STANDBY — it does NOT + * jump to CTRL_ELECTING. + */ +static void test_liveness_timeout_trigger_starts_prevote(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + /* last_activity far enough in the past to exceed heartbeat_timeout_ms. */ + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + ha.last_activity = ts.tv_sec - (time_t)(g_cfg.heartbeat_timeout_ms / 1000) - 10; + + liveness_poll_ctx_t ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.ha = &ha; + ctx.c = NULL; + atomic_init(&ctx.cancelled, 0); + + liveness_poll_fn_test_entry(&ctx); + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 1.1: liveness-timeout failover must start a PreVote round and stay " + "CTRL_STANDBY, not transition straight to CTRL_ELECTING"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 4.4: liveness-timeout failover must not change the term before " + "PreVote wins"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "Req 1.1: liveness-timeout failover must begin a PreVote round"); + + ha_fini(&ha); + printf("test_liveness_timeout_trigger_starts_prevote: PASS\n"); +} + +/* ── Test 3: heartbeat mid-round cancels and re-attaches (Req 4.2) ────────── */ + +/* + * After the failover trigger starts a PreVote round, an incoming primary + * heartbeat means the leader is alive — the campaign must be abandoned: the + * round is cancelled (pre_vote_in_progress cleared), the node re-attaches as + * CTRL_STANDBY, and the persistent term is UNCHANGED (Req 4.2). + */ +static void test_heartbeat_mid_round_cancels_prevote(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + int sv[2]; + TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); + ha.primary_ctrl_fd = sv[0]; + + /* Trigger fires → PreVote round in progress. */ + ha_on_primary_ctrl_fd_rdhup(&ha, NULL); + close(sv[1]); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "precondition: a PreVote round must be in progress before the heartbeat"); + + /* A fresh primary heartbeat arrives mid-round → cancel + re-attach. */ + ha_on_heartbeat_received(&ha); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "Req 4.2: a heartbeat mid-round must CANCEL the PreVote round"); + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 4.2: a mid-round heartbeat must not change the persistent term"); + + ha_fini(&ha); + printf("test_heartbeat_mid_round_cancels_prevote: PASS\n"); +} + +/* ── Test 4: heartbeat with NO round in progress is a plain no-op ─────────── */ + +/* + * The cancel logic must only fire when a round is actually in progress: a + * heartbeat received while no PreVote round is running must just update + * last_activity (the existing behaviour) and never touch role/term. + */ +static void test_heartbeat_without_round_is_plain_noop(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress); + + ha_on_heartbeat_received(&ha); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "no round in progress: heartbeat must leave pre_vote_in_progress = 0"); + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "no round: heartbeat must keep the node CTRL_STANDBY"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "no round: heartbeat must not change the term"); + TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity, + "heartbeat must still update last_activity (existing behaviour preserved)"); + + ha_fini(&ha); + printf("test_heartbeat_without_round_is_plain_noop: PASS\n"); +} + +/* ── Test 5: a genuine failover (PreVote wins) still reaches real election ── */ + +/* + * The whole point of PreVote is to defer, not block. After the RDHUP trigger + * starts the round, a peer GRANT reaching quorum must drive the trigger path + * into the real election (CTRL_ELECTING) — failover still proceeds when the + * primary genuinely died. + */ +extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, + int peer_idx, int granted); + +static void test_failover_prevote_win_reaches_real_election(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + int sv[2]; + TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); + ha.primary_ctrl_fd = sv[0]; + + ha_on_primary_ctrl_fd_rdhup(&ha, NULL); + close(sv[1]); + TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress); + + /* Peer index 1 grants → quorum (2) reached → WON → real election. */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); + + int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); + TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, + "Req 4.3/5.x: when PreVote WINS the trigger path must reach the real " + "election (CTRL_ELECTING) — a genuine failover is not blocked"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "a WON round must clear the in-progress flag"); + + ha_fini(&ha); + printf("test_failover_prevote_win_reaches_real_election: PASS (role=%d)\n", role); +} + +/* ── Test 6: config-less voter-only guard still trips before PreVote ──────── */ + +/* + * A config-less node (config_version_tag == 0, peers > 1) must NOT even start + * a PreVote round on RDHUP — it can never win one (the freshness gate denies + * it), and it must stay a passive voter (CTRL_STANDBY). This preserves the + * ha-config-bootstrap Req 2.8 election-safety guard through the rewire. + * + * Requires a real cluster_t with a tag-0 config so the guard path is reached. + */ +static void test_config_less_node_does_not_start_prevote(void) +{ + cluster_t c; + myconf_t myconf; + memset(&c, 0, sizeof(c)); + memset(&myconf, 0, sizeof(myconf)); + myconf.name = (char *)"ctrl-a"; + myconf.config_version_tag = 0; /* config-less */ + myconf.peers_cfg.candidates_count = 2; /* >1 node */ + snprintf(myconf.peers_cfg.candidates[0].name, + sizeof(myconf.peers_cfg.candidates[0].name), "ctrl-a"); + snprintf(myconf.peers_cfg.candidates[1].name, + sizeof(myconf.peers_cfg.candidates[1].name), "ctrl-b"); + c.myconf = &myconf; + + TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0)); + ha_do_demote(&c.ha); + c.ha.phase2_done = 1; + c.ha.n_peers = 1; + make_three_node(&c.ha); /* widen raft so a round would NOT self-win */ + + int sv[2]; + TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv)); + c.ha.primary_ctrl_fd = sv[0]; + + ha_on_primary_ctrl_fd_rdhup(&c.ha, &c); + close(sv[1]); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, c.ha.raft.pre_vote_in_progress, + "config-less node (tag=0, peers>1) must NOT start a PreVote round on RDHUP"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, + (int)atomic_load_explicit(&c.ha.role, memory_order_acquire), + "config-less RDHUP node must stay CTRL_STANDBY (voter only)"); + + ha_fini(&c.ha); + printf("test_config_less_node_does_not_start_prevote: PASS\n"); +} + +/* ── Suite entry point ──────────────────────────────────────────────────── */ + +void test_ha_failover_prevote(void) +{ + /* Ensure the round LOST/timeout branch never touches a live delaymgr. */ + g_rearm_calls = 0; + ha_prevote_rearm_liveness_hook = stub_rearm; + + RUN_TEST(test_rdhup_trigger_starts_prevote_not_election); + RUN_TEST(test_liveness_timeout_trigger_starts_prevote); + RUN_TEST(test_heartbeat_mid_round_cancels_prevote); + RUN_TEST(test_heartbeat_without_round_is_plain_noop); + RUN_TEST(test_failover_prevote_win_reaches_real_election); + RUN_TEST(test_config_less_node_does_not_start_prevote); + + ha_prevote_rearm_liveness_hook = NULL; +}