/* * test_ha_prevote.c — Unit tests for ha-prevote task 4.1: * controller candidate-side PreVote orchestration in ha.c. * * Scope (task 4.1 only): * - ha_start_prevote_async: starts a side-effect-free round (no persist, no * role change, stays CTRL_STANDBY), arms a round timeout. * - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches — * WON → real-election sequence (truncate + CTRL_ELECTING + start election) * LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment * PENDING → no-op * - round-timeout firing = LOST branch. * * These tests drive the reply path through the XCE_TESTING test seam * (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the * existing vote-reply hook pattern. ha.c is unit-test-linked, so the real * orchestration logic is exercised directly. * * Requirements: 1.1, 1.4, 1.5, 5.4, 7.3 * Boundary: PreVote orchestration (ha.c) * * TDD: written first (RED) to specify the exact behaviour before implementation. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include "ha.h" #include "netio.h" #include "raft/raft.h" #include "xce/cluster_def.h" /* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */ /* * ha_start_prevote_async: start a PreVote round. No persist, no role change. * c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op, * raft_begin_pre_vote's send callback returns early in test mode). */ extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c); /* * pre_vote_reply_dispatch_for_test: test entry point that calls the real * pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without * a live iomgr. Mirrors the production reply path. */ extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, int peer_idx, int granted); /* * ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task * firing before a decision (drives the LOST branch). */ extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c); /* Liveness arm-count hook so tests can prove the watchdog was re-armed on a * LOST outcome without a live delaymgr. */ extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c); /* * liveness_ctx_get_for_test: opaque accessor for ha.c's static s_liveness_ctx * (bug-1483 regression lock — see test below). */ extern void *liveness_ctx_get_for_test(void); /* ── Task 5.1 transport-route seams ──────────────────────────────────────── */ /* * pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL * peer_ctrl_msg_handler decode/route path while CTRL_STANDBY + round in * progress (the production condition). Proves the reply is NOT dropped and * reaches pre_vote_reply_dispatch — end-to-end, not via the test hook. */ extern int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt); /* vote_reply_build_pkt_for_test: encode a RequestVoteReply capnp packet (the * preVoteReply reuses the identical payload / union arm). */ extern io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn); /* vote_ep_ctx_build_for_test: allocate a vote_ep_ctx_t and wire ep->data with * the ABSOLUTE peer_idx (the index pre_vote_reply_dispatch expects). */ extern endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t my_term, uint64_t my_tag, uint64_t my_lsn, const char *my_name); /* ── Test config ─────────────────────────────────────────────────────────── */ static ha_config_t g_cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 6000, .election_timeout_ms = 500, .probe_connect_timeout_ms = 50, .max_promotion_retries = 3, .election_jitter_max_ms = 0, .election_priority_step_ms = 500, }; /* Re-arm tracking */ static int g_rearm_calls = 0; static int stub_rearm(ha_state_t *ha, cluster_t *c) { (void)ha; (void)c; g_rearm_calls++; return 0; } /* * Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer * grant to win (quorum = 2; self-grant = 1). raft_init in ha_init only sets a * 1-node placeholder, so we widen the topology directly on the public struct. */ static void make_three_node(ha_state_t *ha) { ha->raft.n_peers = 3; ha->raft.self_idx = ha->my_priority; ha->raft.role = RAFT_FOLLOWER; ha->n_peers = 3; } /* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */ /* * ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment * the persistent term, and must not flip the raft role to CANDIDATE. It sets * the in-progress flag and self-grant in raft_state_t only. */ static void test_prevote_start_no_mutation(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY */ make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); uint64_t raft_term_before = ha.raft.term; int raft_voted_before = ha.raft.voted_for; ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 1.1: role must stay CTRL_STANDBY during the PreVote round"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 1.2: persistent term must not change during the round"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, "Req 1.2: raft term must not change during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, "Req 1.2: raft voted_for must not change during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role, "Req 1.2: raft role must stay FOLLOWER during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "round must be marked in-progress"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "self-grant must be recorded (1)"); ha_fini(&ha); printf("test_prevote_start_no_mutation: PASS\n"); } /* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */ /* * 3-node: after starting the round, feeding one peer GRANT reaches quorum (2). * The WON branch must run the real-election sequence: role becomes * CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election). */ static void test_prevote_majority_grant_starts_real_election(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha_start_prevote_async(&ha, NULL); /* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); /* WON branch enters the real-election sequence: with c==NULL the election * cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY — * the candidate transitioned to CTRL_ELECTING (real-election path). */ TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 1.5: a PreVote WIN must enter the real-election sequence " "(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "WON must clear the in-progress flag (round over)"); ha_fini(&ha); printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role); } /* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */ /* * 3-node: both peers DENY → no quorum → LOST. The node must stay CTRL_STANDBY, * leave the persistent term unchanged (Req 1.4), and re-arm the liveness * watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still * trigger a new round). */ static void test_prevote_no_majority_stays_standby_and_rearms(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); /* Both non-self peers deny → all replied, no quorum → LOST. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/); ha_prevote_rearm_liveness_hook = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 1.4: no PreVote majority must keep the node CTRL_STANDBY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 1.4: no PreVote majority must leave the persistent term unchanged"); TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a " "later genuine failure still triggers a new round"); ha_fini(&ha); printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n"); } /* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */ /* * If the round times out before a decision, it is treated as a loss: stay * CTRL_STANDBY, term unchanged, re-arm liveness. */ static void test_prevote_round_timeout_is_loss(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); /* No replies arrive; the round timeout fires. */ ha_prevote_round_timeout_fire_for_test(&ha, NULL); ha_prevote_rearm_liveness_hook = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 5.4: a round timeout must leave the persistent term unchanged"); TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 5.4: a timed-out round must re-arm the liveness watchdog"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "round timeout must clear the in-progress flag"); ha_fini(&ha); printf("test_prevote_round_timeout_is_loss: PASS\n"); } /* ── Test 4b: LOST round with no test hook must not leak the liveness ctx * (bug-1483) ──────────────────────────────────────────────────────────── */ /* * Every other LOST-path test in this file stubs ha_prevote_rearm_liveness_hook * to avoid driving the real ha_arm_heartbeat_timeout_vtask(ha, c) with c==NULL * — that unstubbed shape is exactly what bug-1483 found: the real function * callocs a liveness_poll_ctx_t and publishes it to the static s_liveness_ctx, * but with c==NULL there is no delaymgr to schedule liveness_poll_fn on, so * nothing ever frees it — a permanent leak/dangling-publish on every such * call. This test deliberately does NOT stub the hook, driving the real path, * and asserts the ctx is never left published once arming without a * scheduler is impossible. */ static void test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); /* Deliberately leave ha_prevote_rearm_liveness_hook == NULL so the LOST * branch calls the real ha_arm_heartbeat_timeout_vtask(&ha, NULL). */ TEST_ASSERT_NULL_MESSAGE(ha_prevote_rearm_liveness_hook, "precondition: hook must be unstubbed to drive the real path"); ha_start_prevote_async(&ha, NULL); ha_prevote_round_timeout_fire_for_test(&ha, NULL); TEST_ASSERT_NULL_MESSAGE(liveness_ctx_get_for_test(), "bug-1483: arming the liveness watchdog with c==NULL (no delaymgr " "to poll on) must not leave an allocated ctx published as the " "\"active\" s_liveness_ctx -- nothing would ever free it"); ha_fini(&ha); printf("test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx: PASS\n"); } /* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */ /* * Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back * in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round * (self-grant + in-progress), which a peer grant can again win — proving the * candidate machinery is reusable, not one-shot. */ static void test_prevote_reusable_after_loss(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha_prevote_rearm_liveness_hook = stub_rearm; /* Round 1: lost. */ ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); /* Round 2: a fresh round arms cleanly (self-grant + in-progress). */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "a new round after a loss must re-arm (in-progress)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "a new round must reset the tally to the self-grant"); /* Round 2 wins with a peer grant. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "a later genuine failure (new round) must still be able to win and " "start a real election"); ha_prevote_rearm_liveness_hook = NULL; ha_fini(&ha); printf("test_prevote_reusable_after_loss: PASS\n"); } /* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */ static void test_prevote_pending_is_noop(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* 5-node so a single grant is below quorum (3) → PENDING. */ ha.raft.n_peers = 5; ha.raft.self_idx = 0; ha.raft.role = RAFT_FOLLOWER; ha.n_peers = 5; ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "a PENDING (sub-quorum) reply must not change the role"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "a PENDING reply must keep the round in progress"); ha_fini(&ha); printf("test_prevote_pending_is_noop: PASS\n"); } /* ── Test 7: a preVoteReply over the REAL transport reaches the tally (5.1) ─ */ /* * Task 5.1 (Req 7.1): a preVoteReply arriving on the candidate's outbound * PreVote endpoint — while the candidate is still CTRL_STANDBY with a round in * progress — must be ROUTED to the PreVote tally by peer_ctrl_msg_handler (NOT * dropped because the role is STANDBY), and drive the round to a decision * end-to-end. This exercises the production decode/route path, not the * pre_vote_reply_dispatch_for_test hook. */ static void test_prevote_reply_over_real_transport_reaches_tally(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY */ make_three_node(&ha); /* Start a real round: self-grant + in-progress, role stays CTRL_STANDBY. */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: role must be CTRL_STANDBY during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: round must be in progress"); /* Build an endpoint carrying ABSOLUTE peer_idx = 1 (the index the tally * dedups by), exactly as ha_send_pre_vote_request sets it. */ endpoint_t ep; memset(&ep, 0, sizeof(ep)); vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1, /*my_term=*/1, /*my_tag=*/0, /*my_lsn=*/0, "test"); /* Encode a GRANT preVoteReply (same wire payload as a real-vote reply). */ io_packet_t *pkt = vote_reply_build_pkt_for_test(/*term=*/1, /*granted=*/1, /*tag=*/0, /*lsn=*/0); TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "vote_reply_build_pkt_for_test must succeed"); /* Route it through the REAL handler (peer_ctrl_msg_handler) while STANDBY. */ int rc = pre_vote_reply_msg_handler_for_test(&ep, pkt); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handler must return 0"); /* Peer grant → quorum (2 of 3) → WON: the reply was NOT dropped; it reached * the tally and drove the round to a decision (real-election sequence). */ int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 7.1: a preVoteReply over the real transport must reach the tally " "and drive the round to a decision (left CTRL_STANDBY)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the decided round must clear the in-progress flag"); free(ep.data); /* vctx allocated by vote_ep_ctx_build_for_test */ ha_fini(&ha); printf("test_prevote_reply_over_real_transport_reaches_tally: PASS (role=%d)\n", role); } /* ══════════════════════════════════════════════════════════════════════════ * Task 5.2 — Frozen-then-resumed no-op scenario (the headline bug this whole * feature exists to fix). * * 3-node cluster, primary alive. A previously-frozen standby resumes and starts * a PreVote round; both peers DENY (one is the PRIMARY by role, one is a STANDBY * with a fresh heartbeat). With both peers denying, the candidate gets only its * self-grant (1 of 3, below quorum 2) → the round resolves LOST. * * Two complementary angles, tied together as one scenario: * * A. CANDIDATE side (the resumed standby): after the failover trigger starts a * round, feeding it the two healthy-peer DENY replies resolves the round * LOST. Therefore NO real election starts (role never leaves CTRL_STANDBY, * raft.role stays FOLLOWER), the persistent term is byte-for-byte UNCHANGED * (Req 4.4), and the candidate re-arms liveness so the next genuine * heartbeat re-attaches it (Req 4.1 / 4.2-adjacent). * * B. VOTER side (the healthy primary / fresh standby): a node that is PRIMARY, * or STANDBY with a fresh heartbeat, receiving a preVoteRequest from the * resumed node DENIES it and — critically — does NOT adopt the candidate's * (higher) prospective term, does NOT demote, truncate, or self-restart * (Req 4.3). The voter's term/role are unchanged and the reply carries the * voter's OWN term. * * Requirements: 4.1, 4.3, 4.4. * Design: "Testing Strategy → Integration Tests → Frozen-then-resumed no-op". * ════════════════════════════════════════════════════════════════════════════ */ /* ── Voter-side mirror of cluster.c OPCode_preVoteRequest (B) ────────────────── * * cluster.c is NOT linked into the unit-test binary, so we exercise the EXACT * decision its dispatch composes: the raft eligibility gate (role-deny / * fresh-heartbeat-deny, via the registered raft_cb_vote_eligible) plus the * contract that the reply term is the voter's CURRENT term, UNCHANGED, on grant * OR deny. (Freshness floor is irrelevant for the deny cases here — the role / * heartbeat gate fires first — so the freshness check is omitted for clarity.) * * Returns 1 = grant, 0 = deny; *out_reply_term = voter's unchanged term. */ /* raft_on_pre_vote_request is declared in raft.h (now carries the candidate * (lastLogTerm, lastLogIndex) tuple — ha-term-safety task 1.2); no local extern. */ static int voter_pre_vote_decide(ha_state_t *ha, int from_idx, uint64_t cand_term, uint64_t *out_reply_term) { /* Pass (0,0) for the candidate last-log tuple: this PreVote test exercises * the role/heartbeat eligibility gate, not the logOk freshness path; with * the on_get_entry_term stub reading local last-log term as 0, logOk grants * so existing gate behavior is preserved (ha-term-safety task 1.2). */ int granted = (raft_on_pre_vote_request(&ha->raft, from_idx, cand_term, 0, 0) == RAFT_GRANT) ? 1 : 0; if (out_reply_term) *out_reply_term = atomic_load_explicit(&ha->term, memory_order_acquire); return granted; } /* set_last_activity_age: ha->last_activity = "offset_ms ago" (whole seconds, * matching ha_on_heartbeat_received). */ static void set_last_activity_age(ha_state_t *ha, long offset_ms) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); ha->last_activity = now.tv_sec - offset_ms / 1000; } /* * Angle B — the healthy peers DENY without adopting term/role/voted-for. * * Two voters answer the resumed node's preVoteRequest for prospective term 6 * (its inflated current+1): * - voter P: CTRL_PRIMARY, term 5 → role gate denies; term stays 5, no demote. * - voter S: CTRL_STANDBY, term 5, fresh heartbeat → heartbeat gate denies; * term stays 5, stays STANDBY. * Neither adopts term 6; neither mutates voted_for / role. Each reply carries * the voter's OWN unchanged term (Req 4.3). */ static void test_frozen_rejoin_healthy_peers_deny_without_adoption(void) { const uint64_t PROSPECTIVE_TERM = 6ULL; /* resumed node's inflated cur+1 */ /* ── Voter P: the healthy PRIMARY ── */ ha_state_t prim; TEST_ASSERT_EQUAL_INT(0, ha_init(&prim, &g_cfg, 0)); atomic_store_explicit(&prim.role, (int32_t)CTRL_PRIMARY, memory_order_release); atomic_store_explicit(&prim.term, 5ULL, memory_order_release); prim.raft.term = 5ULL; prim.raft.role = RAFT_LEADER; prim.raft.voted_for = -1; prim.voted_for_term = 0; uint64_t reply_term = 0; int granted = voter_pre_vote_decide(&prim, /*from_idx=*/2, PROSPECTIVE_TERM, &reply_term); TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted, "Req 4.3: the healthy PRIMARY must DENY a frozen-rejoin PreVote"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, "Req 4.3: PRIMARY's denial reply carries its OWN term (no adoption of 6)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, atomic_load_explicit(&prim.term, memory_order_acquire), "Req 4.3: PRIMARY must NOT adopt the candidate's higher prospective term"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, prim.raft.term, "Req 4.3: PRIMARY raft term must NOT change"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)atomic_load_explicit(&prim.role, memory_order_acquire), "Req 4.3: PRIMARY must NOT demote in response to a rejoining standby"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_LEADER, (int)prim.raft.role, "Req 4.3: PRIMARY raft role must stay LEADER (no demotion/self-restart)"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, prim.raft.voted_for, "Req 4.3: a PreVote must not record a vote on the PRIMARY"); /* ── Voter S: a healthy STANDBY with a fresh heartbeat ── */ ha_state_t sb; TEST_ASSERT_EQUAL_INT(0, ha_init(&sb, &g_cfg, 0)); atomic_store_explicit(&sb.role, (int32_t)CTRL_STANDBY, memory_order_release); atomic_store_explicit(&sb.term, 5ULL, memory_order_release); sb.raft.term = 5ULL; sb.raft.role = RAFT_FOLLOWER; sb.raft.voted_for = -1; sb.voted_for_term = 0; sb.primary_ctrl_fd = 999; /* live primary connection */ set_last_activity_age(&sb, 500L); /* fresh heartbeat (< heartbeat_timeout) */ reply_term = 0; granted = voter_pre_vote_decide(&sb, /*from_idx=*/2, PROSPECTIVE_TERM, &reply_term); TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted, "Req 4.3: a fresh-heartbeat STANDBY must DENY a frozen-rejoin PreVote"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, "Req 4.3: STANDBY's denial reply carries its OWN term (no adoption of 6)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, atomic_load_explicit(&sb.term, memory_order_acquire), "Req 4.3: STANDBY must NOT adopt the candidate's higher prospective term"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)atomic_load_explicit(&sb.role, memory_order_acquire), "Req 4.3: STANDBY role must NOT change"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, sb.raft.voted_for, "Req 4.3: a PreVote must not record a vote on the STANDBY"); ha_fini(&sb); ha_fini(&prim); printf("test_frozen_rejoin_healthy_peers_deny_without_adoption: PASS\n"); } /* * Angle A (+ the tie-together) — the resumed candidate gets NO majority. * * The resumed standby starts a PreVote round (self-grant, in-progress, still * CTRL_STANDBY, term untouched). It then receives the two DENY replies its * healthy peers send (angle B). With both peers denying, the tally has only the * self-grant (1 of 3) → LOST. Therefore: * - no real election is ever started: role never leaves CTRL_STANDBY and * raft.role stays FOLLOWER (no truncate / electing-transition / self-restart * on the candidate either); * - the candidate's persistent term is byte-for-byte UNCHANGED (Req 4.4); * - the liveness watchdog is re-armed so the next genuine heartbeat re-attaches * the node (Req 4.1 / 4.2-adjacent). * * This is exactly the frozen-rejoin no-op: a transient unreachability event * leaves the cluster term unmoved and the primary untouched. */ static void test_frozen_rejoin_candidate_gets_no_majority_noop(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed standby) */ make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); uint64_t raft_term_before = ha.raft.term; int raft_voted_before = ha.raft.voted_for; g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; /* The resumed node starts its round — side-effect-free precondition. */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: the resumed standby's PreVote round must be in progress"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "precondition: only the self-grant is recorded at round start"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: starting the round leaves the node CTRL_STANDBY"); /* Both healthy peers DENY (PRIMARY by role; STANDBY with fresh heartbeat). */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny — primary peer*/); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny — fresh standby peer*/); ha_prevote_rearm_liveness_hook = NULL; /* No real vote request was sent: the round resolved LOST, so the candidate * never entered the real-election sequence. Role stayed CTRL_STANDBY and * the raft role never left FOLLOWER. */ TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 4.1: no PreVote majority → no real election → node stays CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role, "Req 4.1: with no majority the candidate's raft role must stay FOLLOWER " "(no real election started, no truncate/electing-transition)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the LOST round must be cleared (round over)"); /* Cluster term unchanged across the whole event (Req 4.4). */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.4: the persistent term must be UNCHANGED across the frozen-rejoin event"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, "Req 4.4: the raft term must be UNCHANGED across the frozen-rejoin event"); TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, "Req 4.4: no vote was recorded — the prospective term was never persisted"); /* Re-armed so the next genuine heartbeat re-attaches the node (Req 4.1/4.2). */ TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 4.1: a lost frozen-rejoin round must re-arm liveness so the node " "re-attaches on the next primary heartbeat"); ha_fini(&ha); printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n"); } /* ══════════════════════════════════════════════════════════════════════════ * Task 5.3 — Genuine-failover and re-attach scenario tests. * * The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER * legitimate failover, never BLOCK it. Three scenarios prove the suppressed * cases (5.2) and the genuine cases (5.3) are distinguished by the SAME * machinery — a dead primary elects a new one, a heartbeat mid-round cleanly * re-attaches, and a lost round leaves failover re-armed and reusable. * * A. Genuine failover ELECTS (Req 5.1): * 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE, * so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate * passes). Both angles: * - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the * PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2). * - CANDIDATE side: feeding that grant to the tally reaches quorum → WON, * and the candidate proceeds into the real-election sequence * (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it. * * B. Re-attach on heartbeat mid-round (Req 4.2): * a round is in progress; a primary heartbeat arrives → the round is * cancelled, the node re-attaches CTRL_STANDBY, term unchanged, * pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.) * * C. Liveness re-arm after loss (Req 5.4 robustness): * a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine * failure drives a FRESH round that WINS. Proves a lost round never * permanently wedges failover. * * Requirements: 5.1, 5.4, 4.2. * Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on * heartbeat mid-round; Liveness re-arm after loss); * "System Flows → PreVote round (candidate side)". * ════════════════════════════════════════════════════════════════════════════ */ extern void ha_on_heartbeat_received(ha_state_t *ha); /* * Scenario A — GENUINE FAILOVER ELECTS (Req 5.1). * * Two angles, one scenario: * * A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone * STALE (no primary connection, last_activity older than heartbeat_timeout) * is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the * 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and * it flips the decision grant↔deny. No persistent state is mutated on grant * (PreVote is side-effect-free): term/voted_for/role all unchanged. * * A2 (candidate side): the resumed candidate starts a round and receives that * genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds * into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING / * CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote. */ static void test_genuine_failover_stale_peer_grants_and_elects(void) { /* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */ ha_state_t voter; TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0)); atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release); atomic_store_explicit(&voter.term, 5ULL, memory_order_release); voter.raft.term = 5ULL; voter.raft.role = RAFT_FOLLOWER; voter.raft.voted_for = -1; voter.voted_for_term = 0; voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */ /* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the * voter would itself be eligible to start an election (Req 3.2). */ set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L); uint64_t reply_term = 0; int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1, /*cand_term=*/6ULL, &reply_term); TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted, "Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the " "PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, "Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term " "(PreVote adopts no term)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, atomic_load_explicit(&voter.term, memory_order_acquire), "Req 5.1: granting a PreVote must NOT change the voter's persistent term"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for, "Req 5.1: granting a PreVote must record NO vote (real vote still grantable)"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)atomic_load_explicit(&voter.role, memory_order_acquire), "Req 5.1: granting a PreVote must not change the voter's role"); ha_fini(&voter); /* ── A2: the candidate wins the round on that grant → real election ── */ ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */ make_three_node(&ha); uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire); ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: the failover candidate's PreVote round is in progress"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "precondition: starting the round mutates no term (still side-effect-free)"); /* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new " "primary — the candidate enters the real-election sequence, not stay STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "Req 5.1: the WON round must clear the in-progress flag (real election begun)"); ha_fini(&ha); printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role); } /* * Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2). * * A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when * a primary heartbeat arrives — meaning the leader is in fact alive and this was * NOT a genuine failure. The campaign must be abandoned cleanly: the round is * cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY, * and the persistent term is byte-for-byte UNCHANGED. This is the suppressed * case: the same machinery that elects on a genuine death (scenario A) silently * stands down the instant the primary proves alive. */ static void test_heartbeat_mid_round_clean_reattach(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); uint64_t raft_term_before = ha.raft.term; int raft_voted_before = ha.raft.voted_for; /* A failover round is in progress (self-grant, still STANDBY). */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: a PreVote round must be in progress before the heartbeat"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: the candidate is CTRL_STANDBY during the round"); /* A primary heartbeat arrives mid-round → the leader is alive → cancel. */ ha_on_heartbeat_received(&ha); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role, "Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, "Req 4.2: a mid-round heartbeat cancel must NOT change the raft term"); TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, "Req 4.2: a cancelled round records no vote"); TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity, "Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)"); ha_fini(&ha); printf("test_heartbeat_mid_round_clean_reattach: PASS\n"); } /* * Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4). * * A first PreVote round LOSES (both peers deny — e.g. a brief split where the * primary was momentarily believed alive by a quorum). A lost round must NOT * permanently wedge failover: the liveness watchdog is re-armed. When a * SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to * the self-grant and a peer GRANT this time reaches quorum → WON → real election. * * This is the robustness guarantee that distinguishes 5.3 from a one-shot: the * machinery survives a loss and still elects when the failure is real. */ static void test_lost_round_rearms_then_later_failure_elects(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; /* ── Round 1: LOSES (both peers deny) ── */ ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 5.4: a lost round leaves the node CTRL_STANDBY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 5.4: a lost round leaves the persistent term unchanged"); TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 5.4: a lost round must re-arm the liveness watchdog so a later " "genuine failure can still trigger a fresh round"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the lost round is over (in-progress cleared)"); /* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "Req 5.4: a later genuine failure must arm a FRESH round (in-progress)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "Req 5.4: the fresh round resets the tally to the self-grant " "(the previous loss did not poison the machinery)"); /* This time a peer grants → quorum → WON → real election. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); ha_prevote_rearm_liveness_hook = NULL; int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 5.4: after an earlier lost round, a later genuine failure must still " "win a fresh PreVote and start a real election — failover is not wedged"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the WON second round clears the in-progress flag"); ha_fini(&ha); printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role); } /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_prevote(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_prevote_start_no_mutation); RUN_TEST(test_prevote_majority_grant_starts_real_election); RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms); RUN_TEST(test_prevote_round_timeout_is_loss); RUN_TEST(test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx); RUN_TEST(test_prevote_reusable_after_loss); RUN_TEST(test_prevote_pending_is_noop); RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally); /* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */ RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption); RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop); /* Task 5.3 — genuine-failover + re-attach scenarios */ RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects); RUN_TEST(test_heartbeat_mid_round_clean_reattach); RUN_TEST(test_lost_round_rearms_then_later_failure_elects); }