/* * test_ha_prevote.c — Unit tests for ha-prevote task 4.1: * controller candidate-side PreVote orchestration in ha.c. * * Scope (task 4.1 only): * - ha_start_prevote_async: starts a side-effect-free round (no persist, no * role change, stays CTRL_STANDBY), arms a round timeout. * - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches — * WON → real-election sequence (truncate + CTRL_ELECTING + start election) * LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment * PENDING → no-op * - round-timeout firing = LOST branch. * * These tests drive the reply path through the XCE_TESTING test seam * (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the * existing vote-reply hook pattern. ha.c is unit-test-linked, so the real * orchestration logic is exercised directly. * * Requirements: 1.1, 1.4, 1.5, 5.4, 7.3 * Boundary: PreVote orchestration (ha.c) * * TDD: written first (RED) to specify the exact behaviour before implementation. */ #define _GNU_SOURCE #include #include #include #include #include #include #include #include #include #include #include "ha.h" #include "netio.h" #include "raft/raft.h" #include "vtaskmgr.h" #include "xce/cluster_def.h" /* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */ /* * ha_start_prevote_async: start a PreVote round. No persist, no role change. * c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op, * raft_begin_pre_vote's send callback returns early in test mode). */ extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c); /* * pre_vote_reply_dispatch_for_test: test entry point that calls the real * pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without * a live iomgr. Mirrors the production reply path. */ extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c, int peer_idx, int granted); /* * ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task * firing before a decision (drives the LOST branch). */ extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c); /* Liveness arm-count hook so tests can prove the watchdog was re-armed on a * LOST outcome without a live delaymgr. */ extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c); /* * liveness_ctx_get_for_test: opaque accessor for ha.c's static s_liveness_ctx * (bug-1483 regression lock — see test below). */ extern void *liveness_ctx_get_for_test(void); /* ── Task 5.1 transport-route seams ──────────────────────────────────────── */ /* * pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL * peer_ctrl_msg_handler decode/route path while CTRL_STANDBY + round in * progress (the production condition). Proves the reply is NOT dropped and * reaches pre_vote_reply_dispatch — end-to-end, not via the test hook. */ extern int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt); /* vote_reply_build_pkt_for_test: encode a RequestVoteReply capnp packet (the * preVoteReply reuses the identical payload / union arm). */ extern io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted, uint64_t peer_tag, uint64_t peer_lsn); /* vote_ep_ctx_build_for_test: allocate a vote_ep_ctx_t and wire ep->data with * the ABSOLUTE peer_idx (the index pre_vote_reply_dispatch expects). */ extern endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep, ha_state_t *ha, ha_elect_ctx_t *ctx, int peer_idx, uint64_t my_term, uint64_t my_tag, uint64_t my_lsn, const char *my_name); /* ── bug-1915 follow-up seams (ha.c, XCE_TESTING) ───────────────────────── */ /* Current PreVote round id (ha->pre_vote_round, read under raft_lock). */ extern uint64_t ha_pre_vote_round_for_test(ha_state_t *ha); /* Deliver a PreVote reply stamped with an explicit (possibly stale) round. */ extern void pre_vote_reply_dispatch_round_for_test(ha_state_t *ha, cluster_t *c, uint64_t round, int peer_idx, int granted); /* Fire a round timeout that belongs to an explicit (possibly stale) round. */ extern void ha_prevote_round_timeout_fire_round_for_test(ha_state_t *ha, cluster_t *c, uint64_t round); /* Stamp a test-built endpoint as a PreVote endpoint of `round`. */ extern void vote_ep_ctx_mark_pre_vote_for_test(endpoint_t *ep, uint64_t round); extern int pre_vote_ep_err_handler_for_test(endpoint_t *ep); extern int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt); extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term); /* Number of PreVote rounds decided WON (observation counter). */ extern _Atomic int ha_prevote_won_count_for_test; /* ── Test config ─────────────────────────────────────────────────────────── */ static ha_config_t g_cfg = { .heartbeat_interval_ms = 3000, .heartbeat_timeout_ms = 6000, .election_timeout_ms = 500, .probe_connect_timeout_ms = 50, .max_promotion_retries = 3, .election_jitter_max_ms = 0, .election_priority_step_ms = 500, }; /* Re-arm tracking */ static int g_rearm_calls = 0; static int stub_rearm(ha_state_t *ha, cluster_t *c) { (void)ha; (void)c; g_rearm_calls++; return 0; } /* * Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer * grant to win (quorum = 2; self-grant = 1). raft_init in ha_init only sets a * 1-node placeholder, so we widen the topology directly on the public struct. */ static void make_three_node(ha_state_t *ha) { ha->raft.n_peers = 3; ha->raft.self_idx = ha->my_priority; ha->raft.role = RAFT_FOLLOWER; ha->n_peers = 3; } /* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */ /* * ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment * the persistent term, and must not flip the raft role to CANDIDATE. It sets * the in-progress flag and self-grant in raft_state_t only. */ static void test_prevote_start_no_mutation(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY */ make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); uint64_t raft_term_before = ha.raft.term; int raft_voted_before = ha.raft.voted_for; ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 1.1: role must stay CTRL_STANDBY during the PreVote round"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 1.2: persistent term must not change during the round"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, "Req 1.2: raft term must not change during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, "Req 1.2: raft voted_for must not change during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role, "Req 1.2: raft role must stay FOLLOWER during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "round must be marked in-progress"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "self-grant must be recorded (1)"); ha_fini(&ha); printf("test_prevote_start_no_mutation: PASS\n"); } /* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */ /* * 3-node: after starting the round, feeding one peer GRANT reaches quorum (2). * The WON branch must run the real-election sequence: role becomes * CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election). */ static void test_prevote_majority_grant_starts_real_election(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha_start_prevote_async(&ha, NULL); /* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); /* WON branch enters the real-election sequence: with c==NULL the election * cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY — * the candidate transitioned to CTRL_ELECTING (real-election path). */ TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 1.5: a PreVote WIN must enter the real-election sequence " "(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "WON must clear the in-progress flag (round over)"); ha_fini(&ha); printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role); } /* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */ /* * 3-node: both peers DENY → no quorum → LOST. The node must stay CTRL_STANDBY, * leave the persistent term unchanged (Req 1.4), and re-arm the liveness * watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still * trigger a new round). */ static void test_prevote_no_majority_stays_standby_and_rearms(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); /* Both non-self peers deny → all replied, no quorum → LOST. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/); ha_prevote_rearm_liveness_hook = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 1.4: no PreVote majority must keep the node CTRL_STANDBY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 1.4: no PreVote majority must leave the persistent term unchanged"); TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a " "later genuine failure still triggers a new round"); ha_fini(&ha); printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n"); } /* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */ /* * If the round times out before a decision, it is treated as a loss: stay * CTRL_STANDBY, term unchanged, re-arm liveness. */ static void test_prevote_round_timeout_is_loss(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); /* No replies arrive; the round timeout fires. */ ha_prevote_round_timeout_fire_for_test(&ha, NULL); ha_prevote_rearm_liveness_hook = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 5.4: a round timeout must leave the persistent term unchanged"); TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 5.4: a timed-out round must re-arm the liveness watchdog"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "round timeout must clear the in-progress flag"); ha_fini(&ha); printf("test_prevote_round_timeout_is_loss: PASS\n"); } /* ── Test 4b: LOST round with no test hook must not leak the liveness ctx * (bug-1483) ──────────────────────────────────────────────────────────── */ /* * Every other LOST-path test in this file stubs ha_prevote_rearm_liveness_hook * to avoid driving the real ha_arm_heartbeat_timeout_vtask(ha, c) with c==NULL * — that unstubbed shape is exactly what bug-1483 found: the real function * callocs a liveness_poll_ctx_t and publishes it to the static s_liveness_ctx, * but with c==NULL there is no delaymgr to schedule liveness_poll_fn on, so * nothing ever frees it — a permanent leak/dangling-publish on every such * call. This test deliberately does NOT stub the hook, driving the real path, * and asserts the ctx is never left published once arming without a * scheduler is impossible. */ static void test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); /* Deliberately leave ha_prevote_rearm_liveness_hook == NULL so the LOST * branch calls the real ha_arm_heartbeat_timeout_vtask(&ha, NULL). */ TEST_ASSERT_NULL_MESSAGE(ha_prevote_rearm_liveness_hook, "precondition: hook must be unstubbed to drive the real path"); ha_start_prevote_async(&ha, NULL); ha_prevote_round_timeout_fire_for_test(&ha, NULL); TEST_ASSERT_NULL_MESSAGE(liveness_ctx_get_for_test(), "bug-1483: arming the liveness watchdog with c==NULL (no delaymgr " "to poll on) must not leave an allocated ctx published as the " "\"active\" s_liveness_ctx -- nothing would ever free it"); ha_fini(&ha); printf("test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx: PASS\n"); } /* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */ /* * Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back * in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round * (self-grant + in-progress), which a peer grant can again win — proving the * candidate machinery is reusable, not one-shot. */ static void test_prevote_reusable_after_loss(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha_prevote_rearm_liveness_hook = stub_rearm; /* Round 1: lost. */ ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); /* Round 2: a fresh round arms cleanly (self-grant + in-progress). */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "a new round after a loss must re-arm (in-progress)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "a new round must reset the tally to the self-grant"); /* Round 2 wins with a peer grant. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "a later genuine failure (new round) must still be able to win and " "start a real election"); ha_prevote_rearm_liveness_hook = NULL; ha_fini(&ha); printf("test_prevote_reusable_after_loss: PASS\n"); } /* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */ static void test_prevote_pending_is_noop(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* 5-node so a single grant is below quorum (3) → PENDING. */ ha.raft.n_peers = 5; ha.raft.self_idx = 0; ha.raft.role = RAFT_FOLLOWER; ha.n_peers = 5; ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "a PENDING (sub-quorum) reply must not change the role"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "a PENDING reply must keep the round in progress"); ha_fini(&ha); printf("test_prevote_pending_is_noop: PASS\n"); } /* ── Test 7: a preVoteReply over the REAL transport reaches the tally (5.1) ─ */ /* * Task 5.1 (Req 7.1): a preVoteReply arriving on the candidate's outbound * PreVote endpoint — while the candidate is still CTRL_STANDBY with a round in * progress — must be ROUTED to the PreVote tally by peer_ctrl_msg_handler (NOT * dropped because the role is STANDBY), and drive the round to a decision * end-to-end. This exercises the production decode/route path, not the * pre_vote_reply_dispatch_for_test hook. */ static void test_prevote_reply_over_real_transport_reaches_tally(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY */ make_three_node(&ha); /* Start a real round: self-grant + in-progress, role stays CTRL_STANDBY. */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: role must be CTRL_STANDBY during the round"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: round must be in progress"); /* Build an endpoint carrying ABSOLUTE peer_idx = 1 (the index the tally * dedups by), exactly as ha_send_pre_vote_request sets it. */ endpoint_t ep; memset(&ep, 0, sizeof(ep)); vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1, /*my_term=*/1, /*my_tag=*/0, /*my_lsn=*/0, "test"); /* bug-1915 follow-up: a production PreVote endpoint is stamped with its * kind and round (raft_cb_send_pre_vote_request); do the same here. */ vote_ep_ctx_mark_pre_vote_for_test(&ep, ha_pre_vote_round_for_test(&ha)); /* Encode a GRANT preVoteReply (same wire payload as a real-vote reply). */ io_packet_t *pkt = vote_reply_build_pkt_for_test(/*term=*/1, /*granted=*/1, /*tag=*/0, /*lsn=*/0); TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "vote_reply_build_pkt_for_test must succeed"); /* Route it through the REAL handler (peer_ctrl_msg_handler) while STANDBY. */ int rc = pre_vote_reply_msg_handler_for_test(&ep, pkt); TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handler must return 0"); /* Peer grant → quorum (2 of 3) → WON: the reply was NOT dropped; it reached * the tally and drove the round to a decision (real-election sequence). */ int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 7.1: a preVoteReply over the real transport must reach the tally " "and drive the round to a decision (left CTRL_STANDBY)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the decided round must clear the in-progress flag"); free(ep.data); /* vctx allocated by vote_ep_ctx_build_for_test */ ha_fini(&ha); printf("test_prevote_reply_over_real_transport_reaches_tally: PASS (role=%d)\n", role); } /* ══════════════════════════════════════════════════════════════════════════ * Task 5.2 — Frozen-then-resumed no-op scenario (the headline bug this whole * feature exists to fix). * * 3-node cluster, primary alive. A previously-frozen standby resumes and starts * a PreVote round; both peers DENY (one is the PRIMARY by role, one is a STANDBY * with a fresh heartbeat). With both peers denying, the candidate gets only its * self-grant (1 of 3, below quorum 2) → the round resolves LOST. * * Two complementary angles, tied together as one scenario: * * A. CANDIDATE side (the resumed standby): after the failover trigger starts a * round, feeding it the two healthy-peer DENY replies resolves the round * LOST. Therefore NO real election starts (role never leaves CTRL_STANDBY, * raft.role stays FOLLOWER), the persistent term is byte-for-byte UNCHANGED * (Req 4.4), and the candidate re-arms liveness so the next genuine * heartbeat re-attaches it (Req 4.1 / 4.2-adjacent). * * B. VOTER side (the healthy primary / fresh standby): a node that is PRIMARY, * or STANDBY with a fresh heartbeat, receiving a preVoteRequest from the * resumed node DENIES it and — critically — does NOT adopt the candidate's * (higher) prospective term, does NOT demote, truncate, or self-restart * (Req 4.3). The voter's term/role are unchanged and the reply carries the * voter's OWN term. * * Requirements: 4.1, 4.3, 4.4. * Design: "Testing Strategy → Integration Tests → Frozen-then-resumed no-op". * ════════════════════════════════════════════════════════════════════════════ */ /* ── Voter-side mirror of cluster.c OPCode_preVoteRequest (B) ────────────────── * * cluster.c is NOT linked into the unit-test binary, so we exercise the EXACT * decision its dispatch composes: the raft eligibility gate (role-deny / * fresh-heartbeat-deny, via the registered raft_cb_vote_eligible) plus the * contract that the reply term is the voter's CURRENT term, UNCHANGED, on grant * OR deny. (Freshness floor is irrelevant for the deny cases here — the role / * heartbeat gate fires first — so the freshness check is omitted for clarity.) * * Returns 1 = grant, 0 = deny; *out_reply_term = voter's unchanged term. */ /* raft_on_pre_vote_request is declared in raft.h (now carries the candidate * (lastLogTerm, lastLogIndex) tuple — ha-term-safety task 1.2); no local extern. */ static int voter_pre_vote_decide(ha_state_t *ha, int from_idx, uint64_t cand_term, uint64_t *out_reply_term) { /* Pass (0,0) for the candidate last-log tuple: this PreVote test exercises * the role/heartbeat eligibility gate, not the logOk freshness path; with * the on_get_entry_term stub reading local last-log term as 0, logOk grants * so existing gate behavior is preserved (ha-term-safety task 1.2). */ int granted = (raft_on_pre_vote_request(&ha->raft, from_idx, cand_term, 0, 0) == RAFT_GRANT) ? 1 : 0; if (out_reply_term) *out_reply_term = atomic_load_explicit(&ha->term, memory_order_acquire); return granted; } /* set_last_activity_age: ha->last_activity = "offset_ms ago" (whole seconds, * matching ha_on_heartbeat_received). */ static void set_last_activity_age(ha_state_t *ha, long offset_ms) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); ha->last_activity = now.tv_sec - offset_ms / 1000; } /* * Angle B — the healthy peers DENY without adopting term/role/voted-for. * * Two voters answer the resumed node's preVoteRequest for prospective term 6 * (its inflated current+1): * - voter P: CTRL_PRIMARY, term 5 → role gate denies; term stays 5, no demote. * - voter S: CTRL_STANDBY, term 5, fresh heartbeat → heartbeat gate denies; * term stays 5, stays STANDBY. * Neither adopts term 6; neither mutates voted_for / role. Each reply carries * the voter's OWN unchanged term (Req 4.3). */ static void test_frozen_rejoin_healthy_peers_deny_without_adoption(void) { const uint64_t PROSPECTIVE_TERM = 6ULL; /* resumed node's inflated cur+1 */ /* ── Voter P: the healthy PRIMARY ── */ ha_state_t prim; TEST_ASSERT_EQUAL_INT(0, ha_init(&prim, &g_cfg, 0)); atomic_store_explicit(&prim.role, (int32_t)CTRL_PRIMARY, memory_order_release); atomic_store_explicit(&prim.term, 5ULL, memory_order_release); prim.raft.term = 5ULL; prim.raft.role = RAFT_LEADER; prim.raft.voted_for = -1; prim.voted_for_term = 0; uint64_t reply_term = 0; int granted = voter_pre_vote_decide(&prim, /*from_idx=*/2, PROSPECTIVE_TERM, &reply_term); TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted, "Req 4.3: the healthy PRIMARY must DENY a frozen-rejoin PreVote"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, "Req 4.3: PRIMARY's denial reply carries its OWN term (no adoption of 6)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, atomic_load_explicit(&prim.term, memory_order_acquire), "Req 4.3: PRIMARY must NOT adopt the candidate's higher prospective term"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, prim.raft.term, "Req 4.3: PRIMARY raft term must NOT change"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, (int)atomic_load_explicit(&prim.role, memory_order_acquire), "Req 4.3: PRIMARY must NOT demote in response to a rejoining standby"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_LEADER, (int)prim.raft.role, "Req 4.3: PRIMARY raft role must stay LEADER (no demotion/self-restart)"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, prim.raft.voted_for, "Req 4.3: a PreVote must not record a vote on the PRIMARY"); /* ── Voter S: a healthy STANDBY with a fresh heartbeat ── */ ha_state_t sb; TEST_ASSERT_EQUAL_INT(0, ha_init(&sb, &g_cfg, 0)); atomic_store_explicit(&sb.role, (int32_t)CTRL_STANDBY, memory_order_release); atomic_store_explicit(&sb.term, 5ULL, memory_order_release); sb.raft.term = 5ULL; sb.raft.role = RAFT_FOLLOWER; sb.raft.voted_for = -1; sb.voted_for_term = 0; sb.primary_ctrl_fd = 999; /* live primary connection */ set_last_activity_age(&sb, 500L); /* fresh heartbeat (< heartbeat_timeout) */ reply_term = 0; granted = voter_pre_vote_decide(&sb, /*from_idx=*/2, PROSPECTIVE_TERM, &reply_term); TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted, "Req 4.3: a fresh-heartbeat STANDBY must DENY a frozen-rejoin PreVote"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, "Req 4.3: STANDBY's denial reply carries its OWN term (no adoption of 6)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, atomic_load_explicit(&sb.term, memory_order_acquire), "Req 4.3: STANDBY must NOT adopt the candidate's higher prospective term"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)atomic_load_explicit(&sb.role, memory_order_acquire), "Req 4.3: STANDBY role must NOT change"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, sb.raft.voted_for, "Req 4.3: a PreVote must not record a vote on the STANDBY"); ha_fini(&sb); ha_fini(&prim); printf("test_frozen_rejoin_healthy_peers_deny_without_adoption: PASS\n"); } /* * Angle A (+ the tie-together) — the resumed candidate gets NO majority. * * The resumed standby starts a PreVote round (self-grant, in-progress, still * CTRL_STANDBY, term untouched). It then receives the two DENY replies its * healthy peers send (angle B). With both peers denying, the tally has only the * self-grant (1 of 3) → LOST. Therefore: * - no real election is ever started: role never leaves CTRL_STANDBY and * raft.role stays FOLLOWER (no truncate / electing-transition / self-restart * on the candidate either); * - the candidate's persistent term is byte-for-byte UNCHANGED (Req 4.4); * - the liveness watchdog is re-armed so the next genuine heartbeat re-attaches * the node (Req 4.1 / 4.2-adjacent). * * This is exactly the frozen-rejoin no-op: a transient unreachability event * leaves the cluster term unmoved and the primary untouched. */ static void test_frozen_rejoin_candidate_gets_no_majority_noop(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed standby) */ make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); uint64_t raft_term_before = ha.raft.term; int raft_voted_before = ha.raft.voted_for; g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; /* The resumed node starts its round — side-effect-free precondition. */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: the resumed standby's PreVote round must be in progress"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "precondition: only the self-grant is recorded at round start"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: starting the round leaves the node CTRL_STANDBY"); /* Both healthy peers DENY (PRIMARY by role; STANDBY with fresh heartbeat). */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny — primary peer*/); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny — fresh standby peer*/); ha_prevote_rearm_liveness_hook = NULL; /* No real vote request was sent: the round resolved LOST, so the candidate * never entered the real-election sequence. Role stayed CTRL_STANDBY and * the raft role never left FOLLOWER. */ TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 4.1: no PreVote majority → no real election → node stays CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role, "Req 4.1: with no majority the candidate's raft role must stay FOLLOWER " "(no real election started, no truncate/electing-transition)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the LOST round must be cleared (round over)"); /* Cluster term unchanged across the whole event (Req 4.4). */ TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.4: the persistent term must be UNCHANGED across the frozen-rejoin event"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, "Req 4.4: the raft term must be UNCHANGED across the frozen-rejoin event"); TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, "Req 4.4: no vote was recorded — the prospective term was never persisted"); /* Re-armed so the next genuine heartbeat re-attaches the node (Req 4.1/4.2). */ TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 4.1: a lost frozen-rejoin round must re-arm liveness so the node " "re-attaches on the next primary heartbeat"); ha_fini(&ha); printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n"); } /* ══════════════════════════════════════════════════════════════════════════ * Task 5.3 — Genuine-failover and re-attach scenario tests. * * The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER * legitimate failover, never BLOCK it. Three scenarios prove the suppressed * cases (5.2) and the genuine cases (5.3) are distinguished by the SAME * machinery — a dead primary elects a new one, a heartbeat mid-round cleanly * re-attaches, and a lost round leaves failover re-armed and reusable. * * A. Genuine failover ELECTS (Req 5.1): * 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE, * so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate * passes). Both angles: * - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the * PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2). * - CANDIDATE side: feeding that grant to the tally reaches quorum → WON, * and the candidate proceeds into the real-election sequence * (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it. * * B. Re-attach on heartbeat mid-round (Req 4.2): * a round is in progress; a primary heartbeat arrives → the round is * cancelled, the node re-attaches CTRL_STANDBY, term unchanged, * pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.) * * C. Liveness re-arm after loss (Req 5.4 robustness): * a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine * failure drives a FRESH round that WINS. Proves a lost round never * permanently wedges failover. * * Requirements: 5.1, 5.4, 4.2. * Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on * heartbeat mid-round; Liveness re-arm after loss); * "System Flows → PreVote round (candidate side)". * ════════════════════════════════════════════════════════════════════════════ */ extern void ha_on_heartbeat_received(ha_state_t *ha); /* * Scenario A — GENUINE FAILOVER ELECTS (Req 5.1). * * Two angles, one scenario: * * A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone * STALE (no primary connection, last_activity older than heartbeat_timeout) * is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the * 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and * it flips the decision grant↔deny. No persistent state is mutated on grant * (PreVote is side-effect-free): term/voted_for/role all unchanged. * * A2 (candidate side): the resumed candidate starts a round and receives that * genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds * into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING / * CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote. */ static void test_genuine_failover_stale_peer_grants_and_elects(void) { /* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */ ha_state_t voter; TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0)); atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release); atomic_store_explicit(&voter.term, 5ULL, memory_order_release); voter.raft.term = 5ULL; voter.raft.role = RAFT_FOLLOWER; voter.raft.voted_for = -1; voter.voted_for_term = 0; voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */ /* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the * voter would itself be eligible to start an election (Req 3.2). */ set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L); uint64_t reply_term = 0; int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1, /*cand_term=*/6ULL, &reply_term); TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted, "Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the " "PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, "Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term " "(PreVote adopts no term)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, atomic_load_explicit(&voter.term, memory_order_acquire), "Req 5.1: granting a PreVote must NOT change the voter's persistent term"); TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for, "Req 5.1: granting a PreVote must record NO vote (real vote still grantable)"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, (int)atomic_load_explicit(&voter.role, memory_order_acquire), "Req 5.1: granting a PreVote must not change the voter's role"); ha_fini(&voter); /* ── A2: the candidate wins the round on that grant → real election ── */ ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */ make_three_node(&ha); uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire); ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: the failover candidate's PreVote round is in progress"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "precondition: starting the round mutates no term (still side-effect-free)"); /* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new " "primary — the candidate enters the real-election sequence, not stay STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "Req 5.1: the WON round must clear the in-progress flag (real election begun)"); ha_fini(&ha); printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role); } /* * Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2). * * A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when * a primary heartbeat arrives — meaning the leader is in fact alive and this was * NOT a genuine failure. The campaign must be abandoned cleanly: the round is * cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY, * and the persistent term is byte-for-byte UNCHANGED. This is the suppressed * case: the same machinery that elects on a genuine death (scenario A) silently * stands down the instant the primary proves alive. */ static void test_heartbeat_mid_round_clean_reattach(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); uint64_t raft_term_before = ha.raft.term; int raft_voted_before = ha.raft.voted_for; /* A failover round is in progress (self-grant, still STANDBY). */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: a PreVote round must be in progress before the heartbeat"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: the candidate is CTRL_STANDBY during the round"); /* A primary heartbeat arrives mid-round → the leader is alive → cancel. */ ha_on_heartbeat_received(&ha); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role, "Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, "Req 4.2: a mid-round heartbeat cancel must NOT change the raft term"); TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, "Req 4.2: a cancelled round records no vote"); TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity, "Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)"); ha_fini(&ha); printf("test_heartbeat_mid_round_clean_reattach: PASS\n"); } /* * Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4). * * A first PreVote round LOSES (both peers deny — e.g. a brief split where the * primary was momentarily believed alive by a quorum). A lost round must NOT * permanently wedge failover: the liveness watchdog is re-armed. When a * SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to * the self-grant and a peer GRANT this time reaches quorum → WON → real election. * * This is the robustness guarantee that distinguishes 5.3 from a one-shot: the * machinery survives a loss and still elects when the failure is real. */ static void test_lost_round_rearms_then_later_failure_elects(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; /* ── Round 1: LOSES (both peers deny) ── */ ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "Req 5.4: a lost round leaves the node CTRL_STANDBY"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, atomic_load_explicit(&ha.term, memory_order_acquire), "Req 5.4: a lost round leaves the persistent term unchanged"); TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, "Req 5.4: a lost round must re-arm the liveness watchdog so a later " "genuine failure can still trigger a fresh round"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the lost round is over (in-progress cleared)"); /* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "Req 5.4: a later genuine failure must arm a FRESH round (in-progress)"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "Req 5.4: the fresh round resets the tally to the self-grant " "(the previous loss did not poison the machinery)"); /* This time a peer grants → quorum → WON → real election. */ pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); ha_prevote_rearm_liveness_hook = NULL; int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, "Req 5.4: after an earlier lost round, a later genuine failure must still " "win a fresh PreVote and start a real election — failover is not wedged"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the WON second round clears the in-progress flag"); ha_fini(&ha); printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role); } /* ── bug-1915: a reply that arrives after the round is decided is ignored ──── */ /* * 3-node: peer 1's GRANT wins the round (2/3). Peer 2's GRANT is still in * flight and is delivered afterwards — by then the real election may already * have made this node leader. That late reply must not decide the round a * second time: before the fix raft_on_pre_vote_reply counted it (3/3 >= quorum) * and returned WON again, so the controller re-ran ha_enter_real_election → * ha_start_election_async, which overwrote the role with CTRL_ELECTING and * swapped ha->elect_ctx for a fresh empty carrier (losing the round's vote * endpoints and voter LSNs that raft_cb_become_leader promotes / computes the * safe commit point from). */ static void test_bug1915_late_grant_after_won_is_ignored(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted → WON*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_ELECTING, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "precondition: the first majority grant enters the real election"); /* The real election completes before peer 2's PreVote reply is read. */ atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 1 /*late grant*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "bug-1915: a PreVote reply for an already-WON round must not re-enter " "the real election (it demoted the new leader to CTRL_ELECTING)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the decided round stays decided"); ha_fini(&ha); } /* * 3-node: peer 1's GRANT wins the round; peer 2's DENY arrives afterwards. * pre_votes_granted never decreases, so before the fix the tally still read * 2 >= quorum and returned WON a second time even for a DENY — the same * re-entry into the real election as the late-grant case above. */ static void test_bug1915_late_deny_after_won_is_ignored(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted → WON*/); atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*late deny*/); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "bug-1915: a late DENY for an already-WON round must not re-enter " "the real election"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_rearm_calls, "a late reply for a decided round must not be treated as a LOSS either"); ha_prevote_rearm_liveness_hook = NULL; ha_fini(&ha); } /* ══ bug-1915 follow-up: close the remaining paths to a replaced elect_ctx ══ */ /* A carrier the tests install as "the ctx of the election in flight", with an * extra reference so a (buggy) supersession cannot free it under the test. */ static ha_elect_ctx_t *sentinel_ctx(int n_other) { ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) + (size_t)n_other * sizeof(ctx->votes[0])); TEST_ASSERT_NOT_NULL(ctx); ctx->n_peers = n_other; pthread_mutex_init(&ctx->lock, NULL); sem_init(&ctx->done, 0, 0); atomic_init(&ctx->refcount, 2); /* published + held by the test */ return ctx; } static void drop_published_ctx(ha_state_t *ha, ha_elect_ctx_t *sentinel) { ha_elect_ctx_t *cur = (ha_elect_ctx_t *)atomic_exchange_explicit( &ha->elect_ctx, NULL, memory_order_acq_rel); if (cur && cur != sentinel) { pthread_mutex_destroy(&cur->lock); sem_destroy(&cur->done); free(cur); } if (sentinel) { pthread_mutex_destroy(&sentinel->lock); sem_destroy(&sentinel->done); free(sentinel); } } /* * Gap (a): ha_start_election_async on a node whose raft is already LEADER must * change nothing — not the role, not ha->elect_ctx, not the term. Before the * fix it set CTRL_ELECTING and swapped elect_ctx BEFORE raft_start_election * returned -EALREADY (ignored), demoting a live leader and orphaning its * election's vote endpoints / voter LSNs. */ static void test_gapA_election_start_noop_when_raft_leader(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); ha.raft.role = RAFT_LEADER; ha.raft.term = 5; atomic_store_explicit(&ha.term, 5, memory_order_release); atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); ha_elect_ctx_t *mine = sentinel_ctx(2); atomic_store_explicit(&ha.elect_ctx, (void *)mine, memory_order_release); ha_start_election_async(&ha, NULL, 3, NULL, /*tag=*/1, /*lsn=*/10, "n0"); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "gap (a): a leader must not be moved to CTRL_ELECTING"); TEST_ASSERT_EQUAL_PTR_MESSAGE(mine, atomic_load_explicit(&ha.elect_ctx, memory_order_acquire), "gap (a): the leader's elect_ctx must not be swapped"); TEST_ASSERT_EQUAL_UINT64(5, ha.raft.term); TEST_ASSERT_EQUAL_INT(RAFT_LEADER, ha.raft.role); TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&mine->refcount), "gap (a): the published carrier's reference must not be released"); drop_published_ctx(&ha, mine); ha_fini(&ha); } /* Gap (a): same for a node already CANDIDATE — its in-flight election's ctx * holds the vote endpoints and voter LSNs raft_cb_become_leader needs. */ static void test_gapA_election_start_noop_when_raft_candidate(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); ha.raft.role = RAFT_CANDIDATE; ha.raft.term = 7; atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); ha_elect_ctx_t *mine = sentinel_ctx(2); atomic_store_explicit(&ha.elect_ctx, (void *)mine, memory_order_release); ha_start_election_async(&ha, NULL, 3, NULL, /*tag=*/1, /*lsn=*/10, "n0"); TEST_ASSERT_EQUAL_PTR_MESSAGE(mine, atomic_load_explicit(&ha.elect_ctx, memory_order_acquire), "gap (a): a candidate's elect_ctx must not be swapped"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(7, ha.raft.term, "gap (a): no second term bump"); TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, ha.raft.role); TEST_ASSERT_EQUAL_INT(2, atomic_load(&mine->refcount)); drop_published_ctx(&ha, mine); ha_fini(&ha); } /* * Gap (b): a failover trigger (ha_begin_failover — synchronous in unit mode, * the same ha_start_prevote_async the stagger timer runs) that fires while a * real election is in flight must not open a PreVote round: in production that * swaps ha->elect_ctx for an empty PreVote carrier mid-election. */ static void test_gapB_prevote_refused_during_real_election(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); ha.raft.role = RAFT_CANDIDATE; ha.raft.term = 4; atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); uint64_t round0 = ha_pre_vote_round_for_test(&ha); ha_begin_failover(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "gap (b): no PreVote round may open while a real election is in flight"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(round0, ha_pre_vote_round_for_test(&ha), "gap (b): a refused start must not consume a round id"); TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, ha.raft.role); ha_fini(&ha); } /* Gap (b): a leader never opens a PreVote round either. */ static void test_gapB_prevote_refused_when_leader(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); ha.raft.role = RAFT_LEADER; atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "gap (b): a leader must not open a PreVote round"); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha_fini(&ha); } /* Gap (b): a second start while a round is still undecided must not reset the * round (tally, per-peer replies, round id) — the stagger timer firing while * the RDHUP-triggered round is in flight did exactly that. */ static void test_gapB_prevote_restart_mid_round_keeps_round(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha.raft.n_peers = 5; /* quorum 3: one grant leaves the round pending */ ha.n_peers = 5; ha_start_prevote_async(&ha, NULL); uint64_t r1 = ha_pre_vote_round_for_test(&ha); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, pending*/); TEST_ASSERT_EQUAL_INT(2, ha.raft.pre_votes_granted); ha_start_prevote_async(&ha, NULL); /* e.g. a stagger timer firing now */ TEST_ASSERT_EQUAL_INT_MESSAGE(2, ha.raft.pre_votes_granted, "gap (b): an in-flight round's tally must not be reset"); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_replies[1]); TEST_ASSERT_EQUAL_UINT64(r1, ha_pre_vote_round_for_test(&ha)); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress); ha_fini(&ha); } /* * Gap (d): a reply from an OLDER PreVote round must not be tallied in the * current one (replies carry no round id on the wire; the endpoint's captured * round is the identity). */ static void test_gapD_stale_round_reply_dropped(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha.raft.n_peers = 5; ha.n_peers = 5; g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); uint64_t r1 = ha_pre_vote_round_for_test(&ha); ha_prevote_round_timeout_fire_for_test(&ha, NULL); /* r1 LOST */ TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress); ha_start_prevote_async(&ha, NULL); /* r2 */ TEST_ASSERT_TRUE(ha_pre_vote_round_for_test(&ha) != r1); pre_vote_reply_dispatch_round_for_test(&ha, NULL, r1, 1, 1 /*stale grant*/); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, "gap (d): a round-1 grant must not count toward round 2"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_replies[1], "gap (d): a stale reply must not consume peer 1's round-2 slot"); ha_prevote_rearm_liveness_hook = NULL; ha_fini(&ha); } /* Gap (d): round 1's timeout task firing after round 2 started must not end * round 2. */ static void test_gapD_stale_round_timeout_does_not_end_current_round(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); uint64_t r1 = ha_pre_vote_round_for_test(&ha); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); /* r1 LOST */ TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress); ha_start_prevote_async(&ha, NULL); /* r2 */ int rearms_before = g_rearm_calls; ha_prevote_round_timeout_fire_round_for_test(&ha, NULL, r1); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "gap (d): a stale round timeout must not end the current round"); TEST_ASSERT_EQUAL_INT(rearms_before, g_rearm_calls); ha_prevote_rearm_liveness_hook = NULL; ha_fini(&ha); } /* Gap (d): a PreVote endpoint error from round 1 delivered during round 2 must * not count as peer 1's round-2 DENY — that also burned peer 1's slot, so its * genuine round-2 GRANT was then dropped as a duplicate. */ static void test_gapD_stale_round_endpoint_error_not_counted(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; ha_start_prevote_async(&ha, NULL); uint64_t r1 = ha_pre_vote_round_for_test(&ha); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0); pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); /* r1 LOST */ ha_start_prevote_async(&ha, NULL); /* r2 */ endpoint_t ep; memset(&ep, 0, sizeof(ep)); vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1, 1, 0, 0, "t"); vote_ep_ctx_mark_pre_vote_for_test(&ep, r1); /* round-1 endpoint */ TEST_ASSERT_EQUAL_INT(0, pre_vote_ep_err_handler_for_test(&ep)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_replies[1], "gap (d): a stale endpoint error must not be tallied in round 2"); pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1); /* genuine r2 grant */ TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_ELECTING, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "gap (d): peer 1's genuine round-2 grant must still win the round"); free(ep.data); ha_prevote_rearm_liveness_hook = NULL; ha_fini(&ha); } /* * Gap (d), cross-path: a PreVote reply that arrives on its PreVote endpoint * after the round was WON — when the node is already CTRL_ELECTING — must * never be decoded as a REAL vote. The two replies share one wire payload and * the router chose the decoder by role, so a late PreVote GRANT whose term * equals the new election term was counted by raft_on_vote_reply as a real * vote (a PreVote grant records no vote — Election Safety). */ static void test_gapD_late_prevote_reply_never_counts_as_real_vote(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); uint64_t pv_round = ha_pre_vote_round_for_test(&ha); ha_start_election_async(&ha, NULL, 3, NULL, /*tag=*/1, /*lsn=*/10, "n0"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role, "precondition: a real election is in flight"); uint64_t T = ha.raft.term; endpoint_t ep; memset(&ep, 0, sizeof(ep)); vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*abs peer_idx=*/1, T, 0, 0, "t"); vote_ep_ctx_mark_pre_vote_for_test(&ep, pv_round); io_packet_t *pkt = vote_reply_build_pkt_for_test(T, /*granted=*/1, 0, 0); TEST_ASSERT_NOT_NULL(pkt); TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&ep, pkt)); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role, "gap (d): a PreVote grant must never be counted as a real vote"); TEST_ASSERT_EQUAL_INT(1, ha.raft.votes_granted); TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); free(ep.data); drop_published_ctx(&ha, NULL); ha_fini(&ha); } /* Gap (d), cross-path inverse: a reply on a REAL-vote endpoint (e.g. a lost * election's late RequestVoteReply) arriving while a PreVote round is open * must not be tallied as a PreVote reply. */ static void test_gapD_real_vote_reply_never_feeds_prevote_tally(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha_start_prevote_async(&ha, NULL); endpoint_t ep; memset(&ep, 0, sizeof(ep)); vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*votes_idx=*/1, 1, 0, 0, "t"); io_packet_t *pkt = vote_reply_build_pkt_for_test(1, /*granted=*/1, 0, 0); TEST_ASSERT_NOT_NULL(pkt); TEST_ASSERT_EQUAL_INT(0, pre_vote_reply_msg_handler_for_test(&ep, pkt)); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "gap (d): a real-vote endpoint's reply must not decide a PreVote round"); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_votes_granted); TEST_ASSERT_EQUAL_INT(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); free(ep.data); ha_fini(&ha); } /* * Defensive check in raft_cb_become_leader: promotion computes the safe commit * point from ha->elect_ctx. A PreVote carrier (no real votes), a carrier * stamped for another term, or a stamped carrier without a recorded majority * would yield a self-only safe commit point (n_granted = 1 in a 3-node * cluster) — refuse the promotion loudly instead of becoming CTRL_PRIMARY. */ static void test_become_leader_refuses_prevote_carrier(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); ha_elect_ctx_t *pv = sentinel_ctx(2); pv->is_pre_vote = 1; atomic_store_explicit(&ha.elect_ctx, (void *)pv, memory_order_release); raft_cb_become_leader_for_test(&ha, 1); TEST_ASSERT_TRUE_MESSAGE( atomic_load_explicit(&ha.role, memory_order_acquire) != CTRL_PRIMARY, "promotion must be refused when elect_ctx is a PreVote carrier"); drop_published_ctx(&ha, pv); ha_fini(&ha); } static void test_become_leader_refuses_stamped_carrier_without_majority(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); ha_elect_ctx_t *ec = sentinel_ctx(2); ec->election_term = 3; /* this election's carrier ... */ /* ... but no voter recorded as granted: self-only */ atomic_store_explicit(&ha.elect_ctx, (void *)ec, memory_order_release); raft_cb_become_leader_for_test(&ha, 3); TEST_ASSERT_TRUE_MESSAGE( atomic_load_explicit(&ha.role, memory_order_acquire) != CTRL_PRIMARY, "promotion must be refused when elect_ctx records no voted majority"); drop_published_ctx(&ha, ec); ha_fini(&ha); } static void test_become_leader_refuses_carrier_of_other_term(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); ha_elect_ctx_t *ec = sentinel_ctx(2); ec->election_term = 3; ec->votes[0].vote_granted = 1; ec->votes[1].vote_granted = 1; atomic_store_explicit(&ha.elect_ctx, (void *)ec, memory_order_release); raft_cb_become_leader_for_test(&ha, 4); /* won term 4, ctx is term 3's */ TEST_ASSERT_TRUE_MESSAGE( atomic_load_explicit(&ha.role, memory_order_acquire) != CTRL_PRIMARY, "promotion must be refused when elect_ctx belongs to another election"); drop_published_ctx(&ha, ec); ha_fini(&ha); } /* No false refusal: this election's stamped carrier with a recorded majority * promotes normally. */ static void test_become_leader_accepts_own_carrier_with_majority(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); ha_elect_ctx_t *ec = sentinel_ctx(2); ec->election_term = 3; ec->votes[1].vote_granted = 1; /* self + 1 = 2 of 3 */ atomic_store_explicit(&ha.elect_ctx, (void *)ec, memory_order_release); raft_cb_become_leader_for_test(&ha, 3); TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); drop_published_ctx(&ha, ec); ha_fini(&ha); } /* * Gap (e): real threads. Two peers' GRANTs for the same 3-node round are * dispatched concurrently from two threads released by a barrier (no sleeps). * Whatever the interleaving, the round must be decided WON exactly once. The * pass condition is deterministic; the interleaving is not, so the loop gives * the race many chances to show up if the raft_lock decision check regresses. */ typedef struct { ha_state_t *ha; pthread_barrier_t *bar; int peer; } race_arg_t; static void *race_reply_fn(void *arg) { race_arg_t *a = (race_arg_t *)arg; pthread_barrier_wait(a->bar); pre_vote_reply_dispatch_for_test(a->ha, NULL, a->peer, 1); return NULL; } static void test_gapE_concurrent_grants_decide_round_once(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); for (int iter = 0; iter < 500; iter++) { atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress); int won0 = atomic_load(&ha_prevote_won_count_for_test); pthread_barrier_t bar; pthread_barrier_init(&bar, NULL, 2); race_arg_t a1 = { &ha, &bar, 1 }, a2 = { &ha, &bar, 2 }; pthread_t t1, t2; TEST_ASSERT_EQUAL_INT(0, pthread_create(&t1, NULL, race_reply_fn, &a1)); TEST_ASSERT_EQUAL_INT(0, pthread_create(&t2, NULL, race_reply_fn, &a2)); pthread_join(t1, NULL); pthread_join(t2, NULL); pthread_barrier_destroy(&bar); TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&ha_prevote_won_count_for_test) - won0, "gap (e): two concurrent grants must decide the round WON once"); } ha_fini(&ha); } /* ══ bug-1915 review round 2 ════════════════════════════════════════════════ */ extern void (*ha_prevote_pre_arm_hook)(ha_state_t *ha, cluster_t *c, uint64_t round); extern int (*ha_prevote_round_timer_arm_hook)(ha_state_t *ha, cluster_t *c, uint64_t round, void *tctx); extern int ha_prevote_active_round_timer_for_test(uint64_t *round_out, int *cancelled_out); extern void ha_prevote_round_timer_clear_for_test(void); extern _Atomic int ha_liveness_arm_count_for_test; extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t my_config_tag, uint64_t my_lsn, const char *my_name); /* Round-timer arm stub: records every context it accepted (the test owns and * frees them — no delaymgr ever fires them) and returns g_arm_rc. */ #define MAX_ARMS 16 static void *g_arm_ctx[MAX_ARMS]; static uint64_t g_arm_round[MAX_ARMS]; static int g_arm_n = 0; static int g_arm_rc = 0; static int stub_arm(ha_state_t *ha, cluster_t *c, uint64_t round, void *tctx) { (void)ha; (void)c; if (g_arm_rc < 0) return g_arm_rc; /* scheduling failed: caller keeps ownership */ TEST_ASSERT_TRUE(g_arm_n < MAX_ARMS); g_arm_ctx[g_arm_n] = tctx; g_arm_round[g_arm_n] = round; g_arm_n++; return 0; } static void arm_stub_install(void) { g_arm_n = 0; g_arm_rc = 0; ha_prevote_round_timer_arm_hook = stub_arm; } static void arm_stub_remove(void) { ha_prevote_round_timer_arm_hook = NULL; ha_prevote_pre_arm_hook = NULL; ha_prevote_round_timer_clear_for_test(); for (int i = 0; i < g_arm_n; i++) free(g_arm_ctx[i]); g_arm_n = 0; } /* * Finding 1 (liveness race): ha_start_prevote_async opened round R, then — * before it armed R's timeout — round R closed (here: its timeout path; in * production equally a sync LOST or a heartbeat) and another thread (the * stagger timer) opened R+1 and armed R+1's timeout. The first thread then * cancelled R+1's timer and armed one for R, a no-op by round id: R+1 was left * open with NO timer, and gap (b) refuses every later round, so the node was * stuck until every peer replied or a heartbeat arrived. The interleaving is * injected deterministically at the pre-arm seam (no sleeps, no threads). */ static int g_interleave_done = 0; static void interleave_close_and_reopen(ha_state_t *ha, cluster_t *c, uint64_t round) { if (g_interleave_done) return; g_interleave_done = 1; ha_prevote_round_timeout_fire_round_for_test(ha, c, round); /* R closes */ ha_start_prevote_async(ha, c); /* R+1 opens */ } static void test_r2_f1_open_round_always_has_its_timer(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); cluster_t *c = calloc(1, sizeof(*c)); /* no myconf: sends are no-ops */ TEST_ASSERT_NOT_NULL(c); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; arm_stub_install(); g_interleave_done = 0; ha_prevote_pre_arm_hook = interleave_close_and_reopen; ha_start_prevote_async(&ha, c); uint64_t cur = ha_pre_vote_round_for_test(&ha); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_interleave_done, "precondition: the interleaving ran"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "precondition: round R+1 is open"); uint64_t t_round = 0; int t_cancelled = 1; TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha_prevote_active_round_timer_for_test(&t_round, &t_cancelled), "finding 1: the open round must have a published round timeout"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(cur, t_round, "finding 1: the published round timeout must belong to the OPEN round " "(R+1), not to the already-closed round R"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, t_cancelled, "finding 1: the open round's timeout must not be cancelled"); arm_stub_remove(); ha_prevote_rearm_liveness_hook = NULL; free(c); ha_fini(&ha); } /* * Finding 6: if the round timeout cannot be armed (calloc / delay_task_ms * failure), the round must not stay open with no timer — gap (b) would refuse * every later round. The round is closed and the liveness watchdog re-armed, * so the next failure detection opens a fresh round. */ static void test_r2_f6_round_timer_arm_failure_closes_round(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); cluster_t *c = calloc(1, sizeof(*c)); TEST_ASSERT_NOT_NULL(c); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; arm_stub_install(); g_arm_rc = -ENOMEM; ha_start_prevote_async(&ha, c); uint64_t r1 = ha_pre_vote_round_for_test(&ha); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "finding 6: a round whose timeout could not be armed must be closed"); TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rearm_calls, "finding 6: the liveness watchdog must be re-armed"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_prevote_active_round_timer_for_test(NULL, NULL), "finding 6: no round timeout may stay published after a failed arm"); g_arm_rc = 0; ha_start_prevote_async(&ha, c); /* the next failure detection */ TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "finding 6: a later round must not be refused"); TEST_ASSERT_TRUE(ha_pre_vote_round_for_test(&ha) == r1 + 1); arm_stub_remove(); ha_prevote_rearm_liveness_hook = NULL; free(c); ha_fini(&ha); } /* * Finding 2: a refused promotion (carrier defect) must not leave the node raft * LEADER / CTRL_ELECTING with no timer — every self-recovery path refuses a * LEADER/ELECTING node. It steps down to FOLLOWER in the SAME term (no term * or vote change: it keeps its self-vote, so it cannot vote for another * candidate in this term), becomes CTRL_STANDBY and re-arms the liveness * watchdog, so the next timeout campaigns for term+1. */ static void test_r2_f2_refused_promotion_steps_down_and_rearms(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); /* As raft_on_vote_reply leaves it right before on_become_leader. */ ha.raft.term = 4; ha.raft.voted_for = ha.raft.self_idx; ha.raft.role = RAFT_LEADER; atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); ha_elect_ctx_t *pv = sentinel_ctx(2); pv->is_pre_vote = 1; atomic_store_explicit(&ha.elect_ctx, (void *)pv, memory_order_release); int arms0 = atomic_load(&ha_liveness_arm_count_for_test); raft_cb_become_leader_for_test(&ha, 4); TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, (int)atomic_load_explicit(&ha.role, memory_order_acquire), "finding 2: a refused promotion must leave the node CTRL_STANDBY"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role, "finding 2: raft must step down to FOLLOWER"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(4, ha.raft.term, "finding 2: the step-down keeps the term"); TEST_ASSERT_EQUAL_INT_MESSAGE(ha.raft.self_idx, ha.raft.voted_for, "finding 2: the step-down keeps this term's (self) vote"); TEST_ASSERT_EQUAL_UINT64(4, atomic_load_explicit(&ha.term, memory_order_acquire)); TEST_ASSERT_TRUE_MESSAGE( atomic_load(&ha_liveness_arm_count_for_test) > arms0, "finding 2: the liveness watchdog must be re-armed"); /* ... and the node can campaign again (gap (b) admits a new round). */ ha_start_prevote_async(&ha, NULL); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, "finding 2: the stepped-down node must be able to open a PreVote round"); drop_published_ctx(&ha, pv); ha_fini(&ha); } /* * Finding 3: the library's delayed-election trigger (raft_on_leader_lost -> * on_start_election_after_delay -> ha_dispatch_election_start) called * raft_start_election directly — no ha->elect_ctx carrier stamped for the new * term, no CTRL_ELECTING transition — so every win from it would be refused * by raft_cb_become_leader. It must go through the stamped start path * (ha_enter_real_election -> ha_start_election_async), observed here at its * ha_rdhup_election_hook seam. */ static int g_r2_election_spy_calls = 0; static void r2_election_spy(ha_state_t *ha, cluster_t *c, int n_peers, xce_candidate_t **peers, uint64_t tag, uint64_t lsn, const char *name) { (void)ha; (void)c; (void)n_peers; (void)peers; (void)tag; (void)lsn; (void)name; g_r2_election_spy_calls++; } static void test_r2_f3_delayed_election_uses_stamped_start_path(void) { myconf_t myconf; memset(&myconf, 0, sizeof(myconf)); myconf.name = "ctrl-a"; myconf.config_version_tag = 0x1234; myconf.peers_cfg.candidates_count = 3; const char *names[3] = { "ctrl-a", "ctrl-b", "ctrl-c" }; for (int i = 0; i < 3; i++) { snprintf(myconf.peers_cfg.candidates[i].name, sizeof(myconf.peers_cfg.candidates[i].name), "%s", names[i]); myconf.peers_cfg.candidates[i].port = 7001 + i; } cluster_t *c = calloc(1, sizeof(*c)); TEST_ASSERT_NOT_NULL(c); c->myconf = &myconf; ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0 /* priority 0: no delay */)); ha_do_demote(&ha); make_three_node(&ha); ha.raft.term = 6; ha.raft_c = c; /* production wiring for the delayed trigger */ g_r2_election_spy_calls = 0; ha_rdhup_election_hook = r2_election_spy; pthread_mutex_lock(&ha.raft_lock); raft_on_leader_lost(&ha.raft); pthread_mutex_unlock(&ha.raft_lock); ha_rdhup_election_hook = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_r2_election_spy_calls, "finding 3: the delayed election must go through the stamped start " "path (ha_enter_real_election / ha_start_election_async)"); TEST_ASSERT_EQUAL_UINT64_MESSAGE(6, ha.raft.term, "finding 3: raft_start_election must not be called raw (no carrier)"); TEST_ASSERT_EQUAL_INT(CTRL_ELECTING, (int)atomic_load_explicit(&ha.role, memory_order_acquire)); ha.raft_c = NULL; ha_fini(&ha); free(c); } /* * Finding 5: a PreVote endpoint's error is routed by endpoint KIND, like its * replies: in every role it goes to the PreVote tally (which drops it unless * its round is open), never to the CTRL_PRIMARY promoted-standby cleanup (a * PreVote endpoint is never promoted). */ static void test_r2_f5_prevote_ep_error_routed_by_kind_in_primary(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha.raft.n_peers = 5; /* one deny leaves the round pending */ ha.n_peers = 5; ha_start_prevote_async(&ha, NULL); uint64_t r = ha_pre_vote_round_for_test(&ha); atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release); endpoint_t ep; memset(&ep, 0, sizeof(ep)); vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1, 1, 0, 0, "t"); vote_ep_ctx_mark_pre_vote_for_test(&ep, r); TEST_ASSERT_EQUAL_INT(0, pre_vote_ep_err_handler_for_test(&ep)); TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_replies[1], "finding 5: a PreVote endpoint error must reach the PreVote tally in " "every role (routed by endpoint kind, not by role)"); free(ep.data); ha_fini(&ha); } /* ══ bug-1915 final review ═════════════════════════════════════════════════ */ extern int ha_prevote_round_timeout_fn(void *arg); extern void ha_failover_stagger_fire_for_test(ha_state_t *ha, cluster_t *c); extern _Atomic int ha_prevote_start_vtask_done_for_test; /* Poll an atomic flag for up to `ms` milliseconds; returns its final value. */ static int fr_wait_flag(_Atomic int *flag, int want, int ms) { for (int i = 0; i < ms; i++) { if (atomic_load(flag) >= want) return atomic_load(flag); usleep(1000); } return atomic_load(flag); } /* Holds ha->raft_lock on its own thread until told to release it (a stand-in * for a pool worker parked in an fsync under raft_lock). */ typedef struct { ha_state_t *ha; _Atomic int locked; _Atomic int release; } fr_lock_holder_t; static void *fr_lock_holder_fn(void *arg) { fr_lock_holder_t *h = (fr_lock_holder_t *)arg; pthread_mutex_lock(&h->ha->raft_lock); atomic_store(&h->locked, 1); while (!atomic_load(&h->release)) usleep(1000); pthread_mutex_unlock(&h->ha->raft_lock); return NULL; } /* Fires a round-timeout context on its own thread (the delaymgr thread). */ typedef struct { void *tctx; _Atomic int done; } fr_fire_t; static void *fr_fire_fn(void *arg) { fr_fire_t *f = (fr_fire_t *)arg; ha_prevote_round_timeout_fn(f->tctx); atomic_store(&f->done, 1); return NULL; } /* Fire `tctx` on a "delaymgr" thread while another thread holds raft_lock; * returns 1 if the timer body returned within `ms` while the lock was held. * The lock is always released and both threads joined before returning. */ static int fr_fire_under_held_lock(ha_state_t *ha, void *tctx, int ms) { fr_lock_holder_t h; memset(&h, 0, sizeof(h)); h.ha = ha; pthread_t th, tf; TEST_ASSERT_EQUAL_INT(0, pthread_create(&th, NULL, fr_lock_holder_fn, &h)); TEST_ASSERT_EQUAL_INT(1, fr_wait_flag(&h.locked, 1, 2000)); fr_fire_t f; memset(&f, 0, sizeof(f)); f.tctx = tctx; TEST_ASSERT_EQUAL_INT(0, pthread_create(&tf, NULL, fr_fire_fn, &f)); int returned_while_locked = fr_wait_flag(&f.done, 1, ms); atomic_store(&h.release, 1); pthread_join(th, NULL); pthread_join(tf, NULL); TEST_ASSERT_EQUAL_INT(1, atomic_load(&f.done)); return returned_while_locked; } /* * Finding 1: a CANCELLED round timer must return without taking raft_lock — * the delaymgr thread must never wait behind a raft_lock holder (e.g. a pool * worker in an fsync). A LIVE timer still takes the locked path (it must * decide the round under raft_lock), so it does wait — the contrast proves the * fast path is taken only for a cancelled context. */ static void test_final_f1_cancelled_round_timer_skips_raft_lock(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); cluster_t *c = calloc(1, sizeof(*c)); /* no myconf: sends are no-ops */ TEST_ASSERT_NOT_NULL(c); arm_stub_install(); g_rearm_calls = 0; ha_prevote_rearm_liveness_hook = stub_rearm; /* Round 1 opens with a timer, then is decided LOST (both peers deny): * the decision cancels and unpublishes its timer under raft_lock. */ ha_start_prevote_async(&ha, c); TEST_ASSERT_EQUAL_INT(1, g_arm_n); void *cancelled_tctx = g_arm_ctx[0]; pre_vote_reply_dispatch_for_test(&ha, c, 1, 0); pre_vote_reply_dispatch_for_test(&ha, c, 2, 0); TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha_prevote_active_round_timer_for_test(NULL, NULL), "precondition: the decided round's timer is unpublished"); int fast = fr_fire_under_held_lock(&ha, cancelled_tctx, 1000); TEST_ASSERT_EQUAL_INT_MESSAGE(1, fast, "finding 1: a cancelled round timer must return without waiting for " "raft_lock (delaymgr is trigger-only and must never block)"); /* Contrast: a LIVE timer of the open round 2 must take raft_lock — it * blocks while the lock is held, then decides the round (timeout = LOSS). */ ha_start_prevote_async(&ha, c); TEST_ASSERT_EQUAL_INT(2, g_arm_n); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress); int rearm_before = g_rearm_calls; int live_fast = fr_fire_under_held_lock(&ha, g_arm_ctx[1], 200); TEST_ASSERT_EQUAL_INT_MESSAGE(0, live_fast, "a live round timer must still take raft_lock (locked path)"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, "the live timer, once it got raft_lock, ends its round as a LOSS"); TEST_ASSERT_EQUAL_INT(rearm_before + 1, g_rearm_calls); ha_prevote_rearm_liveness_hook = NULL; arm_stub_remove(); free(c); ha_fini(&ha); } /* * Finding 3: a late RequestVoteReply on an endpoint of an EARLIER election * (vctx->my_term = T-1) from a voter now in term T carries the current term T * and granted=0. It must be dropped — not counted as that peer's deny in the * term-T election — so the peer's genuine term-T reply still counts. */ static void test_final_f3_stale_election_reply_not_counted(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); ha.raft.n_peers = 5; /* one grant does not decide: observe the tally */ ha.n_peers = 5; ha.raft.term = 3; atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); ha_start_election_async(&ha, NULL, 5, NULL, /*tag=*/1, /*lsn=*/10, "n0"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role, "precondition: the term-T election is in flight"); uint64_t T = ha.raft.term; TEST_ASSERT_EQUAL_UINT64(4, T); TEST_ASSERT_EQUAL_INT(0, ha.raft.replies_received); /* Stale: endpoint of the term-(T-1) election; voter already in term T. */ endpoint_t stale; memset(&stale, 0, sizeof(stale)); vote_ep_ctx_build_for_test(&stale, &ha, NULL, /*votes_idx=*/1, T - 1, 0, 0, "t"); io_packet_t *pkt = vote_reply_build_pkt_for_test(T, /*granted=*/0, 0, 0); TEST_ASSERT_NOT_NULL(pkt); TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&stale, pkt)); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.replies_received, "finding 3: a reply from an earlier election's endpoint must not be " "counted in the current election"); TEST_ASSERT_EQUAL_INT(0, ha.raft.vote_replies[2]); TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, ha.raft.role); /* Genuine: the same peer's reply on this election's endpoint counts. */ endpoint_t cur; memset(&cur, 0, sizeof(cur)); vote_ep_ctx_build_for_test(&cur, &ha, NULL, /*votes_idx=*/1, T, 0, 0, "t"); pkt = vote_reply_build_pkt_for_test(T, /*granted=*/1, 0, 0); TEST_ASSERT_NOT_NULL(pkt); TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&cur, pkt)); TEST_ASSERT_EQUAL_INT_MESSAGE(2, ha.raft.votes_granted, "finding 3: the peer's genuine reply must be counted, not dropped as " "a duplicate of the stale one"); TEST_ASSERT_EQUAL_INT(1, ha.raft.replies_received); free(stale.data); free(cur.data); drop_published_ctx(&ha, NULL); ha_fini(&ha); } /* * Finding 3, higher-term exception: a late reply on an earlier election's * endpoint whose term is ABOVE ours must still make this node adopt the term * and step down (Raft: any higher term steps down), but must never be counted * as a vote in the current election. */ static void test_final_f3_stale_higher_term_reply_steps_down_without_vote(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); make_three_node(&ha); ha.raft.n_peers = 5; ha.n_peers = 5; ha.raft.term = 3; atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release); ha_start_election_async(&ha, NULL, 5, NULL, /*tag=*/1, /*lsn=*/10, "n0"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role, "precondition: the term-T election is in flight"); uint64_t T = ha.raft.term; TEST_ASSERT_EQUAL_UINT64(4, T); int granted_before = ha.raft.votes_granted; endpoint_t stale; memset(&stale, 0, sizeof(stale)); vote_ep_ctx_build_for_test(&stale, &ha, NULL, /*votes_idx=*/1, T - 1, 0, 0, "t"); io_packet_t *pkt = vote_reply_build_pkt_for_test(T + 1, /*granted=*/1, 0, 0); TEST_ASSERT_NOT_NULL(pkt); TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&stale, pkt)); TEST_ASSERT_EQUAL_UINT64_MESSAGE(T + 1, ha.raft.term, "a higher-term reply must make the node adopt that term"); TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, ha.raft.role, "a higher-term reply must step the candidate down"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.replies_received, "a stale higher-term reply must never be counted as a reply"); TEST_ASSERT_EQUAL_INT_MESSAGE(granted_before, ha.raft.votes_granted, "a stale higher-term grant must never be counted as a vote"); free(stale.data); drop_published_ctx(&ha, NULL); ha_fini(&ha); } /* Finding 4 probes: where ha_start_prevote_async ran (pre-arm seam fires * inside it, under raft_lock, after the round opened). */ static _Atomic int g_f4_hook_runs = 0; static pthread_t g_f4_hook_thread; static void f4_record_thread(ha_state_t *ha, cluster_t *c, uint64_t round) { (void)ha; (void)c; (void)round; g_f4_hook_thread = pthread_self(); atomic_fetch_add(&g_f4_hook_runs, 1); } /* * Finding 4: the failover-stagger delaymgr body must hand the PreVote start * (per-peer bounded connects under raft_lock) to a VTASK_LIVE_BLOCK vtask, not * run it inline on the delaymgr thread; the vtask then starts the round. */ static void test_final_f4_stagger_hands_prevote_start_to_vtask(void) { vtaskmgr_t pool; TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&pool, 4)); TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&pool)); ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha.last_activity = 0; /* no live primary: the stagger proceeds */ ha.pool = &pool; uint64_t r0 = ha_pre_vote_round_for_test(&ha); int done0 = atomic_load(&ha_prevote_start_vtask_done_for_test); atomic_store(&g_f4_hook_runs, 0); ha_prevote_pre_arm_hook = f4_record_thread; pthread_t caller = pthread_self(); ha_failover_stagger_fire_for_test(&ha, NULL); int runs = fr_wait_flag(&g_f4_hook_runs, 1, 2000); int done = fr_wait_flag(&ha_prevote_start_vtask_done_for_test, done0 + 1, 2000); int inline_run = (runs > 0) && pthread_equal(g_f4_hook_thread, caller); ha_prevote_pre_arm_hook = NULL; TEST_ASSERT_EQUAL_INT_MESSAGE(1, runs, "finding 4: the PreVote round must be started exactly once"); TEST_ASSERT_EQUAL_INT_MESSAGE(0, inline_run, "finding 4: the stagger body must hand the PreVote start to a vtask, " "not connect inline on the delaymgr thread"); TEST_ASSERT_EQUAL_INT(done0 + 1, done); TEST_ASSERT_EQUAL_UINT64(r0 + 1, ha_pre_vote_round_for_test(&ha)); pthread_mutex_lock(&ha.raft_lock); int in_prog = ha.raft.pre_vote_in_progress; pthread_mutex_unlock(&ha.raft_lock); TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_prog, "finding 4: the vtask must start the PreVote round"); vtaskmgr_fini(&pool); ha.pool = NULL; ha_fini(&ha); } /* Finding 4 (skip kept): a live primary seen during the stagger skips the * PreVote on the delaymgr thread — nothing is enqueued, no round opens. */ static void test_final_f4_stagger_skips_when_primary_alive(void) { vtaskmgr_t pool; TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&pool, 4)); TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&pool)); ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); ha.last_activity = now.tv_sec; /* fresh primary heartbeat */ atomic_store_explicit(&ha.leaderless, 0, memory_order_release); ha.pool = &pool; uint64_t r0 = ha_pre_vote_round_for_test(&ha); int done0 = atomic_load(&ha_prevote_start_vtask_done_for_test); ha_failover_stagger_fire_for_test(&ha, NULL); usleep(50 * 1000); TEST_ASSERT_EQUAL_INT(done0, atomic_load(&ha_prevote_start_vtask_done_for_test)); TEST_ASSERT_EQUAL_UINT64(r0, ha_pre_vote_round_for_test(&ha)); TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress); vtaskmgr_fini(&pool); ha.pool = NULL; ha_fini(&ha); } /* Finding 4 (fallback): with no pool, the stagger starts the round inline * (fail loud, stay alive — mirrors ha_dispatch_rdhup_failover). */ static void test_final_f4_stagger_inline_fallback_without_pool(void) { ha_state_t ha; TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); ha_do_demote(&ha); make_three_node(&ha); ha.last_activity = 0; ha.pool = NULL; uint64_t r0 = ha_pre_vote_round_for_test(&ha); ha_failover_stagger_fire_for_test(&ha, NULL); TEST_ASSERT_EQUAL_UINT64(r0 + 1, ha_pre_vote_round_for_test(&ha)); TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress); ha_fini(&ha); } /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_prevote(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_prevote_start_no_mutation); RUN_TEST(test_prevote_majority_grant_starts_real_election); RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms); RUN_TEST(test_prevote_round_timeout_is_loss); RUN_TEST(test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx); RUN_TEST(test_prevote_reusable_after_loss); RUN_TEST(test_prevote_pending_is_noop); RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally); /* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */ RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption); RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop); /* Task 5.3 — genuine-failover + re-attach scenarios */ RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects); RUN_TEST(test_heartbeat_mid_round_clean_reattach); RUN_TEST(test_lost_round_rearms_then_later_failure_elects); /* bug-1915 — replies after the round is decided are dropped */ RUN_TEST(test_bug1915_late_grant_after_won_is_ignored); RUN_TEST(test_bug1915_late_deny_after_won_is_ignored); /* bug-1915 follow-up gaps (a)-(e) + promotion-time carrier check */ RUN_TEST(test_gapA_election_start_noop_when_raft_leader); RUN_TEST(test_gapA_election_start_noop_when_raft_candidate); RUN_TEST(test_gapB_prevote_refused_during_real_election); RUN_TEST(test_gapB_prevote_refused_when_leader); RUN_TEST(test_gapB_prevote_restart_mid_round_keeps_round); RUN_TEST(test_gapD_stale_round_reply_dropped); RUN_TEST(test_gapD_stale_round_timeout_does_not_end_current_round); RUN_TEST(test_gapD_stale_round_endpoint_error_not_counted); RUN_TEST(test_gapD_late_prevote_reply_never_counts_as_real_vote); RUN_TEST(test_gapD_real_vote_reply_never_feeds_prevote_tally); RUN_TEST(test_become_leader_refuses_prevote_carrier); RUN_TEST(test_become_leader_refuses_stamped_carrier_without_majority); RUN_TEST(test_become_leader_refuses_carrier_of_other_term); RUN_TEST(test_become_leader_accepts_own_carrier_with_majority); RUN_TEST(test_gapE_concurrent_grants_decide_round_once); RUN_TEST(test_r2_f1_open_round_always_has_its_timer); RUN_TEST(test_r2_f6_round_timer_arm_failure_closes_round); RUN_TEST(test_r2_f2_refused_promotion_steps_down_and_rearms); RUN_TEST(test_r2_f3_delayed_election_uses_stamped_start_path); RUN_TEST(test_r2_f5_prevote_ep_error_routed_by_kind_in_primary); /* bug-1915 final review */ RUN_TEST(test_final_f1_cancelled_round_timer_skips_raft_lock); RUN_TEST(test_final_f3_stale_election_reply_not_counted); RUN_TEST(test_final_f3_stale_higher_term_reply_steps_down_without_vote); RUN_TEST(test_final_f4_stagger_hands_prevote_start_to_vtask); RUN_TEST(test_final_f4_stagger_skips_when_primary_alive); RUN_TEST(test_final_f4_stagger_inline_fallback_without_pool); }