diff --git a/.kiro/specs/ha-prevote/tasks.md b/.kiro/specs/ha-prevote/tasks.md index 8b3529c4..40f0528c 100644 --- a/.kiro/specs/ha-prevote/tasks.md +++ b/.kiro/specs/ha-prevote/tasks.md @@ -78,7 +78,7 @@ - _Requirements: 4.1, 4.3, 4.4_ - _Depends: 3.1, 4.1, 4.2, 5.1_ -- [ ] 5.3 Genuine-failover and re-attach scenario tests +- [x] 5.3 Genuine-failover and re-attach scenario tests - Add scenario tests: dead primary with stale peer heartbeats → PreVotes granted → a new primary is elected within the existing election-timeout bounds; heartbeat mid-round → clean re-attach; failed PreVote → liveness watchdog re-armed so a later real failure still elects. - Observable: a genuine primary death elects a new primary, and the suppressed cases never do. - _Requirements: 5.1, 5.4, 4.2_ diff --git a/src/controller/testsuite/test_ha_prevote.c b/src/controller/testsuite/test_ha_prevote.c index 8143232f..af4aca85 100644 --- a/src/controller/testsuite/test_ha_prevote.c +++ b/src/controller/testsuite/test_ha_prevote.c @@ -623,6 +623,244 @@ static void test_frozen_rejoin_candidate_gets_no_majority_noop(void) printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n"); } +/* ══════════════════════════════════════════════════════════════════════════ + * Task 5.3 — Genuine-failover and re-attach scenario tests. + * + * The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER + * legitimate failover, never BLOCK it. Three scenarios prove the suppressed + * cases (5.2) and the genuine cases (5.3) are distinguished by the SAME + * machinery — a dead primary elects a new one, a heartbeat mid-round cleanly + * re-attaches, and a lost round leaves failover re-armed and reusable. + * + * A. Genuine failover ELECTS (Req 5.1): + * 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE, + * so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate + * passes). Both angles: + * - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the + * PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2). + * - CANDIDATE side: feeding that grant to the tally reaches quorum → WON, + * and the candidate proceeds into the real-election sequence + * (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it. + * + * B. Re-attach on heartbeat mid-round (Req 4.2): + * a round is in progress; a primary heartbeat arrives → the round is + * cancelled, the node re-attaches CTRL_STANDBY, term unchanged, + * pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.) + * + * C. Liveness re-arm after loss (Req 5.4 robustness): + * a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine + * failure drives a FRESH round that WINS. Proves a lost round never + * permanently wedges failover. + * + * Requirements: 5.1, 5.4, 4.2. + * Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on + * heartbeat mid-round; Liveness re-arm after loss); + * "System Flows → PreVote round (candidate side)". + * ════════════════════════════════════════════════════════════════════════════ */ + +extern void ha_on_heartbeat_received(ha_state_t *ha); + +/* + * Scenario A — GENUINE FAILOVER ELECTS (Req 5.1). + * + * Two angles, one scenario: + * + * A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone + * STALE (no primary connection, last_activity older than heartbeat_timeout) + * is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the + * 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and + * it flips the decision grant↔deny. No persistent state is mutated on grant + * (PreVote is side-effect-free): term/voted_for/role all unchanged. + * + * A2 (candidate side): the resumed candidate starts a round and receives that + * genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds + * into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING / + * CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote. + */ +static void test_genuine_failover_stale_peer_grants_and_elects(void) +{ + /* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */ + ha_state_t voter; + TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0)); + atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release); + atomic_store_explicit(&voter.term, 5ULL, memory_order_release); + voter.raft.term = 5ULL; + voter.raft.role = RAFT_FOLLOWER; + voter.raft.voted_for = -1; + voter.voted_for_term = 0; + voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */ + /* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the + * voter would itself be eligible to start an election (Req 3.2). */ + set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L); + + uint64_t reply_term = 0; + int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1, + /*cand_term=*/6ULL, &reply_term); + + TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted, + "Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the " + "PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, + "Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term " + "(PreVote adopts no term)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, + atomic_load_explicit(&voter.term, memory_order_acquire), + "Req 5.1: granting a PreVote must NOT change the voter's persistent term"); + TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for, + "Req 5.1: granting a PreVote must record NO vote (real vote still grantable)"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, + (int)atomic_load_explicit(&voter.role, memory_order_acquire), + "Req 5.1: granting a PreVote must not change the voter's role"); + ha_fini(&voter); + + /* ── A2: the candidate wins the round on that grant → real election ── */ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */ + make_three_node(&ha); + + uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + ha_start_prevote_async(&ha, NULL); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "precondition: the failover candidate's PreVote round is in progress"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "precondition: starting the round mutates no term (still side-effect-free)"); + + /* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); + + int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); + TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, + "Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new " + "primary — the candidate enters the real-election sequence, not stay STANDBY"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "Req 5.1: the WON round must clear the in-progress flag (real election begun)"); + + ha_fini(&ha); + printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role); +} + +/* + * Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2). + * + * A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when + * a primary heartbeat arrives — meaning the leader is in fact alive and this was + * NOT a genuine failure. The campaign must be abandoned cleanly: the round is + * cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY, + * and the persistent term is byte-for-byte UNCHANGED. This is the suppressed + * case: the same machinery that elects on a genuine death (scenario A) silently + * stands down the instant the primary proves alive. + */ +static void test_heartbeat_mid_round_clean_reattach(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + uint64_t raft_term_before = ha.raft.term; + int raft_voted_before = ha.raft.voted_for; + + /* A failover round is in progress (self-grant, still STANDBY). */ + ha_start_prevote_async(&ha, NULL); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "precondition: a PreVote round must be in progress before the heartbeat"); + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "precondition: the candidate is CTRL_STANDBY during the round"); + + /* A primary heartbeat arrives mid-round → the leader is alive → cancel. */ + ha_on_heartbeat_received(&ha); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round"); + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role, + "Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, + "Req 4.2: a mid-round heartbeat cancel must NOT change the raft term"); + TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, + "Req 4.2: a cancelled round records no vote"); + TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity, + "Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)"); + + ha_fini(&ha); + printf("test_heartbeat_mid_round_clean_reattach: PASS\n"); +} + +/* + * Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4). + * + * A first PreVote round LOSES (both peers deny — e.g. a brief split where the + * primary was momentarily believed alive by a quorum). A lost round must NOT + * permanently wedge failover: the liveness watchdog is re-armed. When a + * SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to + * the self-grant and a peer GRANT this time reaches quorum → WON → real election. + * + * This is the robustness guarantee that distinguishes 5.3 from a one-shot: the + * machinery survives a loss and still elects when the failure is real. + */ +static void test_lost_round_rearms_then_later_failure_elects(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + + g_rearm_calls = 0; + ha_prevote_rearm_liveness_hook = stub_rearm; + + /* ── Round 1: LOSES (both peers deny) ── */ + ha_start_prevote_async(&ha, NULL); + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/); + pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/); + + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 5.4: a lost round leaves the node CTRL_STANDBY"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 5.4: a lost round leaves the persistent term unchanged"); + TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, + "Req 5.4: a lost round must re-arm the liveness watchdog so a later " + "genuine failure can still trigger a fresh round"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "the lost round is over (in-progress cleared)"); + + /* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */ + ha_start_prevote_async(&ha, NULL); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "Req 5.4: a later genuine failure must arm a FRESH round (in-progress)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, + "Req 5.4: the fresh round resets the tally to the self-grant " + "(the previous loss did not poison the machinery)"); + + /* This time a peer grants → quorum → WON → real election. */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/); + + ha_prevote_rearm_liveness_hook = NULL; + + int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire); + TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY, + "Req 5.4: after an earlier lost round, a later genuine failure must still " + "win a fresh PreVote and start a real election — failover is not wedged"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "the WON second round clears the in-progress flag"); + + ha_fini(&ha); + printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role); +} + /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_prevote(void) @@ -638,5 +876,9 @@ void test_ha_prevote(void) /* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */ RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption); RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop); + /* Task 5.3 — genuine-failover + re-attach scenarios */ + RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects); + RUN_TEST(test_heartbeat_mid_round_clean_reattach); + RUN_TEST(test_lost_round_rearms_then_later_failure_elects); UNITY_END(); }