test(ha-prevote): genuine-failover, re-attach, and liveness-rearm scenarios (task 5.3)
Three scenario tests (zero production change): a stale-heartbeat standby GRANTS (grant<->deny flip vs 5.2) and the won round advances to CTRL_ELECTING (PreVote does not block legitimate failover); a heartbeat mid-round cancels and re-attaches as standby with term unchanged; a lost round re-arms liveness and a later genuine failure still wins (reusable, not one-shot). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
86ac7c4863
commit
22fa753fa6
2 changed files with 243 additions and 1 deletions
|
|
@ -78,7 +78,7 @@
|
|||
- _Requirements: 4.1, 4.3, 4.4_
|
||||
- _Depends: 3.1, 4.1, 4.2, 5.1_
|
||||
|
||||
- [ ] 5.3 Genuine-failover and re-attach scenario tests
|
||||
- [x] 5.3 Genuine-failover and re-attach scenario tests
|
||||
- Add scenario tests: dead primary with stale peer heartbeats → PreVotes granted → a new primary is elected within the existing election-timeout bounds; heartbeat mid-round → clean re-attach; failed PreVote → liveness watchdog re-armed so a later real failure still elects.
|
||||
- Observable: a genuine primary death elects a new primary, and the suppressed cases never do.
|
||||
- _Requirements: 5.1, 5.4, 4.2_
|
||||
|
|
|
|||
|
|
@ -623,6 +623,244 @@ static void test_frozen_rejoin_candidate_gets_no_majority_noop(void)
|
|||
printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n");
|
||||
}
|
||||
|
||||
/* ══════════════════════════════════════════════════════════════════════════
|
||||
* Task 5.3 — Genuine-failover and re-attach scenario tests.
|
||||
*
|
||||
* The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER
|
||||
* legitimate failover, never BLOCK it. Three scenarios prove the suppressed
|
||||
* cases (5.2) and the genuine cases (5.3) are distinguished by the SAME
|
||||
* machinery — a dead primary elects a new one, a heartbeat mid-round cleanly
|
||||
* re-attaches, and a lost round leaves failover re-armed and reusable.
|
||||
*
|
||||
* A. Genuine failover ELECTS (Req 5.1):
|
||||
* 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE,
|
||||
* so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate
|
||||
* passes). Both angles:
|
||||
* - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the
|
||||
* PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2).
|
||||
* - CANDIDATE side: feeding that grant to the tally reaches quorum → WON,
|
||||
* and the candidate proceeds into the real-election sequence
|
||||
* (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it.
|
||||
*
|
||||
* B. Re-attach on heartbeat mid-round (Req 4.2):
|
||||
* a round is in progress; a primary heartbeat arrives → the round is
|
||||
* cancelled, the node re-attaches CTRL_STANDBY, term unchanged,
|
||||
* pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.)
|
||||
*
|
||||
* C. Liveness re-arm after loss (Req 5.4 robustness):
|
||||
* a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine
|
||||
* failure drives a FRESH round that WINS. Proves a lost round never
|
||||
* permanently wedges failover.
|
||||
*
|
||||
* Requirements: 5.1, 5.4, 4.2.
|
||||
* Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on
|
||||
* heartbeat mid-round; Liveness re-arm after loss);
|
||||
* "System Flows → PreVote round (candidate side)".
|
||||
* ════════════════════════════════════════════════════════════════════════════ */
|
||||
|
||||
extern void ha_on_heartbeat_received(ha_state_t *ha);
|
||||
|
||||
/*
|
||||
* Scenario A — GENUINE FAILOVER ELECTS (Req 5.1).
|
||||
*
|
||||
* Two angles, one scenario:
|
||||
*
|
||||
* A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone
|
||||
* STALE (no primary connection, last_activity older than heartbeat_timeout)
|
||||
* is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the
|
||||
* 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and
|
||||
* it flips the decision grant↔deny. No persistent state is mutated on grant
|
||||
* (PreVote is side-effect-free): term/voted_for/role all unchanged.
|
||||
*
|
||||
* A2 (candidate side): the resumed candidate starts a round and receives that
|
||||
* genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds
|
||||
* into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING /
|
||||
* CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote.
|
||||
*/
|
||||
static void test_genuine_failover_stale_peer_grants_and_elects(void)
|
||||
{
|
||||
/* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */
|
||||
ha_state_t voter;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0));
|
||||
atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
||||
atomic_store_explicit(&voter.term, 5ULL, memory_order_release);
|
||||
voter.raft.term = 5ULL;
|
||||
voter.raft.role = RAFT_FOLLOWER;
|
||||
voter.raft.voted_for = -1;
|
||||
voter.voted_for_term = 0;
|
||||
voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */
|
||||
/* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the
|
||||
* voter would itself be eligible to start an election (Req 3.2). */
|
||||
set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L);
|
||||
|
||||
uint64_t reply_term = 0;
|
||||
int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1,
|
||||
/*cand_term=*/6ULL, &reply_term);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
||||
"Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the "
|
||||
"PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
|
||||
"Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term "
|
||||
"(PreVote adopts no term)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
|
||||
atomic_load_explicit(&voter.term, memory_order_acquire),
|
||||
"Req 5.1: granting a PreVote must NOT change the voter's persistent term");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for,
|
||||
"Req 5.1: granting a PreVote must record NO vote (real vote still grantable)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&voter.role, memory_order_acquire),
|
||||
"Req 5.1: granting a PreVote must not change the voter's role");
|
||||
ha_fini(&voter);
|
||||
|
||||
/* ── A2: the candidate wins the round on that grant → real election ── */
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */
|
||||
make_three_node(&ha);
|
||||
|
||||
uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
||||
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"precondition: the failover candidate's PreVote round is in progress");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before,
|
||||
atomic_load_explicit(&ha.term, memory_order_acquire),
|
||||
"precondition: starting the round mutates no term (still side-effect-free)");
|
||||
|
||||
/* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
|
||||
|
||||
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
||||
"Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new "
|
||||
"primary — the candidate enters the real-election sequence, not stay STANDBY");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||
"Req 5.1: the WON round must clear the in-progress flag (real election begun)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role);
|
||||
}
|
||||
|
||||
/*
|
||||
* Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2).
|
||||
*
|
||||
* A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when
|
||||
* a primary heartbeat arrives — meaning the leader is in fact alive and this was
|
||||
* NOT a genuine failure. The campaign must be abandoned cleanly: the round is
|
||||
* cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY,
|
||||
* and the persistent term is byte-for-byte UNCHANGED. This is the suppressed
|
||||
* case: the same machinery that elects on a genuine death (scenario A) silently
|
||||
* stands down the instant the primary proves alive.
|
||||
*/
|
||||
static void test_heartbeat_mid_round_clean_reattach(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
||||
uint64_t raft_term_before = ha.raft.term;
|
||||
int raft_voted_before = ha.raft.voted_for;
|
||||
|
||||
/* A failover round is in progress (self-grant, still STANDBY). */
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"precondition: a PreVote round must be in progress before the heartbeat");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"precondition: the candidate is CTRL_STANDBY during the round");
|
||||
|
||||
/* A primary heartbeat arrives mid-round → the leader is alive → cancel. */
|
||||
ha_on_heartbeat_received(&ha);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||
"Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role,
|
||||
"Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
||||
atomic_load_explicit(&ha.term, memory_order_acquire),
|
||||
"Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
|
||||
"Req 4.2: a mid-round heartbeat cancel must NOT change the raft term");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
|
||||
"Req 4.2: a cancelled round records no vote");
|
||||
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity,
|
||||
"Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_heartbeat_mid_round_clean_reattach: PASS\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4).
|
||||
*
|
||||
* A first PreVote round LOSES (both peers deny — e.g. a brief split where the
|
||||
* primary was momentarily believed alive by a quorum). A lost round must NOT
|
||||
* permanently wedge failover: the liveness watchdog is re-armed. When a
|
||||
* SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to
|
||||
* the self-grant and a peer GRANT this time reaches quorum → WON → real election.
|
||||
*
|
||||
* This is the robustness guarantee that distinguishes 5.3 from a one-shot: the
|
||||
* machinery survives a loss and still elects when the failure is real.
|
||||
*/
|
||||
static void test_lost_round_rearms_then_later_failure_elects(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
||||
|
||||
g_rearm_calls = 0;
|
||||
ha_prevote_rearm_liveness_hook = stub_rearm;
|
||||
|
||||
/* ── Round 1: LOSES (both peers deny) ── */
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"Req 5.4: a lost round leaves the node CTRL_STANDBY");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
||||
atomic_load_explicit(&ha.term, memory_order_acquire),
|
||||
"Req 5.4: a lost round leaves the persistent term unchanged");
|
||||
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
||||
"Req 5.4: a lost round must re-arm the liveness watchdog so a later "
|
||||
"genuine failure can still trigger a fresh round");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||
"the lost round is over (in-progress cleared)");
|
||||
|
||||
/* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"Req 5.4: a later genuine failure must arm a FRESH round (in-progress)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
||||
"Req 5.4: the fresh round resets the tally to the self-grant "
|
||||
"(the previous loss did not poison the machinery)");
|
||||
|
||||
/* This time a peer grants → quorum → WON → real election. */
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
|
||||
|
||||
ha_prevote_rearm_liveness_hook = NULL;
|
||||
|
||||
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
||||
"Req 5.4: after an earlier lost round, a later genuine failure must still "
|
||||
"win a fresh PreVote and start a real election — failover is not wedged");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||
"the WON second round clears the in-progress flag");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role);
|
||||
}
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_prevote(void)
|
||||
|
|
@ -638,5 +876,9 @@ void test_ha_prevote(void)
|
|||
/* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */
|
||||
RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption);
|
||||
RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop);
|
||||
/* Task 5.3 — genuine-failover + re-attach scenarios */
|
||||
RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects);
|
||||
RUN_TEST(test_heartbeat_mid_round_clean_reattach);
|
||||
RUN_TEST(test_lost_round_rearms_then_later_failure_elects);
|
||||
UNITY_END();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue