lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(ha-prevote): genuine-failover, re-attach, and liveness-rearm scenarios (task 5.3)

Browse source 
Three scenario tests (zero production change): a stale-heartbeat standby GRANTS
(grant<->deny flip vs 5.2) and the won round advances to CTRL_ELECTING (PreVote
does not block legitimate failover); a heartbeat mid-round cancels and re-attaches
as standby with term unchanged; a lost round re-arms liveness and a later genuine
failure still wins (reusable, not one-shot).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-06-16 07:09:01 +00:00
parent 86ac7c4863
commit 22fa753fa6
2 changed files with 243 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

242
src/controller/testsuite/test_ha_prevote.c
View file

@ -623,6 +623,244 @@ static void test_frozen_rejoin_candidate_gets_no_majority_noop(void)
printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════════
* Task 5.3 — Genuine-failover and re-attach scenario tests.
*
* The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER
* legitimate failover, never BLOCK it. Three scenarios prove the suppressed
* cases (5.2) and the genuine cases (5.3) are distinguished by the SAME
* machinery — a dead primary elects a new one, a heartbeat mid-round cleanly
* re-attaches, and a lost round leaves failover re-armed and reusable.
*
* A. Genuine failover ELECTS (Req 5.1):
* 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE,
* so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate
* passes). Both angles:
* - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the
* PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2).
* - CANDIDATE side: feeding that grant to the tally reaches quorum → WON,
* and the candidate proceeds into the real-election sequence
* (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it.
*
* B. Re-attach on heartbeat mid-round (Req 4.2):
* a round is in progress; a primary heartbeat arrives → the round is
* cancelled, the node re-attaches CTRL_STANDBY, term unchanged,
* pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.)
*
* C. Liveness re-arm after loss (Req 5.4 robustness):
* a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine
* failure drives a FRESH round that WINS. Proves a lost round never
* permanently wedges failover.
*
* Requirements: 5.1, 5.4, 4.2.
* Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on
* heartbeat mid-round; Liveness re-arm after loss);
* "System Flows → PreVote round (candidate side)".
* ════════════════════════════════════════════════════════════════════════════ */
extern void ha_on_heartbeat_received(ha_state_t *ha);
/*
* Scenario A — GENUINE FAILOVER ELECTS (Req 5.1).
*
* Two angles, one scenario:
*
* A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone
* STALE (no primary connection, last_activity older than heartbeat_timeout)
* is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the
* 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and
* it flips the decision grant↔deny. No persistent state is mutated on grant
* (PreVote is side-effect-free): term/voted_for/role all unchanged.
*
* A2 (candidate side): the resumed candidate starts a round and receives that
* genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds
* into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING /
* CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote.
*/
static void test_genuine_failover_stale_peer_grants_and_elects(void)
{
/* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */
ha_state_t voter;
TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0));
atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&voter.term, 5ULL, memory_order_release);
voter.raft.term = 5ULL;
voter.raft.role = RAFT_FOLLOWER;
voter.raft.voted_for = -1;
voter.voted_for_term = 0;
voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */
/* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the
* voter would itself be eligible to start an election (Req 3.2). */
set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L);
uint64_t reply_term = 0;
int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1,
/*cand_term=*/6ULL, &reply_term);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
"Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the "
"PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
"Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term "
"(PreVote adopts no term)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
atomic_load_explicit(&voter.term, memory_order_acquire),
"Req 5.1: granting a PreVote must NOT change the voter's persistent term");
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for,
"Req 5.1: granting a PreVote must record NO vote (real vote still grantable)");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
(int)atomic_load_explicit(&voter.role, memory_order_acquire),
"Req 5.1: granting a PreVote must not change the voter's role");
ha_fini(&voter);
/* ── A2: the candidate wins the round on that grant → real election ── */
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */
make_three_node(&ha);
uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: the failover candidate's PreVote round is in progress");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"precondition: starting the round mutates no term (still side-effect-free)");
/* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new "
"primary — the candidate enters the real-election sequence, not stay STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"Req 5.1: the WON round must clear the in-progress flag (real election begun)");
ha_fini(&ha);
printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role);
}
/*
* Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2).
*
* A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when
* a primary heartbeat arrives — meaning the leader is in fact alive and this was
* NOT a genuine failure. The campaign must be abandoned cleanly: the round is
* cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY,
* and the persistent term is byte-for-byte UNCHANGED. This is the suppressed
* case: the same machinery that elects on a genuine death (scenario A) silently
* stands down the instant the primary proves alive.
*/
static void test_heartbeat_mid_round_clean_reattach(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
uint64_t raft_term_before = ha.raft.term;
int raft_voted_before = ha.raft.voted_for;
/* A failover round is in progress (self-grant, still STANDBY). */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: a PreVote round must be in progress before the heartbeat");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"precondition: the candidate is CTRL_STANDBY during the round");
/* A primary heartbeat arrives mid-round → the leader is alive → cancel. */
ha_on_heartbeat_received(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role,
"Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
"Req 4.2: a mid-round heartbeat cancel must NOT change the raft term");
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
"Req 4.2: a cancelled round records no vote");
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity,
"Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)");
ha_fini(&ha);
printf("test_heartbeat_mid_round_clean_reattach: PASS\n");
}
/*
* Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4).
*
* A first PreVote round LOSES (both peers deny — e.g. a brief split where the
* primary was momentarily believed alive by a quorum). A lost round must NOT
* permanently wedge failover: the liveness watchdog is re-armed. When a
* SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to
* the self-grant and a peer GRANT this time reaches quorum → WON → real election.
*
* This is the robustness guarantee that distinguishes 5.3 from a one-shot: the
* machinery survives a loss and still elects when the failure is real.
*/
static void test_lost_round_rearms_then_later_failure_elects(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
/* ── Round 1: LOSES (both peers deny) ── */
ha_start_prevote_async(&ha, NULL);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 5.4: a lost round leaves the node CTRL_STANDBY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 5.4: a lost round leaves the persistent term unchanged");
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
"Req 5.4: a lost round must re-arm the liveness watchdog so a later "
"genuine failure can still trigger a fresh round");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the lost round is over (in-progress cleared)");
/* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"Req 5.4: a later genuine failure must arm a FRESH round (in-progress)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"Req 5.4: the fresh round resets the tally to the self-grant "
"(the previous loss did not poison the machinery)");
/* This time a peer grants → quorum → WON → real election. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
ha_prevote_rearm_liveness_hook = NULL;
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 5.4: after an earlier lost round, a later genuine failure must still "
"win a fresh PreVote and start a real election — failover is not wedged");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the WON second round clears the in-progress flag");
ha_fini(&ha);
printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role);
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_prevote(void)
@ -638,5 +876,9 @@ void test_ha_prevote(void)
/* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */
RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption);
RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop);
/* Task 5.3 — genuine-failover + re-attach scenarios */
RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects);
RUN_TEST(test_heartbeat_mid_round_clean_reattach);
RUN_TEST(test_lost_round_rearms_then_later_failure_elects);
UNITY_END();
}