From 86ac7c48633798a9a17d457ea4b11784271bfc69 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 16 Jun 2026 06:54:27 +0000 Subject: [PATCH] test(ha-prevote): frozen-then-resumed no-op scenario (task 5.2) Two-angle scenario test (zero production change): a healthy PRIMARY and a fresh-heartbeat STANDBY both DENY a higher-prospective-term preVoteRequest without adopting the term or demoting; and a resumed standby whose round both peers deny gets no majority -> stays CTRL_STANDBY/FOLLOWER, term byte-for-byte unchanged, liveness re-armed. Regresses if the disruptive-server bug returns. Co-Authored-By: Claude Opus 4.8 (1M context) --- .kiro/specs/ha-prevote/tasks.md | 2 +- src/controller/testsuite/test_ha_prevote.c | 221 +++++++++++++++++++++ 2 files changed, 222 insertions(+), 1 deletion(-) diff --git a/.kiro/specs/ha-prevote/tasks.md b/.kiro/specs/ha-prevote/tasks.md index 958ce6b..8b3529c 100644 --- a/.kiro/specs/ha-prevote/tasks.md +++ b/.kiro/specs/ha-prevote/tasks.md @@ -72,7 +72,7 @@ - _Boundary: PreVote dispatch_ - _Depends: 1.1, 3.1, 4.1_ (shares the dispatch file with 3.1; owns only the opcode-to-handler routing, while 4.1 owns the handler) -- [ ] 5.2 Frozen-then-resumed no-op scenario test +- [x] 5.2 Frozen-then-resumed no-op scenario test - Add a 3-node scenario test: primary alive, a resumed standby starts a PreVote round; both peers deny (primary role + fresh heartbeat). - Observable: no real vote request is sent, the cluster term is unchanged across the event, and the primary stays primary (no demotion/truncate/self-restart). - _Requirements: 4.1, 4.3, 4.4_ diff --git a/src/controller/testsuite/test_ha_prevote.c b/src/controller/testsuite/test_ha_prevote.c index b2d0211..8143232 100644 --- a/src/controller/testsuite/test_ha_prevote.c +++ b/src/controller/testsuite/test_ha_prevote.c @@ -405,6 +405,224 @@ static void test_prevote_reply_over_real_transport_reaches_tally(void) printf("test_prevote_reply_over_real_transport_reaches_tally: PASS (role=%d)\n", role); } +/* ══════════════════════════════════════════════════════════════════════════ + * Task 5.2 — Frozen-then-resumed no-op scenario (the headline bug this whole + * feature exists to fix). + * + * 3-node cluster, primary alive. A previously-frozen standby resumes and starts + * a PreVote round; both peers DENY (one is the PRIMARY by role, one is a STANDBY + * with a fresh heartbeat). With both peers denying, the candidate gets only its + * self-grant (1 of 3, below quorum 2) → the round resolves LOST. + * + * Two complementary angles, tied together as one scenario: + * + * A. CANDIDATE side (the resumed standby): after the failover trigger starts a + * round, feeding it the two healthy-peer DENY replies resolves the round + * LOST. Therefore NO real election starts (role never leaves CTRL_STANDBY, + * raft.role stays FOLLOWER), the persistent term is byte-for-byte UNCHANGED + * (Req 4.4), and the candidate re-arms liveness so the next genuine + * heartbeat re-attaches it (Req 4.1 / 4.2-adjacent). + * + * B. VOTER side (the healthy primary / fresh standby): a node that is PRIMARY, + * or STANDBY with a fresh heartbeat, receiving a preVoteRequest from the + * resumed node DENIES it and — critically — does NOT adopt the candidate's + * (higher) prospective term, does NOT demote, truncate, or self-restart + * (Req 4.3). The voter's term/role are unchanged and the reply carries the + * voter's OWN term. + * + * Requirements: 4.1, 4.3, 4.4. + * Design: "Testing Strategy → Integration Tests → Frozen-then-resumed no-op". + * ════════════════════════════════════════════════════════════════════════════ */ + +/* ── Voter-side mirror of cluster.c OPCode_preVoteRequest (B) ────────────────── + * + * cluster.c is NOT linked into the unit-test binary, so we exercise the EXACT + * decision its dispatch composes: the raft eligibility gate (role-deny / + * fresh-heartbeat-deny, via the registered raft_cb_vote_eligible) plus the + * contract that the reply term is the voter's CURRENT term, UNCHANGED, on grant + * OR deny. (Freshness floor is irrelevant for the deny cases here — the role / + * heartbeat gate fires first — so the freshness check is omitted for clarity.) + * + * Returns 1 = grant, 0 = deny; *out_reply_term = voter's unchanged term. */ +extern int raft_on_pre_vote_request(raft_state_t *s, int from_idx, uint64_t cand_term); + +static int voter_pre_vote_decide(ha_state_t *ha, int from_idx, uint64_t cand_term, + uint64_t *out_reply_term) +{ + int granted = + (raft_on_pre_vote_request(&ha->raft, from_idx, cand_term) == RAFT_GRANT) + ? 1 : 0; + if (out_reply_term) + *out_reply_term = atomic_load_explicit(&ha->term, memory_order_acquire); + return granted; +} + +/* set_last_activity_age: ha->last_activity = "offset_ms ago" (whole seconds, + * matching ha_on_heartbeat_received). */ +static void set_last_activity_age(ha_state_t *ha, long offset_ms) +{ + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + ha->last_activity = now.tv_sec - offset_ms / 1000; +} + +/* + * Angle B — the healthy peers DENY without adopting term/role/voted-for. + * + * Two voters answer the resumed node's preVoteRequest for prospective term 6 + * (its inflated current+1): + * - voter P: CTRL_PRIMARY, term 5 → role gate denies; term stays 5, no demote. + * - voter S: CTRL_STANDBY, term 5, fresh heartbeat → heartbeat gate denies; + * term stays 5, stays STANDBY. + * Neither adopts term 6; neither mutates voted_for / role. Each reply carries + * the voter's OWN unchanged term (Req 4.3). + */ +static void test_frozen_rejoin_healthy_peers_deny_without_adoption(void) +{ + const uint64_t PROSPECTIVE_TERM = 6ULL; /* resumed node's inflated cur+1 */ + + /* ── Voter P: the healthy PRIMARY ── */ + ha_state_t prim; + TEST_ASSERT_EQUAL_INT(0, ha_init(&prim, &g_cfg, 0)); + atomic_store_explicit(&prim.role, (int32_t)CTRL_PRIMARY, memory_order_release); + atomic_store_explicit(&prim.term, 5ULL, memory_order_release); + prim.raft.term = 5ULL; + prim.raft.role = RAFT_LEADER; + prim.raft.voted_for = -1; + prim.voted_for_term = 0; + + uint64_t reply_term = 0; + int granted = voter_pre_vote_decide(&prim, /*from_idx=*/2, PROSPECTIVE_TERM, + &reply_term); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted, + "Req 4.3: the healthy PRIMARY must DENY a frozen-rejoin PreVote"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, + "Req 4.3: PRIMARY's denial reply carries its OWN term (no adoption of 6)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, + atomic_load_explicit(&prim.term, memory_order_acquire), + "Req 4.3: PRIMARY must NOT adopt the candidate's higher prospective term"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, prim.raft.term, + "Req 4.3: PRIMARY raft term must NOT change"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY, + (int)atomic_load_explicit(&prim.role, memory_order_acquire), + "Req 4.3: PRIMARY must NOT demote in response to a rejoining standby"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_LEADER, (int)prim.raft.role, + "Req 4.3: PRIMARY raft role must stay LEADER (no demotion/self-restart)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(-1, prim.raft.voted_for, + "Req 4.3: a PreVote must not record a vote on the PRIMARY"); + + /* ── Voter S: a healthy STANDBY with a fresh heartbeat ── */ + ha_state_t sb; + TEST_ASSERT_EQUAL_INT(0, ha_init(&sb, &g_cfg, 0)); + atomic_store_explicit(&sb.role, (int32_t)CTRL_STANDBY, memory_order_release); + atomic_store_explicit(&sb.term, 5ULL, memory_order_release); + sb.raft.term = 5ULL; + sb.raft.role = RAFT_FOLLOWER; + sb.raft.voted_for = -1; + sb.voted_for_term = 0; + sb.primary_ctrl_fd = 999; /* live primary connection */ + set_last_activity_age(&sb, 500L); /* fresh heartbeat (< heartbeat_timeout) */ + + reply_term = 0; + granted = voter_pre_vote_decide(&sb, /*from_idx=*/2, PROSPECTIVE_TERM, + &reply_term); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted, + "Req 4.3: a fresh-heartbeat STANDBY must DENY a frozen-rejoin PreVote"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term, + "Req 4.3: STANDBY's denial reply carries its OWN term (no adoption of 6)"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, + atomic_load_explicit(&sb.term, memory_order_acquire), + "Req 4.3: STANDBY must NOT adopt the candidate's higher prospective term"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, + (int)atomic_load_explicit(&sb.role, memory_order_acquire), + "Req 4.3: STANDBY role must NOT change"); + TEST_ASSERT_EQUAL_INT_MESSAGE(-1, sb.raft.voted_for, + "Req 4.3: a PreVote must not record a vote on the STANDBY"); + + ha_fini(&sb); + ha_fini(&prim); + printf("test_frozen_rejoin_healthy_peers_deny_without_adoption: PASS\n"); +} + +/* + * Angle A (+ the tie-together) — the resumed candidate gets NO majority. + * + * The resumed standby starts a PreVote round (self-grant, in-progress, still + * CTRL_STANDBY, term untouched). It then receives the two DENY replies its + * healthy peers send (angle B). With both peers denying, the tally has only the + * self-grant (1 of 3) → LOST. Therefore: + * - no real election is ever started: role never leaves CTRL_STANDBY and + * raft.role stays FOLLOWER (no truncate / electing-transition / self-restart + * on the candidate either); + * - the candidate's persistent term is byte-for-byte UNCHANGED (Req 4.4); + * - the liveness watchdog is re-armed so the next genuine heartbeat re-attaches + * the node (Req 4.1 / 4.2-adjacent). + * + * This is exactly the frozen-rejoin no-op: a transient unreachability event + * leaves the cluster term unmoved and the primary untouched. + */ +static void test_frozen_rejoin_candidate_gets_no_majority_noop(void) +{ + ha_state_t ha; + TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0)); + ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed standby) */ + make_three_node(&ha); + + uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire); + uint64_t raft_term_before = ha.raft.term; + int raft_voted_before = ha.raft.voted_for; + + g_rearm_calls = 0; + ha_prevote_rearm_liveness_hook = stub_rearm; + + /* The resumed node starts its round — side-effect-free precondition. */ + ha_start_prevote_async(&ha, NULL); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "precondition: the resumed standby's PreVote round must be in progress"); + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted, + "precondition: only the self-grant is recorded at round start"); + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "precondition: starting the round leaves the node CTRL_STANDBY"); + + /* Both healthy peers DENY (PRIMARY by role; STANDBY with fresh heartbeat). */ + pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny — primary peer*/); + pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny — fresh standby peer*/); + + ha_prevote_rearm_liveness_hook = NULL; + + /* No real vote request was sent: the round resolved LOST, so the candidate + * never entered the real-election sequence. Role stayed CTRL_STANDBY and + * the raft role never left FOLLOWER. */ + TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY, + (int)atomic_load_explicit(&ha.role, memory_order_acquire), + "Req 4.1: no PreVote majority → no real election → node stays CTRL_STANDBY"); + TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role, + "Req 4.1: with no majority the candidate's raft role must stay FOLLOWER " + "(no real election started, no truncate/electing-transition)"); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "the LOST round must be cleared (round over)"); + + /* Cluster term unchanged across the whole event (Req 4.4). */ + TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before, + atomic_load_explicit(&ha.term, memory_order_acquire), + "Req 4.4: the persistent term must be UNCHANGED across the frozen-rejoin event"); + TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term, + "Req 4.4: the raft term must be UNCHANGED across the frozen-rejoin event"); + TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for, + "Req 4.4: no vote was recorded — the prospective term was never persisted"); + + /* Re-armed so the next genuine heartbeat re-attaches the node (Req 4.1/4.2). */ + TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls, + "Req 4.1: a lost frozen-rejoin round must re-arm liveness so the node " + "re-attaches on the next primary heartbeat"); + + ha_fini(&ha); + printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n"); +} + /* ── Suite entry point ──────────────────────────────────────────────────── */ void test_ha_prevote(void) @@ -417,5 +635,8 @@ void test_ha_prevote(void) RUN_TEST(test_prevote_reusable_after_loss); RUN_TEST(test_prevote_pending_is_noop); RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally); + /* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */ + RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption); + RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop); UNITY_END(); }