feat(ha-prevote): controller candidate-side PreVote orchestration (task 4.1)
ha.c: raft_cb_send_pre_vote_request (emits preVoteRequest), ha_start_prevote_async (side-effect-free round + bounded round timeout), pre_vote_reply_dispatch (raft_on_pre_vote_reply + branch), ha_enter_real_election helper. WON -> truncate + CTRL_ELECTING + ha_start_election_async (unchanged real-election path); LOST/ timeout -> stay CTRL_STANDBY + re-arm liveness + log abandonment; PENDING -> no-op. Round inert until task 4.2 wires triggers; absolute peer-index keyed tally. 6 new test_ha_prevote tests via reply hook; daemon + suite link clean. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
6f95e4cdfc
commit
7e19bf6110
6 changed files with 801 additions and 1 deletions
|
|
@ -49,6 +49,7 @@ extern void test_ha_repl_push(void);
|
|||
extern void test_ha_connection_reuse(void);
|
||||
extern void test_ha_rdhup_integration(void);
|
||||
extern void test_ha_liveness(void);
|
||||
extern void test_ha_prevote(void);
|
||||
extern void test_commit_engine(void);
|
||||
extern void test_commit_barrier_rollout(void);
|
||||
extern void test_commit_barrier_reply(void);
|
||||
|
|
@ -174,6 +175,7 @@ int main(int argc, char* argv[])
|
|||
TEST(test_ha_connection_reuse);
|
||||
TEST(test_ha_rdhup_integration);
|
||||
TEST(test_ha_liveness);
|
||||
TEST(test_ha_prevote);
|
||||
TEST(test_commit_engine);
|
||||
TEST(test_commit_barrier_rollout);
|
||||
TEST(test_commit_barrier_reply);
|
||||
|
|
|
|||
338
src/controller/testsuite/test_ha_prevote.c
Normal file
338
src/controller/testsuite/test_ha_prevote.c
Normal file
|
|
@ -0,0 +1,338 @@
|
|||
/*
|
||||
* test_ha_prevote.c — Unit tests for ha-prevote task 4.1:
|
||||
* controller candidate-side PreVote orchestration in ha.c.
|
||||
*
|
||||
* Scope (task 4.1 only):
|
||||
* - ha_start_prevote_async: starts a side-effect-free round (no persist, no
|
||||
* role change, stays CTRL_STANDBY), arms a round timeout.
|
||||
* - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches —
|
||||
* WON → real-election sequence (truncate + CTRL_ELECTING + start election)
|
||||
* LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment
|
||||
* PENDING → no-op
|
||||
* - round-timeout firing = LOST branch.
|
||||
*
|
||||
* These tests drive the reply path through the XCE_TESTING test seam
|
||||
* (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the
|
||||
* existing vote-reply hook pattern. ha.c is unit-test-linked, so the real
|
||||
* orchestration logic is exercised directly.
|
||||
*
|
||||
* Requirements: 1.1, 1.4, 1.5, 5.4, 7.3
|
||||
* Boundary: PreVote orchestration (ha.c)
|
||||
*
|
||||
* TDD: written first (RED) to specify the exact behaviour before implementation.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "ha.h"
|
||||
#include "netio.h"
|
||||
#include "raft/raft.h"
|
||||
#include "xce/cluster_def.h"
|
||||
|
||||
/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */
|
||||
|
||||
/*
|
||||
* ha_start_prevote_async: start a PreVote round. No persist, no role change.
|
||||
* c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op,
|
||||
* raft_begin_pre_vote's send callback returns early in test mode).
|
||||
*/
|
||||
extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c);
|
||||
|
||||
/*
|
||||
* pre_vote_reply_dispatch_for_test: test entry point that calls the real
|
||||
* pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without
|
||||
* a live iomgr. Mirrors the production reply path.
|
||||
*/
|
||||
extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
|
||||
int peer_idx, int granted);
|
||||
|
||||
/*
|
||||
* ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task
|
||||
* firing before a decision (drives the LOST branch).
|
||||
*/
|
||||
extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c);
|
||||
|
||||
/* Liveness arm-count hook so tests can prove the watchdog was re-armed on a
|
||||
* LOST outcome without a live delaymgr. */
|
||||
extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);
|
||||
|
||||
/* ── Test config ─────────────────────────────────────────────────────────── */
|
||||
|
||||
static ha_config_t g_cfg = {
|
||||
.heartbeat_interval_ms = 3000,
|
||||
.heartbeat_timeout_ms = 6000,
|
||||
.election_timeout_ms = 500,
|
||||
.probe_connect_timeout_ms = 50,
|
||||
.max_promotion_retries = 3,
|
||||
.election_jitter_max_ms = 0,
|
||||
.election_priority_step_ms = 500,
|
||||
};
|
||||
|
||||
/* Re-arm tracking */
|
||||
static int g_rearm_calls = 0;
|
||||
static int stub_rearm(ha_state_t *ha, cluster_t *c)
|
||||
{
|
||||
(void)ha; (void)c;
|
||||
g_rearm_calls++;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer
|
||||
* grant to win (quorum = 2; self-grant = 1). raft_init in ha_init only sets a
|
||||
* 1-node placeholder, so we widen the topology directly on the public struct.
|
||||
*/
|
||||
static void make_three_node(ha_state_t *ha)
|
||||
{
|
||||
ha->raft.n_peers = 3;
|
||||
ha->raft.self_idx = ha->my_priority;
|
||||
ha->raft.role = RAFT_FOLLOWER;
|
||||
ha->n_peers = 3;
|
||||
}
|
||||
|
||||
/* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */
|
||||
|
||||
/*
|
||||
* ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment
|
||||
* the persistent term, and must not flip the raft role to CANDIDATE. It sets
|
||||
* the in-progress flag and self-grant in raft_state_t only.
|
||||
*/
|
||||
static void test_prevote_start_no_mutation(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha); /* role = CTRL_STANDBY */
|
||||
make_three_node(&ha);
|
||||
|
||||
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
||||
uint64_t raft_term_before = ha.raft.term;
|
||||
int raft_voted_before = ha.raft.voted_for;
|
||||
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"Req 1.1: role must stay CTRL_STANDBY during the PreVote round");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
||||
atomic_load_explicit(&ha.term, memory_order_acquire),
|
||||
"Req 1.2: persistent term must not change during the round");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
|
||||
"Req 1.2: raft term must not change during the round");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
|
||||
"Req 1.2: raft voted_for must not change during the round");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role,
|
||||
"Req 1.2: raft role must stay FOLLOWER during the round");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"round must be marked in-progress");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
||||
"self-grant must be recorded (1)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_prevote_start_no_mutation: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */
|
||||
|
||||
/*
|
||||
* 3-node: after starting the round, feeding one peer GRANT reaches quorum (2).
|
||||
* The WON branch must run the real-election sequence: role becomes
|
||||
* CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election).
|
||||
*/
|
||||
static void test_prevote_majority_grant_starts_real_election(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
|
||||
/* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
|
||||
|
||||
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
||||
/* WON branch enters the real-election sequence: with c==NULL the election
|
||||
* cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY —
|
||||
* the candidate transitioned to CTRL_ELECTING (real-election path). */
|
||||
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
||||
"Req 1.5: a PreVote WIN must enter the real-election sequence "
|
||||
"(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||
"WON must clear the in-progress flag (round over)");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role);
|
||||
}
|
||||
|
||||
/* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */
|
||||
|
||||
/*
|
||||
* 3-node: both peers DENY → no quorum → LOST. The node must stay CTRL_STANDBY,
|
||||
* leave the persistent term unchanged (Req 1.4), and re-arm the liveness
|
||||
* watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still
|
||||
* trigger a new round).
|
||||
*/
|
||||
static void test_prevote_no_majority_stays_standby_and_rearms(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
||||
|
||||
g_rearm_calls = 0;
|
||||
ha_prevote_rearm_liveness_hook = stub_rearm;
|
||||
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
|
||||
/* Both non-self peers deny → all replied, no quorum → LOST. */
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
|
||||
|
||||
ha_prevote_rearm_liveness_hook = NULL;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"Req 1.4: no PreVote majority must keep the node CTRL_STANDBY");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
||||
atomic_load_explicit(&ha.term, memory_order_acquire),
|
||||
"Req 1.4: no PreVote majority must leave the persistent term unchanged");
|
||||
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
||||
"Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a "
|
||||
"later genuine failure still triggers a new round");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */
|
||||
|
||||
/*
|
||||
* If the round times out before a decision, it is treated as a loss: stay
|
||||
* CTRL_STANDBY, term unchanged, re-arm liveness.
|
||||
*/
|
||||
static void test_prevote_round_timeout_is_loss(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
||||
|
||||
g_rearm_calls = 0;
|
||||
ha_prevote_rearm_liveness_hook = stub_rearm;
|
||||
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
|
||||
/* No replies arrive; the round timeout fires. */
|
||||
ha_prevote_round_timeout_fire_for_test(&ha, NULL);
|
||||
|
||||
ha_prevote_rearm_liveness_hook = NULL;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)");
|
||||
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
||||
atomic_load_explicit(&ha.term, memory_order_acquire),
|
||||
"Req 5.4: a round timeout must leave the persistent term unchanged");
|
||||
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
||||
"Req 5.4: a timed-out round must re-arm the liveness watchdog");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||
"round timeout must clear the in-progress flag");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_prevote_round_timeout_is_loss: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */
|
||||
|
||||
/*
|
||||
* Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back
|
||||
* in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round
|
||||
* (self-grant + in-progress), which a peer grant can again win — proving the
|
||||
* candidate machinery is reusable, not one-shot.
|
||||
*/
|
||||
static void test_prevote_reusable_after_loss(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
make_three_node(&ha);
|
||||
|
||||
ha_prevote_rearm_liveness_hook = stub_rearm;
|
||||
|
||||
/* Round 1: lost. */
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0);
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0);
|
||||
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
||||
|
||||
/* Round 2: a fresh round arms cleanly (self-grant + in-progress). */
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"a new round after a loss must re-arm (in-progress)");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
||||
"a new round must reset the tally to the self-grant");
|
||||
|
||||
/* Round 2 wins with a peer grant. */
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1);
|
||||
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
||||
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
||||
"a later genuine failure (new round) must still be able to win and "
|
||||
"start a real election");
|
||||
|
||||
ha_prevote_rearm_liveness_hook = NULL;
|
||||
ha_fini(&ha);
|
||||
printf("test_prevote_reusable_after_loss: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */
|
||||
|
||||
static void test_prevote_pending_is_noop(void)
|
||||
{
|
||||
ha_state_t ha;
|
||||
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
||||
ha_do_demote(&ha);
|
||||
/* 5-node so a single grant is below quorum (3) → PENDING. */
|
||||
ha.raft.n_peers = 5;
|
||||
ha.raft.self_idx = 0;
|
||||
ha.raft.role = RAFT_FOLLOWER;
|
||||
ha.n_peers = 5;
|
||||
|
||||
ha_start_prevote_async(&ha, NULL);
|
||||
|
||||
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/);
|
||||
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
||||
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||
"a PENDING (sub-quorum) reply must not change the role");
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||
"a PENDING reply must keep the round in progress");
|
||||
|
||||
ha_fini(&ha);
|
||||
printf("test_prevote_pending_is_noop: PASS\n");
|
||||
}
|
||||
|
||||
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
||||
|
||||
void test_ha_prevote(void)
|
||||
{
|
||||
UNITY_BEGIN();
|
||||
RUN_TEST(test_prevote_start_no_mutation);
|
||||
RUN_TEST(test_prevote_majority_grant_starts_real_election);
|
||||
RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms);
|
||||
RUN_TEST(test_prevote_round_timeout_is_loss);
|
||||
RUN_TEST(test_prevote_reusable_after_loss);
|
||||
RUN_TEST(test_prevote_pending_is_noop);
|
||||
UNITY_END();
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue