forked from lei.chen/myxce
Rongsong Shen
004a40bb3b
1.1: WalCatchupEntry gains term@4/prevLogIndex@5/prevLogTerm@6 (append-only). 1.2: RequestVote/PreVote gain lastLogTerm@4/lastLogIndex@5; cluster.c vote callers fixed to the extended library arity (RESTORES xce_controller + controller_unit_test compile+link, broken by the raft-commit-safety signature extension); ha.c emits the candidate's last-log tuple on send. 5 minimal stub log/commit callbacks wired into s_real_cbs so raft_init's new non-NULL requirement is satisfied — tasks 2.x replace them with real bodies; the logOk freshness gate is inert until the term seam (1.3) + on_get_entry_term (2.2). Pre-existing test_repl_unit.c wal_catchup -EAGAIN failures + ASAN leaks are an overlayfs /tmp fsync env artifact (bug-850), not from this task (untouched paths; append-only schema cannot affect the @3 entries-list decode). Build/baseline isolation is blocked by a c-capnproto flake-hash drift — see Implementation Notes.
890 lines
42 KiB
C
890 lines
42 KiB
C
/*
|
|
* test_ha_prevote.c — Unit tests for ha-prevote task 4.1:
|
|
* controller candidate-side PreVote orchestration in ha.c.
|
|
*
|
|
* Scope (task 4.1 only):
|
|
* - ha_start_prevote_async: starts a side-effect-free round (no persist, no
|
|
* role change, stays CTRL_STANDBY), arms a round timeout.
|
|
* - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches —
|
|
* WON → real-election sequence (truncate + CTRL_ELECTING + start election)
|
|
* LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment
|
|
* PENDING → no-op
|
|
* - round-timeout firing = LOST branch.
|
|
*
|
|
* These tests drive the reply path through the XCE_TESTING test seam
|
|
* (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the
|
|
* existing vote-reply hook pattern. ha.c is unit-test-linked, so the real
|
|
* orchestration logic is exercised directly.
|
|
*
|
|
* Requirements: 1.1, 1.4, 1.5, 5.4, 7.3
|
|
* Boundary: PreVote orchestration (ha.c)
|
|
*
|
|
* TDD: written first (RED) to specify the exact behaviour before implementation.
|
|
*/
|
|
#define _GNU_SOURCE
|
|
#include <stdatomic.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <time.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "ha.h"
|
|
#include "netio.h"
|
|
#include "raft/raft.h"
|
|
#include "xce/cluster_def.h"
|
|
|
|
/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */
|
|
|
|
/*
|
|
* ha_start_prevote_async: start a PreVote round. No persist, no role change.
|
|
* c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op,
|
|
* raft_begin_pre_vote's send callback returns early in test mode).
|
|
*/
|
|
extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c);
|
|
|
|
/*
|
|
* pre_vote_reply_dispatch_for_test: test entry point that calls the real
|
|
* pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without
|
|
* a live iomgr. Mirrors the production reply path.
|
|
*/
|
|
extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
|
|
int peer_idx, int granted);
|
|
|
|
/*
|
|
* ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task
|
|
* firing before a decision (drives the LOST branch).
|
|
*/
|
|
extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c);
|
|
|
|
/* Liveness arm-count hook so tests can prove the watchdog was re-armed on a
|
|
* LOST outcome without a live delaymgr. */
|
|
extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);
|
|
|
|
/* ── Task 5.1 transport-route seams ──────────────────────────────────────── */
|
|
|
|
/*
|
|
* pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL
|
|
* peer_ctrl_msg_handler decode/route path while CTRL_STANDBY + round in
|
|
* progress (the production condition). Proves the reply is NOT dropped and
|
|
* reaches pre_vote_reply_dispatch — end-to-end, not via the test hook.
|
|
*/
|
|
extern int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt);
|
|
|
|
/* vote_reply_build_pkt_for_test: encode a RequestVoteReply capnp packet (the
|
|
* preVoteReply reuses the identical payload / union arm). */
|
|
extern io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
|
|
uint64_t peer_tag,
|
|
uint64_t peer_lsn);
|
|
|
|
/* vote_ep_ctx_build_for_test: allocate a vote_ep_ctx_t and wire ep->data with
|
|
* the ABSOLUTE peer_idx (the index pre_vote_reply_dispatch expects). */
|
|
extern endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
|
|
ha_state_t *ha,
|
|
ha_elect_ctx_t *ctx,
|
|
int peer_idx,
|
|
uint64_t my_term,
|
|
uint64_t my_tag,
|
|
uint64_t my_lsn,
|
|
const char *my_name);
|
|
|
|
/* ── Test config ─────────────────────────────────────────────────────────── */
|
|
|
|
static ha_config_t g_cfg = {
|
|
.heartbeat_interval_ms = 3000,
|
|
.heartbeat_timeout_ms = 6000,
|
|
.election_timeout_ms = 500,
|
|
.probe_connect_timeout_ms = 50,
|
|
.max_promotion_retries = 3,
|
|
.election_jitter_max_ms = 0,
|
|
.election_priority_step_ms = 500,
|
|
};
|
|
|
|
/* Re-arm tracking */
|
|
static int g_rearm_calls = 0;
|
|
static int stub_rearm(ha_state_t *ha, cluster_t *c)
|
|
{
|
|
(void)ha; (void)c;
|
|
g_rearm_calls++;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer
|
|
* grant to win (quorum = 2; self-grant = 1). raft_init in ha_init only sets a
|
|
* 1-node placeholder, so we widen the topology directly on the public struct.
|
|
*/
|
|
static void make_three_node(ha_state_t *ha)
|
|
{
|
|
ha->raft.n_peers = 3;
|
|
ha->raft.self_idx = ha->my_priority;
|
|
ha->raft.role = RAFT_FOLLOWER;
|
|
ha->n_peers = 3;
|
|
}
|
|
|
|
/* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */
|
|
|
|
/*
|
|
* ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment
|
|
* the persistent term, and must not flip the raft role to CANDIDATE. It sets
|
|
* the in-progress flag and self-grant in raft_state_t only.
|
|
*/
|
|
static void test_prevote_start_no_mutation(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha); /* role = CTRL_STANDBY */
|
|
make_three_node(&ha);
|
|
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
uint64_t raft_term_before = ha.raft.term;
|
|
int raft_voted_before = ha.raft.voted_for;
|
|
|
|
ha_start_prevote_async(&ha, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"Req 1.1: role must stay CTRL_STANDBY during the PreVote round");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"Req 1.2: persistent term must not change during the round");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
|
|
"Req 1.2: raft term must not change during the round");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
|
|
"Req 1.2: raft voted_for must not change during the round");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role,
|
|
"Req 1.2: raft role must stay FOLLOWER during the round");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"round must be marked in-progress");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
|
"self-grant must be recorded (1)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_prevote_start_no_mutation: PASS\n");
|
|
}
|
|
|
|
/* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */
|
|
|
|
/*
|
|
* 3-node: after starting the round, feeding one peer GRANT reaches quorum (2).
|
|
* The WON branch must run the real-election sequence: role becomes
|
|
* CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election).
|
|
*/
|
|
static void test_prevote_majority_grant_starts_real_election(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
make_three_node(&ha);
|
|
|
|
ha_start_prevote_async(&ha, NULL);
|
|
|
|
/* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
|
|
|
|
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
/* WON branch enters the real-election sequence: with c==NULL the election
|
|
* cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY —
|
|
* the candidate transitioned to CTRL_ELECTING (real-election path). */
|
|
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
|
"Req 1.5: a PreVote WIN must enter the real-election sequence "
|
|
"(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"WON must clear the in-progress flag (round over)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role);
|
|
}
|
|
|
|
/* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */
|
|
|
|
/*
|
|
* 3-node: both peers DENY → no quorum → LOST. The node must stay CTRL_STANDBY,
|
|
* leave the persistent term unchanged (Req 1.4), and re-arm the liveness
|
|
* watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still
|
|
* trigger a new round).
|
|
*/
|
|
static void test_prevote_no_majority_stays_standby_and_rearms(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
make_three_node(&ha);
|
|
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
g_rearm_calls = 0;
|
|
ha_prevote_rearm_liveness_hook = stub_rearm;
|
|
|
|
ha_start_prevote_async(&ha, NULL);
|
|
|
|
/* Both non-self peers deny → all replied, no quorum → LOST. */
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
|
|
|
|
ha_prevote_rearm_liveness_hook = NULL;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"Req 1.4: no PreVote majority must keep the node CTRL_STANDBY");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"Req 1.4: no PreVote majority must leave the persistent term unchanged");
|
|
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
|
"Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a "
|
|
"later genuine failure still triggers a new round");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n");
|
|
}
|
|
|
|
/* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */
|
|
|
|
/*
|
|
* If the round times out before a decision, it is treated as a loss: stay
|
|
* CTRL_STANDBY, term unchanged, re-arm liveness.
|
|
*/
|
|
static void test_prevote_round_timeout_is_loss(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
make_three_node(&ha);
|
|
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
g_rearm_calls = 0;
|
|
ha_prevote_rearm_liveness_hook = stub_rearm;
|
|
|
|
ha_start_prevote_async(&ha, NULL);
|
|
|
|
/* No replies arrive; the round timeout fires. */
|
|
ha_prevote_round_timeout_fire_for_test(&ha, NULL);
|
|
|
|
ha_prevote_rearm_liveness_hook = NULL;
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"Req 5.4: a round timeout must leave the persistent term unchanged");
|
|
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
|
"Req 5.4: a timed-out round must re-arm the liveness watchdog");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"round timeout must clear the in-progress flag");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_prevote_round_timeout_is_loss: PASS\n");
|
|
}
|
|
|
|
/* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */
|
|
|
|
/*
|
|
* Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back
|
|
* in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round
|
|
* (self-grant + in-progress), which a peer grant can again win — proving the
|
|
* candidate machinery is reusable, not one-shot.
|
|
*/
|
|
static void test_prevote_reusable_after_loss(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
make_three_node(&ha);
|
|
|
|
ha_prevote_rearm_liveness_hook = stub_rearm;
|
|
|
|
/* Round 1: lost. */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0);
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0);
|
|
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
|
|
|
|
/* Round 2: a fresh round arms cleanly (self-grant + in-progress). */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"a new round after a loss must re-arm (in-progress)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
|
"a new round must reset the tally to the self-grant");
|
|
|
|
/* Round 2 wins with a peer grant. */
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1);
|
|
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
|
"a later genuine failure (new round) must still be able to win and "
|
|
"start a real election");
|
|
|
|
ha_prevote_rearm_liveness_hook = NULL;
|
|
ha_fini(&ha);
|
|
printf("test_prevote_reusable_after_loss: PASS\n");
|
|
}
|
|
|
|
/* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */
|
|
|
|
static void test_prevote_pending_is_noop(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
/* 5-node so a single grant is below quorum (3) → PENDING. */
|
|
ha.raft.n_peers = 5;
|
|
ha.raft.self_idx = 0;
|
|
ha.raft.role = RAFT_FOLLOWER;
|
|
ha.n_peers = 5;
|
|
|
|
ha_start_prevote_async(&ha, NULL);
|
|
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"a PENDING (sub-quorum) reply must not change the role");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"a PENDING reply must keep the round in progress");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_prevote_pending_is_noop: PASS\n");
|
|
}
|
|
|
|
/* ── Test 7: a preVoteReply over the REAL transport reaches the tally (5.1) ─ */
|
|
|
|
/*
|
|
* Task 5.1 (Req 7.1): a preVoteReply arriving on the candidate's outbound
|
|
* PreVote endpoint — while the candidate is still CTRL_STANDBY with a round in
|
|
* progress — must be ROUTED to the PreVote tally by peer_ctrl_msg_handler (NOT
|
|
* dropped because the role is STANDBY), and drive the round to a decision
|
|
* end-to-end. This exercises the production decode/route path, not the
|
|
* pre_vote_reply_dispatch_for_test hook.
|
|
*/
|
|
static void test_prevote_reply_over_real_transport_reaches_tally(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha); /* role = CTRL_STANDBY */
|
|
make_three_node(&ha);
|
|
|
|
/* Start a real round: self-grant + in-progress, role stays CTRL_STANDBY. */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"precondition: role must be CTRL_STANDBY during the round");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"precondition: round must be in progress");
|
|
|
|
/* Build an endpoint carrying ABSOLUTE peer_idx = 1 (the index the tally
|
|
* dedups by), exactly as ha_send_pre_vote_request sets it. */
|
|
endpoint_t ep;
|
|
memset(&ep, 0, sizeof(ep));
|
|
vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1,
|
|
/*my_term=*/1, /*my_tag=*/0, /*my_lsn=*/0, "test");
|
|
|
|
/* Encode a GRANT preVoteReply (same wire payload as a real-vote reply). */
|
|
io_packet_t *pkt = vote_reply_build_pkt_for_test(/*term=*/1, /*granted=*/1,
|
|
/*tag=*/0, /*lsn=*/0);
|
|
TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "vote_reply_build_pkt_for_test must succeed");
|
|
|
|
/* Route it through the REAL handler (peer_ctrl_msg_handler) while STANDBY. */
|
|
int rc = pre_vote_reply_msg_handler_for_test(&ep, pkt);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handler must return 0");
|
|
|
|
/* Peer grant → quorum (2 of 3) → WON: the reply was NOT dropped; it reached
|
|
* the tally and drove the round to a decision (real-election sequence). */
|
|
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
|
"Req 7.1: a preVoteReply over the real transport must reach the tally "
|
|
"and drive the round to a decision (left CTRL_STANDBY)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"the decided round must clear the in-progress flag");
|
|
|
|
free(ep.data); /* vctx allocated by vote_ep_ctx_build_for_test */
|
|
ha_fini(&ha);
|
|
printf("test_prevote_reply_over_real_transport_reaches_tally: PASS (role=%d)\n", role);
|
|
}
|
|
|
|
/* ══════════════════════════════════════════════════════════════════════════
|
|
* Task 5.2 — Frozen-then-resumed no-op scenario (the headline bug this whole
|
|
* feature exists to fix).
|
|
*
|
|
* 3-node cluster, primary alive. A previously-frozen standby resumes and starts
|
|
* a PreVote round; both peers DENY (one is the PRIMARY by role, one is a STANDBY
|
|
* with a fresh heartbeat). With both peers denying, the candidate gets only its
|
|
* self-grant (1 of 3, below quorum 2) → the round resolves LOST.
|
|
*
|
|
* Two complementary angles, tied together as one scenario:
|
|
*
|
|
* A. CANDIDATE side (the resumed standby): after the failover trigger starts a
|
|
* round, feeding it the two healthy-peer DENY replies resolves the round
|
|
* LOST. Therefore NO real election starts (role never leaves CTRL_STANDBY,
|
|
* raft.role stays FOLLOWER), the persistent term is byte-for-byte UNCHANGED
|
|
* (Req 4.4), and the candidate re-arms liveness so the next genuine
|
|
* heartbeat re-attaches it (Req 4.1 / 4.2-adjacent).
|
|
*
|
|
* B. VOTER side (the healthy primary / fresh standby): a node that is PRIMARY,
|
|
* or STANDBY with a fresh heartbeat, receiving a preVoteRequest from the
|
|
* resumed node DENIES it and — critically — does NOT adopt the candidate's
|
|
* (higher) prospective term, does NOT demote, truncate, or self-restart
|
|
* (Req 4.3). The voter's term/role are unchanged and the reply carries the
|
|
* voter's OWN term.
|
|
*
|
|
* Requirements: 4.1, 4.3, 4.4.
|
|
* Design: "Testing Strategy → Integration Tests → Frozen-then-resumed no-op".
|
|
* ════════════════════════════════════════════════════════════════════════════ */
|
|
|
|
/* ── Voter-side mirror of cluster.c OPCode_preVoteRequest (B) ──────────────────
|
|
*
|
|
* cluster.c is NOT linked into the unit-test binary, so we exercise the EXACT
|
|
* decision its dispatch composes: the raft eligibility gate (role-deny /
|
|
* fresh-heartbeat-deny, via the registered raft_cb_vote_eligible) plus the
|
|
* contract that the reply term is the voter's CURRENT term, UNCHANGED, on grant
|
|
* OR deny. (Freshness floor is irrelevant for the deny cases here — the role /
|
|
* heartbeat gate fires first — so the freshness check is omitted for clarity.)
|
|
*
|
|
* Returns 1 = grant, 0 = deny; *out_reply_term = voter's unchanged term. */
|
|
/* raft_on_pre_vote_request is declared in raft.h (now carries the candidate
|
|
* (lastLogTerm, lastLogIndex) tuple — ha-term-safety task 1.2); no local extern. */
|
|
|
|
static int voter_pre_vote_decide(ha_state_t *ha, int from_idx, uint64_t cand_term,
|
|
uint64_t *out_reply_term)
|
|
{
|
|
/* Pass (0,0) for the candidate last-log tuple: this PreVote test exercises
|
|
* the role/heartbeat eligibility gate, not the logOk freshness path; with
|
|
* the on_get_entry_term stub reading local last-log term as 0, logOk grants
|
|
* so existing gate behavior is preserved (ha-term-safety task 1.2). */
|
|
int granted =
|
|
(raft_on_pre_vote_request(&ha->raft, from_idx, cand_term, 0, 0)
|
|
== RAFT_GRANT)
|
|
? 1 : 0;
|
|
if (out_reply_term)
|
|
*out_reply_term = atomic_load_explicit(&ha->term, memory_order_acquire);
|
|
return granted;
|
|
}
|
|
|
|
/* set_last_activity_age: ha->last_activity = "offset_ms ago" (whole seconds,
|
|
* matching ha_on_heartbeat_received). */
|
|
static void set_last_activity_age(ha_state_t *ha, long offset_ms)
|
|
{
|
|
struct timespec now;
|
|
clock_gettime(CLOCK_MONOTONIC, &now);
|
|
ha->last_activity = now.tv_sec - offset_ms / 1000;
|
|
}
|
|
|
|
/*
|
|
* Angle B — the healthy peers DENY without adopting term/role/voted-for.
|
|
*
|
|
* Two voters answer the resumed node's preVoteRequest for prospective term 6
|
|
* (its inflated current+1):
|
|
* - voter P: CTRL_PRIMARY, term 5 → role gate denies; term stays 5, no demote.
|
|
* - voter S: CTRL_STANDBY, term 5, fresh heartbeat → heartbeat gate denies;
|
|
* term stays 5, stays STANDBY.
|
|
* Neither adopts term 6; neither mutates voted_for / role. Each reply carries
|
|
* the voter's OWN unchanged term (Req 4.3).
|
|
*/
|
|
static void test_frozen_rejoin_healthy_peers_deny_without_adoption(void)
|
|
{
|
|
const uint64_t PROSPECTIVE_TERM = 6ULL; /* resumed node's inflated cur+1 */
|
|
|
|
/* ── Voter P: the healthy PRIMARY ── */
|
|
ha_state_t prim;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&prim, &g_cfg, 0));
|
|
atomic_store_explicit(&prim.role, (int32_t)CTRL_PRIMARY, memory_order_release);
|
|
atomic_store_explicit(&prim.term, 5ULL, memory_order_release);
|
|
prim.raft.term = 5ULL;
|
|
prim.raft.role = RAFT_LEADER;
|
|
prim.raft.voted_for = -1;
|
|
prim.voted_for_term = 0;
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = voter_pre_vote_decide(&prim, /*from_idx=*/2, PROSPECTIVE_TERM,
|
|
&reply_term);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"Req 4.3: the healthy PRIMARY must DENY a frozen-rejoin PreVote");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
|
|
"Req 4.3: PRIMARY's denial reply carries its OWN term (no adoption of 6)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
|
|
atomic_load_explicit(&prim.term, memory_order_acquire),
|
|
"Req 4.3: PRIMARY must NOT adopt the candidate's higher prospective term");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, prim.raft.term,
|
|
"Req 4.3: PRIMARY raft term must NOT change");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY,
|
|
(int)atomic_load_explicit(&prim.role, memory_order_acquire),
|
|
"Req 4.3: PRIMARY must NOT demote in response to a rejoining standby");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_LEADER, (int)prim.raft.role,
|
|
"Req 4.3: PRIMARY raft role must stay LEADER (no demotion/self-restart)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, prim.raft.voted_for,
|
|
"Req 4.3: a PreVote must not record a vote on the PRIMARY");
|
|
|
|
/* ── Voter S: a healthy STANDBY with a fresh heartbeat ── */
|
|
ha_state_t sb;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&sb, &g_cfg, 0));
|
|
atomic_store_explicit(&sb.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&sb.term, 5ULL, memory_order_release);
|
|
sb.raft.term = 5ULL;
|
|
sb.raft.role = RAFT_FOLLOWER;
|
|
sb.raft.voted_for = -1;
|
|
sb.voted_for_term = 0;
|
|
sb.primary_ctrl_fd = 999; /* live primary connection */
|
|
set_last_activity_age(&sb, 500L); /* fresh heartbeat (< heartbeat_timeout) */
|
|
|
|
reply_term = 0;
|
|
granted = voter_pre_vote_decide(&sb, /*from_idx=*/2, PROSPECTIVE_TERM,
|
|
&reply_term);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
|
|
"Req 4.3: a fresh-heartbeat STANDBY must DENY a frozen-rejoin PreVote");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
|
|
"Req 4.3: STANDBY's denial reply carries its OWN term (no adoption of 6)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
|
|
atomic_load_explicit(&sb.term, memory_order_acquire),
|
|
"Req 4.3: STANDBY must NOT adopt the candidate's higher prospective term");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&sb.role, memory_order_acquire),
|
|
"Req 4.3: STANDBY role must NOT change");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, sb.raft.voted_for,
|
|
"Req 4.3: a PreVote must not record a vote on the STANDBY");
|
|
|
|
ha_fini(&sb);
|
|
ha_fini(&prim);
|
|
printf("test_frozen_rejoin_healthy_peers_deny_without_adoption: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Angle A (+ the tie-together) — the resumed candidate gets NO majority.
|
|
*
|
|
* The resumed standby starts a PreVote round (self-grant, in-progress, still
|
|
* CTRL_STANDBY, term untouched). It then receives the two DENY replies its
|
|
* healthy peers send (angle B). With both peers denying, the tally has only the
|
|
* self-grant (1 of 3) → LOST. Therefore:
|
|
* - no real election is ever started: role never leaves CTRL_STANDBY and
|
|
* raft.role stays FOLLOWER (no truncate / electing-transition / self-restart
|
|
* on the candidate either);
|
|
* - the candidate's persistent term is byte-for-byte UNCHANGED (Req 4.4);
|
|
* - the liveness watchdog is re-armed so the next genuine heartbeat re-attaches
|
|
* the node (Req 4.1 / 4.2-adjacent).
|
|
*
|
|
* This is exactly the frozen-rejoin no-op: a transient unreachability event
|
|
* leaves the cluster term unmoved and the primary untouched.
|
|
*/
|
|
static void test_frozen_rejoin_candidate_gets_no_majority_noop(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed standby) */
|
|
make_three_node(&ha);
|
|
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
uint64_t raft_term_before = ha.raft.term;
|
|
int raft_voted_before = ha.raft.voted_for;
|
|
|
|
g_rearm_calls = 0;
|
|
ha_prevote_rearm_liveness_hook = stub_rearm;
|
|
|
|
/* The resumed node starts its round — side-effect-free precondition. */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"precondition: the resumed standby's PreVote round must be in progress");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
|
"precondition: only the self-grant is recorded at round start");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"precondition: starting the round leaves the node CTRL_STANDBY");
|
|
|
|
/* Both healthy peers DENY (PRIMARY by role; STANDBY with fresh heartbeat). */
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny — primary peer*/);
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny — fresh standby peer*/);
|
|
|
|
ha_prevote_rearm_liveness_hook = NULL;
|
|
|
|
/* No real vote request was sent: the round resolved LOST, so the candidate
|
|
* never entered the real-election sequence. Role stayed CTRL_STANDBY and
|
|
* the raft role never left FOLLOWER. */
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"Req 4.1: no PreVote majority → no real election → node stays CTRL_STANDBY");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role,
|
|
"Req 4.1: with no majority the candidate's raft role must stay FOLLOWER "
|
|
"(no real election started, no truncate/electing-transition)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"the LOST round must be cleared (round over)");
|
|
|
|
/* Cluster term unchanged across the whole event (Req 4.4). */
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"Req 4.4: the persistent term must be UNCHANGED across the frozen-rejoin event");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
|
|
"Req 4.4: the raft term must be UNCHANGED across the frozen-rejoin event");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
|
|
"Req 4.4: no vote was recorded — the prospective term was never persisted");
|
|
|
|
/* Re-armed so the next genuine heartbeat re-attaches the node (Req 4.1/4.2). */
|
|
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
|
"Req 4.1: a lost frozen-rejoin round must re-arm liveness so the node "
|
|
"re-attaches on the next primary heartbeat");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n");
|
|
}
|
|
|
|
/* ══════════════════════════════════════════════════════════════════════════
|
|
* Task 5.3 — Genuine-failover and re-attach scenario tests.
|
|
*
|
|
* The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER
|
|
* legitimate failover, never BLOCK it. Three scenarios prove the suppressed
|
|
* cases (5.2) and the genuine cases (5.3) are distinguished by the SAME
|
|
* machinery — a dead primary elects a new one, a heartbeat mid-round cleanly
|
|
* re-attaches, and a lost round leaves failover re-armed and reusable.
|
|
*
|
|
* A. Genuine failover ELECTS (Req 5.1):
|
|
* 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE,
|
|
* so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate
|
|
* passes). Both angles:
|
|
* - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the
|
|
* PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2).
|
|
* - CANDIDATE side: feeding that grant to the tally reaches quorum → WON,
|
|
* and the candidate proceeds into the real-election sequence
|
|
* (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it.
|
|
*
|
|
* B. Re-attach on heartbeat mid-round (Req 4.2):
|
|
* a round is in progress; a primary heartbeat arrives → the round is
|
|
* cancelled, the node re-attaches CTRL_STANDBY, term unchanged,
|
|
* pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.)
|
|
*
|
|
* C. Liveness re-arm after loss (Req 5.4 robustness):
|
|
* a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine
|
|
* failure drives a FRESH round that WINS. Proves a lost round never
|
|
* permanently wedges failover.
|
|
*
|
|
* Requirements: 5.1, 5.4, 4.2.
|
|
* Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on
|
|
* heartbeat mid-round; Liveness re-arm after loss);
|
|
* "System Flows → PreVote round (candidate side)".
|
|
* ════════════════════════════════════════════════════════════════════════════ */
|
|
|
|
extern void ha_on_heartbeat_received(ha_state_t *ha);
|
|
|
|
/*
|
|
* Scenario A — GENUINE FAILOVER ELECTS (Req 5.1).
|
|
*
|
|
* Two angles, one scenario:
|
|
*
|
|
* A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone
|
|
* STALE (no primary connection, last_activity older than heartbeat_timeout)
|
|
* is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the
|
|
* 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and
|
|
* it flips the decision grant↔deny. No persistent state is mutated on grant
|
|
* (PreVote is side-effect-free): term/voted_for/role all unchanged.
|
|
*
|
|
* A2 (candidate side): the resumed candidate starts a round and receives that
|
|
* genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds
|
|
* into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING /
|
|
* CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote.
|
|
*/
|
|
static void test_genuine_failover_stale_peer_grants_and_elects(void)
|
|
{
|
|
/* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */
|
|
ha_state_t voter;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0));
|
|
atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release);
|
|
atomic_store_explicit(&voter.term, 5ULL, memory_order_release);
|
|
voter.raft.term = 5ULL;
|
|
voter.raft.role = RAFT_FOLLOWER;
|
|
voter.raft.voted_for = -1;
|
|
voter.voted_for_term = 0;
|
|
voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */
|
|
/* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the
|
|
* voter would itself be eligible to start an election (Req 3.2). */
|
|
set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L);
|
|
|
|
uint64_t reply_term = 0;
|
|
int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1,
|
|
/*cand_term=*/6ULL, &reply_term);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
|
|
"Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the "
|
|
"PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
|
|
"Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term "
|
|
"(PreVote adopts no term)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
|
|
atomic_load_explicit(&voter.term, memory_order_acquire),
|
|
"Req 5.1: granting a PreVote must NOT change the voter's persistent term");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for,
|
|
"Req 5.1: granting a PreVote must record NO vote (real vote still grantable)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&voter.role, memory_order_acquire),
|
|
"Req 5.1: granting a PreVote must not change the voter's role");
|
|
ha_fini(&voter);
|
|
|
|
/* ── A2: the candidate wins the round on that grant → real election ── */
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */
|
|
make_three_node(&ha);
|
|
|
|
uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
ha_start_prevote_async(&ha, NULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"precondition: the failover candidate's PreVote round is in progress");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"precondition: starting the round mutates no term (still side-effect-free)");
|
|
|
|
/* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
|
|
|
|
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
|
"Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new "
|
|
"primary — the candidate enters the real-election sequence, not stay STANDBY");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"Req 5.1: the WON round must clear the in-progress flag (real election begun)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role);
|
|
}
|
|
|
|
/*
|
|
* Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2).
|
|
*
|
|
* A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when
|
|
* a primary heartbeat arrives — meaning the leader is in fact alive and this was
|
|
* NOT a genuine failure. The campaign must be abandoned cleanly: the round is
|
|
* cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY,
|
|
* and the persistent term is byte-for-byte UNCHANGED. This is the suppressed
|
|
* case: the same machinery that elects on a genuine death (scenario A) silently
|
|
* stands down the instant the primary proves alive.
|
|
*/
|
|
static void test_heartbeat_mid_round_clean_reattach(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
make_three_node(&ha);
|
|
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
uint64_t raft_term_before = ha.raft.term;
|
|
int raft_voted_before = ha.raft.voted_for;
|
|
|
|
/* A failover round is in progress (self-grant, still STANDBY). */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"precondition: a PreVote round must be in progress before the heartbeat");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"precondition: the candidate is CTRL_STANDBY during the round");
|
|
|
|
/* A primary heartbeat arrives mid-round → the leader is alive → cancel. */
|
|
ha_on_heartbeat_received(&ha);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role,
|
|
"Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
|
|
"Req 4.2: a mid-round heartbeat cancel must NOT change the raft term");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
|
|
"Req 4.2: a cancelled round records no vote");
|
|
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity,
|
|
"Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_heartbeat_mid_round_clean_reattach: PASS\n");
|
|
}
|
|
|
|
/*
|
|
* Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4).
|
|
*
|
|
* A first PreVote round LOSES (both peers deny — e.g. a brief split where the
|
|
* primary was momentarily believed alive by a quorum). A lost round must NOT
|
|
* permanently wedge failover: the liveness watchdog is re-armed. When a
|
|
* SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to
|
|
* the self-grant and a peer GRANT this time reaches quorum → WON → real election.
|
|
*
|
|
* This is the robustness guarantee that distinguishes 5.3 from a one-shot: the
|
|
* machinery survives a loss and still elects when the failure is real.
|
|
*/
|
|
static void test_lost_round_rearms_then_later_failure_elects(void)
|
|
{
|
|
ha_state_t ha;
|
|
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
|
|
ha_do_demote(&ha);
|
|
make_three_node(&ha);
|
|
|
|
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
|
|
|
|
g_rearm_calls = 0;
|
|
ha_prevote_rearm_liveness_hook = stub_rearm;
|
|
|
|
/* ── Round 1: LOSES (both peers deny) ── */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
|
|
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
|
|
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
|
|
"Req 5.4: a lost round leaves the node CTRL_STANDBY");
|
|
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
|
|
atomic_load_explicit(&ha.term, memory_order_acquire),
|
|
"Req 5.4: a lost round leaves the persistent term unchanged");
|
|
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
|
|
"Req 5.4: a lost round must re-arm the liveness watchdog so a later "
|
|
"genuine failure can still trigger a fresh round");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"the lost round is over (in-progress cleared)");
|
|
|
|
/* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */
|
|
ha_start_prevote_async(&ha, NULL);
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
|
"Req 5.4: a later genuine failure must arm a FRESH round (in-progress)");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
|
|
"Req 5.4: the fresh round resets the tally to the self-grant "
|
|
"(the previous loss did not poison the machinery)");
|
|
|
|
/* This time a peer grants → quorum → WON → real election. */
|
|
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
|
|
|
|
ha_prevote_rearm_liveness_hook = NULL;
|
|
|
|
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
|
|
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
|
|
"Req 5.4: after an earlier lost round, a later genuine failure must still "
|
|
"win a fresh PreVote and start a real election — failover is not wedged");
|
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
|
"the WON second round clears the in-progress flag");
|
|
|
|
ha_fini(&ha);
|
|
printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role);
|
|
}
|
|
|
|
/* ── Suite entry point ──────────────────────────────────────────────────── */
|
|
|
|
void test_ha_prevote(void)
|
|
{
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_prevote_start_no_mutation);
|
|
RUN_TEST(test_prevote_majority_grant_starts_real_election);
|
|
RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms);
|
|
RUN_TEST(test_prevote_round_timeout_is_loss);
|
|
RUN_TEST(test_prevote_reusable_after_loss);
|
|
RUN_TEST(test_prevote_pending_is_noop);
|
|
RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally);
|
|
/* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */
|
|
RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption);
|
|
RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop);
|
|
/* Task 5.3 — genuine-failover + re-attach scenarios */
|
|
RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects);
|
|
RUN_TEST(test_heartbeat_mid_round_clean_reattach);
|
|
RUN_TEST(test_lost_round_rearms_then_later_failure_elects);
|
|
UNITY_END();
|
|
}
|