lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/controller/testsuite/test_ha_prevote.c
Rongsong Shen 88a8bf8a47
fix(controller,lib): decide each PreVote round once and promote only with this election's votes (bug-1915) 
A late PreVote grant after WON re-ran the WON path and started a second
real election that replaced elect_ctx, so the term-2 leader promoted no
vote endpoints. Rounds now carry an id and endpoints a round/kind; stale
or wrong-kind replies, timeouts and errors are dropped; raft ignores
replies with no round in progress. raft_cb_become_leader promotes only
with its own stamped election record (own term, majority), else steps
down and re-arms. raft_on_leader_lost keeps voted_for (one vote per
term) and the delayed election starts only as a follower in the lost
term. The failover stagger hands the PreVote start to a vtask so the
delaymgr stays trigger-only.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-04 09:14:38 +00:00

2184 lines
94 KiB
C
Raw Blame History

/*
* test_ha_prevote.c — Unit tests for ha-prevote task 4.1:
* controller candidate-side PreVote orchestration in ha.c.
*
* Scope (task 4.1 only):
* - ha_start_prevote_async: starts a side-effect-free round (no persist, no
* role change, stays CTRL_STANDBY), arms a round timeout.
* - pre_vote_reply_dispatch: feeds raft_on_pre_vote_reply and branches —
* WON → real-election sequence (truncate + CTRL_ELECTING + start election)
* LOST → stay CTRL_STANDBY, re-arm liveness watchdog, log abandonment
* PENDING → no-op
* - round-timeout firing = LOST branch.
*
* These tests drive the reply path through the XCE_TESTING test seam
* (pre_vote_reply_dispatch_for_test) WITHOUT real transport, mirroring the
* existing vote-reply hook pattern. ha.c is unit-test-linked, so the real
* orchestration logic is exercised directly.
*
* Requirements: 1.1, 1.4, 1.5, 5.4, 7.3
* Boundary: PreVote orchestration (ha.c)
*
* TDD: written first (RED) to specify the exact behaviour before implementation.
*/
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include <unity/unity.h>
#include "ha.h"
#include "netio.h"
#include "raft/raft.h"
#include "vtaskmgr.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */
/*
* ha_start_prevote_async: start a PreVote round. No persist, no role change.
* c may be NULL (unit-test mode — delaymgr round timeout becomes a no-op,
* raft_begin_pre_vote's send callback returns early in test mode).
*/
extern void ha_start_prevote_async(ha_state_t *ha, cluster_t *c);
/*
* pre_vote_reply_dispatch_for_test: test entry point that calls the real
* pre_vote_reply_dispatch with the decoded (peer_idx, granted) fields, without
* a live iomgr. Mirrors the production reply path.
*/
extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted);
/*
* ha_prevote_round_timeout_fire_for_test: simulate the round-timeout task
* firing before a decision (drives the LOST branch).
*/
extern void ha_prevote_round_timeout_fire_for_test(ha_state_t *ha, cluster_t *c);
/* Liveness arm-count hook so tests can prove the watchdog was re-armed on a
* LOST outcome without a live delaymgr. */
extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);
/*
* liveness_ctx_get_for_test: opaque accessor for ha.c's static s_liveness_ctx
* (bug-1483 regression lock — see test below).
*/
extern void *liveness_ctx_get_for_test(void);
/* ── Task 5.1 transport-route seams ──────────────────────────────────────── */
/*
* pre_vote_reply_msg_handler_for_test: drive a preVoteReply through the REAL
* peer_ctrl_msg_handler decode/route path while CTRL_STANDBY + round in
* progress (the production condition). Proves the reply is NOT dropped and
* reaches pre_vote_reply_dispatch — end-to-end, not via the test hook.
*/
extern int pre_vote_reply_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt);
/* vote_reply_build_pkt_for_test: encode a RequestVoteReply capnp packet (the
* preVoteReply reuses the identical payload / union arm). */
extern io_packet_t *vote_reply_build_pkt_for_test(uint64_t term, int vote_granted,
uint64_t peer_tag,
uint64_t peer_lsn);
/* vote_ep_ctx_build_for_test: allocate a vote_ep_ctx_t and wire ep->data with
* the ABSOLUTE peer_idx (the index pre_vote_reply_dispatch expects). */
extern endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
ha_state_t *ha,
ha_elect_ctx_t *ctx,
int peer_idx,
uint64_t my_term,
uint64_t my_tag,
uint64_t my_lsn,
const char *my_name);
/* ── bug-1915 follow-up seams (ha.c, XCE_TESTING) ───────────────────────── */
/* Current PreVote round id (ha->pre_vote_round, read under raft_lock). */
extern uint64_t ha_pre_vote_round_for_test(ha_state_t *ha);
/* Deliver a PreVote reply stamped with an explicit (possibly stale) round. */
extern void pre_vote_reply_dispatch_round_for_test(ha_state_t *ha, cluster_t *c,
uint64_t round, int peer_idx,
int granted);
/* Fire a round timeout that belongs to an explicit (possibly stale) round. */
extern void ha_prevote_round_timeout_fire_round_for_test(ha_state_t *ha,
cluster_t *c,
uint64_t round);
/* Stamp a test-built endpoint as a PreVote endpoint of `round`. */
extern void vote_ep_ctx_mark_pre_vote_for_test(endpoint_t *ep, uint64_t round);
extern int pre_vote_ep_err_handler_for_test(endpoint_t *ep);
extern int peer_ctrl_msg_handler_for_test(endpoint_t *ep, io_packet_t *pkt);
extern void raft_cb_become_leader_for_test(ha_state_t *ha, uint64_t term);
/* Number of PreVote rounds decided WON (observation counter). */
extern _Atomic int ha_prevote_won_count_for_test;
/* ── Test config ─────────────────────────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 6000,
.election_timeout_ms = 500,
.probe_connect_timeout_ms = 50,
.max_promotion_retries = 3,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
};
/* Re-arm tracking */
static int g_rearm_calls = 0;
static int stub_rearm(ha_state_t *ha, cluster_t *c)
{
(void)ha; (void)c;
g_rearm_calls++;
return 0;
}
/*
* Make ha->raft a 3-node cluster (self=0) so the PreVote tally needs a peer
* grant to win (quorum = 2; self-grant = 1). raft_init in ha_init only sets a
* 1-node placeholder, so we widen the topology directly on the public struct.
*/
static void make_three_node(ha_state_t *ha)
{
ha->raft.n_peers = 3;
ha->raft.self_idx = ha->my_priority;
ha->raft.role = RAFT_FOLLOWER;
ha->n_peers = 3;
}
/* ── Test 1: starting a round mutates no persistent state (Req 1.1, 1.2) ──── */
/*
* ha_start_prevote_async must keep the node CTRL_STANDBY, must not increment
* the persistent term, and must not flip the raft role to CANDIDATE. It sets
* the in-progress flag and self-grant in raft_state_t only.
*/
static void test_prevote_start_no_mutation(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
uint64_t raft_term_before = ha.raft.term;
int raft_voted_before = ha.raft.voted_for;
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 1.1: role must stay CTRL_STANDBY during the PreVote round");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 1.2: persistent term must not change during the round");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
"Req 1.2: raft term must not change during the round");
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
"Req 1.2: raft voted_for must not change during the round");
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role,
"Req 1.2: raft role must stay FOLLOWER during the round");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"round must be marked in-progress");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"self-grant must be recorded (1)");
ha_fini(&ha);
printf("test_prevote_start_no_mutation: PASS\n");
}
/* ── Test 2: majority grant WINS → real election started (Req 1.5) ────────── */
/*
* 3-node: after starting the round, feeding one peer GRANT reaches quorum (2).
* The WON branch must run the real-election sequence: role becomes
* CTRL_ELECTING (then PRIMARY/election proceeds via raft_start_election).
*/
static void test_prevote_majority_grant_starts_real_election(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha_start_prevote_async(&ha, NULL);
/* Peer index 1 grants → pre_votes_granted = 2 = quorum → WON. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
/* WON branch enters the real-election sequence: with c==NULL the election
* cannot complete to PRIMARY, but the role MUST have left CTRL_STANDBY —
* the candidate transitioned to CTRL_ELECTING (real-election path). */
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 1.5: a PreVote WIN must enter the real-election sequence "
"(CTRL_ELECTING / CTRL_PRIMARY), not stay CTRL_STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"WON must clear the in-progress flag (round over)");
ha_fini(&ha);
printf("test_prevote_majority_grant_starts_real_election: PASS (role=%d)\n", role);
}
/* ── Test 3: no majority (all DENY) → stay STANDBY, term unchanged, re-arm ─── */
/*
* 3-node: both peers DENY → no quorum → LOST. The node must stay CTRL_STANDBY,
* leave the persistent term unchanged (Req 1.4), and re-arm the liveness
* watchdog (Req 5.4 / 7.3 robustness — a later genuine failure must still
* trigger a new round).
*/
static void test_prevote_no_majority_stays_standby_and_rearms(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
ha_start_prevote_async(&ha, NULL);
/* Both non-self peers deny → all replied, no quorum → LOST. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
ha_prevote_rearm_liveness_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 1.4: no PreVote majority must keep the node CTRL_STANDBY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 1.4: no PreVote majority must leave the persistent term unchanged");
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
"Req 5.4: a lost PreVote round must re-arm the liveness watchdog so a "
"later genuine failure still triggers a new round");
ha_fini(&ha);
printf("test_prevote_no_majority_stays_standby_and_rearms: PASS\n");
}
/* ── Test 4: round-timeout firing = LOSS (Req 5.4) ──────────────────────── */
/*
* If the round times out before a decision, it is treated as a loss: stay
* CTRL_STANDBY, term unchanged, re-arm liveness.
*/
static void test_prevote_round_timeout_is_loss(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
ha_start_prevote_async(&ha, NULL);
/* No replies arrive; the round timeout fires. */
ha_prevote_round_timeout_fire_for_test(&ha, NULL);
ha_prevote_rearm_liveness_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 5.4: a round timeout must be treated as a loss (stay CTRL_STANDBY)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 5.4: a round timeout must leave the persistent term unchanged");
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
"Req 5.4: a timed-out round must re-arm the liveness watchdog");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"round timeout must clear the in-progress flag");
ha_fini(&ha);
printf("test_prevote_round_timeout_is_loss: PASS\n");
}
/* ── Test 4b: LOST round with no test hook must not leak the liveness ctx
* (bug-1483) ──────────────────────────────────────────────────────────── */
/*
* Every other LOST-path test in this file stubs ha_prevote_rearm_liveness_hook
* to avoid driving the real ha_arm_heartbeat_timeout_vtask(ha, c) with c==NULL
* — that unstubbed shape is exactly what bug-1483 found: the real function
* callocs a liveness_poll_ctx_t and publishes it to the static s_liveness_ctx,
* but with c==NULL there is no delaymgr to schedule liveness_poll_fn on, so
* nothing ever frees it — a permanent leak/dangling-publish on every such
* call. This test deliberately does NOT stub the hook, driving the real path,
* and asserts the ctx is never left published once arming without a
* scheduler is impossible.
*/
static void test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
/* Deliberately leave ha_prevote_rearm_liveness_hook == NULL so the LOST
* branch calls the real ha_arm_heartbeat_timeout_vtask(&ha, NULL). */
TEST_ASSERT_NULL_MESSAGE(ha_prevote_rearm_liveness_hook,
"precondition: hook must be unstubbed to drive the real path");
ha_start_prevote_async(&ha, NULL);
ha_prevote_round_timeout_fire_for_test(&ha, NULL);
TEST_ASSERT_NULL_MESSAGE(liveness_ctx_get_for_test(),
"bug-1483: arming the liveness watchdog with c==NULL (no delaymgr "
"to poll on) must not leave an allocated ctx published as the "
"\"active\" s_liveness_ctx -- nothing would ever free it");
ha_fini(&ha);
printf("test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx: PASS\n");
}
/* ── Test 5: a subsequent genuine failure still triggers a new round ─────── */
/*
* Observable robustness (Req 1.4 / 5.4): after a LOST round, the node is back
* in CTRL_STANDBY and a fresh ha_start_prevote_async re-arms a new round
* (self-grant + in-progress), which a peer grant can again win — proving the
* candidate machinery is reusable, not one-shot.
*/
static void test_prevote_reusable_after_loss(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha_prevote_rearm_liveness_hook = stub_rearm;
/* Round 1: lost. */
ha_start_prevote_async(&ha, NULL);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0);
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
/* Round 2: a fresh round arms cleanly (self-grant + in-progress). */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"a new round after a loss must re-arm (in-progress)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"a new round must reset the tally to the self-grant");
/* Round 2 wins with a peer grant. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1);
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"a later genuine failure (new round) must still be able to win and "
"start a real election");
ha_prevote_rearm_liveness_hook = NULL;
ha_fini(&ha);
printf("test_prevote_reusable_after_loss: PASS\n");
}
/* ── Test 6: PENDING reply is a no-op (still STANDBY, round still open) ───── */
static void test_prevote_pending_is_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
/* 5-node so a single grant is below quorum (3) → PENDING. */
ha.raft.n_peers = 5;
ha.raft.self_idx = 0;
ha.raft.role = RAFT_FOLLOWER;
ha.n_peers = 5;
ha_start_prevote_async(&ha, NULL);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, still < quorum*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"a PENDING (sub-quorum) reply must not change the role");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"a PENDING reply must keep the round in progress");
ha_fini(&ha);
printf("test_prevote_pending_is_noop: PASS\n");
}
/* ── Test 7: a preVoteReply over the REAL transport reaches the tally (5.1) ─ */
/*
* Task 5.1 (Req 7.1): a preVoteReply arriving on the candidate's outbound
* PreVote endpoint — while the candidate is still CTRL_STANDBY with a round in
* progress — must be ROUTED to the PreVote tally by peer_ctrl_msg_handler (NOT
* dropped because the role is STANDBY), and drive the round to a decision
* end-to-end. This exercises the production decode/route path, not the
* pre_vote_reply_dispatch_for_test hook.
*/
static void test_prevote_reply_over_real_transport_reaches_tally(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
make_three_node(&ha);
/* Start a real round: self-grant + in-progress, role stays CTRL_STANDBY. */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"precondition: role must be CTRL_STANDBY during the round");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: round must be in progress");
/* Build an endpoint carrying ABSOLUTE peer_idx = 1 (the index the tally
* dedups by), exactly as ha_send_pre_vote_request sets it. */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1,
/*my_term=*/1, /*my_tag=*/0, /*my_lsn=*/0, "test");
/* bug-1915 follow-up: a production PreVote endpoint is stamped with its
* kind and round (raft_cb_send_pre_vote_request); do the same here. */
vote_ep_ctx_mark_pre_vote_for_test(&ep, ha_pre_vote_round_for_test(&ha));
/* Encode a GRANT preVoteReply (same wire payload as a real-vote reply). */
io_packet_t *pkt = vote_reply_build_pkt_for_test(/*term=*/1, /*granted=*/1,
/*tag=*/0, /*lsn=*/0);
TEST_ASSERT_NOT_NULL_MESSAGE(pkt, "vote_reply_build_pkt_for_test must succeed");
/* Route it through the REAL handler (peer_ctrl_msg_handler) while STANDBY. */
int rc = pre_vote_reply_msg_handler_for_test(&ep, pkt);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc, "handler must return 0");
/* Peer grant → quorum (2 of 3) → WON: the reply was NOT dropped; it reached
* the tally and drove the round to a decision (real-election sequence). */
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 7.1: a preVoteReply over the real transport must reach the tally "
"and drive the round to a decision (left CTRL_STANDBY)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the decided round must clear the in-progress flag");
free(ep.data); /* vctx allocated by vote_ep_ctx_build_for_test */
ha_fini(&ha);
printf("test_prevote_reply_over_real_transport_reaches_tally: PASS (role=%d)\n", role);
}
/* ══════════════════════════════════════════════════════════════════════════
* Task 5.2 — Frozen-then-resumed no-op scenario (the headline bug this whole
* feature exists to fix).
*
* 3-node cluster, primary alive. A previously-frozen standby resumes and starts
* a PreVote round; both peers DENY (one is the PRIMARY by role, one is a STANDBY
* with a fresh heartbeat). With both peers denying, the candidate gets only its
* self-grant (1 of 3, below quorum 2) → the round resolves LOST.
*
* Two complementary angles, tied together as one scenario:
*
* A. CANDIDATE side (the resumed standby): after the failover trigger starts a
* round, feeding it the two healthy-peer DENY replies resolves the round
* LOST. Therefore NO real election starts (role never leaves CTRL_STANDBY,
* raft.role stays FOLLOWER), the persistent term is byte-for-byte UNCHANGED
* (Req 4.4), and the candidate re-arms liveness so the next genuine
* heartbeat re-attaches it (Req 4.1 / 4.2-adjacent).
*
* B. VOTER side (the healthy primary / fresh standby): a node that is PRIMARY,
* or STANDBY with a fresh heartbeat, receiving a preVoteRequest from the
* resumed node DENIES it and — critically — does NOT adopt the candidate's
* (higher) prospective term, does NOT demote, truncate, or self-restart
* (Req 4.3). The voter's term/role are unchanged and the reply carries the
* voter's OWN term.
*
* Requirements: 4.1, 4.3, 4.4.
* Design: "Testing Strategy → Integration Tests → Frozen-then-resumed no-op".
* ════════════════════════════════════════════════════════════════════════════ */
/* ── Voter-side mirror of cluster.c OPCode_preVoteRequest (B) ──────────────────
*
* cluster.c is NOT linked into the unit-test binary, so we exercise the EXACT
* decision its dispatch composes: the raft eligibility gate (role-deny /
* fresh-heartbeat-deny, via the registered raft_cb_vote_eligible) plus the
* contract that the reply term is the voter's CURRENT term, UNCHANGED, on grant
* OR deny. (Freshness floor is irrelevant for the deny cases here — the role /
* heartbeat gate fires first — so the freshness check is omitted for clarity.)
*
* Returns 1 = grant, 0 = deny; *out_reply_term = voter's unchanged term. */
/* raft_on_pre_vote_request is declared in raft.h (now carries the candidate
* (lastLogTerm, lastLogIndex) tuple — ha-term-safety task 1.2); no local extern. */
static int voter_pre_vote_decide(ha_state_t *ha, int from_idx, uint64_t cand_term,
uint64_t *out_reply_term)
{
/* Pass (0,0) for the candidate last-log tuple: this PreVote test exercises
* the role/heartbeat eligibility gate, not the logOk freshness path; with
* the on_get_entry_term stub reading local last-log term as 0, logOk grants
* so existing gate behavior is preserved (ha-term-safety task 1.2). */
int granted =
(raft_on_pre_vote_request(&ha->raft, from_idx, cand_term, 0, 0)
== RAFT_GRANT)
? 1 : 0;
if (out_reply_term)
*out_reply_term = atomic_load_explicit(&ha->term, memory_order_acquire);
return granted;
}
/* set_last_activity_age: ha->last_activity = "offset_ms ago" (whole seconds,
* matching ha_on_heartbeat_received). */
static void set_last_activity_age(ha_state_t *ha, long offset_ms)
{
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
ha->last_activity = now.tv_sec - offset_ms / 1000;
}
/*
* Angle B — the healthy peers DENY without adopting term/role/voted-for.
*
* Two voters answer the resumed node's preVoteRequest for prospective term 6
* (its inflated current+1):
* - voter P: CTRL_PRIMARY, term 5 → role gate denies; term stays 5, no demote.
* - voter S: CTRL_STANDBY, term 5, fresh heartbeat → heartbeat gate denies;
* term stays 5, stays STANDBY.
* Neither adopts term 6; neither mutates voted_for / role. Each reply carries
* the voter's OWN unchanged term (Req 4.3).
*/
static void test_frozen_rejoin_healthy_peers_deny_without_adoption(void)
{
const uint64_t PROSPECTIVE_TERM = 6ULL; /* resumed node's inflated cur+1 */
/* ── Voter P: the healthy PRIMARY ── */
ha_state_t prim;
TEST_ASSERT_EQUAL_INT(0, ha_init(&prim, &g_cfg, 0));
atomic_store_explicit(&prim.role, (int32_t)CTRL_PRIMARY, memory_order_release);
atomic_store_explicit(&prim.term, 5ULL, memory_order_release);
prim.raft.term = 5ULL;
prim.raft.role = RAFT_LEADER;
prim.raft.voted_for = -1;
prim.voted_for_term = 0;
uint64_t reply_term = 0;
int granted = voter_pre_vote_decide(&prim, /*from_idx=*/2, PROSPECTIVE_TERM,
&reply_term);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
"Req 4.3: the healthy PRIMARY must DENY a frozen-rejoin PreVote");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
"Req 4.3: PRIMARY's denial reply carries its OWN term (no adoption of 6)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
atomic_load_explicit(&prim.term, memory_order_acquire),
"Req 4.3: PRIMARY must NOT adopt the candidate's higher prospective term");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, prim.raft.term,
"Req 4.3: PRIMARY raft term must NOT change");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_PRIMARY,
(int)atomic_load_explicit(&prim.role, memory_order_acquire),
"Req 4.3: PRIMARY must NOT demote in response to a rejoining standby");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_LEADER, (int)prim.raft.role,
"Req 4.3: PRIMARY raft role must stay LEADER (no demotion/self-restart)");
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, prim.raft.voted_for,
"Req 4.3: a PreVote must not record a vote on the PRIMARY");
/* ── Voter S: a healthy STANDBY with a fresh heartbeat ── */
ha_state_t sb;
TEST_ASSERT_EQUAL_INT(0, ha_init(&sb, &g_cfg, 0));
atomic_store_explicit(&sb.role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&sb.term, 5ULL, memory_order_release);
sb.raft.term = 5ULL;
sb.raft.role = RAFT_FOLLOWER;
sb.raft.voted_for = -1;
sb.voted_for_term = 0;
sb.primary_ctrl_fd = 999; /* live primary connection */
set_last_activity_age(&sb, 500L); /* fresh heartbeat (< heartbeat_timeout) */
reply_term = 0;
granted = voter_pre_vote_decide(&sb, /*from_idx=*/2, PROSPECTIVE_TERM,
&reply_term);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, granted,
"Req 4.3: a fresh-heartbeat STANDBY must DENY a frozen-rejoin PreVote");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
"Req 4.3: STANDBY's denial reply carries its OWN term (no adoption of 6)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
atomic_load_explicit(&sb.term, memory_order_acquire),
"Req 4.3: STANDBY must NOT adopt the candidate's higher prospective term");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
(int)atomic_load_explicit(&sb.role, memory_order_acquire),
"Req 4.3: STANDBY role must NOT change");
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, sb.raft.voted_for,
"Req 4.3: a PreVote must not record a vote on the STANDBY");
ha_fini(&sb);
ha_fini(&prim);
printf("test_frozen_rejoin_healthy_peers_deny_without_adoption: PASS\n");
}
/*
* Angle A (+ the tie-together) — the resumed candidate gets NO majority.
*
* The resumed standby starts a PreVote round (self-grant, in-progress, still
* CTRL_STANDBY, term untouched). It then receives the two DENY replies its
* healthy peers send (angle B). With both peers denying, the tally has only the
* self-grant (1 of 3) → LOST. Therefore:
* - no real election is ever started: role never leaves CTRL_STANDBY and
* raft.role stays FOLLOWER (no truncate / electing-transition / self-restart
* on the candidate either);
* - the candidate's persistent term is byte-for-byte UNCHANGED (Req 4.4);
* - the liveness watchdog is re-armed so the next genuine heartbeat re-attaches
* the node (Req 4.1 / 4.2-adjacent).
*
* This is exactly the frozen-rejoin no-op: a transient unreachability event
* leaves the cluster term unmoved and the primary untouched.
*/
static void test_frozen_rejoin_candidate_gets_no_majority_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed standby) */
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
uint64_t raft_term_before = ha.raft.term;
int raft_voted_before = ha.raft.voted_for;
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
/* The resumed node starts its round — side-effect-free precondition. */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: the resumed standby's PreVote round must be in progress");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"precondition: only the self-grant is recorded at round start");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"precondition: starting the round leaves the node CTRL_STANDBY");
/* Both healthy peers DENY (PRIMARY by role; STANDBY with fresh heartbeat). */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny — primary peer*/);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny — fresh standby peer*/);
ha_prevote_rearm_liveness_hook = NULL;
/* No real vote request was sent: the round resolved LOST, so the candidate
* never entered the real-election sequence. Role stayed CTRL_STANDBY and
* the raft role never left FOLLOWER. */
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 4.1: no PreVote majority → no real election → node stays CTRL_STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role,
"Req 4.1: with no majority the candidate's raft role must stay FOLLOWER "
"(no real election started, no truncate/electing-transition)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the LOST round must be cleared (round over)");
/* Cluster term unchanged across the whole event (Req 4.4). */
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 4.4: the persistent term must be UNCHANGED across the frozen-rejoin event");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
"Req 4.4: the raft term must be UNCHANGED across the frozen-rejoin event");
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
"Req 4.4: no vote was recorded — the prospective term was never persisted");
/* Re-armed so the next genuine heartbeat re-attaches the node (Req 4.1/4.2). */
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
"Req 4.1: a lost frozen-rejoin round must re-arm liveness so the node "
"re-attaches on the next primary heartbeat");
ha_fini(&ha);
printf("test_frozen_rejoin_candidate_gets_no_majority_noop: PASS\n");
}
/* ══════════════════════════════════════════════════════════════════════════
* Task 5.3 — Genuine-failover and re-attach scenario tests.
*
* The mirror image of the task-5.2 frozen-rejoin no-op: PreVote must DEFER
* legitimate failover, never BLOCK it. Three scenarios prove the suppressed
* cases (5.2) and the genuine cases (5.3) are distinguished by the SAME
* machinery — a dead primary elects a new one, a heartbeat mid-round cleanly
* re-attaches, and a lost round leaves failover re-armed and reusable.
*
* A. Genuine failover ELECTS (Req 5.1):
* 3-node, the primary is genuinely dead. Its peers' heartbeats are STALE,
* so each is ELIGIBLE to grant (the role/fresh-heartbeat/catch-up gate
* passes). Both angles:
* - VOTER side: a STANDBY whose heartbeat has gone stale GRANTS the
* PreVote (the exact opposite of the fresh-heartbeat DENY in 5.2).
* - CANDIDATE side: feeding that grant to the tally reaches quorum → WON,
* and the candidate proceeds into the real-election sequence
* (CTRL_ELECTING / raft_start_election) — PreVote did NOT block it.
*
* B. Re-attach on heartbeat mid-round (Req 4.2):
* a round is in progress; a primary heartbeat arrives → the round is
* cancelled, the node re-attaches CTRL_STANDBY, term unchanged,
* pre_vote_in_progress cleared. (Drives task 4.2's cancel-on-heartbeat.)
*
* C. Liveness re-arm after loss (Req 5.4 robustness):
* a round LOSES (no majority) → liveness re-armed → a SUBSEQUENT genuine
* failure drives a FRESH round that WINS. Proves a lost round never
* permanently wedges failover.
*
* Requirements: 5.1, 5.4, 4.2.
* Design: "Testing Strategy → Integration Tests" (Genuine failover; Re-attach on
* heartbeat mid-round; Liveness re-arm after loss);
* "System Flows → PreVote round (candidate side)".
* ════════════════════════════════════════════════════════════════════════════ */
extern void ha_on_heartbeat_received(ha_state_t *ha);
/*
* Scenario A — GENUINE FAILOVER ELECTS (Req 5.1).
*
* Two angles, one scenario:
*
* A1 (voter side): a STANDBY peer of the dead primary whose heartbeat has gone
* STALE (no primary connection, last_activity older than heartbeat_timeout)
* is ELIGIBLE and GRANTS the PreVote. This is the exact counterpart of the
* 5.2 fresh-heartbeat DENY: the ONLY difference is heartbeat staleness, and
* it flips the decision grant↔deny. No persistent state is mutated on grant
* (PreVote is side-effect-free): term/voted_for/role all unchanged.
*
* A2 (candidate side): the resumed candidate starts a round and receives that
* genuine GRANT — reaching quorum (2 of 3) → WON → the candidate proceeds
* into the real-election sequence (leaves CTRL_STANDBY for CTRL_ELECTING /
* CTRL_PRIMARY). A genuine primary death is NOT suppressed by PreVote.
*/
static void test_genuine_failover_stale_peer_grants_and_elects(void)
{
/* ── A1: the dead primary's STANDBY peer has a STALE heartbeat → GRANT ── */
ha_state_t voter;
TEST_ASSERT_EQUAL_INT(0, ha_init(&voter, &g_cfg, 0));
atomic_store_explicit(&voter.role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&voter.term, 5ULL, memory_order_release);
voter.raft.term = 5ULL;
voter.raft.role = RAFT_FOLLOWER;
voter.raft.voted_for = -1;
voter.voted_for_term = 0;
voter.primary_ctrl_fd = -1; /* primary connection gone (dead) */
/* Heartbeat is STALE: last_activity older than heartbeat_timeout_ms → the
* voter would itself be eligible to start an election (Req 3.2). */
set_last_activity_age(&voter, (long)g_cfg.heartbeat_timeout_ms + 5000L);
uint64_t reply_term = 0;
int granted = voter_pre_vote_decide(&voter, /*from_idx=*/1,
/*cand_term=*/6ULL, &reply_term);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
"Req 5.1: a STANDBY with a STALE heartbeat (dead primary) must GRANT the "
"PreVote — the genuine-failover counterpart of the 5.2 fresh-heartbeat DENY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL, reply_term,
"Req 5.1: even on GRANT the voter's reply carries its OWN unchanged term "
"(PreVote adopts no term)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(5ULL,
atomic_load_explicit(&voter.term, memory_order_acquire),
"Req 5.1: granting a PreVote must NOT change the voter's persistent term");
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, voter.raft.voted_for,
"Req 5.1: granting a PreVote must record NO vote (real vote still grantable)");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
(int)atomic_load_explicit(&voter.role, memory_order_acquire),
"Req 5.1: granting a PreVote must not change the voter's role");
ha_fini(&voter);
/* ── A2: the candidate wins the round on that grant → real election ── */
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY (resumed candidate) */
make_three_node(&ha);
uint64_t cand_term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: the failover candidate's PreVote round is in progress");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(cand_term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"precondition: starting the round mutates no term (still side-effect-free)");
/* A genuine GRANT from peer 1 (the stale-heartbeat voter above) → quorum. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 5.1: a genuine primary death (PreVote GRANTED → WON) must elect a new "
"primary — the candidate enters the real-election sequence, not stay STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"Req 5.1: the WON round must clear the in-progress flag (real election begun)");
ha_fini(&ha);
printf("test_genuine_failover_stale_peer_grants_and_elects: PASS (role=%d)\n", role);
}
/*
* Scenario B — RE-ATTACH ON HEARTBEAT MID-ROUND (Req 4.2).
*
* A failover candidate is mid-round (self-grant, in-progress, CTRL_STANDBY) when
* a primary heartbeat arrives — meaning the leader is in fact alive and this was
* NOT a genuine failure. The campaign must be abandoned cleanly: the round is
* cancelled (pre_vote_in_progress cleared), the node re-attaches as CTRL_STANDBY,
* and the persistent term is byte-for-byte UNCHANGED. This is the suppressed
* case: the same machinery that elects on a genuine death (scenario A) silently
* stands down the instant the primary proves alive.
*/
static void test_heartbeat_mid_round_clean_reattach(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
uint64_t raft_term_before = ha.raft.term;
int raft_voted_before = ha.raft.voted_for;
/* A failover round is in progress (self-grant, still STANDBY). */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: a PreVote round must be in progress before the heartbeat");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"precondition: the candidate is CTRL_STANDBY during the round");
/* A primary heartbeat arrives mid-round → the leader is alive → cancel. */
ha_on_heartbeat_received(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"Req 4.2: a heartbeat mid-round must CANCEL the in-progress PreVote round");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)RAFT_FOLLOWER, (int)ha.raft.role,
"Req 4.2: the cancelled candidate's raft role stays FOLLOWER (never campaigned)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 4.2: a mid-round heartbeat cancel must NOT change the persistent term");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(raft_term_before, ha.raft.term,
"Req 4.2: a mid-round heartbeat cancel must NOT change the raft term");
TEST_ASSERT_EQUAL_INT_MESSAGE(raft_voted_before, ha.raft.voted_for,
"Req 4.2: a cancelled round records no vote");
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity,
"Req 4.2: the heartbeat still refreshes last_activity (re-attach signal)");
ha_fini(&ha);
printf("test_heartbeat_mid_round_clean_reattach: PASS\n");
}
/*
* Scenario C — LIVENESS RE-ARM AFTER LOSS, then a LATER FAILURE ELECTS (Req 5.4).
*
* A first PreVote round LOSES (both peers deny — e.g. a brief split where the
* primary was momentarily believed alive by a quorum). A lost round must NOT
* permanently wedge failover: the liveness watchdog is re-armed. When a
* SUBSEQUENT, genuine failure fires a fresh round, that round resets cleanly to
* the self-grant and a peer GRANT this time reaches quorum → WON → real election.
*
* This is the robustness guarantee that distinguishes 5.3 from a one-shot: the
* machinery survives a loss and still elects when the failure is real.
*/
static void test_lost_round_rearms_then_later_failure_elects(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
/* ── Round 1: LOSES (both peers deny) ── */
ha_start_prevote_async(&ha, NULL);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0 /*deny*/);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*deny*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 5.4: a lost round leaves the node CTRL_STANDBY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 5.4: a lost round leaves the persistent term unchanged");
TEST_ASSERT_GREATER_OR_EQUAL_INT_MESSAGE(1, g_rearm_calls,
"Req 5.4: a lost round must re-arm the liveness watchdog so a later "
"genuine failure can still trigger a fresh round");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the lost round is over (in-progress cleared)");
/* ── Round 2: a SUBSEQUENT genuine failure → fresh round → WINS ── */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"Req 5.4: a later genuine failure must arm a FRESH round (in-progress)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"Req 5.4: the fresh round resets the tally to the self-grant "
"(the previous loss did not poison the machinery)");
/* This time a peer grants → quorum → WON → real election. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
ha_prevote_rearm_liveness_hook = NULL;
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 5.4: after an earlier lost round, a later genuine failure must still "
"win a fresh PreVote and start a real election — failover is not wedged");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the WON second round clears the in-progress flag");
ha_fini(&ha);
printf("test_lost_round_rearms_then_later_failure_elects: PASS (role=%d)\n", role);
}
/* ── bug-1915: a reply that arrives after the round is decided is ignored ──── */
/*
* 3-node: peer 1's GRANT wins the round (2/3). Peer 2's GRANT is still in
* flight and is delivered afterwards — by then the real election may already
* have made this node leader. That late reply must not decide the round a
* second time: before the fix raft_on_pre_vote_reply counted it (3/3 >= quorum)
* and returned WON again, so the controller re-ran ha_enter_real_election →
* ha_start_election_async, which overwrote the role with CTRL_ELECTING and
* swapped ha->elect_ctx for a fresh empty carrier (losing the round's vote
* endpoints and voter LSNs that raft_cb_become_leader promotes / computes the
* safe commit point from).
*/
static void test_bug1915_late_grant_after_won_is_ignored(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha_start_prevote_async(&ha, NULL);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted → WON*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_ELECTING,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"precondition: the first majority grant enters the real election");
/* The real election completes before peer 2's PreVote reply is read. */
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 1 /*late grant*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"bug-1915: a PreVote reply for an already-WON round must not re-enter "
"the real election (it demoted the new leader to CTRL_ELECTING)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the decided round stays decided");
ha_fini(&ha);
}
/*
* 3-node: peer 1's GRANT wins the round; peer 2's DENY arrives afterwards.
* pre_votes_granted never decreases, so before the fix the tally still read
* 2 >= quorum and returned WON a second time even for a DENY — the same
* re-entry into the real election as the late-grant case above.
*/
static void test_bug1915_late_deny_after_won_is_ignored(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
ha_start_prevote_async(&ha, NULL);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted → WON*/);
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0 /*late deny*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"bug-1915: a late DENY for an already-WON round must not re-enter "
"the real election");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, g_rearm_calls,
"a late reply for a decided round must not be treated as a LOSS either");
ha_prevote_rearm_liveness_hook = NULL;
ha_fini(&ha);
}
/* ══ bug-1915 follow-up: close the remaining paths to a replaced elect_ctx ══ */
/* A carrier the tests install as "the ctx of the election in flight", with an
* extra reference so a (buggy) supersession cannot free it under the test. */
static ha_elect_ctx_t *sentinel_ctx(int n_other)
{
ha_elect_ctx_t *ctx = calloc(1, sizeof(*ctx) +
(size_t)n_other * sizeof(ctx->votes[0]));
TEST_ASSERT_NOT_NULL(ctx);
ctx->n_peers = n_other;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
atomic_init(&ctx->refcount, 2); /* published + held by the test */
return ctx;
}
static void drop_published_ctx(ha_state_t *ha, ha_elect_ctx_t *sentinel)
{
ha_elect_ctx_t *cur = (ha_elect_ctx_t *)atomic_exchange_explicit(
&ha->elect_ctx, NULL, memory_order_acq_rel);
if (cur && cur != sentinel) {
pthread_mutex_destroy(&cur->lock);
sem_destroy(&cur->done);
free(cur);
}
if (sentinel) {
pthread_mutex_destroy(&sentinel->lock);
sem_destroy(&sentinel->done);
free(sentinel);
}
}
/*
* Gap (a): ha_start_election_async on a node whose raft is already LEADER must
* change nothing — not the role, not ha->elect_ctx, not the term. Before the
* fix it set CTRL_ELECTING and swapped elect_ctx BEFORE raft_start_election
* returned -EALREADY (ignored), demoting a live leader and orphaning its
* election's vote endpoints / voter LSNs.
*/
static void test_gapA_election_start_noop_when_raft_leader(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
ha.raft.role = RAFT_LEADER;
ha.raft.term = 5;
atomic_store_explicit(&ha.term, 5, memory_order_release);
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
ha_elect_ctx_t *mine = sentinel_ctx(2);
atomic_store_explicit(&ha.elect_ctx, (void *)mine, memory_order_release);
ha_start_election_async(&ha, NULL, 3, NULL, /*tag=*/1, /*lsn=*/10, "n0");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"gap (a): a leader must not be moved to CTRL_ELECTING");
TEST_ASSERT_EQUAL_PTR_MESSAGE(mine,
atomic_load_explicit(&ha.elect_ctx, memory_order_acquire),
"gap (a): the leader's elect_ctx must not be swapped");
TEST_ASSERT_EQUAL_UINT64(5, ha.raft.term);
TEST_ASSERT_EQUAL_INT(RAFT_LEADER, ha.raft.role);
TEST_ASSERT_EQUAL_INT_MESSAGE(2, atomic_load(&mine->refcount),
"gap (a): the published carrier's reference must not be released");
drop_published_ctx(&ha, mine);
ha_fini(&ha);
}
/* Gap (a): same for a node already CANDIDATE — its in-flight election's ctx
* holds the vote endpoints and voter LSNs raft_cb_become_leader needs. */
static void test_gapA_election_start_noop_when_raft_candidate(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
ha.raft.role = RAFT_CANDIDATE;
ha.raft.term = 7;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *mine = sentinel_ctx(2);
atomic_store_explicit(&ha.elect_ctx, (void *)mine, memory_order_release);
ha_start_election_async(&ha, NULL, 3, NULL, /*tag=*/1, /*lsn=*/10, "n0");
TEST_ASSERT_EQUAL_PTR_MESSAGE(mine,
atomic_load_explicit(&ha.elect_ctx, memory_order_acquire),
"gap (a): a candidate's elect_ctx must not be swapped");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(7, ha.raft.term,
"gap (a): no second term bump");
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, ha.raft.role);
TEST_ASSERT_EQUAL_INT(2, atomic_load(&mine->refcount));
drop_published_ctx(&ha, mine);
ha_fini(&ha);
}
/*
* Gap (b): a failover trigger (ha_begin_failover — synchronous in unit mode,
* the same ha_start_prevote_async the stagger timer runs) that fires while a
* real election is in flight must not open a PreVote round: in production that
* swaps ha->elect_ctx for an empty PreVote carrier mid-election.
*/
static void test_gapB_prevote_refused_during_real_election(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
ha.raft.role = RAFT_CANDIDATE;
ha.raft.term = 4;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
uint64_t round0 = ha_pre_vote_round_for_test(&ha);
ha_begin_failover(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"gap (b): no PreVote round may open while a real election is in flight");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(round0, ha_pre_vote_round_for_test(&ha),
"gap (b): a refused start must not consume a round id");
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, ha.raft.role);
ha_fini(&ha);
}
/* Gap (b): a leader never opens a PreVote round either. */
static void test_gapB_prevote_refused_when_leader(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
ha.raft.role = RAFT_LEADER;
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"gap (b): a leader must not open a PreVote round");
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha_fini(&ha);
}
/* Gap (b): a second start while a round is still undecided must not reset the
* round (tally, per-peer replies, round id) — the stagger timer firing while
* the RDHUP-triggered round is in flight did exactly that. */
static void test_gapB_prevote_restart_mid_round_keeps_round(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha.raft.n_peers = 5; /* quorum 3: one grant leaves the round pending */
ha.n_peers = 5;
ha_start_prevote_async(&ha, NULL);
uint64_t r1 = ha_pre_vote_round_for_test(&ha);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*grant, pending*/);
TEST_ASSERT_EQUAL_INT(2, ha.raft.pre_votes_granted);
ha_start_prevote_async(&ha, NULL); /* e.g. a stagger timer firing now */
TEST_ASSERT_EQUAL_INT_MESSAGE(2, ha.raft.pre_votes_granted,
"gap (b): an in-flight round's tally must not be reset");
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_replies[1]);
TEST_ASSERT_EQUAL_UINT64(r1, ha_pre_vote_round_for_test(&ha));
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress);
ha_fini(&ha);
}
/*
* Gap (d): a reply from an OLDER PreVote round must not be tallied in the
* current one (replies carry no round id on the wire; the endpoint's captured
* round is the identity).
*/
static void test_gapD_stale_round_reply_dropped(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha.raft.n_peers = 5;
ha.n_peers = 5;
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
ha_start_prevote_async(&ha, NULL);
uint64_t r1 = ha_pre_vote_round_for_test(&ha);
ha_prevote_round_timeout_fire_for_test(&ha, NULL); /* r1 LOST */
TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);
ha_start_prevote_async(&ha, NULL); /* r2 */
TEST_ASSERT_TRUE(ha_pre_vote_round_for_test(&ha) != r1);
pre_vote_reply_dispatch_round_for_test(&ha, NULL, r1, 1, 1 /*stale grant*/);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"gap (d): a round-1 grant must not count toward round 2");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_replies[1],
"gap (d): a stale reply must not consume peer 1's round-2 slot");
ha_prevote_rearm_liveness_hook = NULL;
ha_fini(&ha);
}
/* Gap (d): round 1's timeout task firing after round 2 started must not end
* round 2. */
static void test_gapD_stale_round_timeout_does_not_end_current_round(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
ha_start_prevote_async(&ha, NULL);
uint64_t r1 = ha_pre_vote_round_for_test(&ha);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); /* r1 LOST */
TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);
ha_start_prevote_async(&ha, NULL); /* r2 */
int rearms_before = g_rearm_calls;
ha_prevote_round_timeout_fire_round_for_test(&ha, NULL, r1);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"gap (d): a stale round timeout must not end the current round");
TEST_ASSERT_EQUAL_INT(rearms_before, g_rearm_calls);
ha_prevote_rearm_liveness_hook = NULL;
ha_fini(&ha);
}
/* Gap (d): a PreVote endpoint error from round 1 delivered during round 2 must
* not count as peer 1's round-2 DENY — that also burned peer 1's slot, so its
* genuine round-2 GRANT was then dropped as a duplicate. */
static void test_gapD_stale_round_endpoint_error_not_counted(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
ha_start_prevote_async(&ha, NULL);
uint64_t r1 = ha_pre_vote_round_for_test(&ha);
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 0);
pre_vote_reply_dispatch_for_test(&ha, NULL, 2, 0); /* r1 LOST */
ha_start_prevote_async(&ha, NULL); /* r2 */
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1, 1, 0, 0, "t");
vote_ep_ctx_mark_pre_vote_for_test(&ep, r1); /* round-1 endpoint */
TEST_ASSERT_EQUAL_INT(0, pre_vote_ep_err_handler_for_test(&ep));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_replies[1],
"gap (d): a stale endpoint error must not be tallied in round 2");
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1); /* genuine r2 grant */
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_ELECTING,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"gap (d): peer 1's genuine round-2 grant must still win the round");
free(ep.data);
ha_prevote_rearm_liveness_hook = NULL;
ha_fini(&ha);
}
/*
* Gap (d), cross-path: a PreVote reply that arrives on its PreVote endpoint
* after the round was WON — when the node is already CTRL_ELECTING — must
* never be decoded as a REAL vote. The two replies share one wire payload and
* the router chose the decoder by role, so a late PreVote GRANT whose term
* equals the new election term was counted by raft_on_vote_reply as a real
* vote (a PreVote grant records no vote — Election Safety).
*/
static void test_gapD_late_prevote_reply_never_counts_as_real_vote(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
uint64_t pv_round = ha_pre_vote_round_for_test(&ha);
ha_start_election_async(&ha, NULL, 3, NULL, /*tag=*/1, /*lsn=*/10, "n0");
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role,
"precondition: a real election is in flight");
uint64_t T = ha.raft.term;
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*abs peer_idx=*/1, T, 0, 0, "t");
vote_ep_ctx_mark_pre_vote_for_test(&ep, pv_round);
io_packet_t *pkt = vote_reply_build_pkt_for_test(T, /*granted=*/1, 0, 0);
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&ep, pkt));
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role,
"gap (d): a PreVote grant must never be counted as a real vote");
TEST_ASSERT_EQUAL_INT(1, ha.raft.votes_granted);
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
free(ep.data);
drop_published_ctx(&ha, NULL);
ha_fini(&ha);
}
/* Gap (d), cross-path inverse: a reply on a REAL-vote endpoint (e.g. a lost
* election's late RequestVoteReply) arriving while a PreVote round is open
* must not be tallied as a PreVote reply. */
static void test_gapD_real_vote_reply_never_feeds_prevote_tally(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha_start_prevote_async(&ha, NULL);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*votes_idx=*/1, 1, 0, 0, "t");
io_packet_t *pkt = vote_reply_build_pkt_for_test(1, /*granted=*/1, 0, 0);
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0, pre_vote_reply_msg_handler_for_test(&ep, pkt));
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"gap (d): a real-vote endpoint's reply must not decide a PreVote round");
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_votes_granted);
TEST_ASSERT_EQUAL_INT(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
free(ep.data);
ha_fini(&ha);
}
/*
* Defensive check in raft_cb_become_leader: promotion computes the safe commit
* point from ha->elect_ctx. A PreVote carrier (no real votes), a carrier
* stamped for another term, or a stamped carrier without a recorded majority
* would yield a self-only safe commit point (n_granted = 1 in a 3-node
* cluster) — refuse the promotion loudly instead of becoming CTRL_PRIMARY.
*/
static void test_become_leader_refuses_prevote_carrier(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *pv = sentinel_ctx(2);
pv->is_pre_vote = 1;
atomic_store_explicit(&ha.elect_ctx, (void *)pv, memory_order_release);
raft_cb_become_leader_for_test(&ha, 1);
TEST_ASSERT_TRUE_MESSAGE(
atomic_load_explicit(&ha.role, memory_order_acquire) != CTRL_PRIMARY,
"promotion must be refused when elect_ctx is a PreVote carrier");
drop_published_ctx(&ha, pv);
ha_fini(&ha);
}
static void test_become_leader_refuses_stamped_carrier_without_majority(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ec = sentinel_ctx(2);
ec->election_term = 3; /* this election's carrier ... */
/* ... but no voter recorded as granted: self-only */
atomic_store_explicit(&ha.elect_ctx, (void *)ec, memory_order_release);
raft_cb_become_leader_for_test(&ha, 3);
TEST_ASSERT_TRUE_MESSAGE(
atomic_load_explicit(&ha.role, memory_order_acquire) != CTRL_PRIMARY,
"promotion must be refused when elect_ctx records no voted majority");
drop_published_ctx(&ha, ec);
ha_fini(&ha);
}
static void test_become_leader_refuses_carrier_of_other_term(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ec = sentinel_ctx(2);
ec->election_term = 3;
ec->votes[0].vote_granted = 1;
ec->votes[1].vote_granted = 1;
atomic_store_explicit(&ha.elect_ctx, (void *)ec, memory_order_release);
raft_cb_become_leader_for_test(&ha, 4); /* won term 4, ctx is term 3's */
TEST_ASSERT_TRUE_MESSAGE(
atomic_load_explicit(&ha.role, memory_order_acquire) != CTRL_PRIMARY,
"promotion must be refused when elect_ctx belongs to another election");
drop_published_ctx(&ha, ec);
ha_fini(&ha);
}
/* No false refusal: this election's stamped carrier with a recorded majority
* promotes normally. */
static void test_become_leader_accepts_own_carrier_with_majority(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *ec = sentinel_ctx(2);
ec->election_term = 3;
ec->votes[1].vote_granted = 1; /* self + 1 = 2 of 3 */
atomic_store_explicit(&ha.elect_ctx, (void *)ec, memory_order_release);
raft_cb_become_leader_for_test(&ha, 3);
TEST_ASSERT_EQUAL_INT(CTRL_PRIMARY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
drop_published_ctx(&ha, ec);
ha_fini(&ha);
}
/*
* Gap (e): real threads. Two peers' GRANTs for the same 3-node round are
* dispatched concurrently from two threads released by a barrier (no sleeps).
* Whatever the interleaving, the round must be decided WON exactly once. The
* pass condition is deterministic; the interleaving is not, so the loop gives
* the race many chances to show up if the raft_lock decision check regresses.
*/
typedef struct {
ha_state_t *ha;
pthread_barrier_t *bar;
int peer;
} race_arg_t;
static void *race_reply_fn(void *arg)
{
race_arg_t *a = (race_arg_t *)arg;
pthread_barrier_wait(a->bar);
pre_vote_reply_dispatch_for_test(a->ha, NULL, a->peer, 1);
return NULL;
}
static void test_gapE_concurrent_grants_decide_round_once(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
for (int iter = 0; iter < 500; iter++) {
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY,
memory_order_release);
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress);
int won0 = atomic_load(&ha_prevote_won_count_for_test);
pthread_barrier_t bar;
pthread_barrier_init(&bar, NULL, 2);
race_arg_t a1 = { &ha, &bar, 1 }, a2 = { &ha, &bar, 2 };
pthread_t t1, t2;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&t1, NULL, race_reply_fn, &a1));
TEST_ASSERT_EQUAL_INT(0, pthread_create(&t2, NULL, race_reply_fn, &a2));
pthread_join(t1, NULL);
pthread_join(t2, NULL);
pthread_barrier_destroy(&bar);
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
atomic_load(&ha_prevote_won_count_for_test) - won0,
"gap (e): two concurrent grants must decide the round WON once");
}
ha_fini(&ha);
}
/* ══ bug-1915 review round 2 ════════════════════════════════════════════════ */
extern void (*ha_prevote_pre_arm_hook)(ha_state_t *ha, cluster_t *c,
uint64_t round);
extern int (*ha_prevote_round_timer_arm_hook)(ha_state_t *ha, cluster_t *c,
uint64_t round, void *tctx);
extern int ha_prevote_active_round_timer_for_test(uint64_t *round_out,
int *cancelled_out);
extern void ha_prevote_round_timer_clear_for_test(void);
extern _Atomic int ha_liveness_arm_count_for_test;
extern void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
int n_peers, xce_candidate_t **peers,
uint64_t my_config_tag, uint64_t my_lsn,
const char *my_name);
/* Round-timer arm stub: records every context it accepted (the test owns and
* frees them — no delaymgr ever fires them) and returns g_arm_rc. */
#define MAX_ARMS 16
static void *g_arm_ctx[MAX_ARMS];
static uint64_t g_arm_round[MAX_ARMS];
static int g_arm_n = 0;
static int g_arm_rc = 0;
static int stub_arm(ha_state_t *ha, cluster_t *c, uint64_t round, void *tctx)
{
(void)ha; (void)c;
if (g_arm_rc < 0)
return g_arm_rc; /* scheduling failed: caller keeps ownership */
TEST_ASSERT_TRUE(g_arm_n < MAX_ARMS);
g_arm_ctx[g_arm_n] = tctx;
g_arm_round[g_arm_n] = round;
g_arm_n++;
return 0;
}
static void arm_stub_install(void)
{
g_arm_n = 0;
g_arm_rc = 0;
ha_prevote_round_timer_arm_hook = stub_arm;
}
static void arm_stub_remove(void)
{
ha_prevote_round_timer_arm_hook = NULL;
ha_prevote_pre_arm_hook = NULL;
ha_prevote_round_timer_clear_for_test();
for (int i = 0; i < g_arm_n; i++)
free(g_arm_ctx[i]);
g_arm_n = 0;
}
/*
* Finding 1 (liveness race): ha_start_prevote_async opened round R, then —
* before it armed R's timeout — round R closed (here: its timeout path; in
* production equally a sync LOST or a heartbeat) and another thread (the
* stagger timer) opened R+1 and armed R+1's timeout. The first thread then
* cancelled R+1's timer and armed one for R, a no-op by round id: R+1 was left
* open with NO timer, and gap (b) refuses every later round, so the node was
* stuck until every peer replied or a heartbeat arrived. The interleaving is
* injected deterministically at the pre-arm seam (no sleeps, no threads).
*/
static int g_interleave_done = 0;
static void interleave_close_and_reopen(ha_state_t *ha, cluster_t *c,
uint64_t round)
{
if (g_interleave_done)
return;
g_interleave_done = 1;
ha_prevote_round_timeout_fire_round_for_test(ha, c, round); /* R closes */
ha_start_prevote_async(ha, c); /* R+1 opens */
}
static void test_r2_f1_open_round_always_has_its_timer(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
cluster_t *c = calloc(1, sizeof(*c)); /* no myconf: sends are no-ops */
TEST_ASSERT_NOT_NULL(c);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
arm_stub_install();
g_interleave_done = 0;
ha_prevote_pre_arm_hook = interleave_close_and_reopen;
ha_start_prevote_async(&ha, c);
uint64_t cur = ha_pre_vote_round_for_test(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_interleave_done,
"precondition: the interleaving ran");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: round R+1 is open");
uint64_t t_round = 0;
int t_cancelled = 1;
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
ha_prevote_active_round_timer_for_test(&t_round, &t_cancelled),
"finding 1: the open round must have a published round timeout");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(cur, t_round,
"finding 1: the published round timeout must belong to the OPEN round "
"(R+1), not to the already-closed round R");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, t_cancelled,
"finding 1: the open round's timeout must not be cancelled");
arm_stub_remove();
ha_prevote_rearm_liveness_hook = NULL;
free(c);
ha_fini(&ha);
}
/*
* Finding 6: if the round timeout cannot be armed (calloc / delay_task_ms
* failure), the round must not stay open with no timer — gap (b) would refuse
* every later round. The round is closed and the liveness watchdog re-armed,
* so the next failure detection opens a fresh round.
*/
static void test_r2_f6_round_timer_arm_failure_closes_round(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
cluster_t *c = calloc(1, sizeof(*c));
TEST_ASSERT_NOT_NULL(c);
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
arm_stub_install();
g_arm_rc = -ENOMEM;
ha_start_prevote_async(&ha, c);
uint64_t r1 = ha_pre_vote_round_for_test(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"finding 6: a round whose timeout could not be armed must be closed");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_rearm_calls,
"finding 6: the liveness watchdog must be re-armed");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_prevote_active_round_timer_for_test(NULL, NULL),
"finding 6: no round timeout may stay published after a failed arm");
g_arm_rc = 0;
ha_start_prevote_async(&ha, c); /* the next failure detection */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"finding 6: a later round must not be refused");
TEST_ASSERT_TRUE(ha_pre_vote_round_for_test(&ha) == r1 + 1);
arm_stub_remove();
ha_prevote_rearm_liveness_hook = NULL;
free(c);
ha_fini(&ha);
}
/*
* Finding 2: a refused promotion (carrier defect) must not leave the node raft
* LEADER / CTRL_ELECTING with no timer — every self-recovery path refuses a
* LEADER/ELECTING node. It steps down to FOLLOWER in the SAME term (no term
* or vote change: it keeps its self-vote, so it cannot vote for another
* candidate in this term), becomes CTRL_STANDBY and re-arms the liveness
* watchdog, so the next timeout campaigns for term+1.
*/
static void test_r2_f2_refused_promotion_steps_down_and_rearms(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
/* As raft_on_vote_reply leaves it right before on_become_leader. */
ha.raft.term = 4;
ha.raft.voted_for = ha.raft.self_idx;
ha.raft.role = RAFT_LEADER;
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
ha_elect_ctx_t *pv = sentinel_ctx(2);
pv->is_pre_vote = 1;
atomic_store_explicit(&ha.elect_ctx, (void *)pv, memory_order_release);
int arms0 = atomic_load(&ha_liveness_arm_count_for_test);
raft_cb_become_leader_for_test(&ha, 4);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"finding 2: a refused promotion must leave the node CTRL_STANDBY");
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, (int)ha.raft.role,
"finding 2: raft must step down to FOLLOWER");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(4, ha.raft.term,
"finding 2: the step-down keeps the term");
TEST_ASSERT_EQUAL_INT_MESSAGE(ha.raft.self_idx, ha.raft.voted_for,
"finding 2: the step-down keeps this term's (self) vote");
TEST_ASSERT_EQUAL_UINT64(4, atomic_load_explicit(&ha.term,
memory_order_acquire));
TEST_ASSERT_TRUE_MESSAGE(
atomic_load(&ha_liveness_arm_count_for_test) > arms0,
"finding 2: the liveness watchdog must be re-armed");
/* ... and the node can campaign again (gap (b) admits a new round). */
ha_start_prevote_async(&ha, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"finding 2: the stepped-down node must be able to open a PreVote round");
drop_published_ctx(&ha, pv);
ha_fini(&ha);
}
/*
* Finding 3: the library's delayed-election trigger (raft_on_leader_lost ->
* on_start_election_after_delay -> ha_dispatch_election_start) called
* raft_start_election directly — no ha->elect_ctx carrier stamped for the new
* term, no CTRL_ELECTING transition — so every win from it would be refused
* by raft_cb_become_leader. It must go through the stamped start path
* (ha_enter_real_election -> ha_start_election_async), observed here at its
* ha_rdhup_election_hook seam.
*/
static int g_r2_election_spy_calls = 0;
static void r2_election_spy(ha_state_t *ha, cluster_t *c, int n_peers,
xce_candidate_t **peers, uint64_t tag,
uint64_t lsn, const char *name)
{
(void)ha; (void)c; (void)n_peers; (void)peers; (void)tag; (void)lsn;
(void)name;
g_r2_election_spy_calls++;
}
static void test_r2_f3_delayed_election_uses_stamped_start_path(void)
{
myconf_t myconf;
memset(&myconf, 0, sizeof(myconf));
myconf.name = "ctrl-a";
myconf.config_version_tag = 0x1234;
myconf.peers_cfg.candidates_count = 3;
const char *names[3] = { "ctrl-a", "ctrl-b", "ctrl-c" };
for (int i = 0; i < 3; i++) {
snprintf(myconf.peers_cfg.candidates[i].name,
sizeof(myconf.peers_cfg.candidates[i].name), "%s", names[i]);
myconf.peers_cfg.candidates[i].port = 7001 + i;
}
cluster_t *c = calloc(1, sizeof(*c));
TEST_ASSERT_NOT_NULL(c);
c->myconf = &myconf;
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0 /* priority 0: no delay */));
ha_do_demote(&ha);
make_three_node(&ha);
ha.raft.term = 6;
ha.raft_c = c; /* production wiring for the delayed trigger */
g_r2_election_spy_calls = 0;
ha_rdhup_election_hook = r2_election_spy;
pthread_mutex_lock(&ha.raft_lock);
raft_on_leader_lost(&ha.raft);
pthread_mutex_unlock(&ha.raft_lock);
ha_rdhup_election_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_r2_election_spy_calls,
"finding 3: the delayed election must go through the stamped start "
"path (ha_enter_real_election / ha_start_election_async)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(6, ha.raft.term,
"finding 3: raft_start_election must not be called raw (no carrier)");
TEST_ASSERT_EQUAL_INT(CTRL_ELECTING,
(int)atomic_load_explicit(&ha.role, memory_order_acquire));
ha.raft_c = NULL;
ha_fini(&ha);
free(c);
}
/*
* Finding 5: a PreVote endpoint's error is routed by endpoint KIND, like its
* replies: in every role it goes to the PreVote tally (which drops it unless
* its round is open), never to the CTRL_PRIMARY promoted-standby cleanup (a
* PreVote endpoint is never promoted).
*/
static void test_r2_f5_prevote_ep_error_routed_by_kind_in_primary(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha.raft.n_peers = 5; /* one deny leaves the round pending */
ha.n_peers = 5;
ha_start_prevote_async(&ha, NULL);
uint64_t r = ha_pre_vote_round_for_test(&ha);
atomic_store_explicit(&ha.role, (int32_t)CTRL_PRIMARY, memory_order_release);
endpoint_t ep;
memset(&ep, 0, sizeof(ep));
vote_ep_ctx_build_for_test(&ep, &ha, NULL, /*peer_idx=*/1, 1, 0, 0, "t");
vote_ep_ctx_mark_pre_vote_for_test(&ep, r);
TEST_ASSERT_EQUAL_INT(0, pre_vote_ep_err_handler_for_test(&ep));
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_replies[1],
"finding 5: a PreVote endpoint error must reach the PreVote tally in "
"every role (routed by endpoint kind, not by role)");
free(ep.data);
ha_fini(&ha);
}
/* ══ bug-1915 final review ═════════════════════════════════════════════════ */
extern int ha_prevote_round_timeout_fn(void *arg);
extern void ha_failover_stagger_fire_for_test(ha_state_t *ha, cluster_t *c);
extern _Atomic int ha_prevote_start_vtask_done_for_test;
/* Poll an atomic flag for up to `ms` milliseconds; returns its final value. */
static int fr_wait_flag(_Atomic int *flag, int want, int ms)
{
for (int i = 0; i < ms; i++) {
if (atomic_load(flag) >= want)
return atomic_load(flag);
usleep(1000);
}
return atomic_load(flag);
}
/* Holds ha->raft_lock on its own thread until told to release it (a stand-in
* for a pool worker parked in an fsync under raft_lock). */
typedef struct {
ha_state_t *ha;
_Atomic int locked;
_Atomic int release;
} fr_lock_holder_t;
static void *fr_lock_holder_fn(void *arg)
{
fr_lock_holder_t *h = (fr_lock_holder_t *)arg;
pthread_mutex_lock(&h->ha->raft_lock);
atomic_store(&h->locked, 1);
while (!atomic_load(&h->release))
usleep(1000);
pthread_mutex_unlock(&h->ha->raft_lock);
return NULL;
}
/* Fires a round-timeout context on its own thread (the delaymgr thread). */
typedef struct {
void *tctx;
_Atomic int done;
} fr_fire_t;
static void *fr_fire_fn(void *arg)
{
fr_fire_t *f = (fr_fire_t *)arg;
ha_prevote_round_timeout_fn(f->tctx);
atomic_store(&f->done, 1);
return NULL;
}
/* Fire `tctx` on a "delaymgr" thread while another thread holds raft_lock;
* returns 1 if the timer body returned within `ms` while the lock was held.
* The lock is always released and both threads joined before returning. */
static int fr_fire_under_held_lock(ha_state_t *ha, void *tctx, int ms)
{
fr_lock_holder_t h;
memset(&h, 0, sizeof(h));
h.ha = ha;
pthread_t th, tf;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&th, NULL, fr_lock_holder_fn, &h));
TEST_ASSERT_EQUAL_INT(1, fr_wait_flag(&h.locked, 1, 2000));
fr_fire_t f;
memset(&f, 0, sizeof(f));
f.tctx = tctx;
TEST_ASSERT_EQUAL_INT(0, pthread_create(&tf, NULL, fr_fire_fn, &f));
int returned_while_locked = fr_wait_flag(&f.done, 1, ms);
atomic_store(&h.release, 1);
pthread_join(th, NULL);
pthread_join(tf, NULL);
TEST_ASSERT_EQUAL_INT(1, atomic_load(&f.done));
return returned_while_locked;
}
/*
* Finding 1: a CANCELLED round timer must return without taking raft_lock —
* the delaymgr thread must never wait behind a raft_lock holder (e.g. a pool
* worker in an fsync). A LIVE timer still takes the locked path (it must
* decide the round under raft_lock), so it does wait — the contrast proves the
* fast path is taken only for a cancelled context.
*/
static void test_final_f1_cancelled_round_timer_skips_raft_lock(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
cluster_t *c = calloc(1, sizeof(*c)); /* no myconf: sends are no-ops */
TEST_ASSERT_NOT_NULL(c);
arm_stub_install();
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
/* Round 1 opens with a timer, then is decided LOST (both peers deny):
* the decision cancels and unpublishes its timer under raft_lock. */
ha_start_prevote_async(&ha, c);
TEST_ASSERT_EQUAL_INT(1, g_arm_n);
void *cancelled_tctx = g_arm_ctx[0];
pre_vote_reply_dispatch_for_test(&ha, c, 1, 0);
pre_vote_reply_dispatch_for_test(&ha, c, 2, 0);
TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_prevote_active_round_timer_for_test(NULL, NULL),
"precondition: the decided round's timer is unpublished");
int fast = fr_fire_under_held_lock(&ha, cancelled_tctx, 1000);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, fast,
"finding 1: a cancelled round timer must return without waiting for "
"raft_lock (delaymgr is trigger-only and must never block)");
/* Contrast: a LIVE timer of the open round 2 must take raft_lock — it
* blocks while the lock is held, then decides the round (timeout = LOSS). */
ha_start_prevote_async(&ha, c);
TEST_ASSERT_EQUAL_INT(2, g_arm_n);
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress);
int rearm_before = g_rearm_calls;
int live_fast = fr_fire_under_held_lock(&ha, g_arm_ctx[1], 200);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, live_fast,
"a live round timer must still take raft_lock (locked path)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"the live timer, once it got raft_lock, ends its round as a LOSS");
TEST_ASSERT_EQUAL_INT(rearm_before + 1, g_rearm_calls);
ha_prevote_rearm_liveness_hook = NULL;
arm_stub_remove();
free(c);
ha_fini(&ha);
}
/*
* Finding 3: a late RequestVoteReply on an endpoint of an EARLIER election
* (vctx->my_term = T-1) from a voter now in term T carries the current term T
* and granted=0. It must be dropped — not counted as that peer's deny in the
* term-T election — so the peer's genuine term-T reply still counts.
*/
static void test_final_f3_stale_election_reply_not_counted(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
ha.raft.n_peers = 5; /* one grant does not decide: observe the tally */
ha.n_peers = 5;
ha.raft.term = 3;
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
ha_start_election_async(&ha, NULL, 5, NULL, /*tag=*/1, /*lsn=*/10, "n0");
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role,
"precondition: the term-T election is in flight");
uint64_t T = ha.raft.term;
TEST_ASSERT_EQUAL_UINT64(4, T);
TEST_ASSERT_EQUAL_INT(0, ha.raft.replies_received);
/* Stale: endpoint of the term-(T-1) election; voter already in term T. */
endpoint_t stale;
memset(&stale, 0, sizeof(stale));
vote_ep_ctx_build_for_test(&stale, &ha, NULL, /*votes_idx=*/1, T - 1, 0, 0, "t");
io_packet_t *pkt = vote_reply_build_pkt_for_test(T, /*granted=*/0, 0, 0);
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&stale, pkt));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.replies_received,
"finding 3: a reply from an earlier election's endpoint must not be "
"counted in the current election");
TEST_ASSERT_EQUAL_INT(0, ha.raft.vote_replies[2]);
TEST_ASSERT_EQUAL_INT(RAFT_CANDIDATE, ha.raft.role);
/* Genuine: the same peer's reply on this election's endpoint counts. */
endpoint_t cur;
memset(&cur, 0, sizeof(cur));
vote_ep_ctx_build_for_test(&cur, &ha, NULL, /*votes_idx=*/1, T, 0, 0, "t");
pkt = vote_reply_build_pkt_for_test(T, /*granted=*/1, 0, 0);
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&cur, pkt));
TEST_ASSERT_EQUAL_INT_MESSAGE(2, ha.raft.votes_granted,
"finding 3: the peer's genuine reply must be counted, not dropped as "
"a duplicate of the stale one");
TEST_ASSERT_EQUAL_INT(1, ha.raft.replies_received);
free(stale.data);
free(cur.data);
drop_published_ctx(&ha, NULL);
ha_fini(&ha);
}
/*
* Finding 3, higher-term exception: a late reply on an earlier election's
* endpoint whose term is ABOVE ours must still make this node adopt the term
* and step down (Raft: any higher term steps down), but must never be counted
* as a vote in the current election.
*/
static void test_final_f3_stale_higher_term_reply_steps_down_without_vote(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
make_three_node(&ha);
ha.raft.n_peers = 5;
ha.n_peers = 5;
ha.raft.term = 3;
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
ha_start_election_async(&ha, NULL, 5, NULL, /*tag=*/1, /*lsn=*/10, "n0");
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_CANDIDATE, ha.raft.role,
"precondition: the term-T election is in flight");
uint64_t T = ha.raft.term;
TEST_ASSERT_EQUAL_UINT64(4, T);
int granted_before = ha.raft.votes_granted;
endpoint_t stale;
memset(&stale, 0, sizeof(stale));
vote_ep_ctx_build_for_test(&stale, &ha, NULL, /*votes_idx=*/1, T - 1, 0, 0, "t");
io_packet_t *pkt = vote_reply_build_pkt_for_test(T + 1, /*granted=*/1, 0, 0);
TEST_ASSERT_NOT_NULL(pkt);
TEST_ASSERT_EQUAL_INT(0, peer_ctrl_msg_handler_for_test(&stale, pkt));
TEST_ASSERT_EQUAL_UINT64_MESSAGE(T + 1, ha.raft.term,
"a higher-term reply must make the node adopt that term");
TEST_ASSERT_EQUAL_INT_MESSAGE(RAFT_FOLLOWER, ha.raft.role,
"a higher-term reply must step the candidate down");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.replies_received,
"a stale higher-term reply must never be counted as a reply");
TEST_ASSERT_EQUAL_INT_MESSAGE(granted_before, ha.raft.votes_granted,
"a stale higher-term grant must never be counted as a vote");
free(stale.data);
drop_published_ctx(&ha, NULL);
ha_fini(&ha);
}
/* Finding 4 probes: where ha_start_prevote_async ran (pre-arm seam fires
* inside it, under raft_lock, after the round opened). */
static _Atomic int g_f4_hook_runs = 0;
static pthread_t g_f4_hook_thread;
static void f4_record_thread(ha_state_t *ha, cluster_t *c, uint64_t round)
{
(void)ha; (void)c; (void)round;
g_f4_hook_thread = pthread_self();
atomic_fetch_add(&g_f4_hook_runs, 1);
}
/*
* Finding 4: the failover-stagger delaymgr body must hand the PreVote start
* (per-peer bounded connects under raft_lock) to a VTASK_LIVE_BLOCK vtask, not
* run it inline on the delaymgr thread; the vtask then starts the round.
*/
static void test_final_f4_stagger_hands_prevote_start_to_vtask(void)
{
vtaskmgr_t pool;
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&pool, 4));
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&pool));
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha.last_activity = 0; /* no live primary: the stagger proceeds */
ha.pool = &pool;
uint64_t r0 = ha_pre_vote_round_for_test(&ha);
int done0 = atomic_load(&ha_prevote_start_vtask_done_for_test);
atomic_store(&g_f4_hook_runs, 0);
ha_prevote_pre_arm_hook = f4_record_thread;
pthread_t caller = pthread_self();
ha_failover_stagger_fire_for_test(&ha, NULL);
int runs = fr_wait_flag(&g_f4_hook_runs, 1, 2000);
int done = fr_wait_flag(&ha_prevote_start_vtask_done_for_test, done0 + 1, 2000);
int inline_run = (runs > 0) && pthread_equal(g_f4_hook_thread, caller);
ha_prevote_pre_arm_hook = NULL;
TEST_ASSERT_EQUAL_INT_MESSAGE(1, runs,
"finding 4: the PreVote round must be started exactly once");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, inline_run,
"finding 4: the stagger body must hand the PreVote start to a vtask, "
"not connect inline on the delaymgr thread");
TEST_ASSERT_EQUAL_INT(done0 + 1, done);
TEST_ASSERT_EQUAL_UINT64(r0 + 1, ha_pre_vote_round_for_test(&ha));
pthread_mutex_lock(&ha.raft_lock);
int in_prog = ha.raft.pre_vote_in_progress;
pthread_mutex_unlock(&ha.raft_lock);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, in_prog,
"finding 4: the vtask must start the PreVote round");
vtaskmgr_fini(&pool);
ha.pool = NULL;
ha_fini(&ha);
}
/* Finding 4 (skip kept): a live primary seen during the stagger skips the
* PreVote on the delaymgr thread — nothing is enqueued, no round opens. */
static void test_final_f4_stagger_skips_when_primary_alive(void)
{
vtaskmgr_t pool;
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&pool, 4));
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&pool));
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
struct timespec now;
clock_gettime(CLOCK_MONOTONIC, &now);
ha.last_activity = now.tv_sec; /* fresh primary heartbeat */
atomic_store_explicit(&ha.leaderless, 0, memory_order_release);
ha.pool = &pool;
uint64_t r0 = ha_pre_vote_round_for_test(&ha);
int done0 = atomic_load(&ha_prevote_start_vtask_done_for_test);
ha_failover_stagger_fire_for_test(&ha, NULL);
usleep(50 * 1000);
TEST_ASSERT_EQUAL_INT(done0, atomic_load(&ha_prevote_start_vtask_done_for_test));
TEST_ASSERT_EQUAL_UINT64(r0, ha_pre_vote_round_for_test(&ha));
TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);
vtaskmgr_fini(&pool);
ha.pool = NULL;
ha_fini(&ha);
}
/* Finding 4 (fallback): with no pool, the stagger starts the round inline
* (fail loud, stay alive — mirrors ha_dispatch_rdhup_failover). */
static void test_final_f4_stagger_inline_fallback_without_pool(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
ha.last_activity = 0;
ha.pool = NULL;
uint64_t r0 = ha_pre_vote_round_for_test(&ha);
ha_failover_stagger_fire_for_test(&ha, NULL);
TEST_ASSERT_EQUAL_UINT64(r0 + 1, ha_pre_vote_round_for_test(&ha));
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress);
ha_fini(&ha);
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_prevote(void)
{
UnitySetTestFile(__FILE__);
RUN_TEST(test_prevote_start_no_mutation);
RUN_TEST(test_prevote_majority_grant_starts_real_election);
RUN_TEST(test_prevote_no_majority_stays_standby_and_rearms);
RUN_TEST(test_prevote_round_timeout_is_loss);
RUN_TEST(test_bug1483_lost_round_without_hook_does_not_leak_liveness_ctx);
RUN_TEST(test_prevote_reusable_after_loss);
RUN_TEST(test_prevote_pending_is_noop);
RUN_TEST(test_prevote_reply_over_real_transport_reaches_tally);
/* Task 5.2 — frozen-then-resumed no-op scenario (voter side + candidate side) */
RUN_TEST(test_frozen_rejoin_healthy_peers_deny_without_adoption);
RUN_TEST(test_frozen_rejoin_candidate_gets_no_majority_noop);
/* Task 5.3 — genuine-failover + re-attach scenarios */
RUN_TEST(test_genuine_failover_stale_peer_grants_and_elects);
RUN_TEST(test_heartbeat_mid_round_clean_reattach);
RUN_TEST(test_lost_round_rearms_then_later_failure_elects);
/* bug-1915 — replies after the round is decided are dropped */
RUN_TEST(test_bug1915_late_grant_after_won_is_ignored);
RUN_TEST(test_bug1915_late_deny_after_won_is_ignored);
/* bug-1915 follow-up gaps (a)-(e) + promotion-time carrier check */
RUN_TEST(test_gapA_election_start_noop_when_raft_leader);
RUN_TEST(test_gapA_election_start_noop_when_raft_candidate);
RUN_TEST(test_gapB_prevote_refused_during_real_election);
RUN_TEST(test_gapB_prevote_refused_when_leader);
RUN_TEST(test_gapB_prevote_restart_mid_round_keeps_round);
RUN_TEST(test_gapD_stale_round_reply_dropped);
RUN_TEST(test_gapD_stale_round_timeout_does_not_end_current_round);
RUN_TEST(test_gapD_stale_round_endpoint_error_not_counted);
RUN_TEST(test_gapD_late_prevote_reply_never_counts_as_real_vote);
RUN_TEST(test_gapD_real_vote_reply_never_feeds_prevote_tally);
RUN_TEST(test_become_leader_refuses_prevote_carrier);
RUN_TEST(test_become_leader_refuses_stamped_carrier_without_majority);
RUN_TEST(test_become_leader_refuses_carrier_of_other_term);
RUN_TEST(test_become_leader_accepts_own_carrier_with_majority);
RUN_TEST(test_gapE_concurrent_grants_decide_round_once);
RUN_TEST(test_r2_f1_open_round_always_has_its_timer);
RUN_TEST(test_r2_f6_round_timer_arm_failure_closes_round);
RUN_TEST(test_r2_f2_refused_promotion_steps_down_and_rearms);
RUN_TEST(test_r2_f3_delayed_election_uses_stamped_start_path);
RUN_TEST(test_r2_f5_prevote_ep_error_routed_by_kind_in_primary);
/* bug-1915 final review */
RUN_TEST(test_final_f1_cancelled_round_timer_skips_raft_lock);
RUN_TEST(test_final_f3_stale_election_reply_not_counted);
RUN_TEST(test_final_f3_stale_higher_term_reply_steps_down_without_vote);
RUN_TEST(test_final_f4_stagger_hands_prevote_start_to_vtask);
RUN_TEST(test_final_f4_stagger_skips_when_primary_alive);
RUN_TEST(test_final_f4_stagger_inline_fallback_without_pool);
}
Reference in a new issue View git blame Copy permalink