lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-prevote): rewire failover triggers through PreVote, defer destructive step (task 4.2)

Browse source 
ha_on_primary_ctrl_fd_rdhup + liveness_poll_fn now begin a PreVote round via
ha_begin_failover -> ha_start_prevote_async instead of electing directly; WAL
truncate + CTRL_ELECTING deferred to PreVote WON (ha_enter_real_election).
Connection teardown + config-less voter guard preserved at the trigger.
ha_on_heartbeat_received cancels an in-progress round and re-attaches as standby
with no term change. Single-node immediate-WON synchronously enters the real
election. PreVote now live; full controller suite 563 pass / 0 fail; bug-339/340
preserved.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-06-16 06:29:18 +00:00
parent 7e19bf6110
commit b765fc4705
6 changed files with 492 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

227
src/controller/ha.c
View file

@ -3549,10 +3549,42 @@ static void ha_enter_real_election(ha_state_t *ha, cluster_t *c)
int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
const char *my_name = (c->myconf) ? c->myconf->name : "";
ha->raft_c = c;
#ifdef XCE_TESTING
/* Test seam: the WON-branch real election is the same step the pre-rewire
* rdhup body fired, so the existing rdhup-integration tests intercept it
* here (capturing my_tag/my_lsn / counting election starts) — they now
* drive it through the PreVote→WON path (ha-prevote task 4.2).
*
* Mark the role CTRL_ELECTING before invoking the hook: the real
* ha_start_election_async transitions to CTRL_ELECTING as its first step,
* so a test that intercepts the election must still observe the destructive
* role transition the WON branch performs (the spy hook does not start a
* real election that would set it). */
if (ha_rdhup_election_hook != NULL) {
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
memory_order_release);
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
int np = 0;
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
if (strcmp(cfg->candidates[i].name,
c->myhost ? c->myhost->name : "") != 0)
peer_ptrs[np++] = &cfg->candidates[i];
}
ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn,
my_name);
}
return;
}
#endif
/* ha_start_election_async transitions to CTRL_ELECTING and drives the
* UNCHANGED raft_start_election (term++ + self-vote). Peer connections are
* driven by raft_cb_send_vote_request; the peers[] arg is unused. */
ha->raft_c = c;
ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name);
}
@ -3922,6 +3954,26 @@ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
* 10.5 (INFO logs on FSM transitions)
*/
/*
* ha_begin_failover: the single failover entry point (ha-prevote task 4.2).
*
* Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and
* the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route
* here AFTER their connection-teardown bookkeeping and the config-less
* voter-only guard. It starts a side-effect-free PreVote round and returns;
* it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those
* destructive steps are deferred to PreVote success (ha_enter_real_election,
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
* and term untouched (Req 1.1, 4.3, 4.4).
*/
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
"PreVote wins) — staying CTRL_STANDBY");
ha_start_prevote_async(ha, c);
}
void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
@ -3930,23 +3982,31 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
ha_disarm_heartbeat_timeout_vtask(ha);
/* Close and null the primary control fd */
/* Close and null the primary control fd. Connection-teardown bookkeeping
* stays at the trigger regardless of the PreVote outcome (ha-prevote task
* 4.2). */
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
/* Transition to CTRL_ELECTING (Req 6.2) */
atomic_store_explicit(&ha->role, CTRL_ELECTING, memory_order_release);
INFO("HA: primary_ctrl_fd RDHUP → CTRL_ELECTING (Req 10.5)");
/* ha-prevote task 4.2: the failover trigger NO LONGER transitions to
* CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and
* STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is
* deferred to PreVote success (ha_enter_real_election). The node only ever
* inflates its term / unseats a healthy primary if a PreVote majority is
* first obtained (Req 1.1, 4.3, 4.4). */
INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying "
"CTRL_STANDBY until PreVote wins) (Req 1.1)");
if (!c) return; /* unit test mode: skip persist + election */
/* Truncate uncommitted WAL entries above committed_lsn (Req 6.2) */
uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire);
if (committed > 0)
persist_truncate_wal_above(&c->persist, committed);
if (!c) {
/* Unit-test mode: no persist / no live transport. Still begin the
* round so the deferred-destructive behaviour is observable (the round
* state lives in raft_state_t; ha_enter_real_election with c==NULL only
* flips the role on a WIN). */
ha_begin_failover(ha, c);
return;
}
/* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening
* Req 10.5): a config-less node (config_version_tag == 0) must NEVER start
@ -3956,128 +4016,33 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
* primary over the node that actually holds the config — which then pushes
* an empty config and breaks replication.
*
* Mirror the proven ha_start_election_async voter-only guard: skip the
* elect_ctx alloc + the election trigger (raft_on_leader_lost) entirely and
* stay a passive voter (CTRL_STANDBY). handle_request_vote still GRANTS
* incoming vote requests to the fresher configured peer regardless of role,
* and this node transitions/fetches config when the configured winner
* connects and pushes config via ctrlhello.
* This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a
* config-less node can never win a PreVote (the freshness gate denies it),
* so there is no point starting a round — and starting one would needlessly
* probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote
* still GRANTS incoming vote requests to the fresher configured peer
* regardless of role, and this node transitions/fetches config when the
* configured winner connects and pushes config via ctrlhello.
*
* Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false,
* so a lone node still self-elects regardless of tag. */
* so a lone node still self-elects (its PreVote trivially wins). */
uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0;
int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
if (my_config_tag == 0 && n_peers_guard > 1) {
INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — "
"voter only, not starting RDHUP re-election; awaiting configured "
"primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
"voter only, not starting PreVote/RDHUP re-election; awaiting "
"configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
return;
}
/* Task 6.3 / 6.7 fix: delegate RDHUP election trigger to raft library.
*
* raft_on_leader_lost:
* - clears ha->raft.voted_for = -1 (no direct ha->voted_for_* writes here)
* - sets ha->raft.role = RAFT_FOLLOWER
* - fires on_start_election_after_delay(ctx, priority * election_timeout_ms)
* → raft_cb_start_election_after_delay schedules raft_rdhup_delay_fn
* which calls raft_check_and_start_election after the priority jitter.
*
* The manual nanosleep jitter + voted_for clear + skip-if-already-voted
* logic is now fully owned by the raft state machine.
*
* raft->raft_c must be set before raft_on_leader_lost fires callbacks.
*
* CRITICAL (Task 6.7): raft_cb_send_vote_request reads ha->elect_ctx to
* get my_tag/my_lsn for the RequestVote payload. On a node that entered
* the voter-only path at first boot (my_config_tag==0), ha->elect_ctx was
* set to NULL and never refreshed. Allocate a fresh ha_elect_ctx_t here
* so that the subsequent raft_start_election finds a valid context. */
ha->raft_c = c;
/* Candidate freshness for the RequestVote payload. This MUST mirror the
* voter-side freshness floor (cluster.c OPCode_requestVote): the effective
* floor is max(applied_lsn, last_known_committed_lsn) at the EFFECTIVE
* config tag. A standby never advances persist.committed_lsn (only the
* primary's commit engine writes it) — it advances applied_lsn (WAL
* catch-up) and last_known_committed_lsn (heartbeat replies). Advertising
* persist.committed_lsn here makes every surviving standby advertise a
* stale-low LSN while each peer computes a HIGH floor from applied_lsn;
* the freshness gate (cand_lsn < my_lsn) then trips on both sides, each
* denies the other's vote, no candidate reaches quorum, and the cluster
* spins in an infinite RDHUP→ELECTING→STANDBY loop after the primary dies.
* Using ha_vote_self_freshness makes advertised == voter-floor, so the
* freshest node is always granted by every peer. (Matches the boot-path
* election in myapp_run, which advertises applied_lsn.) */
uint64_t cand_tag = 0, cand_lsn = 0;
if (c->myconf) {
uint64_t local_tag = c->myconf->config_version_tag;
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, &cand_lsn);
}
/* Allocate fresh elect_ctx so raft_cb_send_vote_request can read my_tag/my_lsn */
if (c && c->myconf) {
int n_peers = c->myconf->peers_cfg.candidates_count;
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
ha_elect_ctx_t *new_ctx = calloc(1, sizeof(*new_ctx) +
(size_t)n_other * sizeof(new_ctx->votes[0]));
if (new_ctx) {
uint64_t my_tag = cand_tag;
uint64_t my_lsn = cand_lsn;
new_ctx->n_peers = n_other;
new_ctx->votes_granted = 1; /* self-vote placeholder */
new_ctx->result = -1;
new_ctx->c = c;
new_ctx->self_lsn = my_lsn;
new_ctx->self_tag = my_tag;
pthread_mutex_init(&new_ctx->lock, NULL);
sem_init(&new_ctx->done, 0, 0);
if (n_other > 0) {
new_ctx->votes[0].my_tag = my_tag;
new_ctx->votes[0].my_lsn = my_lsn;
}
/* Replace ha->elect_ctx with the fresh ctx. Do NOT free the old
* ctx here: any live vote_ep_ctx_t objects created in a prior
* election round hold a bare pointer to it (vctx->ctx) and will
* dereference it in vote_ep_err_handler / vote_reply_decode_and_dispatch
* when their connection eventually closes. Freeing here would cause
* a UAF. The old ctx is a small allocation; the leak is bounded to
* one per RDHUP cycle and tolerable. */
atomic_store_explicit(&ha->elect_ctx, new_ctx, memory_order_release);
/* Update n_peers tracking — total cluster size (including self) */
ha->n_peers = n_peers;
}
}
#ifdef XCE_TESTING
if (ha_rdhup_election_hook != NULL) {
/* Test seam: build peer args for hook (same as before) so existing
* tests that capture ha_start_election_async arguments still work. */
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
int np = 0;
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
if (strcmp(cfg->candidates[i].name,
c->myhost ? c->myhost->name : "") != 0)
peer_ptrs[np++] = &cfg->candidates[i];
}
uint64_t my_tag = cand_tag;
uint64_t my_lsn = cand_lsn;
const char *my_name = c->myconf ? c->myconf->name : "";
ha_rdhup_election_hook(ha, c, np, peer_ptrs, my_tag, my_lsn, my_name);
}
return;
}
#endif
raft_on_leader_lost(&ha->raft);
/* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async
* (it snapshots self freshness into ha->elect_ctx so the PreVote request
* payload — and, on a WIN, the deferred real election — advertise the
* effective freshness floor, NOT the stale persist.committed_lsn: see the
* bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */
ha_begin_failover(ha, c);
}
/* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */
@ -4263,6 +4228,22 @@ void ha_on_heartbeat_received(ha_state_t *ha)
clock_gettime(CLOCK_MONOTONIC, &ts);
ha->last_activity = ts.tv_sec;
tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec);
/* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader
* is alive — a PreVote round started by a transient blip must be ABANDONED.
* Cancel the in-progress round (clear the raft round state + the controller
* round-timeout handle) and re-attach as a standby. No persistent term /
* voted_for / role change happens here: the node never left CTRL_STANDBY
* during the round, so re-attaching is a no-op on role/term — exactly the
* non-disruptive rejoin the spec requires. */
if (ha->raft.pre_vote_in_progress) {
ha->raft.pre_vote_in_progress = 0;
ha->raft.pre_votes_granted = 0;
ha_prevote_cancel_round_timeout();
INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round "
"and re-attaching as CTRL_STANDBY (leader alive, no term change) "
"(Req 4.2)");
}
}
void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)

12
src/controller/ha.h
View file

@ -820,6 +820,18 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
*/
void ha_start_prevote_async(ha_state_t *ha, cluster_t *c);
/*
* ha_begin_failover: single failover entry point (ha-prevote task 4.2).
*
* Called by both failover triggers (ha_on_primary_ctrl_fd_rdhup and, through
* it, the liveness-timeout poll) AFTER connection-teardown bookkeeping and the
* config-less voter-only guard. Starts a side-effect-free PreVote round
* (ha_start_prevote_async); does NOT truncate the WAL or transition to
* CTRL_ELECTING — those destructive steps are deferred to PreVote success.
* c may be NULL in unit-test mode. Requirements: 1.1, 4.3, 4.4.
*/
void ha_begin_failover(ha_state_t *ha, cluster_t *c);
/*
* ha_dispatch_repl_action_for_standby: evaluate the replication decision for
* one standby and execute the resulting action.

1
src/controller/meson.build
View file

@ -122,6 +122,7 @@ controller_unit_test_src = [
'testsuite/test_ha_rdhup_integration.c',
'testsuite/test_ha_liveness.c',
'testsuite/test_ha_prevote.c',
'testsuite/test_ha_failover_prevote.c',
'testsuite/test_commit_engine.c',
'testsuite/test_commit_barrier_rollout.c',
'testsuite/test_commit_barrier_reply.c',

2
src/controller/testsuite/controller_test.c
View file

@ -50,6 +50,7 @@ extern void test_ha_connection_reuse(void);
extern void test_ha_rdhup_integration(void);
extern void test_ha_liveness(void);
extern void test_ha_prevote(void);
extern void test_ha_failover_prevote(void);
extern void test_commit_engine(void);
extern void test_commit_barrier_rollout(void);
extern void test_commit_barrier_reply(void);
@ -176,6 +177,7 @@ int main(int argc, char* argv[])
TEST(test_ha_rdhup_integration);
TEST(test_ha_liveness);
TEST(test_ha_prevote);
TEST(test_ha_failover_prevote);
TEST(test_commit_engine);
TEST(test_commit_barrier_rollout);
TEST(test_commit_barrier_reply);

371
src/controller/testsuite/test_ha_failover_prevote.c Normal file
View file

@ -0,0 +1,371 @@
/*
* test_ha_failover_prevote.c — ha-prevote task 4.2:
* route the failover triggers (RDHUP + liveness timeout) through PreVote and
* defer the destructive step (truncate + CTRL_ELECTING) until PreVote WINS;
* cancel an in-progress round when a primary heartbeat arrives mid-round.
*
* This is the rewire that activates PreVote on the LIVE failover path. The
* NEW behaviour these tests pin down (which FAILS before the rewire, when the
* triggers elect directly):
*
* (a) Firing the RDHUP failover trigger does NOT transition to CTRL_ELECTING
* or increment the term — it starts a side-effect-free PreVote round and
* the node stays CTRL_STANDBY with pre_vote_in_progress = 1.
* (b) Firing the liveness-timeout trigger goes through the SAME PreVote-first
* path (it inherits the rewired ha_on_primary_ctrl_fd_rdhup).
* (c) A primary heartbeat arriving mid-round CANCELS the round and re-attaches
* as standby with the persistent term unchanged (Req 4.2).
* (d) When the PreVote round then WINS (a peer grants), the trigger path still
* reaches the real election (CTRL_ELECTING) — failover is not blocked.
* (e) The config-less voter-only guard (tag==0, peers>1) still trips and the
* node does NOT even start a PreVote round (it can never win one).
*
* Boundary: PreVote orchestration (trigger path) — ha.c only.
* Requirements: 1.1, 4.2, 4.3, 4.4.
*
* TDD: written FIRST (RED) before the trigger rewire.
*/
#define _GNU_SOURCE
#include <stdatomic.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <sys/socket.h>
#include <unistd.h>
#include <unity/unity.h>
#include "ha.h"
#include "myconf.h"
#include "raft/raft.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING seams declared in ha.c ──────────────────────────────────── */
extern void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c);
extern void ha_on_heartbeat_received(ha_state_t *ha);
extern int (*ha_prevote_rearm_liveness_hook)(ha_state_t *ha, cluster_t *c);
/*
* liveness_poll_ctx_t: defined privately in ha.c; the layout is mirrored here
* (exactly as test_ha_liveness.c does) so the test can build one for the
* test entry-point without a live delaymgr.
*/
typedef struct {
ha_state_t *ha;
cluster_t *c;
_Atomic int cancelled;
} liveness_poll_ctx_t;
extern void liveness_poll_fn_test_entry(liveness_poll_ctx_t *ctx);
/* ── Test config ─────────────────────────────────────────────────────────── */
static ha_config_t g_cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 6000,
.election_timeout_ms = 500,
.probe_connect_timeout_ms = 50,
.max_promotion_retries = 3,
.election_jitter_max_ms = 0,
.election_priority_step_ms = 500,
};
static int g_rearm_calls = 0;
static int stub_rearm(ha_state_t *ha, cluster_t *c)
{
(void)ha; (void)c;
g_rearm_calls++;
return 0;
}
/*
* Widen the raft topology to a 3-node cluster (self=0) so a PreVote round
* does NOT trivially self-win (quorum = 2; self-grant = 1). This lets the
* round STAY in progress so we can observe the deferred / cancel behaviour.
*/
static void make_three_node(ha_state_t *ha)
{
ha->raft.n_peers = 3;
ha->raft.self_idx = 0;
ha->raft.role = RAFT_FOLLOWER;
ha->n_peers = 3;
}
/* ── Test 1: RDHUP trigger defers the destructive step (Req 1.1, 4.4) ─────── */
/*
* Firing ha_on_primary_ctrl_fd_rdhup must NO LONGER transition straight to
* CTRL_ELECTING / truncate / increment the term. Instead it begins a PreVote
* round and stays CTRL_STANDBY with the round in progress. (Unit-test mode,
* c == NULL: ha_enter_real_election — which contains truncate + CTRL_ELECTING —
* must NOT have run, so the role stays CTRL_STANDBY.)
*/
static void test_rdhup_trigger_starts_prevote_not_election(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha); /* role = CTRL_STANDBY */
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
/* Give the handler an fd to close (teardown bookkeeping still happens). */
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&ha, NULL); /* unit-test mode */
close(sv[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 1.1/4.4: RDHUP trigger must start a PreVote round and STAY "
"CTRL_STANDBY, not transition to CTRL_ELECTING");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 4.4: RDHUP trigger must not change the persistent term before "
"PreVote wins");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"Req 1.1: RDHUP trigger must begin a PreVote round (in-progress)");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_votes_granted,
"PreVote round must record the self-grant (1)");
/* The primary fd must still be closed/cleared — teardown bookkeeping stays
* at the trigger regardless of PreVote outcome. */
TEST_ASSERT_EQUAL_INT_MESSAGE(-1, ha.primary_ctrl_fd,
"RDHUP teardown must still close + clear primary_ctrl_fd");
ha_fini(&ha);
printf("test_rdhup_trigger_starts_prevote_not_election: PASS\n");
}
/* ── Test 2: liveness-timeout trigger goes through the same PreVote path ──── */
/*
* The liveness poll fn, on timeout, must reach the SAME PreVote-first path
* (it calls ha_on_primary_ctrl_fd_rdhup, now rewired). Observable: after the
* timeout, the node is in a PreVote round and STAYS CTRL_STANDBY — it does NOT
* jump to CTRL_ELECTING.
*/
static void test_liveness_timeout_trigger_starts_prevote(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
/* last_activity far enough in the past to exceed heartbeat_timeout_ms. */
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
ha.last_activity = ts.tv_sec - (time_t)(g_cfg.heartbeat_timeout_ms / 1000) - 10;
liveness_poll_ctx_t ctx;
memset(&ctx, 0, sizeof(ctx));
ctx.ha = &ha;
ctx.c = NULL;
atomic_init(&ctx.cancelled, 0);
liveness_poll_fn_test_entry(&ctx);
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 1.1: liveness-timeout failover must start a PreVote round and stay "
"CTRL_STANDBY, not transition straight to CTRL_ELECTING");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 4.4: liveness-timeout failover must not change the term before "
"PreVote wins");
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"Req 1.1: liveness-timeout failover must begin a PreVote round");
ha_fini(&ha);
printf("test_liveness_timeout_trigger_starts_prevote: PASS\n");
}
/* ── Test 3: heartbeat mid-round cancels and re-attaches (Req 4.2) ────────── */
/*
* After the failover trigger starts a PreVote round, an incoming primary
* heartbeat means the leader is alive — the campaign must be abandoned: the
* round is cancelled (pre_vote_in_progress cleared), the node re-attaches as
* CTRL_STANDBY, and the persistent term is UNCHANGED (Req 4.2).
*/
static void test_heartbeat_mid_round_cancels_prevote(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
ha.primary_ctrl_fd = sv[0];
/* Trigger fires → PreVote round in progress. */
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
close(sv[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
"precondition: a PreVote round must be in progress before the heartbeat");
/* A fresh primary heartbeat arrives mid-round → cancel + re-attach. */
ha_on_heartbeat_received(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"Req 4.2: a heartbeat mid-round must CANCEL the PreVote round");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"Req 4.2: after a mid-round heartbeat the node re-attaches as CTRL_STANDBY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"Req 4.2: a mid-round heartbeat must not change the persistent term");
ha_fini(&ha);
printf("test_heartbeat_mid_round_cancels_prevote: PASS\n");
}
/* ── Test 4: heartbeat with NO round in progress is a plain no-op ─────────── */
/*
* The cancel logic must only fire when a round is actually in progress: a
* heartbeat received while no PreVote round is running must just update
* last_activity (the existing behaviour) and never touch role/term.
*/
static void test_heartbeat_without_round_is_plain_noop(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
uint64_t term_before = atomic_load_explicit(&ha.term, memory_order_acquire);
TEST_ASSERT_EQUAL_INT(0, ha.raft.pre_vote_in_progress);
ha_on_heartbeat_received(&ha);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"no round in progress: heartbeat must leave pre_vote_in_progress = 0");
TEST_ASSERT_EQUAL_INT_MESSAGE(CTRL_STANDBY,
(int)atomic_load_explicit(&ha.role, memory_order_acquire),
"no round: heartbeat must keep the node CTRL_STANDBY");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(term_before,
atomic_load_explicit(&ha.term, memory_order_acquire),
"no round: heartbeat must not change the term");
TEST_ASSERT_NOT_EQUAL_MESSAGE(0, ha.last_activity,
"heartbeat must still update last_activity (existing behaviour preserved)");
ha_fini(&ha);
printf("test_heartbeat_without_round_is_plain_noop: PASS\n");
}
/* ── Test 5: a genuine failover (PreVote wins) still reaches real election ── */
/*
* The whole point of PreVote is to defer, not block. After the RDHUP trigger
* starts the round, a peer GRANT reaching quorum must drive the trigger path
* into the real election (CTRL_ELECTING) — failover still proceeds when the
* primary genuinely died.
*/
extern void pre_vote_reply_dispatch_for_test(ha_state_t *ha, cluster_t *c,
int peer_idx, int granted);
static void test_failover_prevote_win_reaches_real_election(void)
{
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &g_cfg, 0));
ha_do_demote(&ha);
make_three_node(&ha);
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
close(sv[1]);
TEST_ASSERT_EQUAL_INT(1, ha.raft.pre_vote_in_progress);
/* Peer index 1 grants → quorum (2) reached → WON → real election. */
pre_vote_reply_dispatch_for_test(&ha, NULL, 1, 1 /*granted*/);
int role = (int)atomic_load_explicit(&ha.role, memory_order_acquire);
TEST_ASSERT_TRUE_MESSAGE(role == CTRL_ELECTING || role == CTRL_PRIMARY,
"Req 4.3/5.x: when PreVote WINS the trigger path must reach the real "
"election (CTRL_ELECTING) — a genuine failover is not blocked");
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
"a WON round must clear the in-progress flag");
ha_fini(&ha);
printf("test_failover_prevote_win_reaches_real_election: PASS (role=%d)\n", role);
}
/* ── Test 6: config-less voter-only guard still trips before PreVote ──────── */
/*
* A config-less node (config_version_tag == 0, peers > 1) must NOT even start
* a PreVote round on RDHUP — it can never win one (the freshness gate denies
* it), and it must stay a passive voter (CTRL_STANDBY). This preserves the
* ha-config-bootstrap Req 2.8 election-safety guard through the rewire.
*
* Requires a real cluster_t with a tag-0 config so the guard path is reached.
*/
static void test_config_less_node_does_not_start_prevote(void)
{
cluster_t c;
myconf_t myconf;
memset(&c, 0, sizeof(c));
memset(&myconf, 0, sizeof(myconf));
myconf.name = (char *)"ctrl-a";
myconf.config_version_tag = 0; /* config-less */
myconf.peers_cfg.candidates_count = 2; /* >1 node */
snprintf(myconf.peers_cfg.candidates[0].name,
sizeof(myconf.peers_cfg.candidates[0].name), "ctrl-a");
snprintf(myconf.peers_cfg.candidates[1].name,
sizeof(myconf.peers_cfg.candidates[1].name), "ctrl-b");
c.myconf = &myconf;
TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
ha_do_demote(&c.ha);
c.ha.phase2_done = 1;
c.ha.n_peers = 1;
make_three_node(&c.ha); /* widen raft so a round would NOT self-win */
int sv[2];
TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
c.ha.primary_ctrl_fd = sv[0];
ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
close(sv[1]);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, c.ha.raft.pre_vote_in_progress,
"config-less node (tag=0, peers>1) must NOT start a PreVote round on RDHUP");
TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY,
(int)atomic_load_explicit(&c.ha.role, memory_order_acquire),
"config-less RDHUP node must stay CTRL_STANDBY (voter only)");
ha_fini(&c.ha);
printf("test_config_less_node_does_not_start_prevote: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_failover_prevote(void)
{
/* Ensure the round LOST/timeout branch never touches a live delaymgr. */
g_rearm_calls = 0;
ha_prevote_rearm_liveness_hook = stub_rearm;
RUN_TEST(test_rdhup_trigger_starts_prevote_not_election);
RUN_TEST(test_liveness_timeout_trigger_starts_prevote);
RUN_TEST(test_heartbeat_mid_round_cancels_prevote);
RUN_TEST(test_heartbeat_without_round_is_plain_noop);
RUN_TEST(test_failover_prevote_win_reaches_real_election);
RUN_TEST(test_config_less_node_does_not_start_prevote);
ha_prevote_rearm_liveness_hook = NULL;
}