lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-election): per-priority failover election stagger + prevote dispatch-gate

Browse source 
Desync FAILOVER re-elections by node priority (ha_election_stagger_ms /
ha_failover_stagger_fn) mirroring the startup stagger, with seed jitter and
async prevote start; allow prevote in voter roles through the dispatch gate.
Adds HA role and vote-gate tests (stagger desync by priority, prevote on dead
primary, stickiness-lapse grants).

Pre-existing uncommitted work, committed to establish a clean baseline before
ha-commit-durability implementation.
This commit is contained in:
Rongsong Shen 2026-06-18 04:16:54 +00:00
parent 223e1b8578
commit 749e194207
5 changed files with 238 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

1
src/controller/cluster.c
View file

@ -523,6 +523,7 @@ static int myreq_handler(vtask_t* task)
if (role == CTRL_PRIMARY)
deny_reason = "primary";
else if (role == CTRL_STANDBY &&
c->ha.primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(&c->ha))
deny_reason = "fresh_heartbeat";
else if (atomic_load_explicit(&c->ha.in_catch_up,

91
src/controller/ha.c
View file

@ -307,8 +307,18 @@ static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term)
if (role == CTRL_PRIMARY)
return 0;
/* Fresh heartbeat on STANDBY → primary is alive, deny */
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha))
/* Fresh heartbeat on STANDBY WITH a live primary connection → primary is
* alive, deny. BOTH conditions are required (mirrors the handle_request_vote
* gate at ~ha.c:2205). Without the primary_ctrl_fd>=0 guard a node whose
* primary has DIED keeps a falsely-"fresh" status: ha_heartbeat_is_fresh
* derives from last_activity, which ha_arm_heartbeat_timeout_vtask re-arms to
* now on every failover retry — so the node would deny every legitimate
* higher-term PreVote for minutes, preventing convergence (bug-660).
* primary_ctrl_fd is -1 after the primary RDHUPs / the liveness watchdog
* fires / on a fresh boot, so a dead-or-absent primary correctly does NOT
* deny. */
if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(ha))
return 0;
/* In catch-up → deny */
@ -1680,6 +1690,7 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
* config — accepting these ensures the node doesn't miss the initial
* bootstrap and can transition to CTRL_STANDBY immediately. */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_heartbeat ||
opcode == OPCode_hello ||
@ -1698,6 +1709,7 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
* resume replication without having missed pushes. Only job/client
* ops are rejected (we're not ready to serve). */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
@ -1725,6 +1737,7 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_requestVote:
case OPCode_preVoteRequest:
case OPCode_probeConfigTag:
case OPCode_walAck:
case OPCode_upgradeConfigAck:
@ -4071,11 +4084,85 @@ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
* and term untouched (Req 1.1, 4.3, 4.4).
*/
/* ha_election_stagger_ms: see ha.h. Same formula as the startup stagger
* (myapp.c) — priority baseline plus fresh random jitter, re-randomized each
* call so repeated split-vote collisions converge (Raft §5.2; bug-606). */
uint32_t ha_election_stagger_ms(ha_state_t *ha)
{
if (!ha) return 0;
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
uint32_t step = ha->config.election_priority_step_ms;
uint32_t jmax = ha->config.election_jitter_max_ms;
uint32_t delay = (uint32_t)ha->my_priority * step;
if (jmax > 0)
delay += (uint32_t)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)jmax + 1));
return delay;
}
/* failover_stagger_ctx_t: carries ha/c into the delaymgr-scheduled PreVote start.
* Freed by ha_failover_stagger_fn (one-shot). */
typedef struct {
ha_state_t *ha;
cluster_t *c;
} failover_stagger_ctx_t;
/* ha_failover_stagger_fn: delaymgr body that starts the PreVote round after the
* per-node stagger (bug-606). Guard: if a primary connected during the stagger
* (we are a STANDBY with a fresh heartbeat), the cluster already has a leader —
* skip the PreVote entirely instead of needlessly probing a live primary. */
static int ha_failover_stagger_fn(void *arg)
{
failover_stagger_ctx_t *ctx = (failover_stagger_ctx_t *)arg;
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
INFO("HA: failover stagger elapsed but a primary is alive — skipping "
"PreVote (bug-606)");
free(ctx);
return 0;
}
ha_start_prevote_async(ha, c);
free(ctx);
return 0;
}
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
"PreVote wins) — staying CTRL_STANDBY");
/* Stagger the PreVote start per node so two survivors that lost the primary
* simultaneously do not campaign in lock-step and split the vote forever
* (Raft §5.2; bug-606). Production only: in unit-test mode (c == NULL) or
* with no running delaymgr, start synchronously so the existing failover /
* PreVote tests observe the unchanged immediate behaviour. */
uint32_t stagger = ha_election_stagger_ms(ha);
if (c && c->delaymgr.running && stagger > 0) {
failover_stagger_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (sctx) {
sctx->ha = ha;
sctx->c = c;
INFO("HA: staggering PreVote start by %ums (priority=%d) to avoid "
"split vote (bug-606)", stagger, ha->my_priority);
int rc = delay_task_ms(&c->delaymgr, (int)stagger, 1,
ha_failover_stagger_fn, sctx);
if (rc >= 0)
return;
/* Scheduling failed — fall through to a synchronous start. */
WARN("HA: ha_begin_failover: stagger schedule failed (%d) — "
"starting PreVote immediately", rc);
free(sctx);
}
}
ha_start_prevote_async(ha, c);
}

12
src/controller/ha.h
View file

@ -837,6 +837,18 @@ void ha_start_prevote_async(ha_state_t *ha, cluster_t *c);
*/
void ha_begin_failover(ha_state_t *ha, cluster_t *c);
/*
* ha_election_stagger_ms: per-node desync delay (ms) for a FAILOVER re-election.
*
* Mirrors the startup stagger (myapp.c): my_priority * election_priority_step_ms
* plus a fresh random jitter in [0, election_jitter_max_ms]. Applied before the
* PreVote start on the failover path so two survivors that lose the primary at
* the same instant do not campaign in lock-step and split the vote forever
* (Raft §5.2 randomized election timeout; bug-606). Re-randomized each call —
* the jitter_seed advances — so repeated collisions still converge.
*/
uint32_t ha_election_stagger_ms(ha_state_t *ha);
/*
* ha_dispatch_repl_action_for_standby: evaluate the replication decision for
* one standby and execute the resulting action.

89
src/controller/testsuite/test_ha_role.c
View file

@ -706,6 +706,91 @@ static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
}
/* ── ha-prevote: dispatch gate must admit OPCode_preVoteRequest wherever it
* admits OPCode_requestVote (CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT).
*
* Regression (bug-605): the PreVote opcode + handler were added but never
* wired into ha_dispatch_gate, so a STANDBY voter rejected incoming PreVote
* straw-polls with -XCE_ERR_STANDBY before the handler ran. No preVoteReply
* was ever sent; the candidate's failed exchange was tallied as a denial,
* so a cold cluster looped PreVote LOST forever and never elected a primary.
*
* PreVote is strictly non-mutating, so it must be at least as admissible as a
* real requestVote in every role. This test asserts that parity. */
static void test_dispatch_gate_allows_prevote_in_voter_roles(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
const int32_t voter_roles[] = { CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT };
for (size_t i = 0; i < sizeof(voter_roles) / sizeof(voter_roles[0]); i++) {
atomic_store_explicit(&ha.role, voter_roles[i], memory_order_release);
/* PreVote must be admitted exactly where the real vote is. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
ha_dispatch_gate(&ha, OPCode_requestVote),
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
"ha_dispatch_gate must admit OPCode_preVoteRequest wherever it "
"admits OPCode_requestVote (bug-605)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
"voter role must admit OPCode_preVoteRequest (bug-605)");
}
ha_fini(&ha);
printf("test_dispatch_gate_allows_prevote_in_voter_roles: PASS\n");
}
/* ── ha-prevote/raft (bug-606): failover election stagger desyncs survivors.
*
* After a primary dies, two surviving standbys must NOT campaign in lock-step
* (split-vote livelock, term climbing unbounded). ha_election_stagger_ms gives
* each node a priority-ordered, jitter-bounded delay before starting its
* failover PreVote so one survivor wins before the other begins. Asserts:
* - bounds: stagger ∈ [my_priority*step, my_priority*step + jitter_max]
* - ordering: a higher-priority-index node's MINIMUM stagger exceeds a
* lower-index node's MAXIMUM (requires step > jitter_max — the desync
* guarantee), so the lower index always campaigns first. */
static void test_election_stagger_desyncs_by_priority(void)
{
const uint32_t step = 500, jmax = 200; /* step > jmax → strict ordering */
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 5000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_priority_step_ms = step,
.election_jitter_max_ms = jmax,
};
for (int prio = 0; prio < 3; prio++) {
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, prio));
uint32_t lo = (uint32_t)prio * step;
uint32_t hi = lo + jmax;
/* Sample several times — jitter is re-randomized each call. */
for (int i = 0; i < 16; i++) {
uint32_t d = ha_election_stagger_ms(&ha);
TEST_ASSERT_TRUE_MESSAGE(d >= lo && d <= hi,
"stagger must be within [prio*step, prio*step+jitter_max] (bug-606)");
}
ha_fini(&ha);
}
/* Desync guarantee: prio 1's max (step+jmax) < prio 2's min (2*step). */
TEST_ASSERT_TRUE_MESSAGE((step + jmax) < (2u * step),
"election_priority_step_ms must exceed election_jitter_max_ms so "
"adjacent priorities never overlap (bug-606 desync guarantee)");
printf("test_election_stagger_desyncs_by_priority: PASS\n");
}
void test_ha_role(void)
{
RUN_TEST(test_ha_init_default_role_is_standby);
@ -746,6 +831,10 @@ void test_ha_role(void)
* config knob + wall-clock watchdog deleted, Req 1.5) */
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
/* ha-prevote (bug-605): dispatch gate must admit PreVote like the real vote */
RUN_TEST(test_dispatch_gate_allows_prevote_in_voter_roles);
/* raft/ha-prevote (bug-606): failover election stagger desyncs survivors */
RUN_TEST(test_election_stagger_desyncs_by_priority);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */

47
src/controller/testsuite/test_ha_vote_gate.c
View file

@ -737,6 +737,50 @@ static void test_pre_vote_granted_when_stickiness_lapsed_and_fresh(void)
printf("test_pre_vote_granted_when_stickiness_lapsed_and_fresh: PASS\n");
}
/* ── bug-660: dead primary (fd<0) must GRANT even when last_activity is fresh ── */
static void test_pre_vote_dead_primary_grants_despite_fresh_activity(void)
{
/* The convergence bug: after the primary DIED (primary_ctrl_fd < 0) the
* standby's last_activity stays falsely "fresh" because the failover
* watchdog (ha_arm_heartbeat_timeout_vtask) re-arms it to now on every retry.
* With NO live primary connection the node must GRANT a fresh higher-term
* PreVote so the cluster converges in ONE round instead of denying with
* reason=fresh_heartbeat for minutes (the observed 11-round / 3-min stall).
* Distinguishes from test_pre_vote_granted_when_stickiness_lapsed_and_fresh
* (which has a STALE heartbeat): here the heartbeat looks FRESH but there is
* no primary connection, so the fix's primary_ctrl_fd>=0 guard is what makes
* it grant. */
ha_state_t ha;
ha_init(&ha, &g_gate_cfg, 0);
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
ha.raft.term = 3ULL;
ha.voted_for_term = 0;
ha.primary_ctrl_fd = -1; /* primary connection dropped (RDHUP) */
set_last_activity_age(&ha, 500L); /* last_activity FALSELY fresh (watchdog re-arm) */
prevote_state_snapshot_t before;
snapshot_prevote_state(&ha, &before);
uint64_t reply_term = 0;
int granted = pre_vote_dispatch_decide(&ha,
4ULL, /* prospective term = current+1 */
100ULL, 50ULL, /* candidate meets the floor */
100ULL, 50ULL,
0, &reply_term, NULL, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
"a STANDBY whose primary connection is gone (fd<0) must GRANT the PreVote "
"even if last_activity looks fresh — else convergence stalls (bug-660)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3ULL, reply_term,
"granted PreVote reply carries the voter's UNCHANGED term (no adoption)");
assert_prevote_state_unchanged(&ha, &before);
ha_fini(&ha);
printf("test_pre_vote_dead_primary_grants_despite_fresh_activity: PASS\n");
}
/* ── Test suite entry point ───────────────────────────────────────────────── */
void test_ha_vote_gate(void)
@ -771,4 +815,7 @@ void test_ha_vote_gate(void)
RUN_TEST(test_pre_vote_stale_freshness_denies);
RUN_TEST(test_pre_vote_stale_term_denies_no_adoption);
RUN_TEST(test_pre_vote_granted_when_stickiness_lapsed_and_fresh);
/* bug-660 — dead primary (fd<0) must GRANT even when last_activity is fresh */
RUN_TEST(test_pre_vote_dead_primary_grants_despite_fresh_activity);
}