lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-election): per-priority failover election stagger + prevote dispatch-gate

Browse source 
Desync FAILOVER re-elections by node priority (ha_election_stagger_ms /
ha_failover_stagger_fn) mirroring the startup stagger, with seed jitter and
async prevote start; allow prevote in voter roles through the dispatch gate.
Adds HA role and vote-gate tests (stagger desync by priority, prevote on dead
primary, stickiness-lapse grants).

Pre-existing uncommitted work, committed to establish a clean baseline before
ha-commit-durability implementation.
This commit is contained in:
Rongsong Shen 2026-06-18 04:16:54 +00:00
parent 223e1b8578
commit 749e194207
5 changed files with 238 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

91
src/controller/ha.c
View file

@ -307,8 +307,18 @@ static int raft_cb_vote_eligible(void *ctx, int from_idx, uint64_t term)
if (role == CTRL_PRIMARY)
return 0;
/* Fresh heartbeat on STANDBY → primary is alive, deny */
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha))
/* Fresh heartbeat on STANDBY WITH a live primary connection → primary is
* alive, deny. BOTH conditions are required (mirrors the handle_request_vote
* gate at ~ha.c:2205). Without the primary_ctrl_fd>=0 guard a node whose
* primary has DIED keeps a falsely-"fresh" status: ha_heartbeat_is_fresh
* derives from last_activity, which ha_arm_heartbeat_timeout_vtask re-arms to
* now on every failover retry — so the node would deny every legitimate
* higher-term PreVote for minutes, preventing convergence (bug-660).
* primary_ctrl_fd is -1 after the primary RDHUPs / the liveness watchdog
* fires / on a fresh boot, so a dead-or-absent primary correctly does NOT
* deny. */
if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &&
ha_heartbeat_is_fresh(ha))
return 0;
/* In catch-up → deny */
@ -1680,6 +1690,7 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
* config — accepting these ensures the node doesn't miss the initial
* bootstrap and can transition to CTRL_STANDBY immediately. */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_heartbeat ||
opcode == OPCode_hello ||
@ -1698,6 +1709,7 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
* resume replication without having missed pushes. Only job/client
* ops are rejected (we're not ready to serve). */
if (opcode == OPCode_requestVote ||
opcode == OPCode_preVoteRequest ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
@ -1725,6 +1737,7 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
case OPCode_replConfig:
case OPCode_replConfigPush:
case OPCode_requestVote:
case OPCode_preVoteRequest:
case OPCode_probeConfigTag:
case OPCode_walAck:
case OPCode_upgradeConfigAck:
@ -4071,11 +4084,85 @@ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
* and term untouched (Req 1.1, 4.3, 4.4).
*/
/* ha_election_stagger_ms: see ha.h. Same formula as the startup stagger
* (myapp.c) — priority baseline plus fresh random jitter, re-randomized each
* call so repeated split-vote collisions converge (Raft §5.2; bug-606). */
uint32_t ha_election_stagger_ms(ha_state_t *ha)
{
if (!ha) return 0;
if (ha->jitter_seed == 0)
ha_seed_jitter(&ha->jitter_seed, ha->my_priority);
uint32_t step = ha->config.election_priority_step_ms;
uint32_t jmax = ha->config.election_jitter_max_ms;
uint32_t delay = (uint32_t)ha->my_priority * step;
if (jmax > 0)
delay += (uint32_t)(xorshift64_next(&ha->jitter_seed)
% ((uint64_t)jmax + 1));
return delay;
}
/* failover_stagger_ctx_t: carries ha/c into the delaymgr-scheduled PreVote start.
* Freed by ha_failover_stagger_fn (one-shot). */
typedef struct {
ha_state_t *ha;
cluster_t *c;
} failover_stagger_ctx_t;
/* ha_failover_stagger_fn: delaymgr body that starts the PreVote round after the
* per-node stagger (bug-606). Guard: if a primary connected during the stagger
* (we are a STANDBY with a fresh heartbeat), the cluster already has a leader —
* skip the PreVote entirely instead of needlessly probing a live primary. */
static int ha_failover_stagger_fn(void *arg)
{
failover_stagger_ctx_t *ctx = (failover_stagger_ctx_t *)arg;
ha_state_t *ha = ctx->ha;
cluster_t *c = ctx->c;
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)) {
INFO("HA: failover stagger elapsed but a primary is alive — skipping "
"PreVote (bug-606)");
free(ctx);
return 0;
}
ha_start_prevote_async(ha, c);
free(ctx);
return 0;
}
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
"PreVote wins) — staying CTRL_STANDBY");
/* Stagger the PreVote start per node so two survivors that lost the primary
* simultaneously do not campaign in lock-step and split the vote forever
* (Raft §5.2; bug-606). Production only: in unit-test mode (c == NULL) or
* with no running delaymgr, start synchronously so the existing failover /
* PreVote tests observe the unchanged immediate behaviour. */
uint32_t stagger = ha_election_stagger_ms(ha);
if (c && c->delaymgr.running && stagger > 0) {
failover_stagger_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (sctx) {
sctx->ha = ha;
sctx->c = c;
INFO("HA: staggering PreVote start by %ums (priority=%d) to avoid "
"split vote (bug-606)", stagger, ha->my_priority);
int rc = delay_task_ms(&c->delaymgr, (int)stagger, 1,
ha_failover_stagger_fn, sctx);
if (rc >= 0)
return;
/* Scheduling failed — fall through to a synchronous start. */
WARN("HA: ha_begin_failover: stagger schedule failed (%d) — "
"starting PreVote immediately", rc);
free(sctx);
}
}
ha_start_prevote_async(ha, c);
}