lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-election): per-priority failover election stagger + prevote dispatch-gate

Browse source 
Desync FAILOVER re-elections by node priority (ha_election_stagger_ms /
ha_failover_stagger_fn) mirroring the startup stagger, with seed jitter and
async prevote start; allow prevote in voter roles through the dispatch gate.
Adds HA role and vote-gate tests (stagger desync by priority, prevote on dead
primary, stickiness-lapse grants).

Pre-existing uncommitted work, committed to establish a clean baseline before
ha-commit-durability implementation.
This commit is contained in:
Rongsong Shen 2026-06-18 04:16:54 +00:00
parent 223e1b8578
commit 749e194207
5 changed files with 238 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

89
src/controller/testsuite/test_ha_role.c
View file

@ -706,6 +706,91 @@ static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
}
/* ── ha-prevote: dispatch gate must admit OPCode_preVoteRequest wherever it
* admits OPCode_requestVote (CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT).
*
* Regression (bug-605): the PreVote opcode + handler were added but never
* wired into ha_dispatch_gate, so a STANDBY voter rejected incoming PreVote
* straw-polls with -XCE_ERR_STANDBY before the handler ran. No preVoteReply
* was ever sent; the candidate's failed exchange was tallied as a denial,
* so a cold cluster looped PreVote LOST forever and never elected a primary.
*
* PreVote is strictly non-mutating, so it must be at least as admissible as a
* real requestVote in every role. This test asserts that parity. */
static void test_dispatch_gate_allows_prevote_in_voter_roles(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
const int32_t voter_roles[] = { CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT };
for (size_t i = 0; i < sizeof(voter_roles) / sizeof(voter_roles[0]); i++) {
atomic_store_explicit(&ha.role, voter_roles[i], memory_order_release);
/* PreVote must be admitted exactly where the real vote is. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
ha_dispatch_gate(&ha, OPCode_requestVote),
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
"ha_dispatch_gate must admit OPCode_preVoteRequest wherever it "
"admits OPCode_requestVote (bug-605)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
"voter role must admit OPCode_preVoteRequest (bug-605)");
}
ha_fini(&ha);
printf("test_dispatch_gate_allows_prevote_in_voter_roles: PASS\n");
}
/* ── ha-prevote/raft (bug-606): failover election stagger desyncs survivors.
*
* After a primary dies, two surviving standbys must NOT campaign in lock-step
* (split-vote livelock, term climbing unbounded). ha_election_stagger_ms gives
* each node a priority-ordered, jitter-bounded delay before starting its
* failover PreVote so one survivor wins before the other begins. Asserts:
* - bounds: stagger ∈ [my_priority*step, my_priority*step + jitter_max]
* - ordering: a higher-priority-index node's MINIMUM stagger exceeds a
* lower-index node's MAXIMUM (requires step > jitter_max — the desync
* guarantee), so the lower index always campaigns first. */
static void test_election_stagger_desyncs_by_priority(void)
{
const uint32_t step = 500, jmax = 200; /* step > jmax → strict ordering */
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 5000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_priority_step_ms = step,
.election_jitter_max_ms = jmax,
};
for (int prio = 0; prio < 3; prio++) {
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, prio));
uint32_t lo = (uint32_t)prio * step;
uint32_t hi = lo + jmax;
/* Sample several times — jitter is re-randomized each call. */
for (int i = 0; i < 16; i++) {
uint32_t d = ha_election_stagger_ms(&ha);
TEST_ASSERT_TRUE_MESSAGE(d >= lo && d <= hi,
"stagger must be within [prio*step, prio*step+jitter_max] (bug-606)");
}
ha_fini(&ha);
}
/* Desync guarantee: prio 1's max (step+jmax) < prio 2's min (2*step). */
TEST_ASSERT_TRUE_MESSAGE((step + jmax) < (2u * step),
"election_priority_step_ms must exceed election_jitter_max_ms so "
"adjacent priorities never overlap (bug-606 desync guarantee)");
printf("test_election_stagger_desyncs_by_priority: PASS\n");
}
void test_ha_role(void)
{
RUN_TEST(test_ha_init_default_role_is_standby);
@ -746,6 +831,10 @@ void test_ha_role(void)
* config knob + wall-clock watchdog deleted, Req 1.5) */
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
/* ha-prevote (bug-605): dispatch gate must admit PreVote like the real vote */
RUN_TEST(test_dispatch_gate_allows_prevote_in_voter_roles);
/* raft/ha-prevote (bug-606): failover election stagger desyncs survivors */
RUN_TEST(test_election_stagger_desyncs_by_priority);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */