lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-election): per-priority failover election stagger + prevote dispatch-gate

Browse source 
Desync FAILOVER re-elections by node priority (ha_election_stagger_ms /
ha_failover_stagger_fn) mirroring the startup stagger, with seed jitter and
async prevote start; allow prevote in voter roles through the dispatch gate.
Adds HA role and vote-gate tests (stagger desync by priority, prevote on dead
primary, stickiness-lapse grants).

Pre-existing uncommitted work, committed to establish a clean baseline before
ha-commit-durability implementation.
This commit is contained in:
Rongsong Shen 2026-06-18 04:16:54 +00:00
parent 223e1b8578
commit 749e194207
5 changed files with 238 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

89
src/controller/testsuite/test_ha_role.c
View file

@ -706,6 +706,91 @@ static void test_dispatch_gate_allows_wal_catchup_in_standby_roles(void)
printf("test_dispatch_gate_allows_wal_catchup_in_standby_roles: PASS\n");
}
/* ── ha-prevote: dispatch gate must admit OPCode_preVoteRequest wherever it
* admits OPCode_requestVote (CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT).
*
* Regression (bug-605): the PreVote opcode + handler were added but never
* wired into ha_dispatch_gate, so a STANDBY voter rejected incoming PreVote
* straw-polls with -XCE_ERR_STANDBY before the handler ran. No preVoteReply
* was ever sent; the candidate's failed exchange was tallied as a denial,
* so a cold cluster looped PreVote LOST forever and never elected a primary.
*
* PreVote is strictly non-mutating, so it must be at least as admissible as a
* real requestVote in every role. This test asserts that parity. */
static void test_dispatch_gate_allows_prevote_in_voter_roles(void)
{
ha_state_t ha;
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 2000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
};
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, 0));
const int32_t voter_roles[] = { CTRL_STANDBY, CTRL_ELECTING, CTRL_INIT };
for (size_t i = 0; i < sizeof(voter_roles) / sizeof(voter_roles[0]); i++) {
atomic_store_explicit(&ha.role, voter_roles[i], memory_order_release);
/* PreVote must be admitted exactly where the real vote is. */
TEST_ASSERT_EQUAL_INT_MESSAGE(
ha_dispatch_gate(&ha, OPCode_requestVote),
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
"ha_dispatch_gate must admit OPCode_preVoteRequest wherever it "
"admits OPCode_requestVote (bug-605)");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
ha_dispatch_gate(&ha, OPCode_preVoteRequest),
"voter role must admit OPCode_preVoteRequest (bug-605)");
}
ha_fini(&ha);
printf("test_dispatch_gate_allows_prevote_in_voter_roles: PASS\n");
}
/* ── ha-prevote/raft (bug-606): failover election stagger desyncs survivors.
*
* After a primary dies, two surviving standbys must NOT campaign in lock-step
* (split-vote livelock, term climbing unbounded). ha_election_stagger_ms gives
* each node a priority-ordered, jitter-bounded delay before starting its
* failover PreVote so one survivor wins before the other begins. Asserts:
* - bounds: stagger ∈ [my_priority*step, my_priority*step + jitter_max]
* - ordering: a higher-priority-index node's MINIMUM stagger exceeds a
* lower-index node's MAXIMUM (requires step > jitter_max — the desync
* guarantee), so the lower index always campaigns first. */
static void test_election_stagger_desyncs_by_priority(void)
{
const uint32_t step = 500, jmax = 200; /* step > jmax → strict ordering */
ha_config_t cfg = {
.heartbeat_interval_ms = 3000,
.heartbeat_timeout_ms = 10000,
.election_timeout_ms = 5000,
.probe_connect_timeout_ms = 500,
.max_promotion_retries = 5,
.election_priority_step_ms = step,
.election_jitter_max_ms = jmax,
};
for (int prio = 0; prio < 3; prio++) {
ha_state_t ha;
TEST_ASSERT_EQUAL_INT(0, ha_init(&ha, &cfg, prio));
uint32_t lo = (uint32_t)prio * step;
uint32_t hi = lo + jmax;
/* Sample several times — jitter is re-randomized each call. */
for (int i = 0; i < 16; i++) {
uint32_t d = ha_election_stagger_ms(&ha);
TEST_ASSERT_TRUE_MESSAGE(d >= lo && d <= hi,
"stagger must be within [prio*step, prio*step+jitter_max] (bug-606)");
}
ha_fini(&ha);
}
/* Desync guarantee: prio 1's max (step+jmax) < prio 2's min (2*step). */
TEST_ASSERT_TRUE_MESSAGE((step + jmax) < (2u * step),
"election_priority_step_ms must exceed election_jitter_max_ms so "
"adjacent priorities never overlap (bug-606 desync guarantee)");
printf("test_election_stagger_desyncs_by_priority: PASS\n");
}
void test_ha_role(void)
{
RUN_TEST(test_ha_init_default_role_is_standby);
@ -746,6 +831,10 @@ void test_ha_role(void)
* config knob + wall-clock watchdog deleted, Req 1.5) */
/* Task 3.3 (ha-wal-replication): dispatch gate standby-role guard */
RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_roles);
/* ha-prevote (bug-605): dispatch gate must admit PreVote like the real vote */
RUN_TEST(test_dispatch_gate_allows_prevote_in_voter_roles);
/* raft/ha-prevote (bug-606): failover election stagger desyncs survivors */
RUN_TEST(test_election_stagger_desyncs_by_priority);
}
/* test_am_i_primary suite deleted — am_i_primary() removed (election-iomgr-refactor) */

47
src/controller/testsuite/test_ha_vote_gate.c
View file

@ -737,6 +737,50 @@ static void test_pre_vote_granted_when_stickiness_lapsed_and_fresh(void)
printf("test_pre_vote_granted_when_stickiness_lapsed_and_fresh: PASS\n");
}
/* ── bug-660: dead primary (fd<0) must GRANT even when last_activity is fresh ── */
static void test_pre_vote_dead_primary_grants_despite_fresh_activity(void)
{
/* The convergence bug: after the primary DIED (primary_ctrl_fd < 0) the
* standby's last_activity stays falsely "fresh" because the failover
* watchdog (ha_arm_heartbeat_timeout_vtask) re-arms it to now on every retry.
* With NO live primary connection the node must GRANT a fresh higher-term
* PreVote so the cluster converges in ONE round instead of denying with
* reason=fresh_heartbeat for minutes (the observed 11-round / 3-min stall).
* Distinguishes from test_pre_vote_granted_when_stickiness_lapsed_and_fresh
* (which has a STALE heartbeat): here the heartbeat looks FRESH but there is
* no primary connection, so the fix's primary_ctrl_fd>=0 guard is what makes
* it grant. */
ha_state_t ha;
ha_init(&ha, &g_gate_cfg, 0);
atomic_store_explicit(&ha.role, (int32_t)CTRL_STANDBY, memory_order_release);
atomic_store_explicit(&ha.term, 3ULL, memory_order_release);
ha.raft.term = 3ULL;
ha.voted_for_term = 0;
ha.primary_ctrl_fd = -1; /* primary connection dropped (RDHUP) */
set_last_activity_age(&ha, 500L); /* last_activity FALSELY fresh (watchdog re-arm) */
prevote_state_snapshot_t before;
snapshot_prevote_state(&ha, &before);
uint64_t reply_term = 0;
int granted = pre_vote_dispatch_decide(&ha,
4ULL, /* prospective term = current+1 */
100ULL, 50ULL, /* candidate meets the floor */
100ULL, 50ULL,
0, &reply_term, NULL, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, granted,
"a STANDBY whose primary connection is gone (fd<0) must GRANT the PreVote "
"even if last_activity looks fresh — else convergence stalls (bug-660)");
TEST_ASSERT_EQUAL_UINT64_MESSAGE(3ULL, reply_term,
"granted PreVote reply carries the voter's UNCHANGED term (no adoption)");
assert_prevote_state_unchanged(&ha, &before);
ha_fini(&ha);
printf("test_pre_vote_dead_primary_grants_despite_fresh_activity: PASS\n");
}
/* ── Test suite entry point ───────────────────────────────────────────────── */
void test_ha_vote_gate(void)
@ -771,4 +815,7 @@ void test_ha_vote_gate(void)
RUN_TEST(test_pre_vote_stale_freshness_denies);
RUN_TEST(test_pre_vote_stale_term_denies_no_adoption);
RUN_TEST(test_pre_vote_granted_when_stickiness_lapsed_and_fresh);
/* bug-660 — dead primary (fd<0) must GRANT even when last_activity is fresh */
RUN_TEST(test_pre_vote_dead_primary_grants_despite_fresh_activity);
}