fix(controller): drop RDHUP while electing — break dueling-candidate term escalation (task 5.6)
After 5.5 let the freshest survivor win PreVote and ENTER the real election, round 4 still stalled: the dead primary's control fd keeps RDHUP'ing on a deterministic ~15s cadence, and ha_on_primary_ctrl_fd_rdhup's 5.2 suppress only catches CTRL_STANDBY — so an ELECTING node fell through to ha_begin_failover and restarted a fresh PreVote, preempting its own in-progress real election. Every cycle bumped the term (21->25->28->29...) and kept two survivors dueling in lockstep, defeating the jittered election-timeout tiebreak. Guard the RDHUP entry: if role==CTRL_ELECTING or a PreVote round is in progress, close the dead control fd and return WITHOUT restarting failover — let the in-progress round finish or time out (its election timeout is jittered, which desynchronizes the duel so one survivor wins). A node not currently electing still takes the normal 5.2-suppress / genuine-loss failover path. RED+PIN in test_ha_core_safety.c (22 Tests / 0 Failures): CORE RED — CTRL_ELECTING + RDHUP must not start a new PreVote (Expected 0 Was 1 on baseline); PIN — genuine loss (STANDBY, not electing) still fails over. Preserves Election Safety, 5.2 suppression, and genuine-loss liveness. Re-running 7.2. Co-Authored-By: Claude <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019dbTHR9CsEYK9RsqmQpQpC
This commit is contained in:
parent
fa53feaf62
commit
81cb09a8db
2 changed files with 107 additions and 0 deletions
|
|
@ -6739,6 +6739,34 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
|
||||||
{
|
{
|
||||||
if (!ha) return;
|
if (!ha) return;
|
||||||
|
|
||||||
|
/* raft-ha-core-safety task 5.6: do NOT restart failover while an election is
|
||||||
|
* already in flight. A won PreVote enters the real election (CTRL_ELECTING),
|
||||||
|
* or a PreVote round may be mid-decision (pre_vote_in_progress). The dead
|
||||||
|
* primary's control fd keeps RDHUP'ing on a deterministic ~15s cadence — if
|
||||||
|
* each RDHUP re-triggers a fresh ha_begin_failover it PREEMPTS the in-progress
|
||||||
|
* real election before it can complete, bumping the term every cycle and
|
||||||
|
* keeping two surviving candidates dueling in lockstep (split vote, unbounded
|
||||||
|
* term escalation; research.md §"Task 5.5 fix VERIFIED"). Drop the RDHUP:
|
||||||
|
* close the dead control fd and let the in-progress round finish or time out —
|
||||||
|
* its election timeout is jittered, which desynchronizes the duel so one
|
||||||
|
* survivor wins. A node NOT currently electing falls through to the normal
|
||||||
|
* 5.2 suppress / genuine-loss failover path below. */
|
||||||
|
{
|
||||||
|
ctrl_role_t cur = (ctrl_role_t)atomic_load_explicit(&ha->role,
|
||||||
|
memory_order_acquire);
|
||||||
|
if (cur == CTRL_ELECTING || ha->raft.pre_vote_in_progress) {
|
||||||
|
if (ha->primary_ctrl_fd >= 0) {
|
||||||
|
close(ha->primary_ctrl_fd);
|
||||||
|
ha->primary_ctrl_fd = -1;
|
||||||
|
}
|
||||||
|
INFO("HA: primary_ctrl_fd RDHUP but an election is already in progress "
|
||||||
|
"(role=%d, prevote=%d) — dropping the RDHUP, not restarting "
|
||||||
|
"failover (task 5.6: don't preempt the in-progress election / "
|
||||||
|
"churn the term)", (int)cur, ha->raft.pre_vote_in_progress);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Cancel the liveness poll task first (Req 5.4): prevents a concurrent
|
/* Cancel the liveness poll task first (Req 5.4): prevents a concurrent
|
||||||
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
|
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
|
||||||
ha_disarm_heartbeat_timeout_vtask(ha);
|
ha_disarm_heartbeat_timeout_vtask(ha);
|
||||||
|
|
|
||||||
|
|
@ -1738,6 +1738,79 @@ static void test_5_3_stale_candidate_denied_even_with_primary_gone(void)
|
||||||
printf("test_5_3_stale_candidate_denied_even_with_primary_gone: PASS (pin)\n");
|
printf("test_5_3_stale_candidate_denied_even_with_primary_gone: PASS (pin)\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* ═══════════════════════════════════════════════════════════════════════════
|
||||||
|
* TASK 5.6 — an RDHUP must not restart failover while an election is in progress
|
||||||
|
*
|
||||||
|
* Surfaced after 5.5 (research.md §"Task 5.5 fix VERIFIED"): with the freshest
|
||||||
|
* survivor now able to win a PreVote and ENTER the real election (CTRL_ELECTING),
|
||||||
|
* round 4 still stalls — the dead primary's control channel keeps RDHUP'ing on a
|
||||||
|
* deterministic ~15s cadence, and each RDHUP re-triggers ha_begin_failover → a
|
||||||
|
* fresh PreVote → real election → term++. That preempts the in-progress real
|
||||||
|
* election before it can complete, churns the term unbounded (21→25→28→29…), and
|
||||||
|
* keeps two surviving candidates dueling in lockstep so the jittered election
|
||||||
|
* timeout never breaks the split vote.
|
||||||
|
*
|
||||||
|
* (CORE RED) test_5_6_rdhup_does_not_restart_election_in_progress: a node that
|
||||||
|
* already entered the real election (CTRL_ELECTING) must DROP an RDHUP —
|
||||||
|
* not start a new PreVote failover. FAILs on the baseline (the RDHUP path's
|
||||||
|
* 5.2 suppress only catches CTRL_STANDBY, so an ELECTING node falls through
|
||||||
|
* to ha_begin_failover and restarts the round).
|
||||||
|
*
|
||||||
|
* (PIN) test_5_6_genuine_loss_still_fails_over_when_not_electing: a CTRL_STANDBY
|
||||||
|
* node with a genuinely-dead primary and NO election in progress must STILL
|
||||||
|
* launch failover — the 5.6 guard only suppresses while already electing.
|
||||||
|
*
|
||||||
|
* Requirements: 4.1, 4.3, 6.1, 6.2, 6.3.
|
||||||
|
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
||||||
|
/* CORE RED — an RDHUP while CTRL_ELECTING must not restart failover. */
|
||||||
|
static void test_5_6_rdhup_does_not_restart_election_in_progress(void)
|
||||||
|
{
|
||||||
|
ha_state_t ha;
|
||||||
|
hcs5_make_standby(&ha); /* 3-node */
|
||||||
|
atomic_store_explicit(&ha.term, 6, memory_order_release);
|
||||||
|
|
||||||
|
/* The node already WON a PreVote and entered the real election. */
|
||||||
|
atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release);
|
||||||
|
ha.raft.pre_vote_in_progress = 0;
|
||||||
|
hcs5_mark_primary_dead(&ha); /* primary genuinely gone (no 5.2 suppress) */
|
||||||
|
|
||||||
|
/* The dead primary's control channel RDHUPs (the ~15s cadence). */
|
||||||
|
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress,
|
||||||
|
"RED: an RDHUP while an election is already in progress must NOT start a "
|
||||||
|
"new PreVote failover — that preempts the in-progress real election and "
|
||||||
|
"churns the term, keeping two survivors dueling (task 5.6). Baseline's "
|
||||||
|
"5.2 suppress only catches CTRL_STANDBY, so an ELECTING node restarts.");
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_ELECTING,
|
||||||
|
(int32_t)atomic_load_explicit(&ha.role, memory_order_acquire),
|
||||||
|
"the in-progress election (CTRL_ELECTING) must be preserved across the RDHUP");
|
||||||
|
|
||||||
|
ha_fini(&ha);
|
||||||
|
printf("test_5_6_rdhup_does_not_restart_election_in_progress: done\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/* PIN — genuine loss with NO election in progress must still fail over. */
|
||||||
|
static void test_5_6_genuine_loss_still_fails_over_when_not_electing(void)
|
||||||
|
{
|
||||||
|
ha_state_t ha;
|
||||||
|
hcs5_make_standby(&ha); /* CTRL_STANDBY, 3-node */
|
||||||
|
atomic_store_explicit(&ha.term, 6, memory_order_release);
|
||||||
|
ha.raft.pre_vote_in_progress = 0;
|
||||||
|
hcs5_mark_primary_dead(&ha); /* primary genuinely gone */
|
||||||
|
|
||||||
|
ha_on_primary_ctrl_fd_rdhup(&ha, NULL);
|
||||||
|
|
||||||
|
TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress,
|
||||||
|
"PIN: a CTRL_STANDBY node with a dead primary and NO election in progress "
|
||||||
|
"must STILL launch a real failover — the 5.6 guard only suppresses an "
|
||||||
|
"RDHUP while ALREADY electing, never genuine loss (no over-suppression)");
|
||||||
|
|
||||||
|
ha_fini(&ha);
|
||||||
|
printf("test_5_6_genuine_loss_still_fails_over_when_not_electing: PASS (pin)\n");
|
||||||
|
}
|
||||||
|
|
||||||
/* ═══════════════════════════════════════════════════════════════════════════
|
/* ═══════════════════════════════════════════════════════════════════════════
|
||||||
* SCAFFOLDING (PASS) — prove each shared seam is usable by later phases.
|
* SCAFFOLDING (PASS) — prove each shared seam is usable by later phases.
|
||||||
* ═══════════════════════════════════════════════════════════════════════════ */
|
* ═══════════════════════════════════════════════════════════════════════════ */
|
||||||
|
|
@ -1903,6 +1976,12 @@ void test_ha_core_safety(void)
|
||||||
RUN_TEST(test_5_3_catchup_survivor_grants_fresh_candidate_when_primary_gone);
|
RUN_TEST(test_5_3_catchup_survivor_grants_fresh_candidate_when_primary_gone);
|
||||||
RUN_TEST(test_5_3_catchup_denies_while_live_primary_feeds);
|
RUN_TEST(test_5_3_catchup_denies_while_live_primary_feeds);
|
||||||
RUN_TEST(test_5_3_stale_candidate_denied_even_with_primary_gone);
|
RUN_TEST(test_5_3_stale_candidate_denied_even_with_primary_gone);
|
||||||
|
/* Task 5.6: an RDHUP must not restart failover while an election is in
|
||||||
|
* progress (dueling-candidate / term-escalation stall):
|
||||||
|
* CORE RED — CTRL_ELECTING + RDHUP must NOT start a new PreVote.
|
||||||
|
* PIN — genuine loss (STANDBY, not electing) still fails over. */
|
||||||
|
RUN_TEST(test_5_6_rdhup_does_not_restart_election_in_progress);
|
||||||
|
RUN_TEST(test_5_6_genuine_loss_still_fails_over_when_not_electing);
|
||||||
/* Scaffolding: each shared seam is usable by 2.3 / 4.1 / 5.1. */
|
/* Scaffolding: each shared seam is usable by 2.3 / 4.1 / 5.1. */
|
||||||
RUN_TEST(test_seam_wal_construct_and_probe);
|
RUN_TEST(test_seam_wal_construct_and_probe);
|
||||||
RUN_TEST(test_seam_divergence_is_observable);
|
RUN_TEST(test_seam_divergence_is_observable);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue