diff --git a/src/controller/ha.c b/src/controller/ha.c index bff52132..82f88679 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -6739,6 +6739,34 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) { if (!ha) return; + /* raft-ha-core-safety task 5.6: do NOT restart failover while an election is + * already in flight. A won PreVote enters the real election (CTRL_ELECTING), + * or a PreVote round may be mid-decision (pre_vote_in_progress). The dead + * primary's control fd keeps RDHUP'ing on a deterministic ~15s cadence — if + * each RDHUP re-triggers a fresh ha_begin_failover it PREEMPTS the in-progress + * real election before it can complete, bumping the term every cycle and + * keeping two surviving candidates dueling in lockstep (split vote, unbounded + * term escalation; research.md §"Task 5.5 fix VERIFIED"). Drop the RDHUP: + * close the dead control fd and let the in-progress round finish or time out — + * its election timeout is jittered, which desynchronizes the duel so one + * survivor wins. A node NOT currently electing falls through to the normal + * 5.2 suppress / genuine-loss failover path below. */ + { + ctrl_role_t cur = (ctrl_role_t)atomic_load_explicit(&ha->role, + memory_order_acquire); + if (cur == CTRL_ELECTING || ha->raft.pre_vote_in_progress) { + if (ha->primary_ctrl_fd >= 0) { + close(ha->primary_ctrl_fd); + ha->primary_ctrl_fd = -1; + } + INFO("HA: primary_ctrl_fd RDHUP but an election is already in progress " + "(role=%d, prevote=%d) — dropping the RDHUP, not restarting " + "failover (task 5.6: don't preempt the in-progress election / " + "churn the term)", (int)cur, ha->raft.pre_vote_in_progress); + return; + } + } + /* Cancel the liveness poll task first (Req 5.4): prevents a concurrent * poll fn from triggering a second failover call after EPOLLRDHUP fires. */ ha_disarm_heartbeat_timeout_vtask(ha); diff --git a/src/controller/testsuite/test_ha_core_safety.c b/src/controller/testsuite/test_ha_core_safety.c index 84d7e8bd..711c6c51 100644 --- a/src/controller/testsuite/test_ha_core_safety.c +++ b/src/controller/testsuite/test_ha_core_safety.c @@ -1738,6 +1738,79 @@ static void test_5_3_stale_candidate_denied_even_with_primary_gone(void) printf("test_5_3_stale_candidate_denied_even_with_primary_gone: PASS (pin)\n"); } +/* ═══════════════════════════════════════════════════════════════════════════ + * TASK 5.6 — an RDHUP must not restart failover while an election is in progress + * + * Surfaced after 5.5 (research.md §"Task 5.5 fix VERIFIED"): with the freshest + * survivor now able to win a PreVote and ENTER the real election (CTRL_ELECTING), + * round 4 still stalls — the dead primary's control channel keeps RDHUP'ing on a + * deterministic ~15s cadence, and each RDHUP re-triggers ha_begin_failover → a + * fresh PreVote → real election → term++. That preempts the in-progress real + * election before it can complete, churns the term unbounded (21→25→28→29…), and + * keeps two surviving candidates dueling in lockstep so the jittered election + * timeout never breaks the split vote. + * + * (CORE RED) test_5_6_rdhup_does_not_restart_election_in_progress: a node that + * already entered the real election (CTRL_ELECTING) must DROP an RDHUP — + * not start a new PreVote failover. FAILs on the baseline (the RDHUP path's + * 5.2 suppress only catches CTRL_STANDBY, so an ELECTING node falls through + * to ha_begin_failover and restarts the round). + * + * (PIN) test_5_6_genuine_loss_still_fails_over_when_not_electing: a CTRL_STANDBY + * node with a genuinely-dead primary and NO election in progress must STILL + * launch failover — the 5.6 guard only suppresses while already electing. + * + * Requirements: 4.1, 4.3, 6.1, 6.2, 6.3. + * ═══════════════════════════════════════════════════════════════════════════ */ + +/* CORE RED — an RDHUP while CTRL_ELECTING must not restart failover. */ +static void test_5_6_rdhup_does_not_restart_election_in_progress(void) +{ + ha_state_t ha; + hcs5_make_standby(&ha); /* 3-node */ + atomic_store_explicit(&ha.term, 6, memory_order_release); + + /* The node already WON a PreVote and entered the real election. */ + atomic_store_explicit(&ha.role, (int32_t)CTRL_ELECTING, memory_order_release); + ha.raft.pre_vote_in_progress = 0; + hcs5_mark_primary_dead(&ha); /* primary genuinely gone (no 5.2 suppress) */ + + /* The dead primary's control channel RDHUPs (the ~15s cadence). */ + ha_on_primary_ctrl_fd_rdhup(&ha, NULL); + + TEST_ASSERT_EQUAL_INT_MESSAGE(0, ha.raft.pre_vote_in_progress, + "RED: an RDHUP while an election is already in progress must NOT start a " + "new PreVote failover — that preempts the in-progress real election and " + "churns the term, keeping two survivors dueling (task 5.6). Baseline's " + "5.2 suppress only catches CTRL_STANDBY, so an ELECTING node restarts."); + TEST_ASSERT_EQUAL_INT_MESSAGE((int32_t)CTRL_ELECTING, + (int32_t)atomic_load_explicit(&ha.role, memory_order_acquire), + "the in-progress election (CTRL_ELECTING) must be preserved across the RDHUP"); + + ha_fini(&ha); + printf("test_5_6_rdhup_does_not_restart_election_in_progress: done\n"); +} + +/* PIN — genuine loss with NO election in progress must still fail over. */ +static void test_5_6_genuine_loss_still_fails_over_when_not_electing(void) +{ + ha_state_t ha; + hcs5_make_standby(&ha); /* CTRL_STANDBY, 3-node */ + atomic_store_explicit(&ha.term, 6, memory_order_release); + ha.raft.pre_vote_in_progress = 0; + hcs5_mark_primary_dead(&ha); /* primary genuinely gone */ + + ha_on_primary_ctrl_fd_rdhup(&ha, NULL); + + TEST_ASSERT_EQUAL_INT_MESSAGE(1, ha.raft.pre_vote_in_progress, + "PIN: a CTRL_STANDBY node with a dead primary and NO election in progress " + "must STILL launch a real failover — the 5.6 guard only suppresses an " + "RDHUP while ALREADY electing, never genuine loss (no over-suppression)"); + + ha_fini(&ha); + printf("test_5_6_genuine_loss_still_fails_over_when_not_electing: PASS (pin)\n"); +} + /* ═══════════════════════════════════════════════════════════════════════════ * SCAFFOLDING (PASS) — prove each shared seam is usable by later phases. * ═══════════════════════════════════════════════════════════════════════════ */ @@ -1903,6 +1976,12 @@ void test_ha_core_safety(void) RUN_TEST(test_5_3_catchup_survivor_grants_fresh_candidate_when_primary_gone); RUN_TEST(test_5_3_catchup_denies_while_live_primary_feeds); RUN_TEST(test_5_3_stale_candidate_denied_even_with_primary_gone); + /* Task 5.6: an RDHUP must not restart failover while an election is in + * progress (dueling-candidate / term-escalation stall): + * CORE RED — CTRL_ELECTING + RDHUP must NOT start a new PreVote. + * PIN — genuine loss (STANDBY, not electing) still fails over. */ + RUN_TEST(test_5_6_rdhup_does_not_restart_election_in_progress); + RUN_TEST(test_5_6_genuine_loss_still_fails_over_when_not_electing); /* Scaffolding: each shared seam is usable by 2.3 / 4.1 / 5.1. */ RUN_TEST(test_seam_wal_construct_and_probe); RUN_TEST(test_seam_divergence_is_observable);