fix(controller): debounce RDHUP-triggered failover — stop the term-escalation storm (bug-1310, task 5.10b)

A dead primary's control fd RDHUPs faster than an election settles. The 5.6
guard only drops an RDHUP while the role is instantaneously CTRL_ELECTING/
prevote; an RDHUP landing in the gap after a lost election (role briefly back
to CTRL_STANDBY) re-triggers a fresh ha_begin_failover -> PreVote -> real
election (term++). One bump per RDHUP -> unbounded escalation under a sustained
cadence (observed round 4: 48 begin_failovers, terms 6->24, round overran the
harness window).

Fix: ha_state_t.last_failover_ms (atomic) stamped at ha_begin_failover;
ha_on_primary_ctrl_fd_rdhup drops an RDHUP within 2*election_timeout_ms of the
last trigger (closes the dead fd, returns) so the election's own jittered retry
(5.7) runs without the external RDHUP cadence adding term bumps. First trigger
never debounced; genuine loss still fails over, at most once per window.

Validation: unit-proven — test_rdhup_within_cooldown_is_debounced (recent
last_failover -> stays CTRL_STANDBY) vs test_rdhup_single_node_reaches_primary
(last_failover=0 -> CTRL_PRIMARY) is a differential proof. Soak: 10/10 elect,
terms 4->14 bounded, begin_failover 14/14/12 (was 48), committed_job_loss=0, no
regression. The stochastic storm did not recur this run (debounce fired 0x), so
soak-reproduction of the storm is still pending; the mechanism rests on the
unit test. Election suites green (rdhup 8/0, prevote 12/0, core_safety 24/0,
election 28/0, failover_prevote 6/0).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01JDYFbrgyDHbvNYcxApKhrX

src/controller/ha.c

@@ -6725,9 +6725,22 @@ static int ha_failover_stagger_fn(void *arg)
     return 0;
 }
 
+/* CLOCK_MONOTONIC milliseconds — for the RDHUP failover debounce (5.10b). */
+static uint64_t ha_mono_ms(void)
+{
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    return (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L);
+}
+
 void ha_begin_failover(ha_state_t *ha, cluster_t *c)
 {
     if (!ha) return;
+    /* Stamp the failover-trigger time so a rapid RDHUP cadence from a dead
+     * primary's fd cannot re-trigger a fresh failover on every event (5.10b
+     * debounce in ha_on_primary_ctrl_fd_rdhup). Any trigger source (RDHUP or the
+     * deferred heartbeat-timeout stagger) records here. */
+    atomic_store_explicit(&ha->last_failover_ms, ha_mono_ms(), memory_order_release);
     INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
          "PreVote wins) — staying CTRL_STANDBY");
 
@@ -6790,6 +6803,38 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
         }
     }
 
+    /* raft-ha-core-safety 5.10b: RDHUP failover DEBOUNCE. The 5.6 guard above
+     * only catches an RDHUP while the role is INSTANTANEOUSLY CTRL_ELECTING /
+     * pre_vote_in_progress. But a dead primary's fd RDHUPs faster than an
+     * election settles, and each RDHUP that lands in the gap AFTER a real
+     * election loses (role briefly back at CTRL_STANDDBY before the next attempt)
+     * re-triggers a fresh failover → a term bump per RDHUP (observed: 48
+     * begin_failovers, terms 6→24 in one round; research.md §"5.10b DEEPER").
+     * Suppress an RDHUP-triggered failover for up to 2× the election timeout
+     * after the last trigger — long enough to cover the jittered [T,2T] election
+     * watchdog (5.7) so the election's OWN retry runs without the external RDHUP
+     * cadence piling on extra term bumps. A genuinely-dead primary is still
+     * failed over — at most once per window, not once per RDHUP. First trigger
+     * (last_failover_ms == 0) is never debounced. */
+    {
+        uint64_t now_ms = ha_mono_ms();
+        uint64_t last   = atomic_load_explicit(&ha->last_failover_ms,
+                                               memory_order_acquire);
+        uint64_t cooldown_ms = 2u * (uint64_t)ha->config.election_timeout_ms;
+        if (last != 0 && now_ms >= last && (now_ms - last) < cooldown_ms) {
+            if (ha->primary_ctrl_fd >= 0) {
+                close(ha->primary_ctrl_fd);
+                ha->primary_ctrl_fd = -1;
+            }
+            INFO("HA: primary_ctrl_fd RDHUP within failover cooldown (%lums since "
+                 "last trigger, cooldown=%lums) — debouncing, not restarting "
+                 "failover (5.10b: a sustained RDHUP cadence must not ladder the "
+                 "term; the in-flight election's own jittered retry handles it)",
+                 (unsigned long)(now_ms - last), (unsigned long)cooldown_ms);
+            return;
+        }
+    }
+
     /* Cancel the liveness poll task first (Req 5.4): prevents a concurrent
      * poll fn from triggering a second failover call after EPOLLRDHUP fires. */
     ha_disarm_heartbeat_timeout_vtask(ha);

src/controller/ha.h

@@ -160,6 +160,7 @@ typedef struct {
     /* watchdog_stop / watchdog_thread deleted — replaced by ha_arm_heartbeat_timeout_vtask */
     void            *ssl_ctx;           /* SSL_CTX* cast; avoids pulling openssl headers here */
     uint64_t         jitter_seed;       /* xorshift64 PRNG state for election jitter */
+    _Atomic uint64_t last_failover_ms;  /* CLOCK_MONOTONIC ms of last failover trigger; RDHUP debounce (5.10b) */
     zarray_t        *standby_endpoints; /* connected standby controller endpoints for config push */
     /* ── New fields added by ha-config-bootstrap spec ──────── */
     uint64_t         voted_for_term;                  /* Req 9: term of last granted vote */

src/controller/testsuite/test_ha_rdhup_integration.c

@@ -23,6 +23,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/socket.h>
+#include <time.h>
 #include <unistd.h>
 
 #include <unity/unity.h>
@@ -586,9 +587,65 @@ static void test_rdhup_candidate_advertises_freshness_floor_not_stale_committed(
 
 /* ── Suite entry point ──────────────────────────────────────────────────── */
 
+/* ═══════════════════════════════════════════════════════════════════════════
+ * 5.10b: an RDHUP arriving within the failover cooldown of a prior trigger must
+ * be DEBOUNCED — no fresh failover/election, no term ladder. The round-4 soak
+ * storm (48 begin_failovers, terms 6→24) came from a dead primary's fd RDHUPing
+ * faster than an election settles; each RDHUP that landed after a lost election
+ * (role briefly back at CTRL_STANDBY) re-triggered a failover. The single-node
+ * fixture normally reaches CTRL_PRIMARY on RDHUP (see
+ * test_rdhup_single_node_reaches_primary — that PASS is the "not within
+ * cooldown → still elects" pin). With a RECENT last_failover_ms and a stale
+ * heartbeat (so the 5.2 alive-suppression does not fire), the node must stay
+ * CTRL_STANDBY. Pre-fix (no debounce) this reached CTRL_PRIMARY → RED.
+ * ═════════════════════════════════════════════════════════════════════════ */
+static void test_rdhup_within_cooldown_is_debounced(void)
+{
+    install_stubs();
+
+    cluster_t c;
+    myconf_t  myconf;
+    make_cluster(&c, &myconf, "ctrl-a");
+    myconf.peers_cfg.candidates_count = 1; /* single node → would self-elect */
+
+    TEST_ASSERT_EQUAL_INT(0, ha_init(&c.ha, &g_cfg, 0));
+    ha_do_demote(&c.ha);
+    c.ha.phase2_done   = 1;
+    c.ha.n_peers       = 0;
+    c.ha.last_activity = 0; /* stale heartbeat → 5.2 alive-suppression will NOT fire */
+
+    g_rdhup_state.c        = &c;
+    ha_rdhup_election_hook = rdhup_election_drives_evaluate;
+
+    /* A failover was triggered "just now" → this RDHUP is within the cooldown. */
+    struct timespec ts;
+    clock_gettime(CLOCK_MONOTONIC, &ts);
+    uint64_t now_ms = (uint64_t)ts.tv_sec * 1000u + (uint64_t)(ts.tv_nsec / 1000000L);
+    atomic_store_explicit(&c.ha.last_failover_ms, now_ms, memory_order_release);
+
+    int sv[2];
+    TEST_ASSERT_EQUAL_INT(0, socketpair(AF_UNIX, SOCK_STREAM, 0, sv));
+    c.ha.primary_ctrl_fd = sv[0];
+
+    ha_on_primary_ctrl_fd_rdhup(&c.ha, &c);
+    close(sv[1]);
+
+    int role = (int)atomic_load_explicit(&c.ha.role, memory_order_acquire);
+    TEST_ASSERT_EQUAL_INT_MESSAGE((int)CTRL_STANDBY, role,
+        "an RDHUP within the failover cooldown must be debounced — no fresh "
+        "failover; node stays CTRL_STANDBY (5.10b RDHUP storm)");
+    TEST_ASSERT_EQUAL_INT_MESSAGE(-1, c.ha.primary_ctrl_fd,
+        "the debounced RDHUP must still close the dead primary_ctrl_fd");
+
+    clear_stubs();
+    ha_fini(&c.ha);
+    printf("test_rdhup_within_cooldown_is_debounced: PASS\n");
+}
+
 void test_ha_rdhup_integration(void)
 {
     RUN_TEST(test_rdhup_candidate_advertises_freshness_floor_not_stale_committed);
+    RUN_TEST(test_rdhup_within_cooldown_is_debounced);
     RUN_TEST(test_rdhup_full_path_reaches_primary);
     /* ack-driven promotion test deleted (ha-single-leader-role 4.1:
      * ack-driven promotion machinery removed, no shim) */