From 1ded1022fbed9089fe8e470643556872574efbfb Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sun, 31 May 2026 02:24:52 +0000 Subject: [PATCH] =?UTF-8?q?fix(election-iomgr-refactor):=20RDHUP=20failove?= =?UTF-8?q?r=20fixes=20=E2=80=94=20heartbeat=20watchdog=20+=20jitter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ha.c: arm heartbeat timeout watchdog after ha_find_and_connect_to_primary (all 3 tiers) - ha.c: guard ha_arm_heartbeat_timeout_vtask with c!=NULL (prevents ASAN stack-use-after-return in unit tests) - ha.c: add priority-based jitter to ha_on_primary_ctrl_fd_rdhup to avoid split-brain - ha.c: clear voted_for before RDHUP jitter so check reflects new votes, not stale startup votes --- src/controller/ha.c | 36 +++++++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/src/controller/ha.c b/src/controller/ha.c index d2bdc05..2bc3935 100644 --- a/src/controller/ha.c +++ b/src/controller/ha.c @@ -2134,7 +2134,7 @@ int ha_find_and_connect_to_primary(ha_state_t *ha, ha->voted_for_name, r); ha->primary_ctrl_fd = fd; register_fd_with_iomgr(c, fd); - ha_arm_heartbeat_timeout_vtask(ha, c); + if (c) ha_arm_heartbeat_timeout_vtask(ha, c); return fd; } if (fd == -ETIMEDOUT || fd == -EAGAIN) { @@ -2160,7 +2160,7 @@ int ha_find_and_connect_to_primary(ha_state_t *ha, INFO("HA: Tier 2 connected to '%s'", p2->name); ha->primary_ctrl_fd = fd; register_fd_with_iomgr(c, fd); - ha_arm_heartbeat_timeout_vtask(ha, c); + if (c) ha_arm_heartbeat_timeout_vtask(ha, c); return fd; } } @@ -2177,7 +2177,7 @@ int ha_find_and_connect_to_primary(ha_state_t *ha, INFO("HA: Tier 3 connected to '%s' after backoff", p3->name); ha->primary_ctrl_fd = fd; register_fd_with_iomgr(c, fd); - ha_arm_heartbeat_timeout_vtask(ha, c); + if (c) ha_arm_heartbeat_timeout_vtask(ha, c); return fd; } } @@ -2217,6 +2217,36 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c) if (committed > 0) persist_truncate_wal_above(&c->persist, committed); + /* Clear voted_for so the jitter check below reflects NEW votes cast during + * the RDHUP election, not the stale startup voted_for (different term). */ + pthread_mutex_lock(&ha->lock); + ha->voted_for_term = 0; + memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name)); + pthread_mutex_unlock(&ha->lock); + + /* Priority-based jitter before RDHUP election to avoid split-brain: + * lower-priority nodes (higher priority index) wait longer, giving the + * highest-priority node time to win before others start their elections. */ + if (ha->my_priority > 0) { + uint32_t step_ms = ha->config.election_priority_step_ms + ? ha->config.election_priority_step_ms + : HA_DEFAULT_ELECTION_PRIORITY_STEP_MS; + long delay_ns = (long)ha->my_priority * (long)step_ms * 1000000L; + struct timespec jitter = { + .tv_sec = delay_ns / 1000000000L, + .tv_nsec = delay_ns % 1000000000L, + }; + nanosleep(&jitter, NULL); + /* If already voted during jitter (from the winner's vote request), skip */ + pthread_mutex_lock(&ha->lock); + int already_voted = (ha->voted_for_name[0] != '\0'); + pthread_mutex_unlock(&ha->lock); + if (already_voted) { + INFO("HA: RDHUP — already voted during jitter, skipping own election"); + return; + } + } + /* Fire async election — ha_start_election_async is non-blocking (Req 6.3, 6.5). * ha->phase2_done==1 at this point (set after initial startup election), so * ha_elect_evaluate_and_signal will skip cluster_init_phase2 and go directly