lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(election-iomgr-refactor): RDHUP failover fixes — heartbeat watchdog + jitter

Browse source 
- ha.c: arm heartbeat timeout watchdog after ha_find_and_connect_to_primary (all 3 tiers)
- ha.c: guard ha_arm_heartbeat_timeout_vtask with c!=NULL (prevents ASAN stack-use-after-return in unit tests)
- ha.c: add priority-based jitter to ha_on_primary_ctrl_fd_rdhup to avoid split-brain
- ha.c: clear voted_for before RDHUP jitter so check reflects new votes, not stale startup votes
This commit is contained in:
Rongsong Shen 2026-05-31 02:24:52 +00:00
parent fd93997f78
commit 1ded1022fb
1 changed files with 33 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

36
src/controller/ha.c
View file

@ -2134,7 +2134,7 @@ int ha_find_and_connect_to_primary(ha_state_t *ha,
ha->voted_for_name, r); ha->voted_for_name, r);
ha->primary_ctrl_fd = fd; ha->primary_ctrl_fd = fd;
register_fd_with_iomgr(c, fd); register_fd_with_iomgr(c, fd);
ha_arm_heartbeat_timeout_vtask(ha, c); if (c) ha_arm_heartbeat_timeout_vtask(ha, c);
return fd; return fd;
} }
if (fd == -ETIMEDOUT || fd == -EAGAIN) { if (fd == -ETIMEDOUT || fd == -EAGAIN) {
@ -2160,7 +2160,7 @@ int ha_find_and_connect_to_primary(ha_state_t *ha,
INFO("HA: Tier 2 connected to '%s'", p2->name); INFO("HA: Tier 2 connected to '%s'", p2->name);
ha->primary_ctrl_fd = fd; ha->primary_ctrl_fd = fd;
register_fd_with_iomgr(c, fd); register_fd_with_iomgr(c, fd);
ha_arm_heartbeat_timeout_vtask(ha, c); if (c) ha_arm_heartbeat_timeout_vtask(ha, c);
return fd; return fd;
} }
} }
@ -2177,7 +2177,7 @@ int ha_find_and_connect_to_primary(ha_state_t *ha,
INFO("HA: Tier 3 connected to '%s' after backoff", p3->name); INFO("HA: Tier 3 connected to '%s' after backoff", p3->name);
ha->primary_ctrl_fd = fd; ha->primary_ctrl_fd = fd;
register_fd_with_iomgr(c, fd); register_fd_with_iomgr(c, fd);
ha_arm_heartbeat_timeout_vtask(ha, c); if (c) ha_arm_heartbeat_timeout_vtask(ha, c);
return fd; return fd;
} }
} }
@ -2217,6 +2217,36 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
if (committed > 0) if (committed > 0)
persist_truncate_wal_above(&c->persist, committed); persist_truncate_wal_above(&c->persist, committed);
/* Clear voted_for so the jitter check below reflects NEW votes cast during
* the RDHUP election, not the stale startup voted_for (different term). */
pthread_mutex_lock(&ha->lock);
ha->voted_for_term = 0;
memset(ha->voted_for_name, 0, sizeof(ha->voted_for_name));
pthread_mutex_unlock(&ha->lock);
/* Priority-based jitter before RDHUP election to avoid split-brain:
* lower-priority nodes (higher priority index) wait longer, giving the
* highest-priority node time to win before others start their elections. */
if (ha->my_priority > 0) {
uint32_t step_ms = ha->config.election_priority_step_ms
? ha->config.election_priority_step_ms
: HA_DEFAULT_ELECTION_PRIORITY_STEP_MS;
long delay_ns = (long)ha->my_priority * (long)step_ms * 1000000L;
struct timespec jitter = {
.tv_sec = delay_ns / 1000000000L,
.tv_nsec = delay_ns % 1000000000L,
};
nanosleep(&jitter, NULL);
/* If already voted during jitter (from the winner's vote request), skip */
pthread_mutex_lock(&ha->lock);
int already_voted = (ha->voted_for_name[0] != '\0');
pthread_mutex_unlock(&ha->lock);
if (already_voted) {
INFO("HA: RDHUP — already voted during jitter, skipping own election");
return;
}
}
/* Fire async election — ha_start_election_async is non-blocking (Req 6.3, 6.5). /* Fire async election — ha_start_election_async is non-blocking (Req 6.3, 6.5).
* ha->phase2_done==1 at this point (set after initial startup election), so * ha->phase2_done==1 at this point (set after initial startup election), so
* ha_elect_evaluate_and_signal will skip cluster_init_phase2 and go directly * ha_elect_evaluate_and_signal will skip cluster_init_phase2 and go directly