lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(ha-prevote): rewire failover triggers through PreVote, defer destructive step (task 4.2)

Browse source 
ha_on_primary_ctrl_fd_rdhup + liveness_poll_fn now begin a PreVote round via
ha_begin_failover -> ha_start_prevote_async instead of electing directly; WAL
truncate + CTRL_ELECTING deferred to PreVote WON (ha_enter_real_election).
Connection teardown + config-less voter guard preserved at the trigger.
ha_on_heartbeat_received cancels an in-progress round and re-attaches as standby
with no term change. Single-node immediate-WON synchronously enters the real
election. PreVote now live; full controller suite 563 pass / 0 fail; bug-339/340
preserved.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-06-16 06:29:18 +00:00
parent 7e19bf6110
commit b765fc4705
6 changed files with 492 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

227
src/controller/ha.c
View file

@ -3549,10 +3549,42 @@ static void ha_enter_real_election(ha_state_t *ha, cluster_t *c)
int n_peers = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
const char *my_name = (c->myconf) ? c->myconf->name : "";
ha->raft_c = c;
#ifdef XCE_TESTING
/* Test seam: the WON-branch real election is the same step the pre-rewire
* rdhup body fired, so the existing rdhup-integration tests intercept it
* here (capturing my_tag/my_lsn / counting election starts) — they now
* drive it through the PreVote→WON path (ha-prevote task 4.2).
*
* Mark the role CTRL_ELECTING before invoking the hook: the real
* ha_start_election_async transitions to CTRL_ELECTING as its first step,
* so a test that intercepts the election must still observe the destructive
* role transition the WON branch performs (the spy hook does not start a
* real election that would set it). */
if (ha_rdhup_election_hook != NULL) {
atomic_store_explicit(&ha->role, (int32_t)CTRL_ELECTING,
memory_order_release);
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
int np = 0;
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
if (strcmp(cfg->candidates[i].name,
c->myhost ? c->myhost->name : "") != 0)
peer_ptrs[np++] = &cfg->candidates[i];
}
ha_rdhup_election_hook(ha, c, np, peer_ptrs, cand_tag, cand_lsn,
my_name);
}
return;
}
#endif
/* ha_start_election_async transitions to CTRL_ELECTING and drives the
* UNCHANGED raft_start_election (term++ + self-vote). Peer connections are
* driven by raft_cb_send_vote_request; the peers[] arg is unused. */
ha->raft_c = c;
ha_start_election_async(ha, c, n_peers, NULL, cand_tag, cand_lsn, my_name);
}
@ -3922,6 +3954,26 @@ void (*ha_rdhup_election_hook)(ha_state_t *ha, cluster_t *c,
* 10.5 (INFO logs on FSM transitions)
*/
/*
* ha_begin_failover: the single failover entry point (ha-prevote task 4.2).
*
* Both failover triggers — connection RDHUP (ha_on_primary_ctrl_fd_rdhup) and
* the heartbeat-timeout poll (liveness_poll_fn, via the rdhup handler) — route
* here AFTER their connection-teardown bookkeeping and the config-less
* voter-only guard. It starts a side-effect-free PreVote round and returns;
* it does NOT itself truncate the WAL or transition to CTRL_ELECTING. Those
* destructive steps are deferred to PreVote success (ha_enter_real_election,
* task 4.1) so a frozen-then-resumed standby that loses PreVote leaves the WAL
* and term untouched (Req 1.1, 4.3, 4.4).
*/
void ha_begin_failover(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
INFO("HA: beginning failover via PreVote (no truncate / no term bump until "
"PreVote wins) — staying CTRL_STANDBY");
ha_start_prevote_async(ha, c);
}
void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
{
if (!ha) return;
@ -3930,23 +3982,31 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
* poll fn from triggering a second failover call after EPOLLRDHUP fires. */
ha_disarm_heartbeat_timeout_vtask(ha);
/* Close and null the primary control fd */
/* Close and null the primary control fd. Connection-teardown bookkeeping
* stays at the trigger regardless of the PreVote outcome (ha-prevote task
* 4.2). */
if (ha->primary_ctrl_fd >= 0) {
close(ha->primary_ctrl_fd);
ha->primary_ctrl_fd = -1;
}
/* Transition to CTRL_ELECTING (Req 6.2) */
atomic_store_explicit(&ha->role, CTRL_ELECTING, memory_order_release);
INFO("HA: primary_ctrl_fd RDHUP → CTRL_ELECTING (Req 10.5)");
/* ha-prevote task 4.2: the failover trigger NO LONGER transitions to
* CTRL_ELECTING or truncates the WAL inline. It starts a PreVote round and
* STAYS CTRL_STANDBY; the destructive truncate + CTRL_ELECTING sequence is
* deferred to PreVote success (ha_enter_real_election). The node only ever
* inflates its term / unseats a healthy primary if a PreVote majority is
* first obtained (Req 1.1, 4.3, 4.4). */
INFO("HA: primary_ctrl_fd RDHUP → begin PreVote failover (staying "
"CTRL_STANDBY until PreVote wins) (Req 1.1)");
if (!c) return; /* unit test mode: skip persist + election */
/* Truncate uncommitted WAL entries above committed_lsn (Req 6.2) */
uint64_t committed = atomic_load_explicit(&ha->last_known_committed_lsn,
memory_order_acquire);
if (committed > 0)
persist_truncate_wal_above(&c->persist, committed);
if (!c) {
/* Unit-test mode: no persist / no live transport. Still begin the
* round so the deferred-destructive behaviour is observable (the round
* state lives in raft_state_t; ha_enter_real_election with c==NULL only
* flips the role on a WIN). */
ha_begin_failover(ha, c);
return;
}
/* Election-SAFETY guard (ha-config-bootstrap Req 2.8 / ha-repl-hardening
* Req 10.5): a config-less node (config_version_tag == 0) must NEVER start
@ -3956,128 +4016,33 @@ void ha_on_primary_ctrl_fd_rdhup(ha_state_t *ha, cluster_t *c)
* primary over the node that actually holds the config — which then pushes
* an empty config and breaks replication.
*
* Mirror the proven ha_start_election_async voter-only guard: skip the
* elect_ctx alloc + the election trigger (raft_on_leader_lost) entirely and
* stay a passive voter (CTRL_STANDBY). handle_request_vote still GRANTS
* incoming vote requests to the fresher configured peer regardless of role,
* and this node transitions/fetches config when the configured winner
* connects and pushes config via ctrlhello.
* This guard stays AT THE TRIGGER and runs BEFORE the PreVote round: a
* config-less node can never win a PreVote (the freshness gate denies it),
* so there is no point starting a round — and starting one would needlessly
* probe peers. Stay a passive voter (CTRL_STANDBY); handle_request_vote
* still GRANTS incoming vote requests to the fresher configured peer
* regardless of role, and this node transitions/fetches config when the
* configured winner connects and pushes config via ctrlhello.
*
* Single-node (candidates_count == 1) is unaffected: n_peers > 1 is false,
* so a lone node still self-elects regardless of tag. */
* so a lone node still self-elects (its PreVote trivially wins). */
uint64_t my_config_tag = (c->myconf) ? c->myconf->config_version_tag : 0;
int n_peers_guard = (c->myconf) ? c->myconf->peers_cfg.candidates_count : 1;
if (my_config_tag == 0 && n_peers_guard > 1) {
INFO("ha_on_primary_ctrl_fd_rdhup: no local config (tag=0, n_peers=%d) — "
"voter only, not starting RDHUP re-election; awaiting configured "
"primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
"voter only, not starting PreVote/RDHUP re-election; awaiting "
"configured primary (ha-config-bootstrap Req 2.8)", n_peers_guard);
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY,
memory_order_release);
return;
}
/* Task 6.3 / 6.7 fix: delegate RDHUP election trigger to raft library.
*
* raft_on_leader_lost:
* - clears ha->raft.voted_for = -1 (no direct ha->voted_for_* writes here)
* - sets ha->raft.role = RAFT_FOLLOWER
* - fires on_start_election_after_delay(ctx, priority * election_timeout_ms)
* → raft_cb_start_election_after_delay schedules raft_rdhup_delay_fn
* which calls raft_check_and_start_election after the priority jitter.
*
* The manual nanosleep jitter + voted_for clear + skip-if-already-voted
* logic is now fully owned by the raft state machine.
*
* raft->raft_c must be set before raft_on_leader_lost fires callbacks.
*
* CRITICAL (Task 6.7): raft_cb_send_vote_request reads ha->elect_ctx to
* get my_tag/my_lsn for the RequestVote payload. On a node that entered
* the voter-only path at first boot (my_config_tag==0), ha->elect_ctx was
* set to NULL and never refreshed. Allocate a fresh ha_elect_ctx_t here
* so that the subsequent raft_start_election finds a valid context. */
ha->raft_c = c;
/* Candidate freshness for the RequestVote payload. This MUST mirror the
* voter-side freshness floor (cluster.c OPCode_requestVote): the effective
* floor is max(applied_lsn, last_known_committed_lsn) at the EFFECTIVE
* config tag. A standby never advances persist.committed_lsn (only the
* primary's commit engine writes it) — it advances applied_lsn (WAL
* catch-up) and last_known_committed_lsn (heartbeat replies). Advertising
* persist.committed_lsn here makes every surviving standby advertise a
* stale-low LSN while each peer computes a HIGH floor from applied_lsn;
* the freshness gate (cand_lsn < my_lsn) then trips on both sides, each
* denies the other's vote, no candidate reaches quorum, and the cluster
* spins in an infinite RDHUP→ELECTING→STANDBY loop after the primary dies.
* Using ha_vote_self_freshness makes advertised == voter-floor, so the
* freshest node is always granted by every peer. (Matches the boot-path
* election in myapp_run, which advertises applied_lsn.) */
uint64_t cand_tag = 0, cand_lsn = 0;
if (c->myconf) {
uint64_t local_tag = c->myconf->config_version_tag;
uint64_t local_lsn = atomic_load_explicit(&ha->applied_lsn,
memory_order_acquire);
ha_vote_self_freshness(ha, local_tag, local_lsn, &cand_tag, &cand_lsn);
}
/* Allocate fresh elect_ctx so raft_cb_send_vote_request can read my_tag/my_lsn */
if (c && c->myconf) {
int n_peers = c->myconf->peers_cfg.candidates_count;
int n_other = (n_peers > 1) ? n_peers - 1 : 0;
ha_elect_ctx_t *new_ctx = calloc(1, sizeof(*new_ctx) +
(size_t)n_other * sizeof(new_ctx->votes[0]));
if (new_ctx) {
uint64_t my_tag = cand_tag;
uint64_t my_lsn = cand_lsn;
new_ctx->n_peers = n_other;
new_ctx->votes_granted = 1; /* self-vote placeholder */
new_ctx->result = -1;
new_ctx->c = c;
new_ctx->self_lsn = my_lsn;
new_ctx->self_tag = my_tag;
pthread_mutex_init(&new_ctx->lock, NULL);
sem_init(&new_ctx->done, 0, 0);
if (n_other > 0) {
new_ctx->votes[0].my_tag = my_tag;
new_ctx->votes[0].my_lsn = my_lsn;
}
/* Replace ha->elect_ctx with the fresh ctx. Do NOT free the old
* ctx here: any live vote_ep_ctx_t objects created in a prior
* election round hold a bare pointer to it (vctx->ctx) and will
* dereference it in vote_ep_err_handler / vote_reply_decode_and_dispatch
* when their connection eventually closes. Freeing here would cause
* a UAF. The old ctx is a small allocation; the leak is bounded to
* one per RDHUP cycle and tolerable. */
atomic_store_explicit(&ha->elect_ctx, new_ctx, memory_order_release);
/* Update n_peers tracking — total cluster size (including self) */
ha->n_peers = n_peers;
}
}
#ifdef XCE_TESTING
if (ha_rdhup_election_hook != NULL) {
/* Test seam: build peer args for hook (same as before) so existing
* tests that capture ha_start_election_async arguments still work. */
if (c->myconf && c->myconf->peers_cfg.candidates_count > 0) {
const xce_cluster_config_t *cfg = &c->myconf->peers_cfg;
int n = cfg->candidates_count > 0 ? cfg->candidates_count - 1 : 0;
xce_candidate_t *peer_ptrs[MAX_HA_CANDIDATES];
int np = 0;
for (int i = 0; i < cfg->candidates_count && np < n; i++) {
if (strcmp(cfg->candidates[i].name,
c->myhost ? c->myhost->name : "") != 0)
peer_ptrs[np++] = &cfg->candidates[i];
}
uint64_t my_tag = cand_tag;
uint64_t my_lsn = cand_lsn;
const char *my_name = c->myconf ? c->myconf->name : "";
ha_rdhup_election_hook(ha, c, np, peer_ptrs, my_tag, my_lsn, my_name);
}
return;
}
#endif
raft_on_leader_lost(&ha->raft);
/* Begin the PreVote round. raft->raft_c is set inside ha_start_prevote_async
* (it snapshots self freshness into ha->elect_ctx so the PreVote request
* payload — and, on a WIN, the deferred real election — advertise the
* effective freshness floor, NOT the stale persist.committed_lsn: see the
* bug-339 note in ha_enter_real_election / ha_vote_self_freshness). */
ha_begin_failover(ha, c);
}
/* ── Standby liveness poll (replaces hb_timeout_thread) ─────────────────── */
@ -4263,6 +4228,22 @@ void ha_on_heartbeat_received(ha_state_t *ha)
clock_gettime(CLOCK_MONOTONIC, &ts);
ha->last_activity = ts.tv_sec;
tlog(TLOG_DEBUG3, "HA: heartbeat received — last_activity updated to %ld", (long)ts.tv_sec);
/* ha-prevote task 4.2 (Req 4.2): a fresh primary heartbeat means the leader
* is alive — a PreVote round started by a transient blip must be ABANDONED.
* Cancel the in-progress round (clear the raft round state + the controller
* round-timeout handle) and re-attach as a standby. No persistent term /
* voted_for / role change happens here: the node never left CTRL_STANDBY
* during the round, so re-attaching is a no-op on role/term — exactly the
* non-disruptive rejoin the spec requires. */
if (ha->raft.pre_vote_in_progress) {
ha->raft.pre_vote_in_progress = 0;
ha->raft.pre_votes_granted = 0;
ha_prevote_cancel_round_timeout();
INFO("HA: primary heartbeat arrived mid-PreVote — cancelling the round "
"and re-attaching as CTRL_STANDBY (leader alive, no term change) "
"(Req 4.2)");
}
}
void ha_disarm_heartbeat_timeout_vtask(ha_state_t *ha)