wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha-wal-replication): HA→replicator standby registration bridge (tasks 2.1-2.3)

Browse source 
Register standbys with replicator on connect (election promotion +
late-joiner), deregister on disconnect (vote err handler + standby err
handler). repl_add_standby now idempotent (duplicate ep returns
existing slot).

Co-Authored-By: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-06-10 08:35:02 +00:00
parent 18e351ee1b
commit 81dbc4d7b4
4 changed files with 542 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

154
src/controller/ha.c
View file

@ -510,6 +510,14 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
/* Reset ack counter for this committing round */
atomic_store(&ha->repl_ack_count, 0);
/* Req 2.1 (ha-repl-hardening): re-initialize per-standby convergence
* tracking for THIS committing round. Stale converged flags from a
* previous term (demote → re-elect cycle) must not satisfy the
* majority-convergence gate before standbys re-report their state. */
memset(ha->standby_reported_tag, 0, sizeof(ha->standby_reported_tag));
memset(ha->standby_reported_lsn, 0, sizeof(ha->standby_reported_lsn));
memset(ha->standby_converged, 0, sizeof(ha->standby_converged));
/* Promote surviving vote endpoints to standby replication channels.
* Iterate ha->elect_ctx->votes[i].ep and add non-NULL entries to
* ha->standby_endpoints so push_config and ctrlhello reach them. */
@ -522,6 +530,20 @@ static void raft_cb_become_leader(void *ctx, uint64_t term)
if (ectx->votes[i].ep != NULL) {
zarray_store(ha->standby_endpoints, ectx->votes[i].ep);
INFO("HA: promoted vote ep[%d] → standby channel", i);
/* Task 2.1: register promoted ep with WAL replicator.
* Req 1.1: primary must register standby with repl_add_standby
* when establishing the standby control channel.
* peer->hostname / peer->port from the vote entry's peer pointer. */
if (c && ectx->votes[i].peer != NULL) {
int rc = repl_add_standby(&c->persist.repl,
ectx->votes[i].ep,
ectx->votes[i].peer->hostname,
(uint32_t)ectx->votes[i].peer->port);
if (rc >= 0)
TRACE("HA: repl registered standby ep[%d] (slot %d)", i, rc);
else
WARN("HA: repl_add_standby ep[%d] failed: %d", i, rc);
}
}
}
pthread_mutex_unlock(&ha->lock);
@ -970,6 +992,9 @@ static void ha_advance_applied_lsn(ha_state_t *ha, uint64_t new_lsn)
}
/* ha_apply_single_entry: apply one decoded wal_catchup_entry_t sub-entry.
* Payload bytes (raw_payload/raw_payload_len, extracted by xce_decode)
* are passed through to apply_wal_entry so entity state is restored —
* not just the LSN chain (task 7.3).
* Returns 0 on success, non-zero on error (caller logs and skips). */
static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e)
{
@ -977,10 +1002,10 @@ static int ha_apply_single_entry(persist_t *p, wal_catchup_entry_t *e)
memset(&hdr, 0, sizeof(hdr));
hdr.lsn = e->lsn;
hdr.type = (uint32_t)e->entry_type;
hdr.payload_len = 0;
hdr.payload_len = (uint32_t)e->raw_payload_len;
hdr.crc = 0;
hdr.prev_crc = 0;
return apply_wal_entry(p, &hdr, NULL);
return apply_wal_entry(p, &hdr, e->raw_payload);
}
int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p,
@ -1051,7 +1076,9 @@ int ha_apply_wal_catchup_entry(ha_state_t *ha, persist_t *p,
* 3. Load the snapshot via persist_load_snapshot().
* 4. Set ha->applied_lsn = p->checkpoint_lsn (in-memory).
*
* On mid-transfer reconnect callers discard snap.part externally (Req 4.4).
* Mid-transfer reconnect (Req 4.4): the primary restarts every transfer at
* chunk 0, so chunk_index == 0 discards any stale snap.part left by an
* interrupted transfer before appending — no cross-chunk state needed.
* Returns 0 on success, negative errno on error.
* Requirements: 4.2, 4.3, 4.4 (ha-repl-hardening spec)
*/
@ -1067,6 +1094,12 @@ int ha_handle_snapshot_chunk(ha_state_t *ha, persist_t *p,
char part_path[4096 + 16];
snprintf(part_path, sizeof(part_path), "%s/snap.part", snap_dir);
/* Req 4.4: a transfer always restarts from chunk 0 — discard any stale
* partial file from an interrupted previous transfer. */
if (chunk->chunk_index == 0 && unlink(part_path) != 0 && errno != ENOENT)
WARN("ha_handle_snapshot_chunk: unlink stale snap.part failed: %s",
strerror(errno));
/* Append chunk payload to snap.part */
if (payload_bytes && payload_len > 0) {
FILE *f = fopen(part_path, "ab");
@ -1258,13 +1291,19 @@ void ha_check_majority_convergence(ha_state_t *ha, cluster_t *c)
n_converged++;
}
/* Majority threshold: all standbys must converge (n_peers - 1).
* Single-node (n_peers=1): threshold=0 — satisfied immediately. */
if (n_converged < n_peers - 1)
/* Majority threshold (Req 2.3): primary + converged standbys must form
* a majority of the cluster — ceil(n_peers/2) - 1 standbys, integer form
* n_peers / 2 (n=3 → 1, n=5 → 2). Safe: committed entries are already
* on a majority (per-entry ack) and safe_committed_lsn derives from a
* majority of voters; a lagging minority standby keeps catching up via
* the heartbeat-driven decision loop after promotion.
* Single-node (n_peers<=1): threshold=0 — satisfied immediately. */
int threshold = n_peers / 2;
if (n_converged < threshold)
return;
INFO("HA: majority converged (%d/%d standbys) → CTRL_PRIMARY",
n_converged, n_peers);
INFO("HA: majority converged (%d/%d standbys, threshold %d) → CTRL_PRIMARY",
n_converged, n_peers - 1, threshold);
if (c) {
/* Full promotion: increments term, sets role=PRIMARY, registers hooks,
@ -1355,6 +1394,14 @@ int ha_dispatch_repl_action_for_standby(ha_state_t *ha,
if (c && c->persist.snap_path[0]) {
extern int ha_send_snapshot_to_standby(endpoint_t *, const char *,
uint64_t);
/* Req 4.1: create a FRESH checkpoint so the transferred snapshot
* reflects current state (also trims WAL + rotates old snaps).
* -EBUSY means another checkpoint is in flight — fall back to
* the existing checkpoint_lsn rather than skipping the cycle. */
int ckpt_rc = persist_checkpoint(&c->persist);
if (ckpt_rc != 0 && ckpt_rc != -EBUSY)
WARN("HA repl: persist_checkpoint before snapshot send "
"failed rc=%d — sending existing checkpoint", ckpt_rc);
ha_send_snapshot_to_standby(ep, c->persist.snap_path,
c->persist.checkpoint_lsn);
}
@ -1419,17 +1466,35 @@ int ha_dispatch_gate(ha_state_t *ha, int opcode)
return 0;
case CTRL_INIT:
/* Allow election probes, config probes, AND heartbeats so the primary's
* periodic heartbeat can reach the handler and post heartbeat_seen,
* triggering the CTRL_INIT → CTRL_STANDBY transition (Req 1.1). */
/* Allow election RPCs AND primary→standby replication messages.
* During the startup stagger a primary may already connect and push
* config — accepting these ensures the node doesn't miss the initial
* bootstrap and can transition to CTRL_STANDBY immediately. */
if (opcode == OPCode_requestVote ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_heartbeat)
opcode == OPCode_heartbeat ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
opcode == OPCode_replConfigPush ||
opcode == OPCode_walCatchupEntry ||
opcode == OPCode_snapshotTransfer)
return 0;
return -XCE_ERR_NOT_READY;
case CTRL_ELECTING:
if (opcode == OPCode_requestVote || opcode == OPCode_probeConfigTag)
/* Allow election RPCs AND primary→standby replication messages.
* A node campaigning for leadership must still accept config, WAL,
* and heartbeats from the current primary — if it wins the election
* it needs current state; if it loses, it becomes STANDBY and must
* resume replication without having missed pushes. Only job/client
* ops are rejected (we're not ready to serve). */
if (opcode == OPCode_requestVote ||
opcode == OPCode_probeConfigTag ||
opcode == OPCode_hello ||
opcode == OPCode_nop ||
opcode == OPCode_replConfigPush ||
opcode == OPCode_walCatchupEntry ||
opcode == OPCode_snapshotTransfer)
return 0;
return -XCE_ERR_NOT_READY;
@ -1789,7 +1854,13 @@ static int ha_heartbeat_is_fresh(ha_state_t *ha)
* 6. grant vote, record voted_for, persist (Req 9.1)
*
* Returns 1 = granted, 0 = denied, <0 = error.
*
* TEST-ONLY (XCE_TESTING): production vote dispatch goes through
* cluster.c → raft_on_vote_request. This function is the legacy
* entry point used by test_ha_vote_gate.c and test_ha_election.c
* to exercise role-gate logic in isolation without a full raft state.
*/
#ifdef XCE_TESTING
int handle_request_vote(ha_state_t *ha,
uint64_t term,
const char *candidate_id,
@ -1887,6 +1958,7 @@ int handle_request_vote(ha_state_t *ha,
return 1;
}
#endif /* XCE_TESTING */
/* ── Test-only hook shims (XCE_TESTING only) ──────────────────────────────
* Type definitions (election_timeout_ctx_t, committing_timeout_ctx_t),
@ -2371,6 +2443,9 @@ static void vote_reply_dispatch(ha_state_t *ha, ha_elect_ctx_t *ctx, int idx,
* to vote_reply_dispatch. Factored from vote_reply_msg_handler so that
* peer_ctrl_msg_handler can invoke the same logic without code duplication.
* Packet ownership: caller passes ep->fsm_in.pkt; this function frees it.
* ep->fsm_in.pkt is cleared HERE, before dispatch — vote_reply_dispatch can
* reach raft_cb_become_follower → endpoint_fini(ep) (loser path), which
* would otherwise double-free the packet via the stale fsm_in.pkt pointer.
*/
static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx)
{
@ -2380,6 +2455,9 @@ static int vote_reply_decode_and_dispatch(endpoint_t *ep, vote_ep_ctx_t *vctx)
io_packet_t *pkt = ep->fsm_in.pkt;
int ok = 0;
ep->fsm_in.pkt = NULL; /* take ownership (same pattern as
* repl_ack_decode_and_dispatch) */
uint64_t vr_term = 0;
int vr_granted = 0;
uint64_t vr_tag = 0;
@ -2966,6 +3044,12 @@ static int vote_ep_err_handler(void *ep_raw)
}
pthread_mutex_unlock(&ha->lock);
}
/* Task 2.2: deregister from WAL replicator.
* Req 1.2: when a standby disconnects, the HA layer shall deregister
* the endpoint from the replicator via repl_remove_standby.
* vctx->c may be NULL in test mode — guard accordingly. */
if (vctx->c)
repl_remove_standby(&vctx->c->persist.repl, ep);
} else {
/* Election phase: count as denied vote */
handle_request_vote_reply(ha, vctx->ctx, vctx->peer_idx,
@ -3455,6 +3539,22 @@ void ha_arm_heartbeat_timeout_vtask(ha_state_t *ha, cluster_t *c)
}
}
/* ha_discover_primary: if CTRL_ELECTING, step down to STANDBY on receiving
* a valid primary message. A campaigning node that sees a live primary must
* defer — continuing the election risks split-brain (bug-353). */
void ha_discover_primary(ha_state_t *ha, cluster_t *c)
{
ctrl_role_t role = (ctrl_role_t)atomic_load_explicit(&ha->role,
memory_order_acquire);
if (role != CTRL_ELECTING)
return;
uint64_t term = atomic_load_explicit(&ha->term, memory_order_acquire);
INFO("HA: discovered live primary while ELECTING (term=%" PRIu64 ") — stepping down",
term);
raft_cb_become_follower(ha, term, -1);
}
void ha_on_heartbeat_received(ha_state_t *ha)
{
if (!ha) return;
@ -3587,6 +3687,25 @@ static void on_standby_connected(endpoint_t *ep)
INFO("HA: outbound standby connection to peer[%d] established, ctrlhello sent",
sctx->peer_idx);
/* Task 2.1: register late-joiner with WAL replicator.
* Req 1.1: primary must register standby with repl_add_standby when
* establishing the standby control channel (late-joiner outbound connect).
* Host/port from candidates[peer_idx] — the peer we connected to. */
if (c && c->myconf &&
sctx->peer_idx >= 0 &&
sctx->peer_idx < c->myconf->peers_cfg.candidates_count) {
const xce_candidate_t *peer =
&c->myconf->peers_cfg.candidates[sctx->peer_idx];
int rc = repl_add_standby(&c->persist.repl, ep,
peer->hostname, (uint32_t)peer->port);
if (rc >= 0)
TRACE("HA: repl registered late-joiner standby peer[%d] (slot %d)",
sctx->peer_idx, rc);
else
WARN("HA: repl_add_standby late-joiner peer[%d] failed: %d",
sctx->peer_idx, rc);
}
/* Set state-aware dispatcher for subsequent messages on this endpoint */
ep->msg_handler = peer_ctrl_msg_handler;
}
@ -3603,6 +3722,8 @@ static int standby_ep_err_handler_primary(void *ep_raw)
if (!sctx) return 0;
ha_state_t *ha = sctx->ha;
cluster_t *c = sctx->c;
if (ha->standby_endpoints) {
pthread_mutex_lock(&ha->lock);
for (int i = 0; i < ha->standby_endpoints->count; i++) {
@ -3613,6 +3734,13 @@ static int standby_ep_err_handler_primary(void *ep_raw)
}
pthread_mutex_unlock(&ha->lock);
}
/* Task 2.2: deregister late-joiner standby from WAL replicator before
* freeing the context. Req 1.2: RDHUP or connection error triggers
* repl_remove_standby so the replicator stops sending WAL entries to it. */
if (c)
repl_remove_standby(&c->persist.repl, ep);
free(sctx);
ep->data = NULL;
return 0;