wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(raft-drive-async): offload delaymgr peer reconnect to slow lane (task 2.1)

Browse source 
Heartbeat timer detects a disconnected peer, CAS-acquires a per-peer
single-flight flag, slowlane_submits the bounded connect, and returns — the
delaymgr never blocks on connect. Lane worker (reconnect_lane_fn) does the
bounded endpoint_connect + lock-guarded iomgr_monitor; flag cleared on all
terminal paths (incl. -EEXIST race fix). Default-ON kill-switch retains the
inline path for TDD-RED/fallback. Never self-terminates on failure. 2.2 adds
the inbound/outbound race + failure hardening (documented seam). 196/0 green.
This commit is contained in:
Rongsong Shen 2026-06-26 08:08:31 +00:00
parent a36234a103
commit e07667ff9b
4 changed files with 569 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

261
src/controller/ha.c
View file

@ -3767,6 +3767,33 @@ int handle_request_vote(ha_state_t *ha,
}
#endif /* XCE_TESTING */
/* ── raft-drive-async Task 2.1: reconnect-offload feature flag (ALL builds) ──
*
* drive_async_reconnect_offload gates the peer reconnect path:
* ON (production default) → primary_heartbeat_fn submits the peer reconnect
* to the slow lane and returns immediately; the delaymgr thread NEVER
* performs the blocking connect inline (ha-async-io.md thread-role
* contract — a blocking connect on the heartbeat/election-timeout thread
* stalls liveness to healthy peers and triggers spurious elections).
* OFF → legacy inline blocking connect on the delaymgr thread; retained only
* so the TDD RED test can demonstrate the pre-change stall.
*
* Compiled in ALL builds (production reads it on every heartbeat tick); the
* setter is the test seam to flip RED↔GREEN. */
static _Atomic int g_drive_async_reconnect_offload = 1;
void ha_set_reconnect_offload_for_test(int on)
{
atomic_store_explicit(&g_drive_async_reconnect_offload, on ? 1 : 0,
memory_order_release);
}
int ha_reconnect_offload_enabled(void)
{
return atomic_load_explicit(&g_drive_async_reconnect_offload,
memory_order_acquire);
}
/* ── Test-only hook shims (XCE_TESTING only) ──────────────────────────────
* Type definitions (election_timeout_ctx_t), extern declarations
* (cluster_init_phase2, push_config_transport_only,
@ -3816,6 +3843,16 @@ int (*ha_heartbeat_req_hook)(endpoint_t *ep, uint64_t term,
uint64_t committed_lsn,
uint64_t committed_config_tag) = NULL;
/*
* ha_reconnect_connect_hook: test-only seam for the BLOCKING half of the
* offloaded peer reconnect (raft-drive-async Task 2.1). When non-NULL, the
* slow-lane reconnect worker calls this INSTEAD OF the real bounded
* endpoint_connect + iomgr_monitor, so a test can drive the reconnect path
* deterministically (e.g. park the worker to simulate an unreachable peer).
* Returns 0 to simulate success, <0 to simulate failure. Req 2.1, 2.2, 2.5.
*/
int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx) = NULL;
/*
* ha_repl_push_config_for_standby_hook: test-only seam for the per-standby
* config push triggered by the heartbeat-reply decision loop (Task 3.3).
@ -6406,24 +6443,43 @@ static int standby_ep_err_handler_primary(void *ep_raw)
}
/*
* ha_connect_to_peer: open a new outbound TLS connection to a peer candidate
* and register it as a standby channel. Called from primary_heartbeat_fn
* for peers that are absent from standby_endpoints.
* reconnect_ctx_t: owns the prepared endpoint across the in-flight reconnect
* window (raft-drive-async Task 2.1). Built on the delaymgr tick by the
* non-blocking prep (ha_connect_prep_peer), consumed once on a slow-lane
* worker by reconnect_lane_fn, then freed. The ctx keeps `ep`/`sctx` valid
* for the whole window so the lane worker never touches freed memory while the
* delaymgr thread races ahead to the next tick (Req 2.3 lifetime).
*/
static void ha_connect_to_peer(cluster_t *c, int peer_idx)
typedef struct {
cluster_t *c;
int peer_idx; /* ABSOLUTE candidate index */
endpoint_t *ep; /* prepared, not yet connected */
standby_ep_ctx_t *sctx; /* == ep->data */
int should_log;
} reconnect_ctx_t;
/*
* ha_connect_prep_peer: the NON-BLOCKING half of the outbound peer connect.
* Allocates the endpoint context, initialises the endpoint, sets the SSL
* client state and the on-connected / err handlers — all of which are
* non-blocking and safe to run on the delaymgr tick. Does NOT perform the
* blocking connect (that is ha_connect_finish_peer, run on the slow lane).
*
* Returns a heap reconnect_ctx_t* (caller owns) on success, or NULL on an
* allocation failure (nothing leaked).
*/
static reconnect_ctx_t *ha_connect_prep_peer(cluster_t *c, int peer_idx)
{
ha_state_t *ha = &c->ha;
const xce_candidate_t *peer =
&c->myconf->peers_cfg.candidates[peer_idx];
ha_state_t *ha = &c->ha;
standby_ep_ctx_t *sctx = calloc(1, sizeof(*sctx));
if (!sctx) return;
if (!sctx) return NULL;
sctx->ha = ha;
sctx->c = c;
sctx->peer_idx = peer_idx;
endpoint_t *ep = endpoint_get();
if (!ep) { free(sctx); return; }
if (!ep) { free(sctx); return NULL; }
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
endpoint_set_peer(ep, c->myhost);
@ -6443,14 +6499,74 @@ static void ha_connect_to_peer(cluster_t *c, int peer_idx)
int n = ++ha->peer_connect_fail_count[peer_idx];
int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0);
if (should_log)
INFO("HA: peer[%d] (%s) not connected — initiating outbound connect (attempt %d)",
peer_idx, peer->hostname, n);
reconnect_ctx_t *rc_ctx = calloc(1, sizeof(*rc_ctx));
if (!rc_ctx) {
ep->data = NULL;
endpoint_fini(ep);
free(sctx);
return NULL;
}
rc_ctx->c = c;
rc_ctx->peer_idx = peer_idx;
rc_ctx->ep = ep;
rc_ctx->sctx = sctx;
rc_ctx->should_log = should_log;
return rc_ctx;
}
/*
* ha_connect_finish_peer: the BLOCKING half of the outbound peer connect —
* the bounded connect + lock-guarded iomgr registration. MUST run on a
* slow-lane worker, NEVER on the delaymgr tick (Req 2.1, 2.2; ha-async-io.md
* thread-role contract).
*
* On success the endpoint is connecting and registered with iomgr; on_connected
* (on_standby_connected) fires later from the iomgr loop. On failure the
* endpoint is torn down here and the controller simply retries on the next
* heartbeat tick (controllers never self-terminate).
*
* Returns 0 on a successful connect+register, <0 otherwise.
*
* NOTE (Task 2.2 seam): this implements the happy-path connect+register +
* teardown-on-failure. The inbound-vs-outbound connection-ownership race
* resolution, the no-lost/no-reordered-message guarantee on success, and the
* richer failure/retry hardening are owned by Task 2.2 — they layer on top of
* this happy path without changing this function's contract (return 0/<0,
* caller clears the per-peer in-flight flag).
*/
static int ha_connect_finish_peer(reconnect_ctx_t *rc_ctx)
{
cluster_t *c = rc_ctx->c;
endpoint_t *ep = rc_ctx->ep;
standby_ep_ctx_t *sctx = rc_ctx->sctx;
int peer_idx = rc_ctx->peer_idx;
const xce_candidate_t *peer = &c->myconf->peers_cfg.candidates[peer_idx];
#ifdef XCE_TESTING
if (ha_reconnect_connect_hook) {
/* Deterministic test path: the hook stands in for the real bounded
* connect + register so a test can park the worker (unreachable peer)
* or return a synthetic success/failure without a live socket. On
* the test path we do NOT touch the real endpoint fd. */
int hrc = ha_reconnect_connect_hook(c, peer_idx);
if (hrc < 0) {
ep->data = NULL;
endpoint_fini(ep);
free(sctx);
}
return hrc;
}
#endif
if (rc_ctx->should_log)
INFO("HA: peer[%d] (%s) not connected — initiating outbound connect "
"(attempt %d)", peer_idx, peer->hostname,
c->ha.peer_connect_fail_count[peer_idx]);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
if (rc < 0) {
if (should_log)
if (rc_ctx->should_log)
WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
peer_idx, peer->hostname, peer->port, rc);
/* endpoint_fini calls FREEUP(ep->data) which frees sctx — do not
@ -6458,17 +6574,130 @@ static void ha_connect_to_peer(cluster_t *c, int peer_idx)
ep->data = NULL;
endpoint_fini(ep);
free(sctx);
return;
return rc;
}
rc = iomgr_monitor(ep);
rc = iomgr_monitor(ep); /* lock-guarded epoll registration (Req 4.4) */
if (rc < 0) {
ep->data = NULL;
endpoint_fini(ep);
free(sctx);
return;
return rc;
}
INFO("HA: connecting outbound to peer[%d] (%s:%u)",
peer_idx, peer->hostname, peer->port);
return 0;
}
/*
* reconnect_lane_fn: the slow-lane worker entry (raft-drive-async Task 2.1).
* Runs the bounded blocking connect off the delaymgr thread, then ALWAYS
* clears the per-peer in-flight flag (success or failure) so the dedup never
* wedges and the next heartbeat tick can retry an unreachable peer (Req 2.5,
* 2.4 — never self-terminates). Frees the reconnect ctx.
*/
static void reconnect_lane_fn(void *arg)
{
reconnect_ctx_t *rc_ctx = (reconnect_ctx_t *)arg;
if (!rc_ctx) return;
cluster_t *c = rc_ctx->c;
int peer_idx = rc_ctx->peer_idx;
(void)ha_connect_finish_peer(rc_ctx);
/* Clear in-flight LAST, after the endpoint is fully connected/registered
* or torn down — so a concurrent heartbeat tick cannot enqueue a second
* reconnect that races this one's endpoint setup (Req 2.5). */
atomic_store_explicit(&c->ha.reconnect_inflight[peer_idx], 0,
memory_order_release);
free(rc_ctx);
}
/*
* ha_connect_to_peer: open a new outbound TLS connection to a peer candidate
* and register it as a standby channel. Called from primary_heartbeat_fn for
* peers that are absent from standby_endpoints.
*
* Offload flag ON (default, Task 2.1): prepares the endpoint (non-blocking)
* and submits the bounded connect to the slow lane with a per-peer dedup key,
* then returns immediately — the delaymgr thread NEVER blocks on connect.
*
* Offload flag OFF (legacy / TDD RED only): runs the blocking connect inline
* on the calling (delaymgr) thread. Retained solely so the RED test can show
* the pre-change stall; production runs with the flag ON.
*/
static void ha_connect_to_peer(cluster_t *c, int peer_idx)
{
ha_state_t *ha = &c->ha;
if (!ha_reconnect_offload_enabled()) {
/* ── Legacy inline path (flag OFF) — blocks the caller. ── */
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
if (!rc_ctx) return;
(void)ha_connect_finish_peer(rc_ctx);
free(rc_ctx);
return;
}
/* ── Offload path (flag ON) — non-blocking on the delaymgr thread. ──
*
* Per-peer single-flight: CAS the in-flight flag 0->1. If it was already
* 1, a reconnect to this peer is already in flight on the lane — enqueue
* nothing (Req 2.5). This guard is the authoritative dedup; the lane's
* own per-key dedup is a redundant backstop. */
int expected = 0;
if (!atomic_compare_exchange_strong_explicit(
&ha->reconnect_inflight[peer_idx], &expected, 1,
memory_order_acq_rel, memory_order_acquire)) {
return; /* already reconnecting this peer */
}
reconnect_ctx_t *rc_ctx = ha_connect_prep_peer(c, peer_idx);
if (!rc_ctx) {
/* prep failed (alloc) — release the in-flight guard so the next tick
* retries; controller stays alive (Req 2.4). */
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* Submit the blocking connect to the slow lane. Dedup key is the
* per-peer absolute index (+1 so it is non-zero; key 0 disables dedup). */
uint64_t key = (uint64_t)(peer_idx + 1);
int srv = slowlane_submit(&c->slowlane, key, reconnect_lane_fn, rc_ctx);
if (srv == -EEXIST) {
/* Lane still holds this peer's dedup key from a PRIOR reconnect task.
* This is the narrow window where that task already cleared our
* reconnect_inflight flag (its last action) but the lane has not yet
* removed its key. Our CAS therefore succeeded (flag was 0) even
* though the lane refuses the submit. We MUST release the flag we just
* set — otherwise no task owns its clear and dedup wedges permanently.
* Tear down our duplicate prep; the next tick retries cleanly. */
rc_ctx->ep->data = NULL;
endpoint_fini(rc_ctx->ep);
free(rc_ctx->sctx);
free(rc_ctx);
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
if (srv < 0 && srv != -EAGAIN) {
/* Hard submit failure (alloc / lane stopping). -EAGAIN is NOT a
* failure: the task is still queued and will run, so we keep the flag
* set and let reconnect_lane_fn clear it. Only on a true failure do
* we tear down and release the guard so the next tick retries. */
WARN("HA: slowlane_submit reconnect peer[%d] failed: %d — retry next tick",
peer_idx, srv);
rc_ctx->ep->data = NULL;
endpoint_fini(rc_ctx->ep);
free(rc_ctx->sctx);
free(rc_ctx);
atomic_store_explicit(&ha->reconnect_inflight[peer_idx], 0,
memory_order_release);
return;
}
/* 0 or -EAGAIN: the task is enqueued and will run exactly once on a lane
* worker; reconnect_lane_fn clears the in-flight flag on completion. */
}
/*