wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(ha): reference-counted ha_elect_ctx_t lifecycle (ha-elect-ctx-lifecycle)

Browse source 
Replace the deliberate election-carrier leak with an atomic refcount:
a bias reference held by ha->elect_ctx plus one reference per captured
vote endpoint, released via a new endpoint_t.on_fini teardown-completion
hook (fired once, before ep->data is freed). Both real-election and
PreVote capture sites (raft_cb_send_vote_request,
raft_cb_send_pre_vote_request) increment on capture; both publish sites
(ha_start_election_async, ha_start_prevote_async) now atomic_exchange
the outgoing carrier and release it instead of leaking or freeing
outright, so a still-outstanding vote endpoint keeps the superseded
carrier alive until its own teardown completes.

Verified: 758+ controller tests green, isolated LeakSanitizer run shows
zero ha_elect_ctx_t leaks across many election/PreVote rounds, no
premature-free/UAF under supersession, and a live 10-round HA soak
(36 jobs, zero loss, all 3 nodes served as primary).
This commit is contained in:
Rongsong Shen 2026-07-12 04:25:42 +00:00
parent 58e047f533
commit 6529dcf08f
14 changed files with 879 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

120
src/controller/ha.c
View file

@ -137,6 +137,55 @@ static int primary_heartbeat_fn(void *arg);
static void on_standby_connected(endpoint_t *ep);
static int standby_ep_err_handler_primary(void *ep_raw);
/* ── ha-elect-ctx-lifecycle task 2.1: election carrier reference-counted
* lifetime ──────────────────────────────────────────────────────────────
*
* ha_elect_ctx_release: decrement ctx->refcount; on reaching zero, destroy
* the carrier's mutex/semaphore and free it. This is the ONLY place
* ha_elect_ctx_t is destroyed/freed — every release path (vote-endpoint
* teardown completion via vote_ep_on_fini, and round supersession in
* ha_start_election_async / ha_start_prevote_async) routes through here, so
* the carrier is freed exactly once, from whichever release is last.
*/
static void ha_elect_ctx_release(ha_elect_ctx_t *ctx)
{
if (ctx == NULL)
return;
if (atomic_fetch_sub_explicit(&ctx->refcount, 1, memory_order_acq_rel) != 1)
return; /* not the last reference */
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
}
#ifdef XCE_TESTING
/* ha_elect_ctx_release_for_test: drive ha_elect_ctx_release directly so unit
* tests can verify the release/free mechanism without staging a full
* election round. */
void ha_elect_ctx_release_for_test(ha_elect_ctx_t *ctx)
{
ha_elect_ctx_release(ctx);
}
#endif
/*
* vote_ep_on_fini: endpoint_t.on_fini completion hook wired onto every vote
* endpoint at capture time (raft_cb_send_vote_request /
* raft_cb_send_pre_vote_request). Fires from inside endpoint_fini()
* immediately before ep->data (the vote_ep_ctx_t) is freed — releases the
* carrier reference this endpoint held.
*/
static void vote_ep_on_fini(endpoint_t *ep)
{
vote_ep_ctx_t *vctx = (vote_ep_ctx_t *)ep->data;
if (vctx == NULL)
return;
ha_elect_ctx_release(vctx->ctx);
}
/* ── Real raft callback implementations (Task 6.2) ─────────────
*
* These are registered as s_real_cbs in ha_init, replacing the no-op stubs.
@ -225,6 +274,13 @@ static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term)
ep->data = vctx;
ep->iomgr = &c->iomgr;
/* ha-elect-ctx-lifecycle task 2.2: this endpoint now references ectx via
* vctx->ctx — take a reference before any path that could reach
* endpoint_fini (connect failure below, and beyond). Released exactly
* once by vote_ep_on_fini (wired below) when this endpoint's teardown
* completes, regardless of which teardown path gets there. */
atomic_fetch_add_explicit(&ectx->refcount, 1, memory_order_relaxed);
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
@ -234,6 +290,7 @@ static int raft_cb_send_vote_request(void *ctx, int peer_idx, uint64_t term)
endpoint_set_on_connected(ep, on_vote_connected);
endpoint_set_err_handler(ep, vote_ep_err_handler);
endpoint_set_on_fini(ep, vote_ep_on_fini);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
@ -358,6 +415,11 @@ static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx,
ep->data = vctx;
ep->iomgr = &c->iomgr;
/* ha-elect-ctx-lifecycle task 2.3: same mechanism as the real-vote send
* path above — take a reference before any path that could reach
* endpoint_fini. */
atomic_fetch_add_explicit(&ectx->refcount, 1, memory_order_relaxed);
if (ha->client_ssl_ctx) {
SSL_CTX *client_ctx = (SSL_CTX *)ha->client_ssl_ctx;
ep->ssl = SSL_new(client_ctx);
@ -367,6 +429,7 @@ static int raft_cb_send_pre_vote_request(void *ctx, int peer_idx,
endpoint_set_on_connected(ep, on_pre_vote_connected);
endpoint_set_err_handler(ep, vote_ep_err_handler);
endpoint_set_on_fini(ep, vote_ep_on_fini);
int rc = endpoint_connect(ep, peer->hostname, peer->port,
DEFAULT_CONNECT_TIMEOUT_MS);
@ -6378,6 +6441,21 @@ endpoint_t *vote_ep_ctx_build_for_test(endpoint_t *ep,
return ep;
}
#ifdef XCE_TESTING
/*
* vote_ep_wire_on_fini_for_test (ha-elect-ctx-lifecycle task 3.3): wire the
* real vote_ep_on_fini hook onto ep->on_fini, exactly as raft_cb_send_vote_request
* / raft_cb_send_pre_vote_request do in production. vote_ep_on_fini is static
* to this TU, so tests that build an endpoint via vote_ep_ctx_build_for_test
* and need to drive a REAL endpoint_fini() (to prove the completion hook
* fires and releases the carrier reference) call this to wire it first.
*/
void vote_ep_wire_on_fini_for_test(endpoint_t *ep)
{
endpoint_set_on_fini(ep, vote_ep_on_fini);
}
#endif
/*
* election_timeout_fn_for_test: test helper that exercises the election timeout
* callback logic with an explicit cancelled_val instead of an atomic field.
@ -6892,13 +6970,20 @@ void ha_start_prevote_async(ha_state_t *ha, cluster_t *c)
mc->self_lsn = cand_lsn;
pthread_mutex_init(&mc->lock, NULL);
sem_init(&mc->done, 0, 0);
atomic_init(&mc->refcount, 1); /* ha-elect-ctx-lifecycle task 2.4: bias reference */
if (n_other > 0) {
mc->votes[0].my_tag = cand_tag;
mc->votes[0].my_lsn = cand_lsn;
}
/* Do NOT free a prior ctx (live vote endpoints may hold a bare
* pointer to it — same UAF caution as the rdhup path). */
atomic_store_explicit(&ha->elect_ctx, mc, memory_order_release);
/* ha-elect-ctx-lifecycle task 2.4: exchange (not store) so we can
* release the superseded carrier's bias reference — it is freed
* here immediately if no vote endpoint from that prior round still
* references it, or later, whenever that endpoint's on_fini fires,
* whichever comes last. Replaces the old deliberate leak (see
* design.md's Architecture §Mechanism). */
ha_elect_ctx_t *old_ctx = (ha_elect_ctx_t *)atomic_exchange_explicit(
&ha->elect_ctx, mc, memory_order_acq_rel);
ha_elect_ctx_release(old_ctx);
}
ha->n_peers = n_peers;
}
@ -7136,6 +7221,7 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
ctx->c = c;
pthread_mutex_init(&ctx->lock, NULL);
sem_init(&ctx->done, 0, 0);
atomic_init(&ctx->refcount, 1); /* ha-elect-ctx-lifecycle task 2.4: bias reference */
/* Store candidate's own tag/lsn unconditionally in self_lsn/self_tag (task 6.3):
* ha_compute_safe_committed_lsn uses these for the majority computation,
@ -7149,8 +7235,13 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
ctx->votes[0].my_lsn = my_lsn;
}
/* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx */
atomic_store_explicit(&ha->elect_ctx, ctx, memory_order_release);
/* Store ctx in ha so raft_cb_send_vote_request can find it via ha->elect_ctx.
* ha-elect-ctx-lifecycle task 2.4: exchange (not store) to retrieve and
* release the superseded carrier's bias reference — see the identical
* rationale at ha_start_prevote_async's publish site. */
ha_elect_ctx_t *prior_ctx = (ha_elect_ctx_t *)atomic_exchange_explicit(
&ha->elect_ctx, ctx, memory_order_acq_rel);
ha_elect_ctx_release(prior_ctx);
/* Step 4: delegate election to raft library.
*
@ -7189,24 +7280,25 @@ void ha_start_election_async(ha_state_t *ha, cluster_t *c,
/* Transition to CTRL_STANDBY so myapp_run schedules fetch_config_from_primary.
* The primary will push config via ctrlhello once it becomes CTRL_PRIMARY. */
atomic_store_explicit(&ha->role, (int32_t)CTRL_STANDBY, memory_order_release);
/* Free ctx — no vote sends will happen from this node this round. */
/* ha-elect-ctx-lifecycle task 2.4: no vote sends will happen from this
* node this round — ctx was never captured by any vote endpoint, so
* this release (refcount 1->0) frees it immediately, same outcome as
* the old direct destroy+free, now routed through the single release
* helper for consistency. */
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
ha_elect_ctx_release(ctx);
return;
}
raft_start_election(&ha->raft);
/* For single-node, raft_start_election fires on_become_leader synchronously.
* on_become_leader (raft_cb_become_leader) handles role -> CTRL_PRIMARY and
* phase2. Free the ctx here since no raft callbacks
* will reference it after raft_start_election returns for single-node. */
* phase2. Release the ctx here since no raft callbacks
* will reference it after raft_start_election returns for single-node
* (no vote endpoint is ever created — votes[] is empty). */
if (n_peers == 1) {
atomic_store_explicit(&ha->elect_ctx, NULL, memory_order_release);
pthread_mutex_destroy(&ctx->lock);
sem_destroy(&ctx->done);
free(ctx);
ha_elect_ctx_release(ctx);
return;
}