wdgzyjx/myxce
1
0
Fork 0
forked from lei.chen/myxce
Code Pull requests Activity

feat(raft-drive-async): resolve inbound/outbound channel ownership on reconnect (task 2.2)

Browse source 
When a reconnect registers a new outbound control channel for peer X,
on_standby_connected now evicts any pre-existing standby_endpoints entry
for the same peer_idx (a promoted vote-ep or a stale prior channel),
keeping exactly one live outbound channel per peer (Req 2.6/4.4). The
stale ep is torn down OUTSIDE ha->lock via standby_ep_err_handler_primary
(which re-takes the lock) and reclaimed; resolution is strictly per-peer.

Also fixes endpoint leaks on the endpoint_get() calloc paths: every
endpoint_fini() error path now free(ep) (fini reclaims inner members
only), closing the task-2.1 GREEN-test leak masked by detect_leaks=0.

Tests 18-19 in test_ha_liveness.c assert single-survivor resolution and
per-peer isolation. 196/196 controller unit tests pass.
This commit is contained in:
Rongsong Shen 2026-06-26 09:54:10 +00:00
parent e07667ff9b
commit e03ab9bd72
2 changed files with 259 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

159
src/controller/testsuite/test_ha_liveness.c
View file

@ -1118,6 +1118,162 @@ static void test_reconnect_offload_dedups_per_peer(void)
printf("test_reconnect_offload_dedups_per_peer: PASS\n");
}
/* ── Tests 18–19: raft-drive-async Task 2.2 — inbound/outbound ownership ─────
*
* Acceptance criteria (Req 2.6, 4.4; design ReconnectOffload race resolution,
* ha-async-io.md connection ownership):
* - When a reconnect succeeds and registers a NEW outbound control channel
* for peer X, any PRE-EXISTING standby_endpoints entry for the SAME peer X
* (a promoted vote-ep or a stale prior channel that has not yet been torn
* down) must be resolved deterministically: exactly ONE live outbound
* channel per peer survives — no dual live connection, no orphaned/leaked
* endpoint, and the survivor is the freshly-connected one so no queued
* message is delivered down a dead duplicate (lost/reordered, Req 2.6).
*
* The collision: BOTH raft_cb_become_leader (promotes vote eps) and
* on_standby_connected (late-joiner reconnect) zarray_store into
* standby_endpoints keyed only by ep-POINTER identity — never by peer_idx.
* So two distinct eps for the same peer can both register → two state-push
* fan-out targets → exactly the bug-851 dual-channel hazard.
*
* Determinism: this is the ownership-DECISION logic (not a real socket-timing
* race); the real concurrent inbound-accept-vs-outbound-connect timing race is
* covered by the partition soak (task 4.1). Here we drive the registration
* point directly with two heap endpoints for the same peer_idx and assert the
* resolution. Heap eps + endpoint_init make the teardown production-faithful
* so LeakSanitizer would flag an orphaned/leaked evicted endpoint. */
extern int on_standby_connected_for_test(endpoint_t *ep);
/* Production teardown of an outbound standby channel (removes from
* standby_endpoints + repl, frees ep->data); exposed for test teardown. */
extern int standby_ep_err_handler_primary_for_test(endpoint_t *ep);
/* Test-observable counter: how many stale same-peer channels on_standby_connected
* evicted while resolving ownership (raft-drive-async Task 2.2). */
extern _Atomic int ha_standby_evicted_count_for_test;
/* Allocate + init a heap endpoint wired as a standby channel for peer_idx, the
* way ha_connect_prep_peer / a promoted vote-ep would leave it (production
* teardown via standby_ep_err_handler_primary frees ep->data + endpoint, so the
* evicted one must be reclaimed without a leak). */
static endpoint_t *own_make_standby_ep(ha_state_t *ha, cluster_t *c, int peer_idx)
{
endpoint_t *ep = endpoint_get();
TEST_ASSERT_NOT_NULL(ep);
endpoint_init(ep, AF_INET, SOCK_STREAM, 0, 0);
/* No live iomgr in this unit fixture: leave ep->iomgr NULL so the eviction
* path skips iomgr_unmonitor (it would epoll_ctl on a bogus fd). The
* ownership-resolution logic under test (array + replicator) is unaffected. */
ep->iomgr = NULL;
standby_ep_ctx_build_for_test(ep, ha, c, peer_idx);
return ep;
}
static void own_setup_primary(cluster_t *c, myconf_t *conf)
{
memset(c, 0, sizeof(*c));
memset(conf, 0, sizeof(*conf));
init_ha_primary_with_standby_eps(&c->ha);
conf->peers_cfg.candidates_count = 3;
for (int i = 0; i < 3; i++) {
snprintf(conf->peers_cfg.candidates[i].name,
sizeof(conf->peers_cfg.candidates[i].name), "ctrl%d", i);
snprintf(conf->peers_cfg.candidates[i].hostname,
sizeof(conf->peers_cfg.candidates[i].hostname), "127.0.0.1");
conf->peers_cfg.candidates[i].port = 19000 + i;
}
c->myconf = conf;
c->ha.my_priority = 0;
c->myhost = &c->myhost_info;
}
/*
* Test 18 (Req 2.6/4.4): registering a NEW outbound channel for a peer that
* ALREADY has a live standby_endpoints entry (different ep, same peer_idx)
* resolves to exactly ONE live channel — the new one — and tears the stale one
* down (no dual connection, no leak).
*
* RED (before the resolution lands): on_standby_connected only de-dups by ep
* pointer, so both eps remain → count == 2 (dual live connection).
*/
static void test_reconnect_resolves_dual_channel_same_peer(void)
{
cluster_t c;
myconf_t conf;
own_setup_primary(&c, &conf);
atomic_store(&ha_standby_evicted_count_for_test, 0);
/* Stale prior channel for peer 2 already registered (e.g. a promoted
* vote-ep whose RDHUP has not yet been processed). */
endpoint_t *stale = own_make_standby_ep(&c.ha, &c, 2);
zarray_store(c.ha.standby_endpoints, stale);
TEST_ASSERT_EQUAL_INT(1, c.ha.standby_endpoints->count);
/* New reconnect to the SAME peer 2 completes and registers. */
endpoint_t *fresh = own_make_standby_ep(&c.ha, &c, 2);
on_standby_connected_for_test(fresh);
/* Exactly one live channel for peer 2 — the fresh one. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, c.ha.standby_endpoints->count,
"Req 2.6/4.4: a reconnect for a peer with an existing channel must "
"leave exactly ONE live outbound channel (no dual connection)");
TEST_ASSERT_EQUAL_PTR_MESSAGE(fresh, zarray_get(c.ha.standby_endpoints, 0),
"Req 2.6: the survivor must be the freshly-connected channel so no "
"queued message is delivered down the dead duplicate");
TEST_ASSERT_EQUAL_INT_MESSAGE(1,
atomic_load(&ha_standby_evicted_count_for_test),
"the stale same-peer channel must be evicted exactly once");
/* fresh is owned by standby_endpoints; tear it down the production way so
* LSan sees the heap ep + ctx reclaimed (the err handler frees both). */
standby_ep_err_handler_primary_for_test(fresh);
endpoint_fini(fresh);
free(fresh);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_reconnect_resolves_dual_channel_same_peer: PASS\n");
}
/*
* Test 19 (Req 2.6): registering a NEW channel for a DIFFERENT peer does NOT
* evict an existing channel for another peer — resolution is strictly
* per-peer, so unrelated peers' channels and their queued messages are
* untouched.
*/
static void test_reconnect_preserves_other_peer_channel(void)
{
cluster_t c;
myconf_t conf;
own_setup_primary(&c, &conf);
atomic_store(&ha_standby_evicted_count_for_test, 0);
endpoint_t *peer1 = own_make_standby_ep(&c.ha, &c, 1);
zarray_store(c.ha.standby_endpoints, peer1);
endpoint_t *peer2 = own_make_standby_ep(&c.ha, &c, 2);
on_standby_connected_for_test(peer2);
TEST_ASSERT_EQUAL_INT_MESSAGE(2, c.ha.standby_endpoints->count,
"Req 2.6: a reconnect to one peer must not evict another peer's "
"live channel");
TEST_ASSERT_EQUAL_INT_MESSAGE(0,
atomic_load(&ha_standby_evicted_count_for_test),
"no eviction when the peer_idx differs");
standby_ep_err_handler_primary_for_test(peer1);
endpoint_fini(peer1);
free(peer1);
standby_ep_err_handler_primary_for_test(peer2);
endpoint_fini(peer2);
free(peer2);
fini_ha_primary_with_standby_eps(&c.ha);
printf("test_reconnect_preserves_other_peer_channel: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_liveness(void)
@ -1145,5 +1301,8 @@ void test_ha_liveness(void)
RUN_TEST(test_reconnect_offload_tick_returns_promptly);
RUN_TEST(test_reconnect_inline_tick_blocks_RED);
RUN_TEST(test_reconnect_offload_dedups_per_peer);
/* raft-drive-async Task 2.2 — inbound/outbound ownership (Req 2.6, 4.4) */
RUN_TEST(test_reconnect_resolves_dual_channel_same_peer);
RUN_TEST(test_reconnect_preserves_other_peer_channel);
UNITY_END();
}