lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(raft-drive-async): offload delaymgr peer reconnect to slow lane (task 2.1)

Browse source 
Heartbeat timer detects a disconnected peer, CAS-acquires a per-peer
single-flight flag, slowlane_submits the bounded connect, and returns — the
delaymgr never blocks on connect. Lane worker (reconnect_lane_fn) does the
bounded endpoint_connect + lock-guarded iomgr_monitor; flag cleared on all
terminal paths (incl. -EEXIST race fix). Default-ON kill-switch retains the
inline path for TDD-RED/fallback. Never self-terminates on failure. 2.2 adds
the inbound/outbound race + failure hardening (documented seam). 196/0 green.
This commit is contained in:
Rongsong Shen 2026-06-26 08:08:31 +00:00
parent a36234a103
commit e07667ff9b
4 changed files with 569 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

280
src/controller/testsuite/test_ha_liveness.c
View file

@ -22,9 +22,12 @@
#include <time.h>
#include <unistd.h>
#include <pthread.h>
#include <unity/unity.h>
#include "ha.h"
#include "netio.h"
#include "slowlane.h"
#include "xce/cluster_def.h"
/* ── XCE_TESTING seam ────────────────────────────────────────────────────── */
@ -842,6 +845,279 @@ static void test_ha_arm_does_not_increase_thread_count(void)
printf("test_ha_arm_does_not_increase_thread_count: PASS\n");
}
/* ── Tests 15–17: raft-drive-async Task 2.1 — reconnect offload ──────────────
*
* Acceptance criteria (Req 2.1, 2.2, 2.5; design ReconnectOffload trigger):
* - the delaymgr heartbeat tick detects a disconnected peer and submits the
* reconnect to the slow lane, then returns immediately — it NEVER performs
* the blocking connect inline (flag OFF = old inline path, which stalls);
* - per-peer dedup: while a reconnect to peer X is in flight, a second tick
* enqueues no duplicate;
* - heartbeats to reachable peers keep flowing while one peer is unreachable.
*
* Determinism: the blocking connect half is replaced by ha_reconnect_connect_hook,
* which parks on a test-owned gate to simulate an unreachable peer. The test
* controls exactly when the "connect" completes, so no real socket/timing is
* involved. primary_heartbeat_fn_for_test is driven on a worker thread so the
* test can assert it returns promptly (offload) vs stalls (inline). */
extern int (*ha_reconnect_connect_hook)(cluster_t *c, int peer_idx);
/* Gate that the reconnect hook parks on (simulates an unreachable peer). */
typedef struct {
pthread_mutex_t mu;
pthread_cond_t release_cv; /* test broadcasts to release the parked hook */
pthread_cond_t entered_cv; /* hook broadcasts when it has entered/parked */
int gate_open;
_Atomic int entered; /* hook invocations that reached the gate */
_Atomic int calls; /* total hook invocations (dedup oracle) */
} rc_gate_t;
static rc_gate_t g_rc_gate;
static void rc_gate_init(rc_gate_t *g)
{
pthread_mutex_init(&g->mu, NULL);
pthread_cond_init(&g->release_cv, NULL);
pthread_cond_init(&g->entered_cv, NULL);
g->gate_open = 0;
atomic_store(&g->entered, 0);
atomic_store(&g->calls, 0);
}
static void rc_gate_destroy(rc_gate_t *g)
{
pthread_mutex_destroy(&g->mu);
pthread_cond_destroy(&g->release_cv);
pthread_cond_destroy(&g->entered_cv);
}
/* The hook stand-in for the blocking connect: count, signal entry, park until
* the test opens the gate, then return success. Runs on a slow-lane worker
* (flag ON) or inline on the caller (flag OFF). */
static int rc_gated_connect_hook(cluster_t *c, int peer_idx)
{
(void)c; (void)peer_idx;
atomic_fetch_add(&g_rc_gate.calls, 1);
pthread_mutex_lock(&g_rc_gate.mu);
atomic_fetch_add(&g_rc_gate.entered, 1);
pthread_cond_broadcast(&g_rc_gate.entered_cv);
while (!g_rc_gate.gate_open)
pthread_cond_wait(&g_rc_gate.release_cv, &g_rc_gate.mu);
pthread_mutex_unlock(&g_rc_gate.mu);
return 0; /* simulate a successful connect */
}
static void rc_gate_open(rc_gate_t *g)
{
pthread_mutex_lock(&g->mu);
g->gate_open = 1;
pthread_cond_broadcast(&g->release_cv);
pthread_mutex_unlock(&g->mu);
}
/* Wait (bounded) until at least n hook invocations have parked at the gate. */
static int rc_gate_wait_entered(rc_gate_t *g, int n, int timeout_ms)
{
struct timespec deadline;
clock_gettime(CLOCK_REALTIME, &deadline);
deadline.tv_sec += timeout_ms / 1000;
deadline.tv_nsec += (long)(timeout_ms % 1000) * 1000000L;
if (deadline.tv_nsec >= 1000000000L) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000L; }
pthread_mutex_lock(&g->mu);
int rc = 0;
while (atomic_load(&g->entered) < n) {
rc = pthread_cond_timedwait(&g->entered_cv, &g->mu, &deadline);
if (rc != 0) break;
}
int entered = atomic_load(&g->entered);
pthread_mutex_unlock(&g->mu);
return entered >= n ? 0 : -1;
}
/* Build a primary cluster_t with: peers_cfg of 3 candidates (self at priority
* 0), one reachable standby already in standby_endpoints (peer 1), and peer 2
* absent (so the reconnect scan targets it). The slow lane is initialised. */
static void rc_setup_primary(cluster_t *c, myconf_t *conf, endpoint_t *standby_ep)
{
memset(c, 0, sizeof(*c));
memset(conf, 0, sizeof(*conf));
init_ha_primary_with_standby_eps(&c->ha);
/* 3-node cluster; self = priority 0. */
conf->peers_cfg.candidates_count = 3;
for (int i = 0; i < 3; i++) {
snprintf(conf->peers_cfg.candidates[i].name,
sizeof(conf->peers_cfg.candidates[i].name), "ctrl%d", i);
snprintf(conf->peers_cfg.candidates[i].hostname,
sizeof(conf->peers_cfg.candidates[i].hostname), "127.0.0.1");
conf->peers_cfg.candidates[i].port = 19000 + i;
}
c->myconf = conf;
c->ha.my_priority = 0;
c->myhost = &c->myhost_info;
/* Slow lane for the offload path. */
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c->slowlane, 2, 4));
/* Reachable standby for peer 1 (absolute index 1), already connected. */
memset(standby_ep, 0, sizeof(*standby_ep));
standby_ep_ctx_build_for_test(standby_ep, &c->ha, c, 1);
standby_ep_ctx_t_test *sctx = (standby_ep_ctx_t_test *)standby_ep->data;
sctx->last_push_time = 0; /* stale → heartbeat will be sent */
zarray_store(c->ha.standby_endpoints, standby_ep);
}
static void rc_teardown_primary(cluster_t *c, endpoint_t *standby_ep)
{
slowlane_fini(&c->slowlane);
free(standby_ep->data);
fini_ha_primary_with_standby_eps(&c->ha);
}
/* Worker thread that drives one heartbeat tick; sets *done=1 when it returns. */
typedef struct { cluster_t *c; _Atomic int done; } rc_tick_arg_t;
static void *rc_tick_thread(void *p)
{
rc_tick_arg_t *a = (rc_tick_arg_t *)p;
primary_heartbeat_fn_for_test(a->c);
atomic_store(&a->done, 1);
return NULL;
}
/*
* Test 15 (Req 2.1, 2.2): with the offload flag ON, a heartbeat tick whose
* reconnect target is unreachable (the connect hook parks) RETURNS PROMPTLY —
* the blocking connect runs on the lane, not on the delaymgr thread — and the
* heartbeat to the reachable standby was still sent.
*
* With the flag OFF (RED) the inline connect runs on the tick thread and the
* tick BLOCKS in the gate, so the same assertion (tick done within the
* deadline) FAILS. The flag toggle is what flips RED↔GREEN.
*/
static void rc_run_offload_tick(int offload_on, const char *label)
{
cluster_t c;
myconf_t conf;
endpoint_t standby_ep;
rc_setup_primary(&c, &conf, &standby_ep);
rc_gate_init(&g_rc_gate);
ha_reconnect_connect_hook = rc_gated_connect_hook;
ha_set_reconnect_offload_for_test(offload_on);
g_nop_calls = 0;
ha_nop_req_hook = stub_nop_req;
rc_tick_arg_t arg = { .c = &c };
atomic_store(&arg.done, 0);
pthread_t th;
pthread_create(&th, NULL, rc_tick_thread, &arg);
/* The tick should return within a generous deadline IFF the connect was
* offloaded. Poll arg.done. */
int done = 0;
for (int i = 0; i < 200; i++) { /* up to ~2s */
if (atomic_load(&arg.done)) { done = 1; break; }
struct timespec ts = { .tv_sec = 0, .tv_nsec = 10 * 1000000L };
nanosleep(&ts, NULL);
}
if (offload_on) {
TEST_ASSERT_EQUAL_INT_MESSAGE(1, done,
"Req 2.1/2.2: tick must return promptly with reconnect offloaded "
"(connect parked on the lane, delaymgr thread free)");
/* The reachable standby still got its heartbeat this tick. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, g_nop_calls,
"Req 2.1: heartbeat to the reachable standby must be sent while a "
"peer is unreachable");
/* The reconnect connect was submitted to the lane (hook entered). */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_gate_wait_entered(&g_rc_gate, 1, 1000),
"Req 2.1: the offloaded reconnect connect must run on the lane");
} else {
/* RED: inline connect blocks the tick thread inside the gate. */
TEST_ASSERT_EQUAL_INT_MESSAGE(0, done,
"RED: with offload OFF the inline connect blocks the delaymgr tick");
}
/* Release the parked hook so the worker/tick thread can finish, then join. */
rc_gate_open(&g_rc_gate);
pthread_join(th, NULL);
ha_nop_req_hook = NULL;
ha_reconnect_connect_hook = NULL;
ha_set_reconnect_offload_for_test(1); /* restore production default */
rc_gate_destroy(&g_rc_gate);
rc_teardown_primary(&c, &standby_ep);
printf("%s: PASS\n", label);
}
static void test_reconnect_offload_tick_returns_promptly(void)
{
rc_run_offload_tick(1, "test_reconnect_offload_tick_returns_promptly");
}
static void test_reconnect_inline_tick_blocks_RED(void)
{
rc_run_offload_tick(0, "test_reconnect_inline_tick_blocks_RED");
}
/*
* Test 17 (Req 2.5): per-peer dedup. While a reconnect to the unreachable
* peer is in flight (parked in the hook), a SECOND heartbeat tick enqueues NO
* duplicate reconnect for that peer — the connect hook is invoked exactly once
* across both ticks.
*/
static void test_reconnect_offload_dedups_per_peer(void)
{
cluster_t c;
myconf_t conf;
endpoint_t standby_ep;
rc_setup_primary(&c, &conf, &standby_ep);
rc_gate_init(&g_rc_gate);
ha_reconnect_connect_hook = rc_gated_connect_hook;
ha_set_reconnect_offload_for_test(1);
ha_nop_req_hook = stub_nop_req;
/* Tick 1: submits the reconnect; the lane worker enters the hook and parks. */
primary_heartbeat_fn_for_test(&c);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc_gate_wait_entered(&g_rc_gate, 1, 1000),
"tick 1 must submit the reconnect (hook parks on the lane)");
/* Tick 2: peer still unreachable (in flight) — must NOT enqueue a duplicate. */
primary_heartbeat_fn_for_test(&c);
/* Give any (erroneously) duplicated task time to reach the gate. */
struct timespec ts = { .tv_sec = 0, .tv_nsec = 150 * 1000000L };
nanosleep(&ts, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_rc_gate.calls),
"Req 2.5: a second tick must not enqueue a duplicate reconnect for an "
"already in-flight peer");
/* Release and let the single in-flight reconnect complete. */
rc_gate_open(&g_rc_gate);
/* Drain: wait for the in-flight flag to clear (lane worker done). */
for (int i = 0; i < 200; i++) {
if (atomic_load_explicit(&c.ha.reconnect_inflight[2],
memory_order_acquire) == 0) break;
nanosleep(&ts, NULL);
}
ha_nop_req_hook = NULL;
ha_reconnect_connect_hook = NULL;
rc_gate_destroy(&g_rc_gate);
rc_teardown_primary(&c, &standby_ep);
printf("test_reconnect_offload_dedups_per_peer: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_liveness(void)
@ -865,5 +1141,9 @@ void test_ha_liveness(void)
RUN_TEST(test_cluster_dispatch_nop_skips_when_primary);
RUN_TEST(test_cluster_dispatch_ctrlhello_resets_activity_when_standby);
RUN_TEST(test_ha_arm_does_not_increase_thread_count);
/* raft-drive-async Task 2.1 — reconnect offload (Req 2.1, 2.2, 2.5) */
RUN_TEST(test_reconnect_offload_tick_returns_promptly);
RUN_TEST(test_reconnect_inline_tick_blocks_RED);
RUN_TEST(test_reconnect_offload_dedups_per_peer);
UNITY_END();
}