lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(raft-drive-async): offload requester-less config-push to slow lane (task 3.3)

Browse source 
push_config_transport_only spawns one thread per standby then joins them with a
CONF_PUSH_TIMEOUT_MS bounded wait — blocking the caller. On the requester-less
drive-thread paths (promotion + heartbeat convergence) that wait stalls the
io_worker. Route those through ha_config_push_offloadable, gated by
g_drive_async_configpush_offload (default OFF → inline, byte-for-byte legacy;
ON → slow lane, non-blocking).

Flag ON deep-copies the config bytes + sha into a lane ctx (caller keeps owning
and freeing its own buffer); a single cluster-wide single-flight guard
(ha_state_t.configpush_inflight scalar CAS + lane key=1) bounds it to one
config-push fan-out in flight; the in-flight flag is cleared on every terminal
path (two alloc failures, -EEXIST, hard submit error, lane completion). Send
body factored into ha_config_push_do_send.

Three requester-less drive-thread call sites converted: raft_cb_become_leader
(promotion), the HA_REPL_PUSH_CONFIG heartbeat case, and the direct-CTRL_PRIMARY
FSM promotion branch. The admin-upgrade barrier (push_config_to_standbys, has a
waiting requester) and join.c are untouched.

Lands DORMANT like tasks 3.1/3.2: the lane reads c->ha.standby_endpoints and
writes those borrowed standby endpoints, racing io_worker RDHUP teardown if run
off-thread (bug-1197 family). Flag OFF keeps push+teardown serialized on the
io_worker. Task 3.4 flips this ON together with the snapshot and peerctrl flags
after establishing endpoint-lifetime discipline.

Tests (controller suite EXIT=0): test_configpush_offload_runs_on_lane,
test_configpush_inline_when_flag_off, test_configpush_offload_dedups.
Adversarial cpp-review: no CRITICAL/HIGH/MEDIUM findings.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01S6fDCtXkkEgupu7rUah1y2
This commit is contained in:
Rongsong Shen 2026-06-26 16:05:54 +00:00
parent 6c0523b69b
commit bd02064ebc
5 changed files with 515 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

208
src/controller/testsuite/test_ha_liveness.c
View file

@ -1969,6 +1969,210 @@ static void test_snapshot_offload_dedups_per_peer(void)
printf("test_snapshot_offload_dedups_per_peer: PASS\n");
}
/* ───────────────────────────────────────────────────────────────────────
* Task 3.3 — requester-less ConfigPush slow-lane offload (Req 1.1, 1.5; bug-1198).
*
* Probe: ha_configpush_probe records the thread the push body ran on and
* supports an optional gate so a test can hold the lane worker in flight while
* it submits a second config push (cluster-wide dedup proof, Req 1.5). The two
* production call sites are #ifndef XCE_TESTING, so the tests drive the router
* directly via ha_config_push_offloadable_for_test(). Mirrors the snapshot
* offload probe/gate above.
* ─────────────────────────────────────────────────────────────────────── */
extern void (*ha_configpush_probe)(void);
extern int ha_config_push_offloadable_for_test(cluster_t *c, const uint8_t *bytes,
size_t blen, const uint8_t sha256[32],
uint64_t new_tag);
static _Atomic int g_cp_runs = 0;
static _Atomic pthread_t g_cp_last_thread;
static _Atomic int g_cp_gate_on = 0;
static pthread_mutex_t g_cp_mu = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t g_cp_release_cv = PTHREAD_COND_INITIALIZER;
static int g_cp_parked = 0;
static int g_cp_release = 0;
static void cp_probe(void)
{
atomic_store_explicit(&g_cp_last_thread, pthread_self(),
memory_order_relaxed);
if (atomic_load(&g_cp_gate_on)) {
pthread_mutex_lock(&g_cp_mu);
g_cp_parked = 1;
while (!g_cp_release)
pthread_cond_wait(&g_cp_release_cv, &g_cp_mu);
pthread_mutex_unlock(&g_cp_mu);
}
atomic_fetch_add(&g_cp_runs, 1);
}
/* Bounded poll until the gated body parks; 0 on park, -1 on timeout. */
static int cp_wait_parked(int timeout_ms)
{
pthread_mutex_lock(&g_cp_mu);
int parked = 0;
for (int i = 0; i < timeout_ms; i++) {
if (g_cp_parked) { parked = 1; break; }
pthread_mutex_unlock(&g_cp_mu);
struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; /* 1 ms */
nanosleep(&ts, NULL);
pthread_mutex_lock(&g_cp_mu);
}
pthread_mutex_unlock(&g_cp_mu);
return parked ? 0 : -1;
}
static int cp_wait_runs(int want, int timeout_ms)
{
for (int i = 0; i < timeout_ms; i++) {
if (atomic_load(&g_cp_runs) >= want) break;
struct timespec ts = { .tv_sec = 0, .tv_nsec = 1000000L }; /* 1 ms */
nanosleep(&ts, NULL);
}
return atomic_load(&g_cp_runs);
}
static void cp_release_gate(void)
{
pthread_mutex_lock(&g_cp_mu);
g_cp_release = 1;
pthread_cond_broadcast(&g_cp_release_cv);
pthread_mutex_unlock(&g_cp_mu);
}
static void cp_reset_probe(void)
{
atomic_store(&g_cp_runs, 0);
atomic_store(&g_cp_gate_on, 0);
pthread_mutex_lock(&g_cp_mu);
g_cp_parked = 0;
g_cp_release = 0;
pthread_mutex_unlock(&g_cp_mu);
}
/*
* Task 3.3 GREEN (Req 1.1, 8.1/8.2): with the offload flag ON, a requester-less
* config push is submitted to the slow lane — the push body runs on a LANE
* worker (a DIFFERENT thread from the caller) and the dispatch returns promptly.
* This proves the config push no longer blocks the io_worker/drive thread.
*
* RED before Task 3.3 lands: the push ran inline on the caller thread, so the
* different-thread assertion fails.
*/
static void test_configpush_offload_runs_on_lane(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
ha_init(&c.ha, &g_cfg, 0);
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c.slowlane, 2, 4));
cp_reset_probe();
ha_configpush_probe = cp_probe;
ha_set_configpush_offload_for_test(1);
pthread_t caller = pthread_self();
int rc = ha_config_push_offloadable_for_test(&c, NULL, 0, NULL, 7);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, rc,
"Req 8.1/8.2: dispatch must return promptly (0) after lane submit");
int runs = cp_wait_runs(1, 2000);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, runs,
"Req 1.1: the config push body must run on the lane exactly once");
TEST_ASSERT_FALSE_MESSAGE(
pthread_equal(atomic_load_explicit(&g_cp_last_thread,
memory_order_relaxed), caller),
"Req 1.1: the config push must run on a LANE worker, NOT the "
"caller's (io_worker/drive) thread");
ha_set_configpush_offload_for_test(0); /* restore production default (OFF) */
ha_configpush_probe = NULL;
slowlane_fini(&c.slowlane);
ha_fini(&c.ha);
printf("test_configpush_offload_runs_on_lane: PASS\n");
}
/*
* Task 3.3 no-regression (Req 6.x): with the offload flag OFF (the production
* default until Task 3.4 resolves the borrowed-endpoint lifetime), the config
* push runs INLINE on the calling thread — byte-for-byte the pre-task behavior.
*/
static void test_configpush_inline_when_flag_off(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
ha_init(&c.ha, &g_cfg, 0);
cp_reset_probe();
ha_configpush_probe = cp_probe;
ha_set_configpush_offload_for_test(0); /* explicit OFF */
pthread_t caller = pthread_self();
int rc = ha_config_push_offloadable_for_test(&c, NULL, 0, NULL, 7);
TEST_ASSERT_EQUAL_INT(0, rc);
/* Synchronous: the body already ran on THIS thread before returning. */
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cp_runs),
"Req 6.x: with the flag OFF the config push must run inline");
TEST_ASSERT_TRUE_MESSAGE(
pthread_equal(atomic_load_explicit(&g_cp_last_thread,
memory_order_relaxed), caller),
"Req 6.x: with the flag OFF the push must run on the caller's thread");
ha_configpush_probe = NULL;
ha_fini(&c.ha);
printf("test_configpush_inline_when_flag_off: PASS\n");
}
/*
* Task 3.3 (Req 1.5): cluster-wide single-flight. While one config push is in
* flight on the lane (parked at the gate), a second push enqueues nothing — the
* CAS on configpush_inflight fails. Proof: with the first body parked, the run
* count stays at 0; after release, exactly ONE body has run (the duplicate was
* deduped, not queued).
*/
static void test_configpush_offload_dedups(void)
{
cluster_t c;
memset(&c, 0, sizeof(c));
ha_init(&c.ha, &g_cfg, 0);
TEST_ASSERT_EQUAL_INT(0, slowlane_init(&c.slowlane, 2, 4));
cp_reset_probe();
atomic_store(&g_cp_gate_on, 1); /* lane body parks at the gate */
ha_configpush_probe = cp_probe;
ha_set_configpush_offload_for_test(1);
/* First config push — submitted; its body parks in the lane. */
TEST_ASSERT_EQUAL_INT(0, ha_config_push_offloadable_for_test(&c, NULL, 0, NULL, 7));
TEST_ASSERT_EQUAL_INT_MESSAGE(0, cp_wait_parked(2000),
"the first config push body must reach the gate (running on the lane)");
/* Second config push while the first is in flight. */
TEST_ASSERT_EQUAL_INT(0, ha_config_push_offloadable_for_test(&c, NULL, 0, NULL, 7));
/* Give a generous window: the duplicate must NOT run. */
struct timespec ts = { .tv_sec = 0, .tv_nsec = 200 * 1000000L }; /* 200 ms */
nanosleep(&ts, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(0, atomic_load(&g_cp_runs),
"Req 1.5: the duplicate config push must be deduped while one is in flight");
/* Release; exactly the first body completes. */
cp_release_gate();
int runs = cp_wait_runs(1, 2000);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, runs,
"Req 1.5: exactly one config push ran (duplicate deduped)");
/* And no second body sneaks in after release. */
nanosleep(&ts, NULL);
TEST_ASSERT_EQUAL_INT_MESSAGE(1, atomic_load(&g_cp_runs),
"Req 1.5: the deduped duplicate was dropped, not queued behind the first");
ha_set_configpush_offload_for_test(0); /* restore production default (OFF) */
ha_configpush_probe = NULL;
slowlane_fini(&c.slowlane);
ha_fini(&c.ha);
printf("test_configpush_offload_dedups: PASS\n");
}
/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_ha_liveness(void)
@ -2010,5 +2214,9 @@ void test_ha_liveness(void)
RUN_TEST(test_snapshot_offload_runs_on_lane);
RUN_TEST(test_snapshot_inline_when_flag_off);
RUN_TEST(test_snapshot_offload_dedups_per_peer);
/* raft-drive-async Task 3.3 — requester-less ConfigPush slow-lane offload (Req 1.1, 1.5, 6.x) */
RUN_TEST(test_configpush_offload_runs_on_lane);
RUN_TEST(test_configpush_inline_when_flag_off);
RUN_TEST(test_configpush_offload_dedups);
UNITY_END();
}