lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(raft-apply-decoupling): genuine multi-threaded concurrency tests for commit resolve (task 1.5)

Browse source 
Closes out task-group 1 (the commit-engine event-driven-resolve
keystone). Pure test-only: real pthreads racing the admission gate,
the sweep thread, and commit_on_ack, not sequential calls dressed as
concurrent. Adds the genuine two-real-thread ack-vs-sweep-thread race
both task-1.3 review rounds flagged as a nice-to-have (the existing
race test only proved the tie-break via single-thread white-box state
manipulation).

Also resolves the bug-1407 saga: task 1.4's "closed, 70/70" and this
task's initial "reopened, 10/10 failing" were both correct,
contradictory measurements of the identical source commit. Root cause
was a stale incremental-build artifact in the shared build/ directory
(ninja -C build rebuilt incrementally across 5 sequential tasks
without ever a fresh meson setup) -- not a source regression. Proven
by isolating each measurement in a throwaway git worktree with a
fresh build. bug-1407 is genuinely closed; the incremental-build
staleness gotcha is now documented in cerebrum.md for the remaining
task series.
This commit is contained in:
Rongsong Shen 2026-07-03 13:07:44 +00:00
parent 9b4b805a27
commit 2d9f1e947a
6 changed files with 586 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

83
src/controller/testsuite/test_demotion_convergence.c
View file

@ -458,6 +458,88 @@ static void test_demotion_from_primary_resolves_pending_continuations_failed(voi
"checked\n");
}
/* raft-apply-decoupling task 1.5 (Req 9.1-9.4): a CONCURRENT burst of
* pending continuations, registered from real pthreads racing each other
* for ce->lock, must ALL resolve exactly once (FAILED/-XCE_ERR_STANDBY)
* through the SAME real demotion detection path the test above exercises
* for a fixed pair -- proving commit_resolve_pending_above's
* snapshot-then-resolve loop holds against a registry populated under
* genuine multi-threaded contention, not just two sequential registrations. */
#define DC_BURST_N 10
typedef struct {
uint64_t lsn;
int rc;
dc_continuation_recorder_t* rec;
} dc_burst_arg_t;
static void* dc_burst_register_thread(void* p)
{
dc_burst_arg_t* a = (dc_burst_arg_t*)p;
a->rc = commit_submit_async(&g_dc.commit, a->lsn, dc_recorder_fn, a->rec);
return NULL;
}
static void test_demotion_concurrent_burst_resolves_all_pending_exactly_once(void)
{
dc_continuation_recorder_t recs[DC_BURST_N] = {0};
dc_burst_arg_t args[DC_BURST_N];
pthread_t tids[DC_BURST_N];
dc_setup("burst", /*n_peers=*/3, /*n_standbys=*/2,
/*appended_lsn=*/DC_BURST_N + 4, /*n_vcpus=*/16);
atomic_store(&g_dc.persist.committed_lsn, 4);
/* Register a burst of DC_BURST_N continuations CONCURRENTLY from real
* threads, all for LSNs above the committed floor (4) -- majority is
* unreachable in this fixture (standbys never ACK), so every one stays
* genuinely pending, not already resolved, by the time demotion fires. */
for (int i = 0; i < DC_BURST_N; i++) {
args[i].lsn = (uint64_t)(5 + i);
args[i].rc = -9999;
args[i].rec = &recs[i];
TEST_ASSERT_EQUAL_INT(
0, pthread_create(&tids[i], NULL, dc_burst_register_thread, &args[i]));
}
for (int i = 0; i < DC_BURST_N; i++) {
pthread_join(tids[i], NULL);
}
for (int i = 0; i < DC_BURST_N; i++) {
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, args[i].rc,
"every concurrent registration ahead of the demotion must succeed");
}
TEST_ASSERT_EQUAL_INT(DC_BURST_N, g_dc.commit.inflight_waits);
/* Act: a single real demotion edge. The burst above is already fully
* registered by the time this fires -- this proves the BULK resolve
* path handles a registry populated by real concurrent registrants
* correctly, not just the pre-existing two-continuation case above. */
raft_cb_become_follower_with_cluster_for_test(&g_dc, /*term=*/9);
for (int i = 0; i < DC_BURST_N; i++) {
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load(&recs[i].calls),
"every continuation in the burst must resolve exactly once on "
"demotion");
TEST_ASSERT_EQUAL_INT_MESSAGE(
(int)COMMIT_OUTCOME_FAILED, (int)recs[i].last_outcome,
"demotion must resolve every pending continuation as FAILED");
TEST_ASSERT_EQUAL_INT_MESSAGE(-XCE_ERR_STANDBY, recs[i].last_rc,
"demotion must use the existing -XCE_ERR_STANDBY vocabulary");
TEST_ASSERT_EQUAL_UINT64_MESSAGE((uint64_t)(5 + i), recs[i].last_lsn,
"the recorder must observe the lsn it was registered for");
}
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, g_dc.commit.inflight_waits,
"every admission slot in the burst must be released exactly once");
dc_teardown();
printf("test_demotion_concurrent_burst_resolves_all_pending_exactly_once: "
"checked\n");
}
/* Regression: demotion from STANDBY must NOT truncate, NOT restart, NOT disable
* hooks beyond today's behavior — byte-equivalent to the pre-spec path. A
* STANDBY has no registered hooks and no convergence work. */
@ -897,6 +979,7 @@ void test_demotion_convergence(void)
RUN_TEST(test_demotion_from_primary_disables_hooks);
RUN_TEST(test_demotion_from_primary_resolves_pending_continuations_failed);
RUN_TEST(test_demotion_concurrent_burst_resolves_all_pending_exactly_once);
RUN_TEST(test_demotion_schedules_convergence_truncate_and_restart);
RUN_TEST(test_self_restart_failure_invokes_fatal_hook);
RUN_TEST(test_demotion_from_standby_no_truncation_no_restart);