lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

test(raft-apply-decoupling): genuine multi-threaded concurrency tests for commit resolve (task 1.5)

Browse source 
Closes out task-group 1 (the commit-engine event-driven-resolve
keystone). Pure test-only: real pthreads racing the admission gate,
the sweep thread, and commit_on_ack, not sequential calls dressed as
concurrent. Adds the genuine two-real-thread ack-vs-sweep-thread race
both task-1.3 review rounds flagged as a nice-to-have (the existing
race test only proved the tie-break via single-thread white-box state
manipulation).

Also resolves the bug-1407 saga: task 1.4's "closed, 70/70" and this
task's initial "reopened, 10/10 failing" were both correct,
contradictory measurements of the identical source commit. Root cause
was a stale incremental-build artifact in the shared build/ directory
(ninja -C build rebuilt incrementally across 5 sequential tasks
without ever a fresh meson setup) -- not a source regression. Proven
by isolating each measurement in a throwaway git worktree with a
fresh build. bug-1407 is genuinely closed; the incremental-build
staleness gotcha is now documented in cerebrum.md for the remaining
task series.
This commit is contained in:
Rongsong Shen 2026-07-03 13:07:44 +00:00
parent 9b4b805a27
commit 2d9f1e947a
6 changed files with 586 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

114
src/controller/testsuite/test_persist_truncate.c
View file

@ -10,6 +10,7 @@
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdio.h>
#include <stdlib.h>
@ -375,6 +376,118 @@ static void test_truncate_noop_does_not_resolve_pending_continuations(void)
printf("test_truncate_noop_does_not_resolve_pending_continuations: PASS\n");
}
/* ── Concurrent burst (raft-apply-decoupling task 1.5, Req 9.1-9.4) ────────
* Many continuations, registered from real threads racing each other for
* ce->lock and split across the truncation cutoff, must ALL resolve exactly
* once through the SAME real persist_truncate_wal_above detection path
* exercised above: entries above the new tail FAILED/-ENOENT, entries at/
* below left untouched -- proving the bulk resolve holds under genuine
* concurrent registration, not just the fixed handful the tests above use. */
#define TRUNC_BURST_ABOVE 6
#define TRUNC_BURST_BELOW 3
#define TRUNC_BURST_TOTAL (TRUNC_BURST_ABOVE + TRUNC_BURST_BELOW)
typedef struct {
commit_engine_t* ce;
uint64_t lsn;
int rc;
trunc_continuation_recorder_t* rec;
} trunc_burst_arg_t;
static void* trunc_burst_register_thread(void* p)
{
trunc_burst_arg_t* a = (trunc_burst_arg_t*)p;
a->rc = commit_submit_async(a->ce, a->lsn, trunc_recorder_fn, a->rec);
return NULL;
}
static void test_truncate_concurrent_burst_resolves_above_cutoff_exactly_once(void)
{
char tmpdir[] = "/tmp/xce-trunc-burst-XXXXXX";
TEST_ASSERT_NOT_NULL(mkdtemp(tmpdir));
persist_t *p = make_fresh_persist(tmpdir);
TEST_ASSERT_NOT_NULL(p);
commit_engine_t ce;
TEST_ASSERT_EQUAL_INT(0,
commit_engine_init(&ce, p, /*n_peers=*/3, /*n_vcpus=*/TRUNC_BURST_TOTAL + 4));
int dummy = 0;
for (int i = 0; i < TRUNC_BURST_TOTAL + 2; i++)
persist_wal_sync_append(p, WAL_JOB_SUBMIT, &dummy, sizeof(dummy), NULL);
/* Cutoff placed so lsn 1..TRUNC_BURST_BELOW sit at/below the new tail
* and lsn (TRUNC_BURST_BELOW+1)..TRUNC_BURST_TOTAL sit above it. */
const uint64_t cutoff = TRUNC_BURST_BELOW;
trunc_continuation_recorder_t recs[TRUNC_BURST_TOTAL] = {0};
trunc_burst_arg_t args[TRUNC_BURST_TOTAL];
pthread_t tids[TRUNC_BURST_TOTAL];
/* Register the whole burst -- below-cutoff and above-cutoff LSNs alike
* -- CONCURRENTLY from real threads. */
for (int i = 0; i < TRUNC_BURST_TOTAL; i++) {
args[i].ce = &ce;
args[i].lsn = (uint64_t)(i + 1);
args[i].rc = -9999;
args[i].rec = &recs[i];
TEST_ASSERT_EQUAL_INT(
0, pthread_create(&tids[i], NULL, trunc_burst_register_thread, &args[i]));
}
for (int i = 0; i < TRUNC_BURST_TOTAL; i++) {
pthread_join(tids[i], NULL);
}
for (int i = 0; i < TRUNC_BURST_TOTAL; i++) {
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, args[i].rc,
"every concurrent registration ahead of the truncation must succeed");
}
TEST_ASSERT_EQUAL_INT(TRUNC_BURST_TOTAL, ce.inflight_waits);
/* Act: a single real truncation above cutoff. The burst above is
* already fully registered by the time this fires -- this proves the
* BULK resolve path handles a registry populated by real concurrent
* registrants correctly, not just the fixed three-continuation case
* above. */
int rc = persist_truncate_wal_above(p, cutoff);
TEST_ASSERT_EQUAL_INT(0, rc);
TEST_ASSERT_EQUAL_UINT64(cutoff + 1, p->current_lsn);
for (int i = 0; i < TRUNC_BURST_TOTAL; i++) {
uint64_t lsn = (uint64_t)(i + 1);
if (lsn > cutoff) {
TEST_ASSERT_EQUAL_INT_MESSAGE(
1, atomic_load(&recs[i].calls),
"every continuation above the new tail must resolve exactly "
"once on truncation, even when the whole burst was "
"registered concurrently");
TEST_ASSERT_EQUAL_INT_MESSAGE(
(int)COMMIT_OUTCOME_FAILED, (int)recs[i].last_outcome,
"truncation must resolve as FAILED");
TEST_ASSERT_EQUAL_INT_MESSAGE(-ENOENT, recs[i].last_rc,
"truncation must use the existing -ENOENT vocabulary");
TEST_ASSERT_EQUAL_UINT64(lsn, recs[i].last_lsn);
} else {
TEST_ASSERT_EQUAL_INT_MESSAGE(
0, atomic_load(&recs[i].calls),
"a continuation at/below the new tail must NOT be resolved "
"by truncation, even under concurrent registration");
}
}
TEST_ASSERT_EQUAL_INT_MESSAGE(
TRUNC_BURST_BELOW, ce.inflight_waits,
"only the above-cutoff admission slots are released by truncation; "
"the below-cutoff ones remain pending");
commit_engine_fini(&ce);
teardown_persist(p);
printf("test_truncate_concurrent_burst_resolves_above_cutoff_exactly_once: "
"PASS\n");
}
/* ── Test group entry point ────────────────────────────────────────────────── */
void test_persist_truncate(void)
@ -387,5 +500,6 @@ void test_persist_truncate(void)
RUN_TEST(test_truncate_noop_at_current_lsn);
RUN_TEST(test_truncate_above_resolves_pending_continuations_as_failed);
RUN_TEST(test_truncate_noop_does_not_resolve_pending_continuations);
RUN_TEST(test_truncate_concurrent_burst_resolves_above_cutoff_exactly_once);
UNITY_END();
}