lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix(controller): stop a failed test from hanging the unit suite (bug-1429); mkstemp for upgrade temp file (bug-1861)

Browse source 
A failing TEST_ASSERT longjmps out and skips the fixture teardown, leaving
the commit engine's sweep thread and persist's fsync_worker asleep on a
static struct. The next test re-inits it underneath them, corrupting the
condvar, and a later teardown blocks in pthread_cond_destroy/join until
the 300s meson timeout. The failure line itself was lost because stdout
was fully buffered.

- controller_test: line-buffer stdout; tearDown() reaps what a failed test
  leaked via new XCE_TESTING registries (persist_test_reap_unstopped, then
  commit_test_reap_unfinalized_engines -- persist_stop before engine fini).
  Clean run reaps 0; an injected failure now fails in 2s instead of hanging.
- persist: flusher_flush_once reports only its own durable advance, so
  fsync_worker no longer re-fires the durable callback for an advance a
  concurrent persist_flush made (flaky "must not re-fire" test). Regression
  test test_flusher_pass_does_not_claim_concurrent_flush_advance.
- tests: jap observer is atomic and waits (bounded) for a continuation that
  may resolve on the flusher thread; pump catch-up tests wait for trailing
  replies instead of a single scan.
- config_repl: upgrade decode temp file via mkstemp instead of a
  pthread_self()-derived /tmp name (same value across processes, opened
  without O_EXCL -- collisions and a symlink target).

Verified: controller_unit_test 1222/1222; 24 parallel runs (6x4), no
hangs or ASAN errors.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-09-25 02:54:09 +00:00
parent 44c1981cff
commit b5d59253e5
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
9 changed files with 316 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

115
src/controller/persist.c
View file

@ -745,8 +745,9 @@ static bool flusher_work_pending_locked(persist_t* p)
* for the whole snapshot+sync+advance so no concurrent append can extend
* the segment between the tip read and the fdatasync. Returns the new
* durable_lsn (unchanged if there was nothing to sync). */
static uint64_t flusher_flush_once(persist_t* p)
static uint64_t flusher_flush_once(persist_t* p, bool* advanced)
{
*advanced = false;
#ifdef XCE_TESTING
atomic_fetch_add_explicit(&p->test_flusher_flush_attempt_count, 1,
memory_order_relaxed);
@ -800,8 +801,13 @@ static uint64_t flusher_flush_once(persist_t* p)
* durable_lsn <= current_lsn - 1 is preserved because tip == current_lsn-1
* was read under this same lock). One fdatasync advances durable_lsn past
* however many LSNs were appended since the last sync (batching). */
if (tip > dur)
/* bug-1429: report the advance only if THIS pass made it, decided under
* p->lock -- a concurrent persist_flush() may already have moved
* durable_lsn, and that advance is its to report, not ours */
if (tip > dur) {
atomic_store_explicit(&p->durable_lsn, tip, memory_order_release);
*advanced = true;
}
#ifdef XCE_TESTING
/* Record which thread ran this successful fdatasync (task 5.1 / Req 1.6):
@ -856,9 +862,10 @@ static void* fsync_worker(void* arg)
* landed during the flush, the level is still pending and we flush
* again without needing its signal (lost-wakeup-safe). */
pthread_mutex_unlock(&p->fsync_mutex);
uint64_t before = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
uint64_t after = flusher_flush_once(p);
bool advanced = false;
uint64_t before = atomic_load_explicit(&p->durable_lsn,
memory_order_acquire);
uint64_t after = flusher_flush_once(p, &advanced);
/* COMMIT-EVAL WAKE (task 2.2 consumer wired): durable_lsn is advanced
* INSIDE flusher_flush_once (atomic store with release) BEFORE we
@ -873,9 +880,11 @@ static void* fsync_worker(void* arg)
* the advanced watermark on its own re-check, or (b) is blocked in
* pthread_cond_wait and is woken by this broadcast; it can never both
* miss the watermark AND miss the wake. We only fire the callback when
* durable_lsn actually advanced (after > before): a no-op flush (or a
* failed fdatasync that held the watermark) needs no commit re-eval,
* avoiding spurious wakes.
* THIS pass advanced durable_lsn: a no-op flush (or a failed fdatasync
* that held the watermark) needs no commit re-eval, and an advance a
* concurrent persist_flush() made is fired by persist_flush itself
* (bug-1429: a before/after snapshot here credited that advance to
* the flusher too and fired the callback twice).
*
* LOCK ORDERING (cycle-free): the flusher releases p->lock inside
* flusher_flush_once, THEN the callback takes ce->lock (commit side).
@ -883,7 +892,7 @@ static void* fsync_worker(void* arg)
* it never takes p->lock while holding ce->lock. So the only ordering
* is p->lock → ce->lock; there is no ce->lock → p->lock edge, hence no
* deadlock cycle. The callback explicitly MUST NOT take p->lock. */
if (after > before && p->on_durable_advance != NULL) {
if (advanced && p->on_durable_advance != NULL) {
p->on_durable_advance(p->durable_cb_ctx);
}
@ -1286,6 +1295,80 @@ int persist_init(persist_t* p, persist_config_t* cfg, cluster_t* c)
return 0;
}
#ifdef XCE_TESTING
/* bug-1429: registry of started-but-not-stopped persist instances, see
* persist_test_reap_unstopped() in persist.h. */
#define PERSIST_TEST_MAX_LIVE 64
static pthread_mutex_t g_live_persist_lock = PTHREAD_MUTEX_INITIALIZER;
static persist_t* g_live_persist[PERSIST_TEST_MAX_LIVE];
static int g_n_live_persist;
static void live_persist_add(persist_t* p)
{
pthread_mutex_lock(&g_live_persist_lock);
if (g_n_live_persist < PERSIST_TEST_MAX_LIVE) {
g_live_persist[g_n_live_persist++] = p;
}
pthread_mutex_unlock(&g_live_persist_lock);
}
static void live_persist_del(persist_t* p)
{
int i;
pthread_mutex_lock(&g_live_persist_lock);
for (i = 0; i < g_n_live_persist; i++) {
if (g_live_persist[i] == p) {
g_live_persist[i] = g_live_persist[--g_n_live_persist];
break;
}
}
pthread_mutex_unlock(&g_live_persist_lock);
}
static bool persist_on_own_stack(const void* addr)
{
pthread_attr_t attr;
void* base = NULL;
size_t size = 0;
bool in = false;
if (pthread_getattr_np(pthread_self(), &attr) == 0) {
if (pthread_attr_getstack(&attr, &base, &size) == 0) {
in = (const char*)addr >= (const char*)base
&& (const char*)addr < (const char*)base + size;
}
pthread_attr_destroy(&attr);
}
return in;
}
int persist_test_reap_unstopped(void)
{
persist_t* live[PERSIST_TEST_MAX_LIVE];
int n, i, reaped = 0;
pthread_mutex_lock(&g_live_persist_lock);
n = g_n_live_persist;
memcpy(live, g_live_persist, sizeof(live[0]) * (size_t)n);
g_n_live_persist = 0;
pthread_mutex_unlock(&g_live_persist_lock);
for (i = 0; i < n; i++) {
if (persist_on_own_stack(live[i])) {
WARN("persist: test left a stack-local persist %p running "
"-- its flusher may outlive the frame", (void*)live[i]);
continue;
}
WARN("persist: stopping persist %p the test left running",
(void*)live[i]);
persist_stop(live[i]);
reaped++;
}
return reaped;
}
#endif
int persist_start(persist_t* p)
{
ctrl_metrics_init();
@ -1302,13 +1385,20 @@ int persist_start(persist_t* p)
atomic_store_explicit(&p->fsync_stop, 0, memory_order_release);
int rc = pthread_create(&p->fsync_thread, NULL, fsync_worker, p);
if (rc == 0)
if (rc == 0) {
atomic_store_explicit(&p->fsync_thread_running, 1, memory_order_release);
#ifdef XCE_TESTING
live_persist_add(p);
#endif
}
return rc;
}
void persist_stop(persist_t* p)
{
#ifdef XCE_TESTING
live_persist_del(p);
#endif
if (atomic_load_explicit(&p->fsync_thread_running, memory_order_acquire)) {
/* Set the stop flag and wake the timer wait under fsync_mutex so the
* worker cannot miss the signal between its flag check and the wait. */
@ -1997,6 +2087,11 @@ void persist_set_truncate_cb(persist_t* p, void (*fn)(void* ctx, uint64_t cutoff
}
#ifdef XCE_TESTING
uint64_t persist_test_flusher_flush_once(persist_t* p, bool* advanced)
{
return flusher_flush_once(p, advanced);
}
void persist_test_set_force_fsync_fail(persist_t* p, int fail)
{
if (p)