From 0f3919f195b696e4055520412b2e563582287c5f Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 15 Sep 2026 04:13:53 +0000 Subject: [PATCH] fix(lib): stop cbuf_put/get orphaning their mutex on cancellation, hanging vtaskmgr_fini pthread_cond_wait() reacquires its mutex before honoring a cancellation, but mylock.c installed no cleanup handler to release it again -- a thread cancelled mid-wait terminated holding the lock forever. vtaskmgr's pthread_cancel()-based teardown could land there from three angles (dispatcher cancelled while polling the shared queue, or a vcpu racing back into it after its own shutdown signal), permanently deadlocking any other thread that touched the same queue and hanging vtaskmgr_fini()'s pthread_join() indefinitely. Reproduced live as xce_test hanging on Ctrl-D exit once a connection context was entered; gdb confirmed a vcpu thread blocked forever on a mutex owned by an already-exited thread ID. Fix: cbuf_put()/cbuf_get()/cbuf_timedget() now unlock via a pthread_cleanup_push/pop handler around the wait, so a cancelled waiter always releases the mutex. vcpu_idle()/vcpu_entry() also recognize the teardown poison pill and exit immediately instead of needlessly re-announcing into the already-dead shared queue. New test_vtaskmgr_teardown.c reproduces the race deterministically (self-pinned to one CPU, since true multi-core parallelism rarely hits the window) and bounds vtaskmgr_fini() with a timed join so a regression fails fast instead of hanging the suite. Logged as bug-1810. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC --- .wolf/anatomy-index.json | 55 ++++++- .wolf/anatomy.md | 9 +- .wolf/buglog.json | 31 ++++ .wolf/memory.md | 23 +++ src/lib/meson.build | 1 + src/lib/mylock.c | 33 ++++- src/lib/testsuite/test.c | 2 + src/lib/testsuite/test_vtaskmgr_teardown.c | 158 +++++++++++++++++++++ src/lib/vtaskmgr.c | 32 +++++ 9 files changed, 335 insertions(+), 9 deletions(-) create mode 100644 src/lib/testsuite/test_vtaskmgr_teardown.c diff --git a/.wolf/anatomy-index.json b/.wolf/anatomy-index.json index be9545d9..f8aa9191 100644 --- a/.wolf/anatomy-index.json +++ b/.wolf/anatomy-index.json @@ -1,13 +1,13 @@ { "version": 1, "meta": { - "lastScanned": "2026-09-15T02:34:37.112Z", - "fileCount": 562, + "lastScanned": "2026-09-15T03:56:28.973Z", + "fileCount": 567, "hits": 0, "misses": 0, - "renderedHash": "54b2e66594a769b4176755032afcc65e9009684d6c9b649e2e021d28c2b7f11f", - "storeUpdatedAt": "2026-09-15T02:34:37.121Z", - "rootHash": "cda4e7e066f4e166" + "renderedHash": "a9ac7e8c7eca3b5e2b406896f2877eaecf4b21d63a5ffa783b18205d4002256c", + "storeUpdatedAt": "2026-09-15T03:56:28.981Z", + "rootHash": "7500ad02fc80875a" }, "files": { ".agent-shell/transcripts/2026-08-11-11-39-59.md": { @@ -10264,6 +10264,51 @@ "mtimeMs": 1789428659757.0122, "updatedAt": "2026-09-14T23:30:59.805Z", "source": "hook" + }, + "src/lib/testsuite/test_vtaskmgr_teardown.c": { + "description": "Declares int", + "tokens": 1668, + "hash": "190f0dd0bf0c65c9", + "size": 6255, + "mtimeMs": 1789442667259.7947, + "updatedAt": "2026-09-15T03:24:27.304Z", + "source": "hook" + }, + "src/lib/testsuite/test.c": { + "description": "include ", + "tokens": 1359, + "hash": "329a9fe468acd8ad", + "size": 5093, + "mtimeMs": 1789441323455.6272, + "updatedAt": "2026-09-15T03:02:03.502Z", + "source": "hook" + }, + "src/lib/meson.build": { + "description": "", + "tokens": 2428, + "hash": "61df53cb05f969ce", + "size": 9104, + "mtimeMs": 1789441310019.5066, + "updatedAt": "2026-09-15T03:01:50.064Z", + "source": "hook" + }, + "src/lib/vtaskmgr.c": { + "description": "include ", + "tokens": 4446, + "hash": "928364b0c99d4805", + "size": 16685, + "mtimeMs": 1789444588936.162, + "updatedAt": "2026-09-15T03:56:28.973Z", + "source": "hook" + }, + "src/lib/mylock.c": { + "description": "include ", + "tokens": 1632, + "hash": "ddf11711b4ca77c6", + "size": 6119, + "mtimeMs": 1789443750926.8965, + "updatedAt": "2026-09-15T03:42:30.970Z", + "source": "hook" } } } \ No newline at end of file diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index ec54805a..e23360a3 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,7 +1,7 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T02:34:37.112Z -> Files: 562 tracked | Anatomy hits: 0 | Misses: 0 +> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T03:56:28.973Z +> Files: 567 tracked | Anatomy hits: 0 | Misses: 0 ## ./ @@ -891,13 +891,18 @@ ## src/lib/ +- `meson.build` (~2428 tok) - `misc.c` — include (~486 tok) +- `mylock.c` — include (~1632 tok) - `resreq.c` — define _XOPEN_SOURCE 700 (~6663 tok) - `utils.c` — include (~7015 tok) +- `vtaskmgr.c` — include (~4446 tok) ## src/lib/testsuite/ +- `test_vtaskmgr_teardown.c` — Declares int (~1668 tok) - `test_xcealloc.c` — include (~4979 tok) +- `test.c` — include (~1359 tok) ## src/scheduler/include/ diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 4e4c7c04..3466bb3c 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -21976,6 +21976,37 @@ ], "occurrences": 1, "last_seen": "2026-09-14T16:20:00Z" + }, + { + "id": "bug-1810", + "timestamp": "2026-09-15T03:55:00Z", + "error_message": "User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live connection context (client/worker/scheduler). Live gdb attach on the user's machine showed the main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(vcpu->vcpu, ...); a sibling vcpu thread was simultaneously deadlocked in cbuf_put() at mylock.c:101's pthread_mutex_lock(&(b->mutex)) on vtaskmgr->wbuf.mutex, whose __owner TID had already exited.", + "file": "src/lib/mylock.c (cbuf_put/cbuf_get/cbuf_timedget), src/lib/vtaskmgr.c (vcpu_idle, vcpu_entry)", + "root_cause": "Two compounding defects in src/lib's shared task-pool primitives (used by every daemon + xce_test's own client/worker/scheduler REPL contexts, each of which owns a vtaskmgr_t for async I/O dispatch). (1) mylock.c's cbuf_put()/cbuf_get()/cbuf_timedget() lock b->mutex, then block in pthread_cond_wait()/pthread_cond_timedwait() with NO pthread_cleanup_push() handler installed. POSIX mandates pthread_cond_wait() reacquire its mutex before honoring a pending cancellation and running cleanup handlers -- with no handler registered, a thread cancelled while parked there terminates HOLDING the mutex, orphaning it locked forever; every other thread that later calls pthread_mutex_lock() on that same cbuf deadlocks uninterruptibly (plain mutex lock is not a cancellation point in glibc's NPTL). (2) vtaskmgr.c's teardown protocol races directly into that hazard from up to three independent angles, all torn down via pthread_cancel()+pthread_join(): (a) vtaskmgr_fini() cancels the dispatcher thread FIRST -- if it lands while the dispatcher is blocked in pick_idle_vcpu()'s cbuf_get() on the shared vtaskmgr->wbuf (empty, e.g. because real I/O work had every vcpu momentarily busy), that orphans wbuf's mutex immediately, before any vcpu teardown even starts; (b) vcpu_fini() then wakes each idle vcpu via bind_vtask(NULL, vcpu) (a 'poison pill' through the vcpu's own 1-capacity wbuf) and immediately pthread_cancel()s it, but vcpu_entry()'s loop could not distinguish that pill from 'no task yet' -- it looped straight back into vcpu_idle(), which unconditionally re-announced the vcpu into the shared (by then permanently undrained, since the dispatcher is already dead) vtaskmgr->wbuf, racing the pending cancellation against that second cbuf_put(); (c) the same poison pill arriving while a vcpu was still mid-execution of a real task queued it silently for the NEXT vcpu_idle() call, which still re-announced into the shared queue first before ever consuming it. Any of the three could orphan vtaskmgr->wbuf's mutex; every vcpu processed afterward that still touched the shared queue then deadlocked forever on pthread_mutex_lock(), and vtaskmgr_fini()'s pthread_join() for it never returned -- hanging the whole process (xce_test on Ctrl-D quit; identically reachable from any daemon that tears down a vtaskmgr_t under real concurrent I/O, e.g. controller/scheduler/worker shutdown).", + "fix": "FIXED (2026-09-15), TDD RED->GREEN. Two-part fix, both required (confirmed empirically -- the vtaskmgr.c-only fix alone left the dispatcher-orphan path (2a above) unaddressed and still reproduced the hang under stress). (1) mylock.c: wrapped the pthread_cond_wait()/pthread_cond_timedwait() loops in cbuf_put()/cbuf_get()/cbuf_timedget() with pthread_cleanup_push(pthread_mutex_unlock, &b->mutex) / pthread_cleanup_pop(0) -- a cancelled waiter now always releases the mutex on unwind, so no cbuf mutex can ever be left orphaned-locked by a cancellation, regardless of which caller (dispatcher or vcpu) hits it. cbuf_timedget()'s early ETIMEDOUT return was restructured to break+flag instead of returning from inside the pushed region, since push/pop must be a matched pair at the same lexical nesting level. (2) vtaskmgr.c, defense-in-depth / closes the underlying logic bug independently of (1): vcpu_idle() now does a non-blocking cbuf_tryget() on its own wbuf BEFORE announcing itself into the shared vtaskmgr->wbuf, so an already-queued poison pill (queued while the vcpu was mid-task) is consumed directly instead of triggering a needless re-announce; vcpu_entry()'s loop now checks vcpu->task after vcpu_idle() returns and breaks (exits the thread) on NULL instead of looping back into idle -- a NULL task is unambiguously vcpu_fini()'s shutdown signal, since the dispatcher (vdispatcher_entry) never calls bind_vtask() with a NULL task. New regression test src/lib/testsuite/test_vtaskmgr_teardown.c (wired into src/lib/meson.build + testsuite/test.c): floods a burst of real VTASK_LIVENESS tasks across a 16-vcpu vtaskmgr then calls vtaskmgr_fini() on a helper thread with a pthread_timedjoin_np() 2s bounded-timeout join, self-pinning the test to a single CPU (sched_setaffinity, inherited by every spawned vtaskmgr thread at clone()) since the race needs preemptive time-slicing to reproduce reliably -- confirmed 0/hundreds of hangs on stock multi-core scheduling (true parallelism lets a vcpu almost always win the race to its own pthread_testcancel() before cancellation is even issued) vs. reliable reproduction (hung on iteration 0) once pinned to one core. Verified via: (a) this new test, RED before the fix (Expected 0 Was 110/ETIMEDOUT) and GREEN after; (b) a standalone single-core-pinned stress harness outside the Unity suite, 2000/2000 iterations clean post-fix (vs. hanging on iteration 0 with no fix and again on iteration 3 with only the vtaskmgr.c half of the fix, confirming both parts were needed); (c) full libxce test suite (314/314) and full harness suite (132/132, including the pty-based clitest tests that exercise real client-context connect+quit) both green; (d) a live pty-driven repro of the ORIGINAL user report -- launching the real xce_test binary, entering the client context, sending a real Ctrl-D (EOT) byte -- now exits in <0.1s (previously would hang indefinitely); (e) full project meson test matrix (98/98 suites) green on a clean run -- two earlier full-matrix runs showed unrelated, non-reproducing timeouts (one in harness's own quit-wait, one in controller_unit_test) that did not reproduce on isolated reruns, consistent with this sandbox's pre-existing parallel-test-suite CPU-contention flakiness (see bug-1429/bug-1580), not a regression from this fix.", + "tags": [ + "libxce", + "mylock", + "vtaskmgr", + "cbuf", + "pthread_cancel", + "pthread_cond_wait", + "cleanup-handler", + "orphaned-mutex", + "deadlock", + "hang", + "xce_test", + "harness", + "ctrl-d", + "eof", + "quit", + "teardown", + "race-condition", + "tdd-red-green", + "live-gdb-repro" + ], + "occurrences": 1, + "last_seen": "2026-09-15T03:55:00Z" } ] } diff --git a/.wolf/memory.md b/.wolf/memory.md index ff8f17b8..3745fd28 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -4721,3 +4721,26 @@ | 16:30 | Edited src/lib/utils.c | modified hash_string() | ~24 | | 19:34 | Edited src/include/utils.h | expanded (+6 lines) | ~113 | | 19:34 | Edited src/scheduler/src/core_events.c | DEBUG() → DBG() | ~325 | +| 19:46 | Session end: 8 writes across 3 files (utils.h, core_events.c, utils.c) | 2 reads | ~14326 tok | + +## Session: 2026-09-15 19:46 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 20:00 | Created src/lib/testsuite/test_vtaskmgr_teardown.c | — | ~1070 | +| 20:01 | Edited src/lib/testsuite/test.c | 2→3 lines | ~32 | +| 20:01 | Edited src/lib/meson.build | 2→3 lines | ~44 | +| 20:02 | Edited src/lib/testsuite/test.c | 3→4 lines | ~33 | +| 20:09 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | 2→2 lines | ~17 | +| 20:12 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | modified struct() | ~489 | +| 20:24 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | expanded (+10 lines) | ~296 | +| 20:24 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | modified test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang() | ~214 | +| 20:24 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | added 1 condition(s) | ~47 | +| 20:28 | Edited src/lib/vtaskmgr.c | added 1 condition(s) | ~292 | +| 20:28 | Edited src/lib/vtaskmgr.c | added 1 condition(s) | ~265 | +| 20:37 | Edited src/lib/mylock.c | modified while() | ~255 | +| 20:37 | Edited src/lib/mylock.c | modified while() | ~188 | +| 20:38 | Edited src/lib/mylock.c | modified while() | ~185 | +| 20:42 | Edited src/lib/mylock.c | added 1 condition(s) | ~303 | +| 20:56 | Edited src/lib/vtaskmgr.c | 11→13 lines | ~236 | +| 20:56 | Edited src/lib/vtaskmgr.c | 5→4 lines | ~74 | diff --git a/src/lib/meson.build b/src/lib/meson.build index 28aa725f..7eb79a38 100644 --- a/src/lib/meson.build +++ b/src/lib/meson.build @@ -226,6 +226,7 @@ libxce_test_src = ['testsuite/test.c', 'testsuite/test_ctrlstatus.c', 'testsuite/test_vtask_admission.c', 'testsuite/test_vtask_enq_race.c', + 'testsuite/test_vtaskmgr_teardown.c', 'testsuite/test_xcealloc.c', 'raft/raft_sim.c', ] diff --git a/src/lib/mylock.c b/src/lib/mylock.c index 52617047..a48de08f 100644 --- a/src/lib/mylock.c +++ b/src/lib/mylock.c @@ -99,9 +99,18 @@ int cbuf_put(cbuf_t* b, void* it) } pthread_mutex_lock(&(b->mutex)); + /* pthread_cond_wait() reacquires the mutex before honoring a pending + * cancellation (POSIX-mandated), so a caller cancelled while parked + * here would otherwise terminate holding b->mutex forever -- no other + * thread could ever lock it again. The cleanup handler releases it on + * that unwind; on the normal (non-cancelled) path pop(0) just removes + * the handler without running it, since the explicit unlock below + * still runs. */ + pthread_cleanup_push((void (*)(void*))pthread_mutex_unlock, &(b->mutex)); while (b->len == b->ncap) { pthread_cond_wait(&(b->condput), &(b->mutex)); } + pthread_cleanup_pop(0); i = (b->head + b->len) % b->ncap; @@ -148,9 +157,14 @@ int cbuf_get(cbuf_t* b, void* it) } pthread_mutex_lock(&(b->mutex)); + /* See cbuf_put()'s comment on the same pattern: without this cleanup + * handler, a caller cancelled while parked in pthread_cond_wait() + * would reacquire and then abandon b->mutex locked forever. */ + pthread_cleanup_push((void (*)(void*))pthread_mutex_unlock, &(b->mutex)); while (b->len == 0) { pthread_cond_wait(&(b->condget), &(b->mutex)); } + pthread_cleanup_pop(0); memcpy(it, (void*)((char*)b->data) + b->isize * b->head, b->isize); @@ -189,18 +203,33 @@ int cbuf_tryget(cbuf_t* b, void* it) int cbuf_timedget(cbuf_t* b, void* it, const struct timespec* deadline) { + int timed_out = 0; + if ((b == NULL) || (it == NULL) || (deadline == NULL)) { return -EINVAL; } pthread_mutex_lock(&(b->mutex)); + /* See cbuf_put()'s comment on the same pattern: without this cleanup + * handler, a caller cancelled while parked in pthread_cond_timedwait() + * would reacquire and then abandon b->mutex locked forever. + * pthread_cleanup_push()/pop() must be a matched pair at the same + * lexical nesting level -- break out of the loop on timeout instead + * of returning from inside it, so pop() always runs right after. */ + pthread_cleanup_push((void (*)(void*))pthread_mutex_unlock, &(b->mutex)); while (b->len == 0) { int rc = pthread_cond_timedwait(&(b->condget), &(b->mutex), deadline); if (rc == ETIMEDOUT) { - pthread_mutex_unlock(&(b->mutex)); - return -ETIMEDOUT; + timed_out = 1; + break; } } + pthread_cleanup_pop(0); + + if (timed_out) { + pthread_mutex_unlock(&(b->mutex)); + return -ETIMEDOUT; + } memcpy(it, (void*)((char*)b->data) + b->isize * b->head, b->isize); diff --git a/src/lib/testsuite/test.c b/src/lib/testsuite/test.c index bedbd4b9..3a9e1c54 100644 --- a/src/lib/testsuite/test.c +++ b/src/lib/testsuite/test.c @@ -55,6 +55,7 @@ extern void test_wal_wire(void); extern void test_ctrlstatus(void); extern void test_vtask_admission(void); extern void test_vtask_enq_race(void); +extern void test_vtaskmgr_teardown(void); extern void test_xcealloc(void); void setUp(void) { return; } @@ -180,6 +181,7 @@ int main(int argc, char* argv[]) TEST(test_ctrlstatus); TEST(test_vtask_admission); TEST(test_vtask_enq_race); + TEST(test_vtaskmgr_teardown); TEST(test_xcealloc); return UNITY_END(); diff --git a/src/lib/testsuite/test_vtaskmgr_teardown.c b/src/lib/testsuite/test_vtaskmgr_teardown.c new file mode 100644 index 00000000..1c9954b2 --- /dev/null +++ b/src/lib/testsuite/test_vtaskmgr_teardown.c @@ -0,0 +1,158 @@ +/* + * test_vtaskmgr_teardown.c — vtaskmgr_fini() must never hang tearing down + * idle vcpus. + * + * Root cause under test: vcpu_fini() (vtaskmgr.c) wakes a parked vcpu via + * bind_vtask(NULL, vcpu) (a "poison pill" through the vcpu's own per-vcpu + * cbuf) and then pthread_cancel()s + pthread_join()s it. vcpu_entry()'s loop + * cannot distinguish that poison pill from "no task yet" -- it loops right + * back into vcpu_idle(), which unconditionally re-announces the vcpu into + * the SHARED vtaskmgr->wbuf cbuf (step A) before waiting again (step B). + * By the time any vcpu is torn down, vtaskmgr_fini() has already + * cancelled+joined the dispatcher thread -- the only consumer of + * vtaskmgr->wbuf -- so that shared queue is never drained again. If + * pthread_cancel() lands while the vcpu is blocked inside step A's + * cbuf_put()'s pthread_cond_wait() on that now-permanently-unconsumed + * queue, the thread is torn down without releasing vtaskmgr->wbuf's mutex + * (mylock.c's cbuf_put/cbuf_get install no pthread_cleanup_push() handler), + * orphaning it locked forever. Any OTHER still-alive vcpu that later loops + * back into vcpu_idle() then deadlocks uninterruptibly on + * pthread_mutex_lock() (not a cancellation point in glibc's NPTL), and + * vtaskmgr_fini()'s pthread_join() for it never returns. + * + * Reproduced live: xce_test (harness REPL) hung on Ctrl-D (EOF) exit, its + * main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> + * xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(). A gdb + * attach confirmed a sibling vcpu thread deadlocked in cbuf_put()'s + * pthread_mutex_lock() against vtaskmgr->wbuf.mutex, owned by a thread ID + * that had already exited. + * + * The race is timing-dependent: on a machine with more CPUs than + * NVCPUS+2, the dispatcher/vcpu threads run in true parallel and a vcpu + * almost always reaches its own pthread_testcancel() before + * vtaskmgr_fini()'s pthread_cancel() is even issued, so the hazardous + * second vcpu_idle() call is rarely reached -- confirmed empirically (0 + * hangs across hundreds of multi-core iterations). Pinning this test's + * thread (and therefore, by Linux's affinity-inheritance-at-clone() + * rule, every vtaskmgr thread it spawns) to a SINGLE CPU forces + * preemptive time-slicing instead of true parallelism, which reproduces + * the hang deterministically -- confirmed on iteration 0 of a + * single-core standalone repro. The loop below still runs several + * iterations, with a bounded-timeout join, as a safety margin -- it + * FAILS (rather than hanging the whole suite) the first time any + * iteration doesn't tear down within the deadline. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +#include + +#include "vtaskmgr.h" + +typedef struct { + vtaskmgr_t* mgr; +} fini_arg_t; + +static void* fini_thread(void* p) +{ + fini_arg_t* a = (fini_arg_t*)p; + + vtaskmgr_fini(a->mgr); + return NULL; +} + +static int noop_entry(void* p) +{ + (void)p; + return 0; +} + +static vtask_t* make_noop_task(void) +{ + vtask_data_t* d = calloc(1, sizeof(*d)); + vtask_t* t = vtask_alloc(); + + d->vtype = VTASK_GENERIC; + d->u.generic = NULL; + vtask_init(t, noop_entry, d); + return t; +} + +static void test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang(void) +{ + const int ITERATIONS = 10; + const int NVCPUS = 16; + const int BURST = NVCPUS * 40; + + cpu_set_t saved_affinity; + cpu_set_t single_cpu; + int have_saved_affinity + = (sched_getaffinity(0, sizeof(saved_affinity), &saved_affinity) == 0); + + CPU_ZERO(&single_cpu); + CPU_SET(0, &single_cpu); + /* Threads spawned after this point (vtaskmgr's dispatcher/vcpus, and + * this test's own fini_thread) inherit the calling thread's affinity + * mask at clone() time -- pinning here forces the preemptive + * time-slicing that makes the race reproducible (see file header). */ + sched_setaffinity(0, sizeof(single_cpu), &single_cpu); + + for (int iter = 0; iter < ITERATIONS; iter++) { + vtaskmgr_t mgr; + + TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, NVCPUS)); + TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&mgr)); + + /* Flood a burst of trivial real tasks so the dispatcher is + * ACTIVELY cycling through pick_idle_vcpu()/bind_vtask() -- not + * settled in steady idle -- when fini() below tears it down. This + * mirrors the live scenario (an xce_test client connection with + * genuine in-flight I/O) rather than a purely idle REPL: the + * dispatcher revisiting pick_idle_vcpu()'s blocking cbuf_get() on + * vtaskmgr->wbuf is exactly the window where a cancel can orphan + * that shared mutex before any vcpu teardown even begins. No + * settle wait -- fini() is called immediately, mid-burst. */ + for (int k = 0; k < BURST; k++) { + vtask_t* t = make_noop_task(); + vtask_enq_classified(&mgr, t, VTASK_LIVENESS); + } + + fini_arg_t farg = { .mgr = &mgr }; + pthread_t t; + TEST_ASSERT_EQUAL_INT(0, + pthread_create(&t, NULL, fini_thread, &farg)); + + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += 2; + + int jrc = pthread_timedjoin_np(t, NULL, &deadline); + TEST_ASSERT_EQUAL_INT_MESSAGE(0, jrc, + "vtaskmgr_fini() must not hang tearing down idle vcpus -- an " + "orphaned vtaskmgr->wbuf mutex from a mid-cancellation " + "cbuf_put() leaves a later vcpu deadlocked forever " + "(reproduces the live xce_test Ctrl-D hang)"); + + if (jrc != 0) { + /* Don't let a hung teardown thread corrupt later iterations -- + * detach it (it will never finish) and stop looping; the mgr + * itself is intentionally leaked, since its vcpus may still be + * deadlocked mid-teardown. */ + pthread_detach(t); + break; + } + } + + if (have_saved_affinity) { + sched_setaffinity(0, sizeof(saved_affinity), &saved_affinity); + } +} + +void test_vtaskmgr_teardown(void) +{ + RUN_TEST(test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang); +} diff --git a/src/lib/vtaskmgr.c b/src/lib/vtaskmgr.c index 3eb708e3..de0b56cf 100644 --- a/src/lib/vtaskmgr.c +++ b/src/lib/vtaskmgr.c @@ -48,6 +48,24 @@ void vcpu_idle(vcpu_t* vcpu) vtaskmgr_t* vtaskmgr = (vtaskmgr_t*)vcpu->owner; vcpu->task = NULL; + + /* vcpu_fini()'s teardown protocol (bind_vtask(NULL, vcpu)) can queue a + * shutdown "poison pill" into this vcpu's OWN wbuf while it is busy + * running a real task -- the pill then sits there until the next + * vcpu_idle() call. Consume it directly (non-blocking) instead of + * first re-announcing this vcpu into the SHARED vtaskmgr->wbuf: by + * the time any vcpu_fini() runs, vtaskmgr_fini() has already + * cancelled+joined the dispatcher thread, the sole consumer of + * vtaskmgr->wbuf, so re-announcing here is pure wasted work at best + * (bug-1810) -- mylock.c's cbuf_put()/cbuf_get() now install a + * cancellation cleanup handler so a cancel landing there can no + * longer orphan the mutex, but skipping the needless re-announce + * entirely is still the correct, minimal behavior once a shutdown has + * already been signaled. */ + if (cbuf_tryget(&(vcpu->wbuf), (void*)&(vcpu->task)) == 0) { + return; + } + cbuf_put(&(vtaskmgr->wbuf), (void*)&(vcpu->cpuid)); cbuf_get(&(vcpu->wbuf), (void*)&(vcpu->task)); return; @@ -112,6 +130,20 @@ static void* vcpu_entry(void* arg) pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL); } else { vcpu_idle(my); + + /* vcpu_fini()'s bind_vtask(NULL, vcpu) is the ONLY caller + * that ever delivers a NULL task through this vcpu's own + * wbuf -- the dispatcher (vdispatcher_entry()) only calls + * bind_vtask() when pick_eligible_task() returned non-NULL. + * A NULL task observed here is therefore unambiguously the + * shutdown signal: exit immediately rather than looping back + * into vcpu_idle(), which would re-announce this vcpu into + * the shared vtaskmgr->wbuf after vtaskmgr_fini() has + * already torn down its sole consumer (the dispatcher + * thread) -- see vcpu_idle()'s own comment (bug-1810). */ + if (my->task == NULL) { + break; + } } pthread_testcancel(); }