fix(lib): stop cbuf_put/get orphaning their mutex on cancellation, hanging vtaskmgr_fini
pthread_cond_wait() reacquires its mutex before honoring a cancellation, but mylock.c installed no cleanup handler to release it again -- a thread cancelled mid-wait terminated holding the lock forever. vtaskmgr's pthread_cancel()-based teardown could land there from three angles (dispatcher cancelled while polling the shared queue, or a vcpu racing back into it after its own shutdown signal), permanently deadlocking any other thread that touched the same queue and hanging vtaskmgr_fini()'s pthread_join() indefinitely. Reproduced live as xce_test hanging on Ctrl-D exit once a connection context was entered; gdb confirmed a vcpu thread blocked forever on a mutex owned by an already-exited thread ID. Fix: cbuf_put()/cbuf_get()/cbuf_timedget() now unlock via a pthread_cleanup_push/pop handler around the wait, so a cancelled waiter always releases the mutex. vcpu_idle()/vcpu_entry() also recognize the teardown poison pill and exit immediately instead of needlessly re-announcing into the already-dead shared queue. New test_vtaskmgr_teardown.c reproduces the race deterministically (self-pinned to one CPU, since true multi-core parallelism rarely hits the window) and bounds vtaskmgr_fini() with a timed join so a regression fails fast instead of hanging the suite. Logged as bug-1810. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC
This commit is contained in:
parent
b7250c2ec6
commit
0f3919f195
9 changed files with 335 additions and 9 deletions
|
|
@ -1,13 +1,13 @@
|
|||
{
|
||||
"version": 1,
|
||||
"meta": {
|
||||
"lastScanned": "2026-09-15T02:34:37.112Z",
|
||||
"fileCount": 562,
|
||||
"lastScanned": "2026-09-15T03:56:28.973Z",
|
||||
"fileCount": 567,
|
||||
"hits": 0,
|
||||
"misses": 0,
|
||||
"renderedHash": "54b2e66594a769b4176755032afcc65e9009684d6c9b649e2e021d28c2b7f11f",
|
||||
"storeUpdatedAt": "2026-09-15T02:34:37.121Z",
|
||||
"rootHash": "cda4e7e066f4e166"
|
||||
"renderedHash": "a9ac7e8c7eca3b5e2b406896f2877eaecf4b21d63a5ffa783b18205d4002256c",
|
||||
"storeUpdatedAt": "2026-09-15T03:56:28.981Z",
|
||||
"rootHash": "7500ad02fc80875a"
|
||||
},
|
||||
"files": {
|
||||
".agent-shell/transcripts/2026-08-11-11-39-59.md": {
|
||||
|
|
@ -10264,6 +10264,51 @@
|
|||
"mtimeMs": 1789428659757.0122,
|
||||
"updatedAt": "2026-09-14T23:30:59.805Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/testsuite/test_vtaskmgr_teardown.c": {
|
||||
"description": "Declares int",
|
||||
"tokens": 1668,
|
||||
"hash": "190f0dd0bf0c65c9",
|
||||
"size": 6255,
|
||||
"mtimeMs": 1789442667259.7947,
|
||||
"updatedAt": "2026-09-15T03:24:27.304Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/testsuite/test.c": {
|
||||
"description": "include <argp.h>",
|
||||
"tokens": 1359,
|
||||
"hash": "329a9fe468acd8ad",
|
||||
"size": 5093,
|
||||
"mtimeMs": 1789441323455.6272,
|
||||
"updatedAt": "2026-09-15T03:02:03.502Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/meson.build": {
|
||||
"description": "",
|
||||
"tokens": 2428,
|
||||
"hash": "61df53cb05f969ce",
|
||||
"size": 9104,
|
||||
"mtimeMs": 1789441310019.5066,
|
||||
"updatedAt": "2026-09-15T03:01:50.064Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/vtaskmgr.c": {
|
||||
"description": "include <errno.h>",
|
||||
"tokens": 4446,
|
||||
"hash": "928364b0c99d4805",
|
||||
"size": 16685,
|
||||
"mtimeMs": 1789444588936.162,
|
||||
"updatedAt": "2026-09-15T03:56:28.973Z",
|
||||
"source": "hook"
|
||||
},
|
||||
"src/lib/mylock.c": {
|
||||
"description": "include <errno.h>",
|
||||
"tokens": 1632,
|
||||
"hash": "ddf11711b4ca77c6",
|
||||
"size": 6119,
|
||||
"mtimeMs": 1789443750926.8965,
|
||||
"updatedAt": "2026-09-15T03:42:30.970Z",
|
||||
"source": "hook"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
# anatomy.md
|
||||
|
||||
> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T02:34:37.112Z
|
||||
> Files: 562 tracked | Anatomy hits: 0 | Misses: 0
|
||||
> Auto-maintained by OpenWolf. Last scanned: 2026-09-15T03:56:28.973Z
|
||||
> Files: 567 tracked | Anatomy hits: 0 | Misses: 0
|
||||
|
||||
## ./
|
||||
|
||||
|
|
@ -891,13 +891,18 @@
|
|||
|
||||
## src/lib/
|
||||
|
||||
- `meson.build` (~2428 tok)
|
||||
- `misc.c` — include <errno.h> (~486 tok)
|
||||
- `mylock.c` — include <errno.h> (~1632 tok)
|
||||
- `resreq.c` — define _XOPEN_SOURCE 700 (~6663 tok)
|
||||
- `utils.c` — include <ctype.h> (~7015 tok)
|
||||
- `vtaskmgr.c` — include <errno.h> (~4446 tok)
|
||||
|
||||
## src/lib/testsuite/
|
||||
|
||||
- `test_vtaskmgr_teardown.c` — Declares int (~1668 tok)
|
||||
- `test_xcealloc.c` — include <stdio.h> (~4979 tok)
|
||||
- `test.c` — include <argp.h> (~1359 tok)
|
||||
|
||||
## src/scheduler/include/
|
||||
|
||||
|
|
|
|||
|
|
@ -21976,6 +21976,37 @@
|
|||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-09-14T16:20:00Z"
|
||||
},
|
||||
{
|
||||
"id": "bug-1810",
|
||||
"timestamp": "2026-09-15T03:55:00Z",
|
||||
"error_message": "User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live connection context (client/worker/scheduler). Live gdb attach on the user's machine showed the main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(vcpu->vcpu, ...); a sibling vcpu thread was simultaneously deadlocked in cbuf_put() at mylock.c:101's pthread_mutex_lock(&(b->mutex)) on vtaskmgr->wbuf.mutex, whose __owner TID had already exited.",
|
||||
"file": "src/lib/mylock.c (cbuf_put/cbuf_get/cbuf_timedget), src/lib/vtaskmgr.c (vcpu_idle, vcpu_entry)",
|
||||
"root_cause": "Two compounding defects in src/lib's shared task-pool primitives (used by every daemon + xce_test's own client/worker/scheduler REPL contexts, each of which owns a vtaskmgr_t for async I/O dispatch). (1) mylock.c's cbuf_put()/cbuf_get()/cbuf_timedget() lock b->mutex, then block in pthread_cond_wait()/pthread_cond_timedwait() with NO pthread_cleanup_push() handler installed. POSIX mandates pthread_cond_wait() reacquire its mutex before honoring a pending cancellation and running cleanup handlers -- with no handler registered, a thread cancelled while parked there terminates HOLDING the mutex, orphaning it locked forever; every other thread that later calls pthread_mutex_lock() on that same cbuf deadlocks uninterruptibly (plain mutex lock is not a cancellation point in glibc's NPTL). (2) vtaskmgr.c's teardown protocol races directly into that hazard from up to three independent angles, all torn down via pthread_cancel()+pthread_join(): (a) vtaskmgr_fini() cancels the dispatcher thread FIRST -- if it lands while the dispatcher is blocked in pick_idle_vcpu()'s cbuf_get() on the shared vtaskmgr->wbuf (empty, e.g. because real I/O work had every vcpu momentarily busy), that orphans wbuf's mutex immediately, before any vcpu teardown even starts; (b) vcpu_fini() then wakes each idle vcpu via bind_vtask(NULL, vcpu) (a 'poison pill' through the vcpu's own 1-capacity wbuf) and immediately pthread_cancel()s it, but vcpu_entry()'s loop could not distinguish that pill from 'no task yet' -- it looped straight back into vcpu_idle(), which unconditionally re-announced the vcpu into the shared (by then permanently undrained, since the dispatcher is already dead) vtaskmgr->wbuf, racing the pending cancellation against that second cbuf_put(); (c) the same poison pill arriving while a vcpu was still mid-execution of a real task queued it silently for the NEXT vcpu_idle() call, which still re-announced into the shared queue first before ever consuming it. Any of the three could orphan vtaskmgr->wbuf's mutex; every vcpu processed afterward that still touched the shared queue then deadlocked forever on pthread_mutex_lock(), and vtaskmgr_fini()'s pthread_join() for it never returned -- hanging the whole process (xce_test on Ctrl-D quit; identically reachable from any daemon that tears down a vtaskmgr_t under real concurrent I/O, e.g. controller/scheduler/worker shutdown).",
|
||||
"fix": "FIXED (2026-09-15), TDD RED->GREEN. Two-part fix, both required (confirmed empirically -- the vtaskmgr.c-only fix alone left the dispatcher-orphan path (2a above) unaddressed and still reproduced the hang under stress). (1) mylock.c: wrapped the pthread_cond_wait()/pthread_cond_timedwait() loops in cbuf_put()/cbuf_get()/cbuf_timedget() with pthread_cleanup_push(pthread_mutex_unlock, &b->mutex) / pthread_cleanup_pop(0) -- a cancelled waiter now always releases the mutex on unwind, so no cbuf mutex can ever be left orphaned-locked by a cancellation, regardless of which caller (dispatcher or vcpu) hits it. cbuf_timedget()'s early ETIMEDOUT return was restructured to break+flag instead of returning from inside the pushed region, since push/pop must be a matched pair at the same lexical nesting level. (2) vtaskmgr.c, defense-in-depth / closes the underlying logic bug independently of (1): vcpu_idle() now does a non-blocking cbuf_tryget() on its own wbuf BEFORE announcing itself into the shared vtaskmgr->wbuf, so an already-queued poison pill (queued while the vcpu was mid-task) is consumed directly instead of triggering a needless re-announce; vcpu_entry()'s loop now checks vcpu->task after vcpu_idle() returns and breaks (exits the thread) on NULL instead of looping back into idle -- a NULL task is unambiguously vcpu_fini()'s shutdown signal, since the dispatcher (vdispatcher_entry) never calls bind_vtask() with a NULL task. New regression test src/lib/testsuite/test_vtaskmgr_teardown.c (wired into src/lib/meson.build + testsuite/test.c): floods a burst of real VTASK_LIVENESS tasks across a 16-vcpu vtaskmgr then calls vtaskmgr_fini() on a helper thread with a pthread_timedjoin_np() 2s bounded-timeout join, self-pinning the test to a single CPU (sched_setaffinity, inherited by every spawned vtaskmgr thread at clone()) since the race needs preemptive time-slicing to reproduce reliably -- confirmed 0/hundreds of hangs on stock multi-core scheduling (true parallelism lets a vcpu almost always win the race to its own pthread_testcancel() before cancellation is even issued) vs. reliable reproduction (hung on iteration 0) once pinned to one core. Verified via: (a) this new test, RED before the fix (Expected 0 Was 110/ETIMEDOUT) and GREEN after; (b) a standalone single-core-pinned stress harness outside the Unity suite, 2000/2000 iterations clean post-fix (vs. hanging on iteration 0 with no fix and again on iteration 3 with only the vtaskmgr.c half of the fix, confirming both parts were needed); (c) full libxce test suite (314/314) and full harness suite (132/132, including the pty-based clitest tests that exercise real client-context connect+quit) both green; (d) a live pty-driven repro of the ORIGINAL user report -- launching the real xce_test binary, entering the client context, sending a real Ctrl-D (EOT) byte -- now exits in <0.1s (previously would hang indefinitely); (e) full project meson test matrix (98/98 suites) green on a clean run -- two earlier full-matrix runs showed unrelated, non-reproducing timeouts (one in harness's own quit-wait, one in controller_unit_test) that did not reproduce on isolated reruns, consistent with this sandbox's pre-existing parallel-test-suite CPU-contention flakiness (see bug-1429/bug-1580), not a regression from this fix.",
|
||||
"tags": [
|
||||
"libxce",
|
||||
"mylock",
|
||||
"vtaskmgr",
|
||||
"cbuf",
|
||||
"pthread_cancel",
|
||||
"pthread_cond_wait",
|
||||
"cleanup-handler",
|
||||
"orphaned-mutex",
|
||||
"deadlock",
|
||||
"hang",
|
||||
"xce_test",
|
||||
"harness",
|
||||
"ctrl-d",
|
||||
"eof",
|
||||
"quit",
|
||||
"teardown",
|
||||
"race-condition",
|
||||
"tdd-red-green",
|
||||
"live-gdb-repro"
|
||||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-09-15T03:55:00Z"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4721,3 +4721,26 @@
|
|||
| 16:30 | Edited src/lib/utils.c | modified hash_string() | ~24 |
|
||||
| 19:34 | Edited src/include/utils.h | expanded (+6 lines) | ~113 |
|
||||
| 19:34 | Edited src/scheduler/src/core_events.c | DEBUG() → DBG() | ~325 |
|
||||
| 19:46 | Session end: 8 writes across 3 files (utils.h, core_events.c, utils.c) | 2 reads | ~14326 tok |
|
||||
|
||||
## Session: 2026-09-15 19:46
|
||||
|
||||
| Time | Action | File(s) | Outcome | ~Tokens |
|
||||
|------|--------|---------|---------|--------|
|
||||
| 20:00 | Created src/lib/testsuite/test_vtaskmgr_teardown.c | — | ~1070 |
|
||||
| 20:01 | Edited src/lib/testsuite/test.c | 2→3 lines | ~32 |
|
||||
| 20:01 | Edited src/lib/meson.build | 2→3 lines | ~44 |
|
||||
| 20:02 | Edited src/lib/testsuite/test.c | 3→4 lines | ~33 |
|
||||
| 20:09 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | 2→2 lines | ~17 |
|
||||
| 20:12 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | modified struct() | ~489 |
|
||||
| 20:24 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | expanded (+10 lines) | ~296 |
|
||||
| 20:24 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | modified test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang() | ~214 |
|
||||
| 20:24 | Edited src/lib/testsuite/test_vtaskmgr_teardown.c | added 1 condition(s) | ~47 |
|
||||
| 20:28 | Edited src/lib/vtaskmgr.c | added 1 condition(s) | ~292 |
|
||||
| 20:28 | Edited src/lib/vtaskmgr.c | added 1 condition(s) | ~265 |
|
||||
| 20:37 | Edited src/lib/mylock.c | modified while() | ~255 |
|
||||
| 20:37 | Edited src/lib/mylock.c | modified while() | ~188 |
|
||||
| 20:38 | Edited src/lib/mylock.c | modified while() | ~185 |
|
||||
| 20:42 | Edited src/lib/mylock.c | added 1 condition(s) | ~303 |
|
||||
| 20:56 | Edited src/lib/vtaskmgr.c | 11→13 lines | ~236 |
|
||||
| 20:56 | Edited src/lib/vtaskmgr.c | 5→4 lines | ~74 |
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ libxce_test_src = ['testsuite/test.c',
|
|||
'testsuite/test_ctrlstatus.c',
|
||||
'testsuite/test_vtask_admission.c',
|
||||
'testsuite/test_vtask_enq_race.c',
|
||||
'testsuite/test_vtaskmgr_teardown.c',
|
||||
'testsuite/test_xcealloc.c',
|
||||
'raft/raft_sim.c',
|
||||
]
|
||||
|
|
|
|||
|
|
@ -99,9 +99,18 @@ int cbuf_put(cbuf_t* b, void* it)
|
|||
}
|
||||
|
||||
pthread_mutex_lock(&(b->mutex));
|
||||
/* pthread_cond_wait() reacquires the mutex before honoring a pending
|
||||
* cancellation (POSIX-mandated), so a caller cancelled while parked
|
||||
* here would otherwise terminate holding b->mutex forever -- no other
|
||||
* thread could ever lock it again. The cleanup handler releases it on
|
||||
* that unwind; on the normal (non-cancelled) path pop(0) just removes
|
||||
* the handler without running it, since the explicit unlock below
|
||||
* still runs. */
|
||||
pthread_cleanup_push((void (*)(void*))pthread_mutex_unlock, &(b->mutex));
|
||||
while (b->len == b->ncap) {
|
||||
pthread_cond_wait(&(b->condput), &(b->mutex));
|
||||
}
|
||||
pthread_cleanup_pop(0);
|
||||
|
||||
i = (b->head + b->len) % b->ncap;
|
||||
|
||||
|
|
@ -148,9 +157,14 @@ int cbuf_get(cbuf_t* b, void* it)
|
|||
}
|
||||
|
||||
pthread_mutex_lock(&(b->mutex));
|
||||
/* See cbuf_put()'s comment on the same pattern: without this cleanup
|
||||
* handler, a caller cancelled while parked in pthread_cond_wait()
|
||||
* would reacquire and then abandon b->mutex locked forever. */
|
||||
pthread_cleanup_push((void (*)(void*))pthread_mutex_unlock, &(b->mutex));
|
||||
while (b->len == 0) {
|
||||
pthread_cond_wait(&(b->condget), &(b->mutex));
|
||||
}
|
||||
pthread_cleanup_pop(0);
|
||||
|
||||
memcpy(it, (void*)((char*)b->data) + b->isize * b->head, b->isize);
|
||||
|
||||
|
|
@ -189,18 +203,33 @@ int cbuf_tryget(cbuf_t* b, void* it)
|
|||
|
||||
int cbuf_timedget(cbuf_t* b, void* it, const struct timespec* deadline)
|
||||
{
|
||||
int timed_out = 0;
|
||||
|
||||
if ((b == NULL) || (it == NULL) || (deadline == NULL)) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
pthread_mutex_lock(&(b->mutex));
|
||||
/* See cbuf_put()'s comment on the same pattern: without this cleanup
|
||||
* handler, a caller cancelled while parked in pthread_cond_timedwait()
|
||||
* would reacquire and then abandon b->mutex locked forever.
|
||||
* pthread_cleanup_push()/pop() must be a matched pair at the same
|
||||
* lexical nesting level -- break out of the loop on timeout instead
|
||||
* of returning from inside it, so pop() always runs right after. */
|
||||
pthread_cleanup_push((void (*)(void*))pthread_mutex_unlock, &(b->mutex));
|
||||
while (b->len == 0) {
|
||||
int rc = pthread_cond_timedwait(&(b->condget), &(b->mutex), deadline);
|
||||
if (rc == ETIMEDOUT) {
|
||||
pthread_mutex_unlock(&(b->mutex));
|
||||
return -ETIMEDOUT;
|
||||
timed_out = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
pthread_cleanup_pop(0);
|
||||
|
||||
if (timed_out) {
|
||||
pthread_mutex_unlock(&(b->mutex));
|
||||
return -ETIMEDOUT;
|
||||
}
|
||||
|
||||
memcpy(it, (void*)((char*)b->data) + b->isize * b->head, b->isize);
|
||||
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ extern void test_wal_wire(void);
|
|||
extern void test_ctrlstatus(void);
|
||||
extern void test_vtask_admission(void);
|
||||
extern void test_vtask_enq_race(void);
|
||||
extern void test_vtaskmgr_teardown(void);
|
||||
extern void test_xcealloc(void);
|
||||
|
||||
void setUp(void) { return; }
|
||||
|
|
@ -180,6 +181,7 @@ int main(int argc, char* argv[])
|
|||
TEST(test_ctrlstatus);
|
||||
TEST(test_vtask_admission);
|
||||
TEST(test_vtask_enq_race);
|
||||
TEST(test_vtaskmgr_teardown);
|
||||
TEST(test_xcealloc);
|
||||
|
||||
return UNITY_END();
|
||||
|
|
|
|||
158
src/lib/testsuite/test_vtaskmgr_teardown.c
Normal file
158
src/lib/testsuite/test_vtaskmgr_teardown.c
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
/*
|
||||
* test_vtaskmgr_teardown.c — vtaskmgr_fini() must never hang tearing down
|
||||
* idle vcpus.
|
||||
*
|
||||
* Root cause under test: vcpu_fini() (vtaskmgr.c) wakes a parked vcpu via
|
||||
* bind_vtask(NULL, vcpu) (a "poison pill" through the vcpu's own per-vcpu
|
||||
* cbuf) and then pthread_cancel()s + pthread_join()s it. vcpu_entry()'s loop
|
||||
* cannot distinguish that poison pill from "no task yet" -- it loops right
|
||||
* back into vcpu_idle(), which unconditionally re-announces the vcpu into
|
||||
* the SHARED vtaskmgr->wbuf cbuf (step A) before waiting again (step B).
|
||||
* By the time any vcpu is torn down, vtaskmgr_fini() has already
|
||||
* cancelled+joined the dispatcher thread -- the only consumer of
|
||||
* vtaskmgr->wbuf -- so that shared queue is never drained again. If
|
||||
* pthread_cancel() lands while the vcpu is blocked inside step A's
|
||||
* cbuf_put()'s pthread_cond_wait() on that now-permanently-unconsumed
|
||||
* queue, the thread is torn down without releasing vtaskmgr->wbuf's mutex
|
||||
* (mylock.c's cbuf_put/cbuf_get install no pthread_cleanup_push() handler),
|
||||
* orphaning it locked forever. Any OTHER still-alive vcpu that later loops
|
||||
* back into vcpu_idle() then deadlocks uninterruptibly on
|
||||
* pthread_mutex_lock() (not a cancellation point in glibc's NPTL), and
|
||||
* vtaskmgr_fini()'s pthread_join() for it never returns.
|
||||
*
|
||||
* Reproduced live: xce_test (harness REPL) hung on Ctrl-D (EOF) exit, its
|
||||
* main thread parked forever in leave_ctx() -> xcecli_ctx_fini() ->
|
||||
* xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(). A gdb
|
||||
* attach confirmed a sibling vcpu thread deadlocked in cbuf_put()'s
|
||||
* pthread_mutex_lock() against vtaskmgr->wbuf.mutex, owned by a thread ID
|
||||
* that had already exited.
|
||||
*
|
||||
* The race is timing-dependent: on a machine with more CPUs than
|
||||
* NVCPUS+2, the dispatcher/vcpu threads run in true parallel and a vcpu
|
||||
* almost always reaches its own pthread_testcancel() before
|
||||
* vtaskmgr_fini()'s pthread_cancel() is even issued, so the hazardous
|
||||
* second vcpu_idle() call is rarely reached -- confirmed empirically (0
|
||||
* hangs across hundreds of multi-core iterations). Pinning this test's
|
||||
* thread (and therefore, by Linux's affinity-inheritance-at-clone()
|
||||
* rule, every vtaskmgr thread it spawns) to a SINGLE CPU forces
|
||||
* preemptive time-slicing instead of true parallelism, which reproduces
|
||||
* the hang deterministically -- confirmed on iteration 0 of a
|
||||
* single-core standalone repro. The loop below still runs several
|
||||
* iterations, with a bounded-timeout join, as a safety margin -- it
|
||||
* FAILS (rather than hanging the whole suite) the first time any
|
||||
* iteration doesn't tear down within the deadline.
|
||||
*/
|
||||
#define _GNU_SOURCE
|
||||
#include <pthread.h>
|
||||
#include <sched.h>
|
||||
#include <stdlib.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <unity/unity.h>
|
||||
|
||||
#include "vtaskmgr.h"
|
||||
|
||||
typedef struct {
|
||||
vtaskmgr_t* mgr;
|
||||
} fini_arg_t;
|
||||
|
||||
static void* fini_thread(void* p)
|
||||
{
|
||||
fini_arg_t* a = (fini_arg_t*)p;
|
||||
|
||||
vtaskmgr_fini(a->mgr);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static int noop_entry(void* p)
|
||||
{
|
||||
(void)p;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static vtask_t* make_noop_task(void)
|
||||
{
|
||||
vtask_data_t* d = calloc(1, sizeof(*d));
|
||||
vtask_t* t = vtask_alloc();
|
||||
|
||||
d->vtype = VTASK_GENERIC;
|
||||
d->u.generic = NULL;
|
||||
vtask_init(t, noop_entry, d);
|
||||
return t;
|
||||
}
|
||||
|
||||
static void test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang(void)
|
||||
{
|
||||
const int ITERATIONS = 10;
|
||||
const int NVCPUS = 16;
|
||||
const int BURST = NVCPUS * 40;
|
||||
|
||||
cpu_set_t saved_affinity;
|
||||
cpu_set_t single_cpu;
|
||||
int have_saved_affinity
|
||||
= (sched_getaffinity(0, sizeof(saved_affinity), &saved_affinity) == 0);
|
||||
|
||||
CPU_ZERO(&single_cpu);
|
||||
CPU_SET(0, &single_cpu);
|
||||
/* Threads spawned after this point (vtaskmgr's dispatcher/vcpus, and
|
||||
* this test's own fini_thread) inherit the calling thread's affinity
|
||||
* mask at clone() time -- pinning here forces the preemptive
|
||||
* time-slicing that makes the race reproducible (see file header). */
|
||||
sched_setaffinity(0, sizeof(single_cpu), &single_cpu);
|
||||
|
||||
for (int iter = 0; iter < ITERATIONS; iter++) {
|
||||
vtaskmgr_t mgr;
|
||||
|
||||
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, NVCPUS));
|
||||
TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&mgr));
|
||||
|
||||
/* Flood a burst of trivial real tasks so the dispatcher is
|
||||
* ACTIVELY cycling through pick_idle_vcpu()/bind_vtask() -- not
|
||||
* settled in steady idle -- when fini() below tears it down. This
|
||||
* mirrors the live scenario (an xce_test client connection with
|
||||
* genuine in-flight I/O) rather than a purely idle REPL: the
|
||||
* dispatcher revisiting pick_idle_vcpu()'s blocking cbuf_get() on
|
||||
* vtaskmgr->wbuf is exactly the window where a cancel can orphan
|
||||
* that shared mutex before any vcpu teardown even begins. No
|
||||
* settle wait -- fini() is called immediately, mid-burst. */
|
||||
for (int k = 0; k < BURST; k++) {
|
||||
vtask_t* t = make_noop_task();
|
||||
vtask_enq_classified(&mgr, t, VTASK_LIVENESS);
|
||||
}
|
||||
|
||||
fini_arg_t farg = { .mgr = &mgr };
|
||||
pthread_t t;
|
||||
TEST_ASSERT_EQUAL_INT(0,
|
||||
pthread_create(&t, NULL, fini_thread, &farg));
|
||||
|
||||
struct timespec deadline;
|
||||
clock_gettime(CLOCK_REALTIME, &deadline);
|
||||
deadline.tv_sec += 2;
|
||||
|
||||
int jrc = pthread_timedjoin_np(t, NULL, &deadline);
|
||||
TEST_ASSERT_EQUAL_INT_MESSAGE(0, jrc,
|
||||
"vtaskmgr_fini() must not hang tearing down idle vcpus -- an "
|
||||
"orphaned vtaskmgr->wbuf mutex from a mid-cancellation "
|
||||
"cbuf_put() leaves a later vcpu deadlocked forever "
|
||||
"(reproduces the live xce_test Ctrl-D hang)");
|
||||
|
||||
if (jrc != 0) {
|
||||
/* Don't let a hung teardown thread corrupt later iterations --
|
||||
* detach it (it will never finish) and stop looping; the mgr
|
||||
* itself is intentionally leaked, since its vcpus may still be
|
||||
* deadlocked mid-teardown. */
|
||||
pthread_detach(t);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (have_saved_affinity) {
|
||||
sched_setaffinity(0, sizeof(saved_affinity), &saved_affinity);
|
||||
}
|
||||
}
|
||||
|
||||
void test_vtaskmgr_teardown(void)
|
||||
{
|
||||
RUN_TEST(test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang);
|
||||
}
|
||||
|
|
@ -48,6 +48,24 @@ void vcpu_idle(vcpu_t* vcpu)
|
|||
vtaskmgr_t* vtaskmgr = (vtaskmgr_t*)vcpu->owner;
|
||||
|
||||
vcpu->task = NULL;
|
||||
|
||||
/* vcpu_fini()'s teardown protocol (bind_vtask(NULL, vcpu)) can queue a
|
||||
* shutdown "poison pill" into this vcpu's OWN wbuf while it is busy
|
||||
* running a real task -- the pill then sits there until the next
|
||||
* vcpu_idle() call. Consume it directly (non-blocking) instead of
|
||||
* first re-announcing this vcpu into the SHARED vtaskmgr->wbuf: by
|
||||
* the time any vcpu_fini() runs, vtaskmgr_fini() has already
|
||||
* cancelled+joined the dispatcher thread, the sole consumer of
|
||||
* vtaskmgr->wbuf, so re-announcing here is pure wasted work at best
|
||||
* (bug-1810) -- mylock.c's cbuf_put()/cbuf_get() now install a
|
||||
* cancellation cleanup handler so a cancel landing there can no
|
||||
* longer orphan the mutex, but skipping the needless re-announce
|
||||
* entirely is still the correct, minimal behavior once a shutdown has
|
||||
* already been signaled. */
|
||||
if (cbuf_tryget(&(vcpu->wbuf), (void*)&(vcpu->task)) == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
cbuf_put(&(vtaskmgr->wbuf), (void*)&(vcpu->cpuid));
|
||||
cbuf_get(&(vcpu->wbuf), (void*)&(vcpu->task));
|
||||
return;
|
||||
|
|
@ -112,6 +130,20 @@ static void* vcpu_entry(void* arg)
|
|||
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, NULL);
|
||||
} else {
|
||||
vcpu_idle(my);
|
||||
|
||||
/* vcpu_fini()'s bind_vtask(NULL, vcpu) is the ONLY caller
|
||||
* that ever delivers a NULL task through this vcpu's own
|
||||
* wbuf -- the dispatcher (vdispatcher_entry()) only calls
|
||||
* bind_vtask() when pick_eligible_task() returned non-NULL.
|
||||
* A NULL task observed here is therefore unambiguously the
|
||||
* shutdown signal: exit immediately rather than looping back
|
||||
* into vcpu_idle(), which would re-announce this vcpu into
|
||||
* the shared vtaskmgr->wbuf after vtaskmgr_fini() has
|
||||
* already torn down its sole consumer (the dispatcher
|
||||
* thread) -- see vcpu_idle()'s own comment (bug-1810). */
|
||||
if (my->task == NULL) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
pthread_testcancel();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue