fix(lib): stop cbuf_put/get orphaning their mutex on cancellation, hanging vtaskmgr_fini

pthread_cond_wait() reacquires its mutex before honoring a cancellation,
but mylock.c installed no cleanup handler to release it again -- a
thread cancelled mid-wait terminated holding the lock forever. vtaskmgr's
pthread_cancel()-based teardown could land there from three angles
(dispatcher cancelled while polling the shared queue, or a vcpu racing
back into it after its own shutdown signal), permanently deadlocking any
other thread that touched the same queue and hanging vtaskmgr_fini()'s
pthread_join() indefinitely.

Reproduced live as xce_test hanging on Ctrl-D exit once a connection
context was entered; gdb confirmed a vcpu thread blocked forever on a
mutex owned by an already-exited thread ID.

Fix: cbuf_put()/cbuf_get()/cbuf_timedget() now unlock via a
pthread_cleanup_push/pop handler around the wait, so a cancelled waiter
always releases the mutex. vcpu_idle()/vcpu_entry() also recognize the
teardown poison pill and exit immediately instead of needlessly
re-announcing into the already-dead shared queue.

New test_vtaskmgr_teardown.c reproduces the race deterministically
(self-pinned to one CPU, since true multi-core parallelism rarely hits
the window) and bounds vtaskmgr_fini() with a timed join so a
regression fails fast instead of hanging the suite. Logged as bug-1810.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RLSNJhNoULvVcUhXEFafQC

src/lib/testsuite/test.c

@@ -55,6 +55,7 @@ extern void test_wal_wire(void);
 extern void test_ctrlstatus(void);
 extern void test_vtask_admission(void);
 extern void test_vtask_enq_race(void);
+extern void test_vtaskmgr_teardown(void);
 extern void test_xcealloc(void);
 
 void setUp(void) { return; }
@@ -180,6 +181,7 @@ int main(int argc, char* argv[])
     TEST(test_ctrlstatus);
     TEST(test_vtask_admission);
     TEST(test_vtask_enq_race);
+    TEST(test_vtaskmgr_teardown);
     TEST(test_xcealloc);
 
     return UNITY_END();

src/lib/testsuite/test_vtaskmgr_teardown.c

@@ -0,0 +1,158 @@
+/*
+ * test_vtaskmgr_teardown.c — vtaskmgr_fini() must never hang tearing down
+ * idle vcpus.
+ *
+ * Root cause under test: vcpu_fini() (vtaskmgr.c) wakes a parked vcpu via
+ * bind_vtask(NULL, vcpu) (a "poison pill" through the vcpu's own per-vcpu
+ * cbuf) and then pthread_cancel()s + pthread_join()s it. vcpu_entry()'s loop
+ * cannot distinguish that poison pill from "no task yet" -- it loops right
+ * back into vcpu_idle(), which unconditionally re-announces the vcpu into
+ * the SHARED vtaskmgr->wbuf cbuf (step A) before waiting again (step B).
+ * By the time any vcpu is torn down, vtaskmgr_fini() has already
+ * cancelled+joined the dispatcher thread -- the only consumer of
+ * vtaskmgr->wbuf -- so that shared queue is never drained again. If
+ * pthread_cancel() lands while the vcpu is blocked inside step A's
+ * cbuf_put()'s pthread_cond_wait() on that now-permanently-unconsumed
+ * queue, the thread is torn down without releasing vtaskmgr->wbuf's mutex
+ * (mylock.c's cbuf_put/cbuf_get install no pthread_cleanup_push() handler),
+ * orphaning it locked forever. Any OTHER still-alive vcpu that later loops
+ * back into vcpu_idle() then deadlocks uninterruptibly on
+ * pthread_mutex_lock() (not a cancellation point in glibc's NPTL), and
+ * vtaskmgr_fini()'s pthread_join() for it never returns.
+ *
+ * Reproduced live: xce_test (harness REPL) hung on Ctrl-D (EOF) exit, its
+ * main thread parked forever in leave_ctx() -> xcecli_ctx_fini() ->
+ * xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(). A gdb
+ * attach confirmed a sibling vcpu thread deadlocked in cbuf_put()'s
+ * pthread_mutex_lock() against vtaskmgr->wbuf.mutex, owned by a thread ID
+ * that had already exited.
+ *
+ * The race is timing-dependent: on a machine with more CPUs than
+ * NVCPUS+2, the dispatcher/vcpu threads run in true parallel and a vcpu
+ * almost always reaches its own pthread_testcancel() before
+ * vtaskmgr_fini()'s pthread_cancel() is even issued, so the hazardous
+ * second vcpu_idle() call is rarely reached -- confirmed empirically (0
+ * hangs across hundreds of multi-core iterations). Pinning this test's
+ * thread (and therefore, by Linux's affinity-inheritance-at-clone()
+ * rule, every vtaskmgr thread it spawns) to a SINGLE CPU forces
+ * preemptive time-slicing instead of true parallelism, which reproduces
+ * the hang deterministically -- confirmed on iteration 0 of a
+ * single-core standalone repro. The loop below still runs several
+ * iterations, with a bounded-timeout join, as a safety margin -- it
+ * FAILS (rather than hanging the whole suite) the first time any
+ * iteration doesn't tear down within the deadline.
+ */
+#define _GNU_SOURCE
+#include <pthread.h>
+#include <sched.h>
+#include <stdlib.h>
+#include <time.h>
+#include <unistd.h>
+
+#include <unity/unity.h>
+
+#include "vtaskmgr.h"
+
+typedef struct {
+    vtaskmgr_t* mgr;
+} fini_arg_t;
+
+static void* fini_thread(void* p)
+{
+    fini_arg_t* a = (fini_arg_t*)p;
+
+    vtaskmgr_fini(a->mgr);
+    return NULL;
+}
+
+static int noop_entry(void* p)
+{
+    (void)p;
+    return 0;
+}
+
+static vtask_t* make_noop_task(void)
+{
+    vtask_data_t* d = calloc(1, sizeof(*d));
+    vtask_t*      t = vtask_alloc();
+
+    d->vtype     = VTASK_GENERIC;
+    d->u.generic = NULL;
+    vtask_init(t, noop_entry, d);
+    return t;
+}
+
+static void test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang(void)
+{
+    const int ITERATIONS = 10;
+    const int NVCPUS     = 16;
+    const int BURST      = NVCPUS * 40;
+
+    cpu_set_t saved_affinity;
+    cpu_set_t single_cpu;
+    int       have_saved_affinity
+        = (sched_getaffinity(0, sizeof(saved_affinity), &saved_affinity) == 0);
+
+    CPU_ZERO(&single_cpu);
+    CPU_SET(0, &single_cpu);
+    /* Threads spawned after this point (vtaskmgr's dispatcher/vcpus, and
+     * this test's own fini_thread) inherit the calling thread's affinity
+     * mask at clone() time -- pinning here forces the preemptive
+     * time-slicing that makes the race reproducible (see file header). */
+    sched_setaffinity(0, sizeof(single_cpu), &single_cpu);
+
+    for (int iter = 0; iter < ITERATIONS; iter++) {
+        vtaskmgr_t mgr;
+
+        TEST_ASSERT_EQUAL_INT(0, vtaskmgr_init(&mgr, NVCPUS));
+        TEST_ASSERT_EQUAL_INT(0, vtaskmgr_start(&mgr));
+
+        /* Flood a burst of trivial real tasks so the dispatcher is
+         * ACTIVELY cycling through pick_idle_vcpu()/bind_vtask() -- not
+         * settled in steady idle -- when fini() below tears it down. This
+         * mirrors the live scenario (an xce_test client connection with
+         * genuine in-flight I/O) rather than a purely idle REPL: the
+         * dispatcher revisiting pick_idle_vcpu()'s blocking cbuf_get() on
+         * vtaskmgr->wbuf is exactly the window where a cancel can orphan
+         * that shared mutex before any vcpu teardown even begins. No
+         * settle wait -- fini() is called immediately, mid-burst. */
+        for (int k = 0; k < BURST; k++) {
+            vtask_t* t = make_noop_task();
+            vtask_enq_classified(&mgr, t, VTASK_LIVENESS);
+        }
+
+        fini_arg_t farg = { .mgr = &mgr };
+        pthread_t  t;
+        TEST_ASSERT_EQUAL_INT(0,
+            pthread_create(&t, NULL, fini_thread, &farg));
+
+        struct timespec deadline;
+        clock_gettime(CLOCK_REALTIME, &deadline);
+        deadline.tv_sec += 2;
+
+        int jrc = pthread_timedjoin_np(t, NULL, &deadline);
+        TEST_ASSERT_EQUAL_INT_MESSAGE(0, jrc,
+            "vtaskmgr_fini() must not hang tearing down idle vcpus -- an "
+            "orphaned vtaskmgr->wbuf mutex from a mid-cancellation "
+            "cbuf_put() leaves a later vcpu deadlocked forever "
+            "(reproduces the live xce_test Ctrl-D hang)");
+
+        if (jrc != 0) {
+            /* Don't let a hung teardown thread corrupt later iterations --
+             * detach it (it will never finish) and stop looping; the mgr
+             * itself is intentionally leaked, since its vcpus may still be
+             * deadlocked mid-teardown. */
+            pthread_detach(t);
+            break;
+        }
+    }
+
+    if (have_saved_affinity) {
+        sched_setaffinity(0, sizeof(saved_affinity), &saved_affinity);
+    }
+}
+
+void test_vtaskmgr_teardown(void)
+{
+    RUN_TEST(test_vtaskmgr_fini_idle_multi_vcpu_does_not_hang);
+}