/* * test_interruptcall.c -- proves xcetest_run_interruptible()'s (task 1.4, * Requirements 1.1, 1.2, 1.3, 1.4) heap-block CAS ownership handoff: a * normal/fast call returns its real result and frees cleanly, and a * forced-abandonment call returns -EINTR immediately with no double-free * and no leak -- the abandoned worker thread finishes in the background * and frees the block itself, proven via a side channel rather than * assumed from a fixed sleep. * * ALSO proves (task 1.5, Requirements 1.1, 1.5) the per-connection lock * registry added on top of that same call: two calls sharing a * connection identity (set via xcetest_set_active_connection()) never * run their fn bodies concurrently -- including a call issued right * after a prior one on that same connection was abandoned, which must * wait (interruptibly) for the abandoned worker to finish and release * the lock -- while two calls against DIFFERENT connection identities * run independently/concurrently. A third scenario proves that a call * interrupted WHILE waiting for a lock still held by a prior abandoned * call returns -EINTR without ever acquiring the lock or spawning a * worker for itself. * * xcetest_interrupted() (declared in xcetest_interrupt.h) is normally * defined in test.c, over test.c's own private break_out flag -- but * test.c itself is never linked into controller_unit_test (see * xcetest_interrupt.h's own header comment: test.c pulls in libedit/ * cJSON/libxce/Cap'n Proto/OpenSSL and defines its own main(), both of * which make it impractical to link into a small Unity binary). So this * file provides its OWN local fake definition of xcetest_interrupted(), * over its own local flag -- the same "local fake standing in for a * real dependency this binary never links" pattern already used * elsewhere in this directory (e.g. test_commit_barrier_reply.c's/ * test_worker_join.c's own fakes for on_scheduler_join()/notify()/ * collect_confvars(), per src/controller/meson.build's comment on * sched_push_unit_test_src). interruptcall.c only ever calls * xcetest_interrupted() through its documented pure-read contract, so a * fake reading a test-local flag exercises interruptcall.c's real CAS * state machine exactly as production would. * * The forced-abandonment test triggers a REAL, OS-delivered SIGINT (via * kill(getpid(), SIGINT), sent from a small helper thread once the * worker has had time to start) rather than flipping the fake flag * directly by hand, and installs its handler via xcetest_install_sigint() * (task 1.3) -- the exact installer test.c's own main() uses -- so this * test also doubles as a real, signal-level exercise of the same * interrupt path a live xce_test process would take, not a purely * hand-simulated one. This is choice (ii) from the task brief (a real * SIGINT), preferred over choice (i) (a direct break_out setter) because * no such setter exists on the REAL break_out anyway (xcetest_interrupt.h * deliberately exposes none) -- and a real signal exercises the actual * delivery path end-to-end, the same thing test_xcetest_interrupt.c * already does for xcetest_install_sigint() itself. */ #include #include #include #include #include #include #include #include "interruptcall.h" #include "xcetest_interrupt.h" /* --- local fake for xcetest_interrupted() (see file header) --- */ static volatile sig_atomic_t g_test_break_out = 0; int xcetest_interrupted(void) { return g_test_break_out; } static void test_sigint_handler(int signo) { (void)signo; g_test_break_out = 1; } /* ---- Scenario (a): normal, uninterrupted completion ---- */ #define NORMAL_FN_RC 7 static int fast_fn(void* ctx) { (void)ctx; return NORMAL_FN_RC; } static void test_normal_completion_returns_real_result(void) { static int conn_normal; /* private fake connection key for this test */ g_test_break_out = 0; xcetest_set_active_connection(&conn_normal); int rc = xcetest_run_interruptible(fast_fn, NULL); TEST_ASSERT_EQUAL_INT(NORMAL_FN_RC, rc); /* No side channel to poll here: on this path the CALLING thread's own * poll loop reads block->rc and frees the block synchronously, before * xcetest_run_interruptible() ever returns -- so by the time the * TEST_ASSERT_EQUAL_INT above runs, cleanup has unconditionally * already happened. The remaining proof is this whole suite running * clean under ASAN (no leak, no double-free) in the validating * build+test run. */ } /* ---- Scenario (b): forced abandonment via a real SIGINT ---- */ #define SLOW_FN_RC 99 #define SLOW_FN_SLEEP_MS 900 #define SIGNAL_DELAY_MS 150 #define WAIT_FOR_FREE_MS 3000 #define POLL_STEP_MS 20 static int slow_fn(void* ctx) { (void)ctx; struct timespec ts; ts.tv_sec = 0; ts.tv_nsec = (long)SLOW_FN_SLEEP_MS * 1000L * 1000L; nanosleep(&ts, NULL); return SLOW_FN_RC; } /* Runs on its own helper thread so it can deliver SIGINT to the process * WHILE the main test thread is blocked inside * xcetest_run_interruptible()'s poll loop below. */ static void* send_sigint_after_delay(void* arg) { (void)arg; struct timespec ts; ts.tv_sec = 0; ts.tv_nsec = (long)SIGNAL_DELAY_MS * 1000L * 1000L; nanosleep(&ts, NULL); kill(getpid(), SIGINT); return NULL; } /* Polls the XCE_TESTING-only xcetest_test_abandoned_free_count side * channel (NOT part of the production API) until it reaches `expected` * or a generous timeout elapses, then asserts it landed exactly on * `expected` -- so a test proves an abandoned worker's asynchronous * cleanup (free + lock release, see interruptcall.c's abandoned path) * actually completed, rather than merely hoping enough wall-clock time * passed before the process moves on. Shared by every scenario below * that produces an abandoned-but-still-running worker. */ static void wait_for_abandoned_free_count(int expected) { int waited_ms = 0; while (atomic_load(&xcetest_test_abandoned_free_count) < expected && waited_ms < WAIT_FOR_FREE_MS) { struct timespec poll; poll.tv_sec = 0; poll.tv_nsec = (long)POLL_STEP_MS * 1000L * 1000L; nanosleep(&poll, NULL); waited_ms += POLL_STEP_MS; } TEST_ASSERT_EQUAL_INT(expected, atomic_load(&xcetest_test_abandoned_free_count)); } static void test_forced_abandonment_returns_eintr_and_worker_cleans_up(void) { static int conn_abandon; /* private fake connection key for this test */ g_test_break_out = 0; atomic_store(&xcetest_test_abandoned_free_count, 0); xcetest_install_sigint(test_sigint_handler); xcetest_set_active_connection(&conn_abandon); pthread_t signaler; TEST_ASSERT_EQUAL_INT( 0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL)); /* SLOW_FN_SLEEP_MS (900ms) comfortably exceeds SIGNAL_DELAY_MS (150ms) * plus several 50ms poll intervals, so the interrupt is reliably * noticed well before slow_fn() itself would have returned -- this * call is expected to return -EINTR, not SLOW_FN_RC. */ int rc = xcetest_run_interruptible(slow_fn, NULL); TEST_ASSERT_EQUAL_INT(-EINTR, rc); TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL)); /* The abandoned worker is still running slow_fn() in the background * at this point; wait_for_abandoned_free_count() proves its * asynchronous cleanup actually happened -- confirming no leak -- * rather than merely hoping enough wall-clock time passed before the * process exits. */ wait_for_abandoned_free_count(1); } /* ---- Scenario (c): per-connection lock registry (task 1.5) ---- * * Requirements 1.1, 1.5. Three scenarios, per the task's Observable * Completion line: * (i) two overlapping calls sharing the SAME fake connection key * (one abandoned, one issued immediately after) never run their * fn bodies concurrently -- the second waits for the first's * still-running abandoned worker to release the lock; * (ii) two calls against DIFFERENT fake connection keys run * independently -- one's abandonment does not block the other; * (iii) a call interrupted WHILE waiting for a lock still held by a * prior abandoned call returns -EINTR without ever acquiring * the lock or spawning a worker for itself. * * All three reuse the same real-SIGINT-delivery pattern as scenario (b) * above (xcetest_install_sigint() + a helper thread's kill(getpid(), * SIGINT)) rather than flipping g_test_break_out by hand, for the same * reason given in this file's header comment. g_test_break_out is * intentionally reset to 0 between two calls in the same scenario * whenever the SECOND call is expected to complete normally (not be * interrupted itself) -- xcetest_interrupted() is a pure, non-clearing * read (see xcetest_interrupt.h's CONTRACT), so a stale "interrupted" * flag left set by an earlier call in the same test would otherwise be * observed by the second call's own lock-wait or completion-wait poll * and spuriously abandon it too. */ /* Shared probe: counts how many fn instances are concurrently "inside" * their body right now, and how many times an instance started while * that count was already nonzero. Used two ways below: scenario (i) * asserts overlap_count stays 0 (same key must serialize); scenario * (ii) asserts it becomes nonzero (different keys must NOT serialize). */ typedef struct { _Atomic int running; _Atomic int overlap_count; _Atomic int ran; } overlap_probe_t; static int probe_enter(overlap_probe_t* p) { int prev = atomic_fetch_add(&p->running, 1); if (prev != 0) atomic_fetch_add(&p->overlap_count, 1); atomic_store(&p->ran, 1); return prev; } static void probe_leave(overlap_probe_t* p) { atomic_fetch_sub(&p->running, 1); } static void sleep_ms(long ms) { struct timespec ts; ts.tv_sec = ms / 1000; ts.tv_nsec = (ms % 1000) * 1000L * 1000L; nanosleep(&ts, NULL); } static long elapsed_ms_since(const struct timespec* start) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); return (now.tv_sec - start->tv_sec) * 1000L + (now.tv_nsec - start->tv_nsec) / 1000000L; } /* -- (i) same connection key serializes -- */ #define SAME_KEY_SLOW_SLEEP_MS 900 #define SAME_KEY_FAST_SLEEP_MS 50 #define SAME_KEY_FAST_RC 42 #define SAME_KEY_MIN_WAIT_MS 300 /* well below the ~700ms this call * should actually wait for the * first call's abandoned worker, * comfortably above scheduling * noise if the lock were NOT * enforced (that failure mode * returns in tens of ms). */ static int same_key_slow_fn(void* arg) { overlap_probe_t* p = arg; probe_enter(p); sleep_ms(SAME_KEY_SLOW_SLEEP_MS); probe_leave(p); return SLOW_FN_RC; } static int same_key_fast_fn(void* arg) { overlap_probe_t* p = arg; probe_enter(p); sleep_ms(SAME_KEY_FAST_SLEEP_MS); probe_leave(p); return SAME_KEY_FAST_RC; } static void test_same_connection_key_serializes_overlapping_calls(void) { static int conn_a; /* address used purely as an opaque fake key */ overlap_probe_t probe = {0}; pthread_t signaler; struct timespec wait_start; g_test_break_out = 0; atomic_store(&xcetest_test_abandoned_free_count, 0); xcetest_install_sigint(test_sigint_handler); xcetest_set_active_connection(&conn_a); TEST_ASSERT_EQUAL_INT( 0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL)); int rc1 = xcetest_run_interruptible(same_key_slow_fn, &probe); TEST_ASSERT_EQUAL_INT(-EINTR, rc1); TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL)); TEST_ASSERT_TRUE_MESSAGE(atomic_load(&probe.ran), "the slow call must have started running " "before it was abandoned"); /* conn_a's lock is now held by the still-running abandoned worker. * Reset the fake interrupt flag -- this second call must NOT be * treated as interrupted itself; it is expected to wait for, then * acquire, the same lock once the first worker releases it. */ g_test_break_out = 0; clock_gettime(CLOCK_MONOTONIC, &wait_start); int rc2 = xcetest_run_interruptible(same_key_fast_fn, &probe); long waited_ms = elapsed_ms_since(&wait_start); TEST_ASSERT_EQUAL_INT(SAME_KEY_FAST_RC, rc2); TEST_ASSERT_EQUAL_INT(0, atomic_load(&probe.overlap_count)); TEST_ASSERT_TRUE_MESSAGE( waited_ms >= SAME_KEY_MIN_WAIT_MS, "second call on the same connection returned too quickly -- the " "per-connection lock did not make it wait for the first call's " "abandoned worker to finish and release it"); /* By the time rc2's call could acquire the lock, the abandoned * worker's cleanup (free + counter increment, THEN lock release -- * see interruptcall.c's abandoned path) must already have happened. */ TEST_ASSERT_EQUAL_INT(1, atomic_load(&xcetest_test_abandoned_free_count)); } /* -- (ii) different connection keys run independently -- */ #define DIFF_KEY_SLOW_SLEEP_MS 900 #define DIFF_KEY_FAST_SLEEP_MS 20 #define DIFF_KEY_FAST_RC 77 static int diff_key_slow_fn(void* arg) { overlap_probe_t* p = arg; probe_enter(p); sleep_ms(DIFF_KEY_SLOW_SLEEP_MS); probe_leave(p); return SLOW_FN_RC; } static int diff_key_fast_fn(void* arg) { overlap_probe_t* p = arg; probe_enter(p); sleep_ms(DIFF_KEY_FAST_SLEEP_MS); probe_leave(p); return DIFF_KEY_FAST_RC; } static void test_different_connection_keys_run_independently(void) { static int conn_b, conn_c; /* two distinct opaque fake keys */ overlap_probe_t probe = {0}; pthread_t signaler; g_test_break_out = 0; atomic_store(&xcetest_test_abandoned_free_count, 0); xcetest_install_sigint(test_sigint_handler); xcetest_set_active_connection(&conn_b); TEST_ASSERT_EQUAL_INT( 0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL)); int rc1 = xcetest_run_interruptible(diff_key_slow_fn, &probe); TEST_ASSERT_EQUAL_INT(-EINTR, rc1); TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL)); /* Reset the fake flag before the second call, same reasoning as * scenario (i) -- this call is on a DIFFERENT connection and must * complete normally. */ g_test_break_out = 0; xcetest_set_active_connection(&conn_c); int rc2 = xcetest_run_interruptible(diff_key_fast_fn, &probe); TEST_ASSERT_EQUAL_INT(DIFF_KEY_FAST_RC, rc2); TEST_ASSERT_TRUE_MESSAGE( atomic_load(&probe.overlap_count) >= 1, "different connection keys must not serialize against each " "other -- expected the second call to run concurrently with the " "first call's still-running abandoned worker"); wait_for_abandoned_free_count(1); } /* -- (iii) interrupted while waiting for a held lock: no worker spawned -- */ #define WAIT_KEY_SLOW_SLEEP_MS 900 static int wait_key_slow_fn(void* arg) { (void)arg; sleep_ms(WAIT_KEY_SLOW_SLEEP_MS); return SLOW_FN_RC; } static _Atomic int g_wait_key_fn_invoked = 0; static int wait_key_never_should_run_fn(void* arg) { (void)arg; atomic_store(&g_wait_key_fn_invoked, 1); return 0; } static void test_interrupted_while_waiting_for_lock_never_spawns_worker(void) { static int conn_d; pthread_t signaler1, signaler2; struct timespec t0; g_test_break_out = 0; atomic_store(&xcetest_test_abandoned_free_count, 0); atomic_store(&g_wait_key_fn_invoked, 0); xcetest_install_sigint(test_sigint_handler); xcetest_set_active_connection(&conn_d); TEST_ASSERT_EQUAL_INT( 0, pthread_create(&signaler1, NULL, send_sigint_after_delay, NULL)); int rc1 = xcetest_run_interruptible(wait_key_slow_fn, NULL); TEST_ASSERT_EQUAL_INT(-EINTR, rc1); TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler1, NULL)); /* conn_d's lock is now held by the still-running abandoned worker * (up to ~750ms left of its 900ms sleep). Reset the fake flag, then * arm a SECOND delayed SIGINT that fires while THIS next call is * still waiting to acquire that same lock -- well before the * abandoned worker could ever release it. */ g_test_break_out = 0; TEST_ASSERT_EQUAL_INT( 0, pthread_create(&signaler2, NULL, send_sigint_after_delay, NULL)); clock_gettime(CLOCK_MONOTONIC, &t0); int rc2 = xcetest_run_interruptible(wait_key_never_should_run_fn, NULL); long elapsed_ms = elapsed_ms_since(&t0); TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler2, NULL)); TEST_ASSERT_EQUAL_INT(-EINTR, rc2); TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load(&g_wait_key_fn_invoked), "a call interrupted while waiting for a held connection lock " "must never spawn a worker / invoke fn for itself"); TEST_ASSERT_TRUE_MESSAGE( elapsed_ms < (WAIT_KEY_SLOW_SLEEP_MS - SIGNAL_DELAY_MS), "the interrupted lock-wait should return promptly, well before " "the held lock would ever naturally free"); /* The first call's abandoned worker is still running in the * background; wait for it to finish so this test leaves no live * thread behind when it returns. */ wait_for_abandoned_free_count(1); } /* -- (iv) xcetest_wait_connection_idle() (task 3.2 remediation round 1, * bug-1795) -- deterministic proof of the exact mechanism leave_ctx() now * relies on to close the connection-teardown use-after-free. * * test_test_cli.c's black-box test_interrupt_then_immediate_teardown_no_uaf * exercises the SAME fix end-to-end through a real xce_test subprocess, but * the actual race window there (between an abandoned worker being woken by * xcecli_wakeup() and leave_ctx()'s teardown proceeding) is sub-millisecond * and timing-dependent -- it did not reliably reproduce as a failing test * even with the fix disabled in this environment (matching the original * round-1 review's own observation that a clean ASAN run is evidence the * race didn't fire on THAT run's timing, not evidence the gap doesn't * exist). This test instead proves xcetest_wait_connection_idle() itself, * in isolation, with a WIDE, deterministic timing margin (a 700ms abandoned * worker, not a sub-millisecond one) -- a direct, reliable RED (this * function returning early would fail it) / GREEN (the shipped * implementation passes it) demonstration of the invariant leave_ctx() * depends on. */ #define WCI_SLOW_SLEEP_MS 700 #define WCI_MIN_WAIT_MS 400 /* comfortably below the ~700ms - \ * SIGNAL_DELAY_MS remaining sleep once the \ * call is abandoned, comfortably above \ * scheduling noise if \ * xcetest_wait_connection_idle() did NOT \ * actually wait (that failure mode returns \ * in ~0ms). */ static int wci_slow_fn(void* arg) { (void)arg; sleep_ms(WCI_SLOW_SLEEP_MS); return SLOW_FN_RC; } static void test_wait_connection_idle_blocks_until_abandoned_worker_finishes( void) { static int conn_e; /* private fake connection key for this test */ pthread_t signaler; struct timespec wait_start; long waited_ms; int rc2; g_test_break_out = 0; atomic_store(&xcetest_test_abandoned_free_count, 0); xcetest_install_sigint(test_sigint_handler); xcetest_set_active_connection(&conn_e); TEST_ASSERT_EQUAL_INT( 0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL)); int rc = xcetest_run_interruptible(wci_slow_fn, NULL); TEST_ASSERT_EQUAL_INT(-EINTR, rc); TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL)); /* At this instant the abandoned worker is still running wci_slow_fn() * -- it has NOT yet freed its call block or released conn_e's lock * (interruptcall.c's abandoned path). This is exactly the production * race leave_ctx() closes: xcetest_wait_connection_idle() below must * genuinely block until that worker finishes, not return immediately. */ TEST_ASSERT_EQUAL_INT_MESSAGE( 0, atomic_load(&xcetest_test_abandoned_free_count), "the abandoned worker must still be running at this point for " "this test to actually exercise the wait -- if this fires, the " "worker finished implausibly fast and the timing margin above " "needs to grow, not shrink"); clock_gettime(CLOCK_MONOTONIC, &wait_start); xcetest_wait_connection_idle(&conn_e); waited_ms = elapsed_ms_since(&wait_start); TEST_ASSERT_TRUE_MESSAGE( waited_ms >= WCI_MIN_WAIT_MS, "xcetest_wait_connection_idle() returned too quickly -- it must " "genuinely block until the abandoned worker finishes, not return " "immediately (this is precisely the bug-1795 use-after-free gap: " "a caller proceeding to tear down the connection before this " "point would race the still-running worker)"); TEST_ASSERT_EQUAL_INT_MESSAGE( 1, atomic_load(&xcetest_test_abandoned_free_count), "the abandoned worker's cleanup must have genuinely completed by " "the time xcetest_wait_connection_idle() returns"); /* The semaphore must be left available (posted back), not held -- * proven by a normal subsequent call on the SAME connection key * completing without waiting at all. Reset the fake interrupt flag * first -- xcetest_interrupted() is a pure, non-clearing read (see * xcetest_interrupt.h's CONTRACT), so the flag the earlier SIGINT set * is still "on" here and would otherwise spuriously abandon this call * too, same reasoning as every other multi-call scenario in this * file. */ g_test_break_out = 0; clock_gettime(CLOCK_MONOTONIC, &wait_start); rc2 = xcetest_run_interruptible(fast_fn, NULL); waited_ms = elapsed_ms_since(&wait_start); TEST_ASSERT_EQUAL_INT(NORMAL_FN_RC, rc2); TEST_ASSERT_TRUE_MESSAGE( waited_ms < WCI_MIN_WAIT_MS, "xcetest_wait_connection_idle() must release the semaphore back " "immediately after proving idleness, not hold it -- a subsequent " "call on the same connection should not itself have to wait"); } void test_interruptcall(void) { UnitySetTestFile(__FILE__); RUN_TEST(test_normal_completion_returns_real_result); RUN_TEST(test_forced_abandonment_returns_eintr_and_worker_cleans_up); RUN_TEST(test_same_connection_key_serializes_overlapping_calls); RUN_TEST(test_different_connection_keys_run_independently); RUN_TEST(test_interrupted_while_waiting_for_lock_never_spawns_worker); RUN_TEST(test_wait_connection_idle_blocks_until_abandoned_worker_finishes); }