myxce/src/controller/testsuite/test_interruptcall.c

/*
 * test_interruptcall.c -- proves xcetest_run_interruptible()'s (task 1.4,
 * Requirements 1.1, 1.2, 1.3, 1.4) heap-block CAS ownership handoff: a
 * normal/fast call returns its real result and frees cleanly, and a
 * forced-abandonment call returns -EINTR immediately with no double-free
 * and no leak -- the abandoned worker thread finishes in the background
 * and frees the block itself, proven via a side channel rather than
 * assumed from a fixed sleep.
 *
 * ALSO proves (task 1.5, Requirements 1.1, 1.5) the per-connection lock
 * registry added on top of that same call: two calls sharing a
 * connection identity (set via xcetest_set_active_connection()) never
 * run their fn bodies concurrently -- including a call issued right
 * after a prior one on that same connection was abandoned, which must
 * wait (interruptibly) for the abandoned worker to finish and release
 * the lock -- while two calls against DIFFERENT connection identities
 * run independently/concurrently. A third scenario proves that a call
 * interrupted WHILE waiting for a lock still held by a prior abandoned
 * call returns -EINTR without ever acquiring the lock or spawning a
 * worker for itself.
 *
 * xcetest_interrupted() (declared in xcetest_interrupt.h) is normally
 * defined in test.c, over test.c's own private break_out flag -- but
 * test.c itself is never linked into controller_unit_test (see
 * xcetest_interrupt.h's own header comment: test.c pulls in libedit/
 * cJSON/libxce/Cap'n Proto/OpenSSL and defines its own main(), both of
 * which make it impractical to link into a small Unity binary). So this
 * file provides its OWN local fake definition of xcetest_interrupted(),
 * over its own local flag -- the same "local fake standing in for a
 * real dependency this binary never links" pattern already used
 * elsewhere in this directory (e.g. test_commit_barrier_reply.c's/
 * test_worker_join.c's own fakes for on_scheduler_join()/notify()/
 * collect_confvars(), per src/controller/meson.build's comment on
 * sched_push_unit_test_src). interruptcall.c only ever calls
 * xcetest_interrupted() through its documented pure-read contract, so a
 * fake reading a test-local flag exercises interruptcall.c's real CAS
 * state machine exactly as production would.
 *
 * The forced-abandonment test triggers a REAL, OS-delivered SIGINT (via
 * kill(getpid(), SIGINT), sent from a small helper thread once the
 * worker has had time to start) rather than flipping the fake flag
 * directly by hand, and installs its handler via xcetest_install_sigint()
 * (task 1.3) -- the exact installer test.c's own main() uses -- so this
 * test also doubles as a real, signal-level exercise of the same
 * interrupt path a live xce_test process would take, not a purely
 * hand-simulated one. This is choice (ii) from the task brief (a real
 * SIGINT), preferred over choice (i) (a direct break_out setter) because
 * no such setter exists on the REAL break_out anyway (xcetest_interrupt.h
 * deliberately exposes none) -- and a real signal exercises the actual
 * delivery path end-to-end, the same thing test_xcetest_interrupt.c
 * already does for xcetest_install_sigint() itself.
 */
#include <errno.h>
#include <pthread.h>
#include <signal.h>
#include <stdatomic.h>
#include <time.h>
#include <unistd.h>

#include <unity/unity.h>

#include "interruptcall.h"
#include "xcetest_interrupt.h"

/* --- local fake for xcetest_interrupted() (see file header) --- */
static volatile sig_atomic_t g_test_break_out = 0;

int xcetest_interrupted(void) { return g_test_break_out; }

static void test_sigint_handler(int signo)
{
    (void)signo;
    g_test_break_out = 1;
}

/* ---- Scenario (a): normal, uninterrupted completion ---- */

#define NORMAL_FN_RC 7

static int fast_fn(void* ctx)
{
    (void)ctx;
    return NORMAL_FN_RC;
}

static void test_normal_completion_returns_real_result(void)
{
    static int conn_normal; /* private fake connection key for this test */

    g_test_break_out = 0;
    xcetest_set_active_connection(&conn_normal);

    int rc = xcetest_run_interruptible(fast_fn, NULL);

    TEST_ASSERT_EQUAL_INT(NORMAL_FN_RC, rc);
    /* No side channel to poll here: on this path the CALLING thread's own
     * poll loop reads block->rc and frees the block synchronously, before
     * xcetest_run_interruptible() ever returns -- so by the time the
     * TEST_ASSERT_EQUAL_INT above runs, cleanup has unconditionally
     * already happened. The remaining proof is this whole suite running
     * clean under ASAN (no leak, no double-free) in the validating
     * build+test run. */
}

/* ---- Scenario (b): forced abandonment via a real SIGINT ---- */

#define SLOW_FN_RC       99
#define SLOW_FN_SLEEP_MS 900
#define SIGNAL_DELAY_MS  150
#define WAIT_FOR_FREE_MS 3000
#define POLL_STEP_MS     20

static int slow_fn(void* ctx)
{
    (void)ctx;
    struct timespec ts;
    ts.tv_sec  = 0;
    ts.tv_nsec = (long)SLOW_FN_SLEEP_MS * 1000L * 1000L;
    nanosleep(&ts, NULL);
    return SLOW_FN_RC;
}

/* Runs on its own helper thread so it can deliver SIGINT to the process
 * WHILE the main test thread is blocked inside
 * xcetest_run_interruptible()'s poll loop below. */
static void* send_sigint_after_delay(void* arg)
{
    (void)arg;
    struct timespec ts;
    ts.tv_sec  = 0;
    ts.tv_nsec = (long)SIGNAL_DELAY_MS * 1000L * 1000L;
    nanosleep(&ts, NULL);
    kill(getpid(), SIGINT);
    return NULL;
}

/* Polls the XCE_TESTING-only xcetest_test_abandoned_free_count side
 * channel (NOT part of the production API) until it reaches `expected`
 * or a generous timeout elapses, then asserts it landed exactly on
 * `expected` -- so a test proves an abandoned worker's asynchronous
 * cleanup (free + lock release, see interruptcall.c's abandoned path)
 * actually completed, rather than merely hoping enough wall-clock time
 * passed before the process moves on. Shared by every scenario below
 * that produces an abandoned-but-still-running worker. */
static void wait_for_abandoned_free_count(int expected)
{
    int waited_ms = 0;
    while (atomic_load(&xcetest_test_abandoned_free_count) < expected
           && waited_ms < WAIT_FOR_FREE_MS) {
        struct timespec poll;
        poll.tv_sec  = 0;
        poll.tv_nsec = (long)POLL_STEP_MS * 1000L * 1000L;
        nanosleep(&poll, NULL);
        waited_ms += POLL_STEP_MS;
    }
    TEST_ASSERT_EQUAL_INT(expected, atomic_load(&xcetest_test_abandoned_free_count));
}

static void test_forced_abandonment_returns_eintr_and_worker_cleans_up(void)
{
    static int conn_abandon; /* private fake connection key for this test */

    g_test_break_out = 0;
    atomic_store(&xcetest_test_abandoned_free_count, 0);
    xcetest_install_sigint(test_sigint_handler);
    xcetest_set_active_connection(&conn_abandon);

    pthread_t signaler;
    TEST_ASSERT_EQUAL_INT(
        0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL));

    /* SLOW_FN_SLEEP_MS (900ms) comfortably exceeds SIGNAL_DELAY_MS (150ms)
     * plus several 50ms poll intervals, so the interrupt is reliably
     * noticed well before slow_fn() itself would have returned -- this
     * call is expected to return -EINTR, not SLOW_FN_RC. */
    int rc = xcetest_run_interruptible(slow_fn, NULL);

    TEST_ASSERT_EQUAL_INT(-EINTR, rc);

    TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL));

    /* The abandoned worker is still running slow_fn() in the background
     * at this point; wait_for_abandoned_free_count() proves its
     * asynchronous cleanup actually happened -- confirming no leak --
     * rather than merely hoping enough wall-clock time passed before the
     * process exits. */
    wait_for_abandoned_free_count(1);
}

/* ---- Scenario (c): per-connection lock registry (task 1.5) ----
 *
 * Requirements 1.1, 1.5. Three scenarios, per the task's Observable
 * Completion line:
 *   (i)   two overlapping calls sharing the SAME fake connection key
 *         (one abandoned, one issued immediately after) never run their
 *         fn bodies concurrently -- the second waits for the first's
 *         still-running abandoned worker to release the lock;
 *   (ii)  two calls against DIFFERENT fake connection keys run
 *         independently -- one's abandonment does not block the other;
 *   (iii) a call interrupted WHILE waiting for a lock still held by a
 *         prior abandoned call returns -EINTR without ever acquiring
 *         the lock or spawning a worker for itself.
 *
 * All three reuse the same real-SIGINT-delivery pattern as scenario (b)
 * above (xcetest_install_sigint() + a helper thread's kill(getpid(),
 * SIGINT)) rather than flipping g_test_break_out by hand, for the same
 * reason given in this file's header comment. g_test_break_out is
 * intentionally reset to 0 between two calls in the same scenario
 * whenever the SECOND call is expected to complete normally (not be
 * interrupted itself) -- xcetest_interrupted() is a pure, non-clearing
 * read (see xcetest_interrupt.h's CONTRACT), so a stale "interrupted"
 * flag left set by an earlier call in the same test would otherwise be
 * observed by the second call's own lock-wait or completion-wait poll
 * and spuriously abandon it too.
 */

/* Shared probe: counts how many fn instances are concurrently "inside"
 * their body right now, and how many times an instance started while
 * that count was already nonzero. Used two ways below: scenario (i)
 * asserts overlap_count stays 0 (same key must serialize); scenario
 * (ii) asserts it becomes nonzero (different keys must NOT serialize). */
typedef struct {
    _Atomic int running;
    _Atomic int overlap_count;
    _Atomic int ran;
} overlap_probe_t;

static int probe_enter(overlap_probe_t* p)
{
    int prev = atomic_fetch_add(&p->running, 1);
    if (prev != 0)
        atomic_fetch_add(&p->overlap_count, 1);
    atomic_store(&p->ran, 1);
    return prev;
}

static void probe_leave(overlap_probe_t* p)
{
    atomic_fetch_sub(&p->running, 1);
}

static void sleep_ms(long ms)
{
    struct timespec ts;
    ts.tv_sec  = ms / 1000;
    ts.tv_nsec = (ms % 1000) * 1000L * 1000L;
    nanosleep(&ts, NULL);
}

static long elapsed_ms_since(const struct timespec* start)
{
    struct timespec now;
    clock_gettime(CLOCK_MONOTONIC, &now);
    return (now.tv_sec - start->tv_sec) * 1000L
           + (now.tv_nsec - start->tv_nsec) / 1000000L;
}

/* -- (i) same connection key serializes -- */

#define SAME_KEY_SLOW_SLEEP_MS 900
#define SAME_KEY_FAST_SLEEP_MS 50
#define SAME_KEY_FAST_RC       42
#define SAME_KEY_MIN_WAIT_MS   300 /* well below the ~700ms this call
                                     * should actually wait for the
                                     * first call's abandoned worker,
                                     * comfortably above scheduling
                                     * noise if the lock were NOT
                                     * enforced (that failure mode
                                     * returns in tens of ms). */

static int same_key_slow_fn(void* arg)
{
    overlap_probe_t* p = arg;
    probe_enter(p);
    sleep_ms(SAME_KEY_SLOW_SLEEP_MS);
    probe_leave(p);
    return SLOW_FN_RC;
}

static int same_key_fast_fn(void* arg)
{
    overlap_probe_t* p = arg;
    probe_enter(p);
    sleep_ms(SAME_KEY_FAST_SLEEP_MS);
    probe_leave(p);
    return SAME_KEY_FAST_RC;
}

static void test_same_connection_key_serializes_overlapping_calls(void)
{
    static int        conn_a; /* address used purely as an opaque fake key */
    overlap_probe_t   probe = {0};
    pthread_t         signaler;
    struct timespec   wait_start;

    g_test_break_out = 0;
    atomic_store(&xcetest_test_abandoned_free_count, 0);
    xcetest_install_sigint(test_sigint_handler);
    xcetest_set_active_connection(&conn_a);

    TEST_ASSERT_EQUAL_INT(
        0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL));

    int rc1 = xcetest_run_interruptible(same_key_slow_fn, &probe);

    TEST_ASSERT_EQUAL_INT(-EINTR, rc1);
    TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL));
    TEST_ASSERT_TRUE_MESSAGE(atomic_load(&probe.ran),
                              "the slow call must have started running "
                              "before it was abandoned");

    /* conn_a's lock is now held by the still-running abandoned worker.
     * Reset the fake interrupt flag -- this second call must NOT be
     * treated as interrupted itself; it is expected to wait for, then
     * acquire, the same lock once the first worker releases it. */
    g_test_break_out = 0;

    clock_gettime(CLOCK_MONOTONIC, &wait_start);
    int rc2 = xcetest_run_interruptible(same_key_fast_fn, &probe);
    long waited_ms = elapsed_ms_since(&wait_start);

    TEST_ASSERT_EQUAL_INT(SAME_KEY_FAST_RC, rc2);
    TEST_ASSERT_EQUAL_INT(0, atomic_load(&probe.overlap_count));
    TEST_ASSERT_TRUE_MESSAGE(
        waited_ms >= SAME_KEY_MIN_WAIT_MS,
        "second call on the same connection returned too quickly -- the "
        "per-connection lock did not make it wait for the first call's "
        "abandoned worker to finish and release it");

    /* By the time rc2's call could acquire the lock, the abandoned
     * worker's cleanup (free + counter increment, THEN lock release --
     * see interruptcall.c's abandoned path) must already have happened. */
    TEST_ASSERT_EQUAL_INT(1, atomic_load(&xcetest_test_abandoned_free_count));
}

/* -- (ii) different connection keys run independently -- */

#define DIFF_KEY_SLOW_SLEEP_MS 900
#define DIFF_KEY_FAST_SLEEP_MS 20
#define DIFF_KEY_FAST_RC       77

static int diff_key_slow_fn(void* arg)
{
    overlap_probe_t* p = arg;
    probe_enter(p);
    sleep_ms(DIFF_KEY_SLOW_SLEEP_MS);
    probe_leave(p);
    return SLOW_FN_RC;
}

static int diff_key_fast_fn(void* arg)
{
    overlap_probe_t* p = arg;
    probe_enter(p);
    sleep_ms(DIFF_KEY_FAST_SLEEP_MS);
    probe_leave(p);
    return DIFF_KEY_FAST_RC;
}

static void test_different_connection_keys_run_independently(void)
{
    static int       conn_b, conn_c; /* two distinct opaque fake keys */
    overlap_probe_t  probe = {0};
    pthread_t        signaler;

    g_test_break_out = 0;
    atomic_store(&xcetest_test_abandoned_free_count, 0);
    xcetest_install_sigint(test_sigint_handler);
    xcetest_set_active_connection(&conn_b);

    TEST_ASSERT_EQUAL_INT(
        0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL));

    int rc1 = xcetest_run_interruptible(diff_key_slow_fn, &probe);

    TEST_ASSERT_EQUAL_INT(-EINTR, rc1);
    TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL));

    /* Reset the fake flag before the second call, same reasoning as
     * scenario (i) -- this call is on a DIFFERENT connection and must
     * complete normally. */
    g_test_break_out = 0;
    xcetest_set_active_connection(&conn_c);

    int rc2 = xcetest_run_interruptible(diff_key_fast_fn, &probe);

    TEST_ASSERT_EQUAL_INT(DIFF_KEY_FAST_RC, rc2);
    TEST_ASSERT_TRUE_MESSAGE(
        atomic_load(&probe.overlap_count) >= 1,
        "different connection keys must not serialize against each "
        "other -- expected the second call to run concurrently with the "
        "first call's still-running abandoned worker");

    wait_for_abandoned_free_count(1);
}

/* -- (iii) interrupted while waiting for a held lock: no worker spawned -- */

#define WAIT_KEY_SLOW_SLEEP_MS 900

static int wait_key_slow_fn(void* arg)
{
    (void)arg;
    sleep_ms(WAIT_KEY_SLOW_SLEEP_MS);
    return SLOW_FN_RC;
}

static _Atomic int g_wait_key_fn_invoked = 0;

static int wait_key_never_should_run_fn(void* arg)
{
    (void)arg;
    atomic_store(&g_wait_key_fn_invoked, 1);
    return 0;
}

static void test_interrupted_while_waiting_for_lock_never_spawns_worker(void)
{
    static int      conn_d;
    pthread_t       signaler1, signaler2;
    struct timespec t0;

    g_test_break_out = 0;
    atomic_store(&xcetest_test_abandoned_free_count, 0);
    atomic_store(&g_wait_key_fn_invoked, 0);
    xcetest_install_sigint(test_sigint_handler);
    xcetest_set_active_connection(&conn_d);

    TEST_ASSERT_EQUAL_INT(
        0, pthread_create(&signaler1, NULL, send_sigint_after_delay, NULL));

    int rc1 = xcetest_run_interruptible(wait_key_slow_fn, NULL);

    TEST_ASSERT_EQUAL_INT(-EINTR, rc1);
    TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler1, NULL));

    /* conn_d's lock is now held by the still-running abandoned worker
     * (up to ~750ms left of its 900ms sleep). Reset the fake flag, then
     * arm a SECOND delayed SIGINT that fires while THIS next call is
     * still waiting to acquire that same lock -- well before the
     * abandoned worker could ever release it. */
    g_test_break_out = 0;
    TEST_ASSERT_EQUAL_INT(
        0, pthread_create(&signaler2, NULL, send_sigint_after_delay, NULL));

    clock_gettime(CLOCK_MONOTONIC, &t0);
    int rc2 = xcetest_run_interruptible(wait_key_never_should_run_fn, NULL);
    long elapsed_ms = elapsed_ms_since(&t0);

    TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler2, NULL));
    TEST_ASSERT_EQUAL_INT(-EINTR, rc2);
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        0, atomic_load(&g_wait_key_fn_invoked),
        "a call interrupted while waiting for a held connection lock "
        "must never spawn a worker / invoke fn for itself");
    TEST_ASSERT_TRUE_MESSAGE(
        elapsed_ms < (WAIT_KEY_SLOW_SLEEP_MS - SIGNAL_DELAY_MS),
        "the interrupted lock-wait should return promptly, well before "
        "the held lock would ever naturally free");

    /* The first call's abandoned worker is still running in the
     * background; wait for it to finish so this test leaves no live
     * thread behind when it returns. */
    wait_for_abandoned_free_count(1);
}

/* -- (iv) xcetest_wait_connection_idle() (task 3.2 remediation round 1,
 * bug-1795) -- deterministic proof of the exact mechanism leave_ctx() now
 * relies on to close the connection-teardown use-after-free.
 *
 * test_test_cli.c's black-box test_interrupt_then_immediate_teardown_no_uaf
 * exercises the SAME fix end-to-end through a real xce_test subprocess, but
 * the actual race window there (between an abandoned worker being woken by
 * xcecli_wakeup() and leave_ctx()'s teardown proceeding) is sub-millisecond
 * and timing-dependent -- it did not reliably reproduce as a failing test
 * even with the fix disabled in this environment (matching the original
 * round-1 review's own observation that a clean ASAN run is evidence the
 * race didn't fire on THAT run's timing, not evidence the gap doesn't
 * exist). This test instead proves xcetest_wait_connection_idle() itself,
 * in isolation, with a WIDE, deterministic timing margin (a 700ms abandoned
 * worker, not a sub-millisecond one) -- a direct, reliable RED (this
 * function returning early would fail it) / GREEN (the shipped
 * implementation passes it) demonstration of the invariant leave_ctx()
 * depends on. */

#define WCI_SLOW_SLEEP_MS 700
#define WCI_MIN_WAIT_MS   400 /* comfortably below the ~700ms - \
                                * SIGNAL_DELAY_MS remaining sleep once the \
                                * call is abandoned, comfortably above \
                                * scheduling noise if \
                                * xcetest_wait_connection_idle() did NOT \
                                * actually wait (that failure mode returns \
                                * in ~0ms). */

static int wci_slow_fn(void* arg)
{
    (void)arg;
    sleep_ms(WCI_SLOW_SLEEP_MS);
    return SLOW_FN_RC;
}

static void test_wait_connection_idle_blocks_until_abandoned_worker_finishes(
    void)
{
    static int      conn_e; /* private fake connection key for this test */
    pthread_t       signaler;
    struct timespec wait_start;
    long            waited_ms;
    int             rc2;

    g_test_break_out = 0;
    atomic_store(&xcetest_test_abandoned_free_count, 0);
    xcetest_install_sigint(test_sigint_handler);
    xcetest_set_active_connection(&conn_e);

    TEST_ASSERT_EQUAL_INT(
        0, pthread_create(&signaler, NULL, send_sigint_after_delay, NULL));

    int rc = xcetest_run_interruptible(wci_slow_fn, NULL);

    TEST_ASSERT_EQUAL_INT(-EINTR, rc);
    TEST_ASSERT_EQUAL_INT(0, pthread_join(signaler, NULL));

    /* At this instant the abandoned worker is still running wci_slow_fn()
     * -- it has NOT yet freed its call block or released conn_e's lock
     * (interruptcall.c's abandoned path). This is exactly the production
     * race leave_ctx() closes: xcetest_wait_connection_idle() below must
     * genuinely block until that worker finishes, not return immediately. */
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        0, atomic_load(&xcetest_test_abandoned_free_count),
        "the abandoned worker must still be running at this point for "
        "this test to actually exercise the wait -- if this fires, the "
        "worker finished implausibly fast and the timing margin above "
        "needs to grow, not shrink");

    clock_gettime(CLOCK_MONOTONIC, &wait_start);
    xcetest_wait_connection_idle(&conn_e);
    waited_ms = elapsed_ms_since(&wait_start);

    TEST_ASSERT_TRUE_MESSAGE(
        waited_ms >= WCI_MIN_WAIT_MS,
        "xcetest_wait_connection_idle() returned too quickly -- it must "
        "genuinely block until the abandoned worker finishes, not return "
        "immediately (this is precisely the bug-1795 use-after-free gap: "
        "a caller proceeding to tear down the connection before this "
        "point would race the still-running worker)");
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        1, atomic_load(&xcetest_test_abandoned_free_count),
        "the abandoned worker's cleanup must have genuinely completed by "
        "the time xcetest_wait_connection_idle() returns");

    /* The semaphore must be left available (posted back), not held --
     * proven by a normal subsequent call on the SAME connection key
     * completing without waiting at all. Reset the fake interrupt flag
     * first -- xcetest_interrupted() is a pure, non-clearing read (see
     * xcetest_interrupt.h's CONTRACT), so the flag the earlier SIGINT set
     * is still "on" here and would otherwise spuriously abandon this call
     * too, same reasoning as every other multi-call scenario in this
     * file. */
    g_test_break_out = 0;
    clock_gettime(CLOCK_MONOTONIC, &wait_start);
    rc2       = xcetest_run_interruptible(fast_fn, NULL);
    waited_ms = elapsed_ms_since(&wait_start);

    TEST_ASSERT_EQUAL_INT(NORMAL_FN_RC, rc2);
    TEST_ASSERT_TRUE_MESSAGE(
        waited_ms < WCI_MIN_WAIT_MS,
        "xcetest_wait_connection_idle() must release the semaphore back "
        "immediately after proving idleness, not hold it -- a subsequent "
        "call on the same connection should not itself have to wait");
}

void test_interruptcall(void)
{
    UnitySetTestFile(__FILE__);
    RUN_TEST(test_normal_completion_returns_real_result);
    RUN_TEST(test_forced_abandonment_returns_eintr_and_worker_cleans_up);
    RUN_TEST(test_same_connection_key_serializes_overlapping_calls);
    RUN_TEST(test_different_connection_keys_run_independently);
    RUN_TEST(test_interrupted_while_waiting_for_lock_never_spawns_worker);
    RUN_TEST(test_wait_connection_idle_blocks_until_abandoned_worker_finishes);
}