myxce/src/controller/testsuite/test_fetchfile_disconnect.c

/*
 * test_fetchfile_disconnect.c — bug-1693 regression: a worker disconnect
 * mid-wait wakes on_fetchfile()'s pending waiter immediately (Decision Log
 * fidelity fix).
 *
 * Background: cluster.c's OPCode_schedAdmin dispatch case documents itself
 * as mirroring on_fetchfile's proxy pattern (job.c) -- register a waiter,
 * forward async, block on a condvar with a deadline. An invariant audit
 * found BOTH sites violated the same recorded Decision Log entry: "No
 * client reply-wait timeout as a fix for a hung request -- fail loud at the
 * real transport/source instead of masking with a deadline backstop."
 * Before this fix, handle_worker_leave() (worker.c) only ever nulled w->ep
 * on a worker disconnect -- it never woke a pending on_fetchfile() waiter
 * proxied to that worker, so a disconnect mid-wait could only ever be
 * observed at on_fetchfile's own 10s deadline.
 *
 * The fix: fetchfile_signal_worker_disconnect(worker_t*) (job.c) walks
 * every registered fetchfile_pending_t and wakes each one whose ->worker
 * matches the disconnected worker with a distinguishable -ECONNRESET
 * status (not the deadline's -ETIMEDOUT). handle_worker_leave() now calls
 * it immediately, right after nulling w->ep.
 *
 * Test strategy for THIS file: on_fetchfile() itself needs a real,
 * RTTI-registered job_t/worker_t pair reachable through c->jobmgr/
 * c->workermgr (key_find + lock()/reference() -- see on_fetchfile's own
 * body, job.c) to run end-to-end; building that fixture is squarely a
 * job/worker-lookup concern this fix's scope explicitly does not touch
 * (only the disconnect-to-waiter SIGNAL path is in scope). So this test
 * exercises the real fetchfile_signal_worker_disconnect() function against
 * a real fetchfile_pending_t wait, using job.c's XCE_TESTING-only seam
 * fetchfile_wait_for_test() (mirrors on_fetchfile's own register/wait/
 * unregister sequence exactly, including a real pthread_cond_timedwait
 * deadline) instead of the full on_fetchfile()/handle_worker_leave() call
 * chain -- a "direct unit test of the disconnect-handler's new signal call
 * reaching the right wait context", per this task's own stated minimum bar
 * for whichever of the two sites doesn't get the full live two-thread
 * reproduction (that's test_sched_admin_dispatch.c's
 * test_schedadmin_disconnect_wakes_immediately, which does drive the real
 * handle_scheduler_leave() end-to-end against a real dispatch wait).
 * worker_t itself is a plain, stack-owned struct here (not routed through
 * makeWorker()/wobj.c's RTTI machinery) -- unneeded, since neither
 * fetchfile_signal_worker_disconnect() nor fetchfile_wait_for_test() ever
 * locks/references it; only pointer identity (matching ->worker) matters,
 * mirroring test_sched_worker_change.c's identical "plain struct, not
 * through the real object constructor" rationale.
 */
#define _GNU_SOURCE
#include <errno.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include <unity/unity.h>

#include "cluster.h"
#include "xce/cluster_def.h"

/* Test-only seam (job.c, XCE_TESTING-only) -- see this file's header
 * comment above for why it's used instead of a full on_fetchfile()
 * fixture. */
extern int fetchfile_wait_for_test(worker_t* w, long timeout_ms,
                                   int* out_status);

/* Monotonic milliseconds -- same construction as test_sched_admin_
 * dispatch.c's sa_mono_ms() / test_commit_resolve.c's mono_ms_r(). */
static uint64_t ff_mono_ms(void)
{
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
}

typedef struct {
    worker_t* w;
} ff_disconnect_arg_t;

static void* ff_deliver_disconnect(void* argp)
{
    ff_disconnect_arg_t* arg = (ff_disconnect_arg_t*)argp;

    /* Give the main thread time to register the pending waiter and start
     * blocking on the condvar first -- comfortably inside the backstop
     * deadline below, same margin convention as test_sched_admin_
     * dispatch.c's sa_deliver_scheduler_reply. */
    usleep(200000);

    fetchfile_signal_worker_disconnect(arg->w);
    return NULL;
}

/* ── Regression: disconnect wakes the waiter immediately, not at the
 * deadline ──────────────────────────────────────────────────────────────── */
static void test_fetchfile_worker_disconnect_wakes_immediately(void)
{
    worker_t w;
    memset(&w, 0, sizeof(w));

    ff_disconnect_arg_t arg = { .w = &w };
    pthread_t            tid;
    TEST_ASSERT_EQUAL_INT(0,
                          pthread_create(&tid, NULL, ff_deliver_disconnect, &arg));

    /* Backstop deadline: much larger than the ~200ms disconnect delay above,
     * so a genuine wake is trivially distinguishable from a regression that
     * falls through to the deadline (asserted on below via elapsed time) --
     * but far shorter than production's real 10s on_fetchfile deadline, so
     * a regression fails this test in bounded time rather than hanging for
     * 10s. */
    const long timeout_ms = 5000;
    int        status     = 12345; /* poison -- must be overwritten */

    uint64_t t0   = ff_mono_ms();
    int      woke = fetchfile_wait_for_test(&w, timeout_ms, &status);
    uint64_t took = ff_mono_ms() - t0;

    pthread_join(tid, NULL);

    TEST_ASSERT_TRUE_MESSAGE(
        woke,
        "must be woken by the disconnect signal, not merely reach the "
        "backstop deadline with pending.done still 0");
    TEST_ASSERT_EQUAL_INT_MESSAGE(
        -ECONNRESET, status,
        "a worker disconnect must be relayed with a distinguishable "
        "-ECONNRESET status, never the deadline's -ETIMEDOUT");
    TEST_ASSERT_TRUE_MESSAGE(
        took < 1000,
        "a worker disconnect mid-wait must wake the waiter immediately "
        "(well under the 5s backstop used by this test, and far under "
        "production's real 10s on_fetchfile deadline) -- this is the "
        "regression this test locks: pre-fix, handle_worker_leave() never "
        "signalled fetchfile waiters at all, so this would only ever "
        "complete at the deadline with status -ETIMEDOUT");

    printf("test_fetchfile_worker_disconnect_wakes_immediately: PASS\n");
}

/* ── A disconnect on an UNRELATED worker must not wake this waiter ────────
 *
 * Proves fetchfile_signal_worker_disconnect() actually matches by worker
 * identity (the ->worker field on fetchfile_pending_t) rather than waking
 * every registered waiter indiscriminately -- multiple fetchfile requests
 * against different workers must not cross-signal each other. */
static void test_fetchfile_worker_disconnect_does_not_wake_other_worker(void)
{
    worker_t w_target, w_other;
    memset(&w_target, 0, sizeof(w_target));
    memset(&w_other, 0, sizeof(w_other));

    ff_disconnect_arg_t arg = { .w = &w_other };
    pthread_t            tid;
    TEST_ASSERT_EQUAL_INT(0,
                          pthread_create(&tid, NULL, ff_deliver_disconnect, &arg));

    /* Short backstop -- this test's whole point is that NOTHING wakes it
     * early, so the wait genuinely runs out; keep it small to keep the test
     * fast rather than proving a large deadline elapses. */
    const long timeout_ms = 400;
    int        status     = 12345;

    int woke = fetchfile_wait_for_test(&w_target, timeout_ms, &status);

    pthread_join(tid, NULL);

    TEST_ASSERT_FALSE_MESSAGE(
        woke,
        "a disconnect on a DIFFERENT worker must not wake this waiter -- "
        "fetchfile_signal_worker_disconnect() must match by worker "
        "identity, not signal indiscriminately");
    TEST_ASSERT_EQUAL_INT_MESSAGE(-ETIMEDOUT, status,
                                  "an unmatched waiter must fall through to "
                                  "the ordinary deadline outcome");

    printf(
        "test_fetchfile_worker_disconnect_does_not_wake_other_worker: PASS\n");
}

/* ── Suite entry point ──────────────────────────────────────────────────── */
void test_fetchfile_disconnect(void)
{
    UnitySetTestFile(__FILE__);
    RUN_TEST(test_fetchfile_worker_disconnect_wakes_immediately);
    RUN_TEST(test_fetchfile_worker_disconnect_does_not_wake_other_worker);
}