myxce/src/scheduler/testsuite/test_builtin_fcfs_cursor_uaf.c

/*
 * test_builtin_fcfs_cursor_uaf.c - white-box regression test for bug-1686: a
 * live SIGSEGV in fcfs_cursor_enter_queue() (../src/plugins/builtin_fcfs.c),
 * reached from builtin_fcfs_next_job() <- sched_job_picker() (scheduler.c)
 * <- picker thread, captured from a real 3-node HA deployment.
 *
 * Root cause: fcfs_plugin_data_t.cursor/job_cursor are raw ordq_item_t*
 * values that PERSIST in fcfs_plugin_data_t across SEPARATE
 * builtin_fcfs_next_job() calls, each with its own independent
 * fcfs_plugin_data_t.lock acquisition/release (see that field's own comment
 * and the file's header "Concurrency" comment). fcfs_plugin_data_t.lock
 * correctly makes each individual hook call atomic with respect to the
 * others, but it cannot protect a pointer VALUE that outlives the call that
 * read it: a properly-locked builtin_fcfs_on_queue_update() call for a
 * queue that already has a registered fcfs_queue_t re-sorts it by removing
 * the queue's existing ordq_item_t node (ordq_get + FREEUP(it)) and
 * inserting a brand-new one (ordq_put, fresh calloc). If priv->cursor is
 * resting on that exact node -- which it normally is, since the cursor
 * stays parked on a queue for as long as it keeps yielding ready jobs
 * (sched_queue_is_ready()/sched_job_is_ready() are unconditional `return 1`
 * stubs today in scheduler.c, so a queue's cursor never advances away from
 * it while jobs remain) -- the NEXT next_job() call dereferences
 * priv->cursor->data from freed memory: a real heap-use-after-free, live-
 * reproduced as the SIGSEGV this bug report captured (queue_item was a
 * dangling, freed ordq_item_t*).
 *
 * Critically, this is deterministically reproducible SINGLE-THREADED: since
 * next_job() releases priv->lock before returning, calling next_job(), then
 * on_queue_update() for the SAME queue name, then next_job() again -- all
 * sequentially on one thread, exactly as this test does -- hits the same
 * use-after-free with no real concurrency required. ASAN catches it as a
 * genuine heap-use-after-free even single-threaded.
 *
 * Fix (already applied, three sites, see builtin_fcfs.c's own file header
 * comment and each site's comment for the full mechanism):
 *   1. builtin_fcfs_on_queue_update()'s reuse branch: before FREEUP(it), if
 *      priv->cursor == it, NULLs both priv->cursor and priv->job_cursor.
 *   2. builtin_fcfs_on_job_state_change()'s !JOB_IS_PEND removal branch:
 *      ordq_peek()s the about-to-be-deleted job node first; if it equals
 *      priv->job_cursor, NULLs priv->job_cursor before ordq_del().
 *   3. fcfs_remove_job_everywhere(): same peek-then-invalidate-then-delete
 *      pattern inside its loop over all queues.
 * Always safe: next_job() already treats a NULL cursor as "no more
 * candidates this round" (a pre-existing, unrelated-to-this-bug branch --
 * see test_picker_fcfs_session.c's own no-queues/empty-queue coverage), and
 * begin_session() unconditionally resets both cursors fresh every round, so
 * the worst case of an invalidation is a bounded one-round delay for the
 * jobs on the invalidated queue, never a crash or a permanently lost job.
 * This test exercises exactly site 1 (on_queue_update's reuse-and-resort
 * path), the one the bug report's own crash actually walked into.
 *
 *   RED (fix sites reverted): builtin_fcfs_next_job()'s second call
 *   dereferences the freed ordq_item_t node priv->cursor still points at --
 *   a genuine ASAN heap-use-after-free abort, not a clean TEST_ASSERT
 *   failure, matching bug-1686's captured live SIGSEGV signature
 *   (fcfs_cursor_enter_queue <- builtin_fcfs_next_job, dereferencing a
 *   freed/dangling ordq_item_t*).
 *   GREEN (fix applied): priv->cursor/job_cursor are observably NULLed by
 *   on_queue_update()'s reuse branch before the freed node's lifetime ends;
 *   the next next_job() call returns NULL cleanly (no crash); and a
 *   subsequent begin_session()/next_job() round finds every job again,
 *   proving no job is permanently lost -- only delayed one round.
 *
 * #include's the real ../src/plugins/builtin_fcfs.c (white-box, same
 * convention as test_picker_fcfs.c/test_picker_fcfs_session.c) so this test
 * exercises the real cursor logic and the real private fcfs_plugin_data_t
 * layout -- including direct field access to priv->cursor/priv->job_cursor,
 * which only a white-box #include can give a test -- not a re-implementation
 * of any of it.
 *
 * builtin_fcfs.c's next_job() calls sched_queue_is_ready()/
 * sched_job_is_ready() (declared in scheduler.h, defined in scheduler.c).
 * This translation unit does not link scheduler.c (same as
 * test_picker_fcfs_session.c), so it supplies its own always-ready test
 * doubles below -- this test never needs to force a skip, unlike that
 * file's coverage of the skip branches.
 */
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <unity/unity.h>

#include "plugin.h"
#include "sched/types.h"

/* Always-ready test doubles -- see file header. Neither queue nor job
 * readiness is under test here, only cursor lifetime across separate
 * next_job()/on_queue_update() calls. */
int sched_queue_is_ready(xce_sched_queue_t* q)
{
    (void)q;
    return 1;
}

int sched_job_is_ready(xce_sched_job_t* j)
{
    (void)j;
    return 1;
}

/* register_builtin_fcfs() (bottom of builtin_fcfs.c) calls plugin_register()
 * (plugin.c, not linked into this white-box TU). Stub to satisfy the
 * linker -- this file's test drives the hooks directly, never
 * register_builtin_fcfs() itself. Same convention as
 * test_picker_fcfs_session.c's own stub. */
int plugin_register(void* arg1, void* arg2)
{
    (void)arg1;
    (void)arg2;
    return 0;
}

#include "../src/plugins/builtin_fcfs.c"

void setUp(void) { return; }
void tearDown(void) { return; }

/* ---------------------------------------------------------------------- */
/* Fixture -- same convention as test_picker_fcfs_session.c's fixture_init:
 * "picker" is passed to every hook as this struct's address. Nothing is
 * torn down between tests (suite convention, see meson.build:
 * detect_leaks=0). */
static void fixture_init(xce_sched_plugin_t* plugin)
{
    memset(plugin, 0, sizeof(*plugin));
}

/*
 * THE regression: reproduces bug-1686's exact use-after-free sequence
 * single-threaded (see file header for why single-threaded is sufficient) --
 * next_job() once, then on_queue_update() re-sorting the SAME queue (the
 * live crash's own trigger), then next_job() again.
 */
static void test_cursor_invalidated_on_queue_resort_no_uaf(void)
{
    xce_sched_plugin_t  plugin;
    xce_sched_queue_t   q1 = { .name = "Q1", .priority = 5 };
    xce_sched_job_t     j1 = { .jobid      = 1,
                               .priority   = 5,
                               .queue_time = 1,
                               .state      = JOB_STAT_PEND,
                               .queue      = &q1 };
    xce_sched_job_t     j2 = { .jobid      = 2,
                               .priority   = 1,
                               .queue_time = 2,
                               .state      = JOB_STAT_PEND,
                               .queue      = &q1 };
    fcfs_plugin_data_t* priv;
    xce_sched_job_t*    got;

    fixture_init(&plugin);
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_init(&plugin, NULL));
    priv = (fcfs_plugin_data_t*)plugin.data;
    TEST_ASSERT_NOT_NULL(priv);

    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_queue_update(&plugin, &q1));
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_job_push(&plugin, &j1));
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_job_push(&plugin, &j2));

    /* (b) */
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_begin_session(&plugin));

    /* (c) dispatch job 1; cursor stays parked on Q1's queue node (only
     * queue, and it still has job 2 pending -- see fcfs_cursor_enter_queue's
     * own doc comment: cursor only advances when a queue's job_cursor runs
     * out or the queue itself is skipped, neither of which happens here). */
    got = builtin_fcfs_next_job(&plugin);
    TEST_ASSERT_NOT_NULL(got);
    TEST_ASSERT_EQUAL_UINT64(1, got->jobid);

    TEST_ASSERT_NOT_NULL(priv->cursor);
    TEST_ASSERT_EQUAL_PTR(&q1, ((fcfs_queue_t*)priv->cursor->data)->queue);
    TEST_ASSERT_NOT_NULL(priv->job_cursor);

    /* (d) re-sort the SAME queue (different priority genuinely exercises
     * the reuse-and-resort path) -- this is what frees the OLD ordq_item_t
     * node priv->cursor is resting on. */
    q1.priority = 9;
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_queue_update(&plugin, &q1));

    /* (e) post-fix: both cursors observably NULLed before the free. */
    TEST_ASSERT_NULL(priv->cursor);
    TEST_ASSERT_NULL(priv->job_cursor);

    /* (f) THE use-after-free reproduction point: pre-fix, priv->cursor
     * still points at the node FREEUP(it) already freed in (d), and this
     * call dereferences priv->cursor->data from freed memory -- a genuine
     * ASAN heap-use-after-free abort matching bug-1686's live SIGSEGV.
     * Post-fix, priv->cursor is NULL, so next_job()'s outer while loop never
     * executes and this returns NULL cleanly: "no more candidates this
     * round" for the (invalidated) Q1. */
    got = builtin_fcfs_next_job(&plugin);
    TEST_ASSERT_NULL(got);
    /* stays NULL on a repeated call, same as ordinary exhaustion */
    TEST_ASSERT_NULL(builtin_fcfs_next_job(&plugin));

    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_end_session(&plugin));

    /* (g) no job is permanently lost -- the next round's begin_session()
     * resets the cursor to priv->queues.head fresh, regardless of the
     * invalidation, and finds every pending job again (job 1, never
     * on_dispatch()'d, is still a candidate; job 2 was never reached before
     * the invalidation). */
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_begin_session(&plugin));

    got = builtin_fcfs_next_job(&plugin);
    TEST_ASSERT_NOT_NULL(got);
    TEST_ASSERT_EQUAL_UINT64(1, got->jobid);

    got = builtin_fcfs_next_job(&plugin);
    TEST_ASSERT_NOT_NULL(got);
    TEST_ASSERT_EQUAL_UINT64(2, got->jobid);

    TEST_ASSERT_NULL(builtin_fcfs_next_job(&plugin));
    TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_end_session(&plugin));

    builtin_fcfs_fini(&plugin);
}

int main(void)
{
    UNITY_BEGIN();
    RUN_TEST(test_cursor_invalidated_on_queue_resort_no_uaf);
    return UNITY_END();
}