/* * test_builtin_fcfs_cursor_uaf.c - white-box regression test for bug-1686: a * live SIGSEGV in fcfs_cursor_enter_queue() (../src/plugins/builtin_fcfs.c), * reached from builtin_fcfs_next_job() <- sched_job_picker() (scheduler.c) * <- picker thread, captured from a real 3-node HA deployment. * * Root cause: fcfs_plugin_data_t.cursor/job_cursor are raw ordq_item_t* * values that PERSIST in fcfs_plugin_data_t across SEPARATE * builtin_fcfs_next_job() calls, each with its own independent * fcfs_plugin_data_t.lock acquisition/release (see that field's own comment * and the file's header "Concurrency" comment). fcfs_plugin_data_t.lock * correctly makes each individual hook call atomic with respect to the * others, but it cannot protect a pointer VALUE that outlives the call that * read it: a properly-locked builtin_fcfs_on_queue_update() call for a * queue that already has a registered fcfs_queue_t re-sorts it by removing * the queue's existing ordq_item_t node (ordq_get + FREEUP(it)) and * inserting a brand-new one (ordq_put, fresh calloc). If priv->cursor is * resting on that exact node -- which it normally is, since the cursor * stays parked on a queue for as long as it keeps yielding ready jobs * (sched_queue_is_ready()/sched_job_is_ready() are unconditional `return 1` * stubs today in scheduler.c, so a queue's cursor never advances away from * it while jobs remain) -- the NEXT next_job() call dereferences * priv->cursor->data from freed memory: a real heap-use-after-free, live- * reproduced as the SIGSEGV this bug report captured (queue_item was a * dangling, freed ordq_item_t*). * * Critically, this is deterministically reproducible SINGLE-THREADED: since * next_job() releases priv->lock before returning, calling next_job(), then * on_queue_update() for the SAME queue name, then next_job() again -- all * sequentially on one thread, exactly as this test does -- hits the same * use-after-free with no real concurrency required. ASAN catches it as a * genuine heap-use-after-free even single-threaded. * * Fix (already applied, three sites, see builtin_fcfs.c's own file header * comment and each site's comment for the full mechanism): * 1. builtin_fcfs_on_queue_update()'s reuse branch: before FREEUP(it), if * priv->cursor == it, NULLs both priv->cursor and priv->job_cursor. * 2. builtin_fcfs_on_job_state_change()'s !JOB_IS_PEND removal branch: * ordq_peek()s the about-to-be-deleted job node first; if it equals * priv->job_cursor, NULLs priv->job_cursor before ordq_del(). * 3. fcfs_remove_job_everywhere(): same peek-then-invalidate-then-delete * pattern inside its loop over all queues. * Always safe: next_job() already treats a NULL cursor as "no more * candidates this round" (a pre-existing, unrelated-to-this-bug branch -- * see test_picker_fcfs_session.c's own no-queues/empty-queue coverage), and * begin_session() unconditionally resets both cursors fresh every round, so * the worst case of an invalidation is a bounded one-round delay for the * jobs on the invalidated queue, never a crash or a permanently lost job. * This test exercises exactly site 1 (on_queue_update's reuse-and-resort * path), the one the bug report's own crash actually walked into. * * RED (fix sites reverted): builtin_fcfs_next_job()'s second call * dereferences the freed ordq_item_t node priv->cursor still points at -- * a genuine ASAN heap-use-after-free abort, not a clean TEST_ASSERT * failure, matching bug-1686's captured live SIGSEGV signature * (fcfs_cursor_enter_queue <- builtin_fcfs_next_job, dereferencing a * freed/dangling ordq_item_t*). * GREEN (fix applied): priv->cursor/job_cursor are observably NULLed by * on_queue_update()'s reuse branch before the freed node's lifetime ends; * the next next_job() call returns NULL cleanly (no crash); and a * subsequent begin_session()/next_job() round finds every job again, * proving no job is permanently lost -- only delayed one round. * * #include's the real ../src/plugins/builtin_fcfs.c (white-box, same * convention as test_picker_fcfs.c/test_picker_fcfs_session.c) so this test * exercises the real cursor logic and the real private fcfs_plugin_data_t * layout -- including direct field access to priv->cursor/priv->job_cursor, * which only a white-box #include can give a test -- not a re-implementation * of any of it. * * builtin_fcfs.c's next_job() calls sched_queue_is_ready()/ * sched_job_is_ready() (declared in scheduler.h, defined in scheduler.c). * This translation unit does not link scheduler.c (same as * test_picker_fcfs_session.c), so it supplies its own always-ready test * doubles below -- this test never needs to force a skip, unlike that * file's coverage of the skip branches. */ #include #include #include #include #include #include "plugin.h" #include "sched/types.h" /* Always-ready test doubles -- see file header. Neither queue nor job * readiness is under test here, only cursor lifetime across separate * next_job()/on_queue_update() calls. */ int sched_queue_is_ready(xce_sched_queue_t* q) { (void)q; return 1; } int sched_job_is_ready(xce_sched_job_t* j) { (void)j; return 1; } /* register_builtin_fcfs() (bottom of builtin_fcfs.c) calls plugin_register() * (plugin.c, not linked into this white-box TU). Stub to satisfy the * linker -- this file's test drives the hooks directly, never * register_builtin_fcfs() itself. Same convention as * test_picker_fcfs_session.c's own stub. */ int plugin_register(void* arg1, void* arg2) { (void)arg1; (void)arg2; return 0; } #include "../src/plugins/builtin_fcfs.c" void setUp(void) { return; } void tearDown(void) { return; } /* ---------------------------------------------------------------------- */ /* Fixture -- same convention as test_picker_fcfs_session.c's fixture_init: * "picker" is passed to every hook as this struct's address. Nothing is * torn down between tests (suite convention, see meson.build: * detect_leaks=0). */ static void fixture_init(xce_sched_plugin_t* plugin) { memset(plugin, 0, sizeof(*plugin)); } /* * THE regression: reproduces bug-1686's exact use-after-free sequence * single-threaded (see file header for why single-threaded is sufficient) -- * next_job() once, then on_queue_update() re-sorting the SAME queue (the * live crash's own trigger), then next_job() again. */ static void test_cursor_invalidated_on_queue_resort_no_uaf(void) { xce_sched_plugin_t plugin; xce_sched_queue_t q1 = { .name = "Q1", .priority = 5 }; xce_sched_job_t j1 = { .jobid = 1, .priority = 5, .queue_time = 1, .state = JOB_STAT_PEND, .queue = &q1 }; xce_sched_job_t j2 = { .jobid = 2, .priority = 1, .queue_time = 2, .state = JOB_STAT_PEND, .queue = &q1 }; fcfs_plugin_data_t* priv; xce_sched_job_t* got; fixture_init(&plugin); TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_init(&plugin, NULL)); priv = (fcfs_plugin_data_t*)plugin.data; TEST_ASSERT_NOT_NULL(priv); TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_queue_update(&plugin, &q1)); TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_job_push(&plugin, &j1)); TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_job_push(&plugin, &j2)); /* (b) */ TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_begin_session(&plugin)); /* (c) dispatch job 1; cursor stays parked on Q1's queue node (only * queue, and it still has job 2 pending -- see fcfs_cursor_enter_queue's * own doc comment: cursor only advances when a queue's job_cursor runs * out or the queue itself is skipped, neither of which happens here). */ got = builtin_fcfs_next_job(&plugin); TEST_ASSERT_NOT_NULL(got); TEST_ASSERT_EQUAL_UINT64(1, got->jobid); TEST_ASSERT_NOT_NULL(priv->cursor); TEST_ASSERT_EQUAL_PTR(&q1, ((fcfs_queue_t*)priv->cursor->data)->queue); TEST_ASSERT_NOT_NULL(priv->job_cursor); /* (d) re-sort the SAME queue (different priority genuinely exercises * the reuse-and-resort path) -- this is what frees the OLD ordq_item_t * node priv->cursor is resting on. */ q1.priority = 9; TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_on_queue_update(&plugin, &q1)); /* (e) post-fix: both cursors observably NULLed before the free. */ TEST_ASSERT_NULL(priv->cursor); TEST_ASSERT_NULL(priv->job_cursor); /* (f) THE use-after-free reproduction point: pre-fix, priv->cursor * still points at the node FREEUP(it) already freed in (d), and this * call dereferences priv->cursor->data from freed memory -- a genuine * ASAN heap-use-after-free abort matching bug-1686's live SIGSEGV. * Post-fix, priv->cursor is NULL, so next_job()'s outer while loop never * executes and this returns NULL cleanly: "no more candidates this * round" for the (invalidated) Q1. */ got = builtin_fcfs_next_job(&plugin); TEST_ASSERT_NULL(got); /* stays NULL on a repeated call, same as ordinary exhaustion */ TEST_ASSERT_NULL(builtin_fcfs_next_job(&plugin)); TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_end_session(&plugin)); /* (g) no job is permanently lost -- the next round's begin_session() * resets the cursor to priv->queues.head fresh, regardless of the * invalidation, and finds every pending job again (job 1, never * on_dispatch()'d, is still a candidate; job 2 was never reached before * the invalidation). */ TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_begin_session(&plugin)); got = builtin_fcfs_next_job(&plugin); TEST_ASSERT_NOT_NULL(got); TEST_ASSERT_EQUAL_UINT64(1, got->jobid); got = builtin_fcfs_next_job(&plugin); TEST_ASSERT_NOT_NULL(got); TEST_ASSERT_EQUAL_UINT64(2, got->jobid); TEST_ASSERT_NULL(builtin_fcfs_next_job(&plugin)); TEST_ASSERT_EQUAL_INT(0, builtin_fcfs_end_session(&plugin)); builtin_fcfs_fini(&plugin); } int main(void) { UNITY_BEGIN(); RUN_TEST(test_cursor_invalidated_on_queue_resort_no_uaf); return UNITY_END(); }