Implements the scheduler's first request-reply opcode (OPCode_schedAdmin). sched_handle_admin_request() (handler.c) decodes SchedAdminRequest and dispatches by action: recordStatus answers synchronously (pure read); recordStart/recordStop are deferred onto the existing pend_updates queue (XCE_SCHED_OP_ADMIN_RECORD) since sched_record_start()'s snapshot call requires the session_active==0 precondition only guaranteed on the driver thread, not the async request-handling thread. driver.c's xce_sched_driver_translate_pending_events() performs the deferred action and sends the reply via a new xcecli_reply_schedadmin() (client.c/client.h), correlated by seq copied before the original request is freed. Adversarial review found a genuine cross-thread race: sched_record_status() read driver-owned recording state (including a pointer the driver thread frees) from the vtask thread with no lock -- the codebase's first such race, missed by ASan (doesn't detect races). Fixed with a dedicated record_lock scoped strictly to in-memory field access, never spanning file I/O; sched_record_status() now strdup()s under the lock instead of returning a borrowed live pointer. Verified with a real two-thread lock-contention test (independently confirmed non-vacuous by removing the lock and observing the test fail) plus a deterministic use-after-free reproduction. 26/26 scheduler suite, clang-format clean.
366 lines
12 KiB
C
366 lines
12 KiB
C
/*
|
|
* test_driver_decision_cb.c - contract test for the normal-mode driver's
|
|
* decision-publish callback (scheduler-core spec, task 5.2, Req 5 AC 6 /
|
|
* AC 9's publishing half).
|
|
*
|
|
* Observable under test: xce_sched_driver_on_decision() -- registered via
|
|
* xce_core_set_decision_cb() in main.c and matching xce_core_decision_cb_t's
|
|
* signature exactly -- forwards each decision to the right publish call:
|
|
*
|
|
* - XCE_DECISION_DISPATCH -> sched_publish_dispatch() called exactly
|
|
* once, with njobs=1 and allocs[0] == the SAME xce_job_alloc_t* the
|
|
* decision carried (not a copy).
|
|
* - XCE_DECISION_PEND -> sched_publish_pend_reason() called exactly
|
|
* once, with nreasons=1 and reasons[0] == the SAME xce_pendreason_t*
|
|
* the decision carried.
|
|
* - XCE_DECISION_PREEMPT -> NEITHER publish function is called (no live
|
|
* allocator path produces this decision type today, and no
|
|
* xcecli_preempt() client API exists to publish it through even if one
|
|
* did -- see driver.c's own header comment on this function for the
|
|
* full reasoning); returns -ENOSYS, never crashes, never silently
|
|
* drops without a trace (verified via the ERROR log call count below).
|
|
* - NULL driver_ctx / NULL decision -> -EINVAL, no publish calls.
|
|
*
|
|
* White-box #include of ../src/core.c, ../src/core_clock.c,
|
|
* ../src/core_events.c, ../src/driver.c, and ../src/decision.c -- same
|
|
* convention (and same file set) as test_driver_translate.c (task 5.1),
|
|
* since driver.c's xce_sched_driver_translate_pending_events() -- unused by
|
|
* this file's tests but part of the same TU -- still references
|
|
* xce_core_push_events()/xce_core_now()/sched_get_job() at link time.
|
|
* decision.c is added (over test_driver_translate.c's set) for the real
|
|
* make_dispatch_decision()/make_pend_decision()/make_preempt_decision()
|
|
* constructors this file's fixtures use to build realistic xce_decision_t
|
|
* values, same convention as test_core_session.c.
|
|
*
|
|
* sched_publish_dispatch()/sched_publish_pend_reason() below are SPY test
|
|
* doubles (not the real scheduler.c/xcecli_ctx_t-backed versions -- this
|
|
* file never links scheduler.c, same rationale as test_driver_translate.c's
|
|
* own stubs) that record every call's arguments so this file can assert on
|
|
* the driver's call CONTRACT (Req 5 AC 9: "publish core-emitted decisions
|
|
* back to the controller via the existing xcecli_ctx_t API") without a real
|
|
* or mocked network connection.
|
|
*/
|
|
#include <errno.h>
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "controller.capnp.h"
|
|
#include "kvmgr.h"
|
|
#include "preason.h"
|
|
#include "scheduler.h"
|
|
#include "xstring.h"
|
|
|
|
/* Pull in the real xce_sched_driver_on_decision() (and the rest of
|
|
* driver.c's TU -- see file header comment) under test. */
|
|
#include "../src/core.c"
|
|
#include "../src/core_clock.c"
|
|
#include "../src/core_events.c"
|
|
#include "../src/decision.c"
|
|
#include "../src/driver.c"
|
|
|
|
void kill_sched_job(void* arg) { (void)arg; }
|
|
void kill_sched_queue(void* arg) { (void)arg; }
|
|
void kill_sched_worker(void* arg) { (void)arg; }
|
|
void kill_sched_user(void* arg) { (void)arg; }
|
|
void kill_sched_project(void* arg) { (void)arg; }
|
|
void kill_sched_resval(xce_val_t* val) { (void)val; }
|
|
void plugin_unregister(void* arg) { (void)arg; }
|
|
|
|
/* Link-satisfying stubs: as of scheduler-core spec task 5.2,
|
|
* xce_sched_driver_run_normal_mode_round() (driver.c, real, #included
|
|
* above, alongside xce_sched_driver_on_decision() this file DOES exercise)
|
|
* references xce_core_begin_session()/xce_core_signal_go()/
|
|
* xce_core_end_session() (core_session.c) and plugin_begin_session()/
|
|
* plugin_end_session() (plugin.c) / sched_account_jobs() (scheduler.c).
|
|
* This file never exercises xce_sched_driver_run_normal_mode_round() itself
|
|
* (out of THIS file's scope -- see test_driver_session_ordering.c) and
|
|
* never links core_session.c/plugin.c/scheduler.c. Never actually invoked
|
|
* at runtime by any test in this file. */
|
|
int xce_core_begin_session(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int xce_core_signal_go(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int xce_core_end_session(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int plugin_begin_session(void* arg)
|
|
{
|
|
(void)arg;
|
|
return 0;
|
|
}
|
|
|
|
int plugin_end_session(void* arg)
|
|
{
|
|
(void)arg;
|
|
return 0;
|
|
}
|
|
|
|
int sched_account_jobs(xce_scheduler_t* sched)
|
|
{
|
|
(void)sched;
|
|
return 0;
|
|
}
|
|
|
|
/* Link-satisfying stubs for recording.c/recording_events.c/
|
|
* recording_session.c's real functions -- as of scheduler-core spec task
|
|
* 9.2, driver.c's (real, #included above)
|
|
* xce_sched_driver_translate_pending_events()/
|
|
* xce_sched_driver_run_normal_mode_round() reference these. Neither is
|
|
* exercised by this file's tests (out of THIS file's scope -- see
|
|
* test_driver_translate.c/test_driver_session_ordering.c), and this file
|
|
* never links recording.c's real implementations (they need
|
|
* xce_sched_core_save_snapshot(), snapshot.c). Never actually invoked at
|
|
* runtime by any test in this file. */
|
|
int sched_record_write_events(xce_scheduler_t* sched, xce_core_event_t* events,
|
|
int n)
|
|
{
|
|
(void)sched;
|
|
(void)events;
|
|
(void)n;
|
|
return 0;
|
|
}
|
|
|
|
/* sched_record_start/sched_record_stop (scheduler-core spec, task 9.4) --
|
|
* driver.c's handle_deferred_admin_record() (new this task) calls both;
|
|
* this file never enqueues an XCE_SCHED_OP_ADMIN_RECORD request, so
|
|
* neither is actually invoked at runtime, same rationale as
|
|
* sched_record_write_events above. */
|
|
int sched_record_start(xce_scheduler_t* sched, const char* record_file)
|
|
{
|
|
(void)sched;
|
|
(void)record_file;
|
|
return 0;
|
|
}
|
|
|
|
int sched_record_stop(xce_scheduler_t* sched)
|
|
{
|
|
(void)sched;
|
|
return 0;
|
|
}
|
|
|
|
sched_record_session_ctx_t* sched_record_session_begin(xce_scheduler_t* sched,
|
|
uint64_t begin_round_ts)
|
|
{
|
|
(void)sched;
|
|
(void)begin_round_ts;
|
|
return NULL;
|
|
}
|
|
|
|
void sched_record_session_add_decision(sched_record_session_ctx_t* ctx,
|
|
uint64_t ts, xce_decision_t* decision)
|
|
{
|
|
(void)ctx;
|
|
(void)ts;
|
|
(void)decision;
|
|
}
|
|
|
|
int sched_record_session_end(xce_scheduler_t* sched,
|
|
sched_record_session_ctx_t* ctx, uint64_t end_ts)
|
|
{
|
|
(void)sched;
|
|
(void)ctx;
|
|
(void)end_ts;
|
|
return 0;
|
|
}
|
|
|
|
/* Real reimplementation of schedobj.c's sched_resolve_job_picker -- same
|
|
* rationale as test_driver_translate.c's own copy (this file never links
|
|
* schedobj.c/scheduler.c). Never actually exercised (no test here pushes
|
|
* core events), present only to satisfy the linker for core_events.c. */
|
|
int sched_resolve_job_picker(xce_sched_core_t* core, xce_sched_plugin_t** out)
|
|
{
|
|
(void)core;
|
|
(void)out;
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* Spy test doubles for scheduler.c's sched_publish_dispatch()/ */
|
|
/* sched_publish_pend_reason() -- see file header comment. */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static int g_dispatch_calls;
|
|
static xce_job_dispatch_t g_last_dispatch;
|
|
static xce_job_alloc_t* g_last_dispatch_alloc;
|
|
static xce_scheduler_t* g_last_dispatch_sched;
|
|
|
|
static int g_pend_calls;
|
|
static xce_preason_set_t g_last_pend;
|
|
static xce_pendreason_t* g_last_pend_reason;
|
|
static xce_scheduler_t* g_last_pend_sched;
|
|
|
|
static void reset_spies(void)
|
|
{
|
|
g_dispatch_calls = 0;
|
|
g_last_dispatch_alloc = NULL;
|
|
g_last_dispatch_sched = NULL;
|
|
memset(&g_last_dispatch, 0, sizeof(g_last_dispatch));
|
|
|
|
g_pend_calls = 0;
|
|
g_last_pend_reason = NULL;
|
|
g_last_pend_sched = NULL;
|
|
memset(&g_last_pend, 0, sizeof(g_last_pend));
|
|
}
|
|
|
|
int sched_publish_dispatch(xce_scheduler_t* sched, xce_job_dispatch_t* d)
|
|
{
|
|
g_dispatch_calls++;
|
|
g_last_dispatch_sched = sched;
|
|
g_last_dispatch = *d;
|
|
g_last_dispatch_alloc = (d->njobs > 0) ? d->allocs[0] : NULL;
|
|
return 0;
|
|
}
|
|
|
|
int sched_publish_pend_reason(xce_scheduler_t* sched, xce_preason_set_t* r)
|
|
{
|
|
g_pend_calls++;
|
|
g_last_pend_sched = sched;
|
|
g_last_pend = *r;
|
|
g_last_pend_reason = (r->nreasons > 0) ? r->reasons[0] : NULL;
|
|
return 0;
|
|
}
|
|
|
|
void setUp(void) { reset_spies(); }
|
|
void tearDown(void) { return; }
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* XCE_DECISION_DISPATCH */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_dispatch_decision_publishes_via_sched_publish_dispatch(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_job_alloc_t* alloc = (xce_job_alloc_t*)calloc(1, sizeof(*alloc));
|
|
xce_decision_t* d;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
alloc->jobid = 42;
|
|
|
|
d = make_dispatch_decision(alloc);
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(1, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_pend_calls);
|
|
TEST_ASSERT_EQUAL_PTR(&sched, g_last_dispatch_sched);
|
|
TEST_ASSERT_EQUAL_INT(1, g_last_dispatch.njobs);
|
|
TEST_ASSERT_EQUAL_PTR(alloc, g_last_dispatch_alloc);
|
|
TEST_ASSERT_EQUAL_UINT64(42, g_last_dispatch_alloc->jobid);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* XCE_DECISION_PEND */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_pend_decision_publishes_via_sched_publish_pend_reason(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_pendreason_t* reason = make_pendreason(7);
|
|
xce_decision_t* d;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
TEST_ASSERT_NOT_NULL(reason);
|
|
|
|
d = make_pend_decision(reason);
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(1, g_pend_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_PTR(&sched, g_last_pend_sched);
|
|
TEST_ASSERT_EQUAL_INT(1, g_last_pend.nreasons);
|
|
TEST_ASSERT_EQUAL_PTR(reason, g_last_pend_reason);
|
|
TEST_ASSERT_EQUAL_UINT64(7, g_last_pend_reason->jobid);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* XCE_DECISION_PREEMPT -- documented gap, must fail loudly, not crash */
|
|
/* or fabricate a call to a nonexistent client API. */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_preempt_decision_fails_loudly_publishes_nothing(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_preempt_t preempt;
|
|
xce_decision_t* d;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
memset(&preempt, 0, sizeof(preempt));
|
|
preempt.jobid = 99;
|
|
preempt.nvictims = 0;
|
|
preempt.victims = NULL;
|
|
preempt.reason = make_pendreason(99);
|
|
|
|
d = make_preempt_decision(&preempt);
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-ENOSYS, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_pend_calls);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* NULL argument guards */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_null_driver_ctx_returns_einval_no_publish(void)
|
|
{
|
|
xce_job_alloc_t* alloc = (xce_job_alloc_t*)calloc(1, sizeof(*alloc));
|
|
xce_decision_t* d = make_dispatch_decision(alloc);
|
|
int rc;
|
|
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(NULL, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
}
|
|
|
|
static void test_null_decision_returns_einval_no_publish(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_pend_calls);
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_dispatch_decision_publishes_via_sched_publish_dispatch);
|
|
RUN_TEST(test_pend_decision_publishes_via_sched_publish_pend_reason);
|
|
RUN_TEST(test_preempt_decision_fails_loudly_publishes_nothing);
|
|
RUN_TEST(test_null_driver_ctx_returns_einval_no_publish);
|
|
RUN_TEST(test_null_decision_returns_einval_no_publish);
|
|
return UNITY_END();
|
|
}
|