The recorder encoded each controller event batch after xce_core_push_events() had applied it, and applying moves payload fields out of the events: a WORKERS_PUSH's resvals (core_apply_worker_resvals MOVE_PTR), a JOBS_PUSH's allocs, a JOB_RUSAGE's rusages, and a CONFIG_PUSH's shared resvals (sched_apply_pending_config recorded after sched_init_config). Recordings silently lost exactly those fields -- a worker's resval names were written with NULL values, so --sim-verify replayed "no slots" and turned 3 live DISPATCHes into PENDs. driver.c now records the batch before the push; the periodic in-stream SnapshotEntry moves out of sched_record_write_events() into new sched_record_checkpoint_after_apply(), called after the push so a checkpoint still reflects the events written ahead of it. The config push is likewise recorded before sched_init_config(). Regression test: test_recorded_workers_push_keeps_resval_values (RED: recorded value NULL; GREEN after). Periodic-checkpoint tests updated to the write-then-checkpoint contract. meson test 109/109; 25 live soak recordings replay with 0 mismatches. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
467 lines
16 KiB
C
467 lines
16 KiB
C
/*
|
|
* test_driver_decision_cb.c - contract test for the normal-mode driver's
|
|
* decision-publish callback (scheduler-core spec, task 5.2, Req 5 AC 6 /
|
|
* AC 9's publishing half).
|
|
*
|
|
* Observable under test: xce_sched_driver_on_decision() -- registered via
|
|
* xce_core_set_decision_cb() in main.c and matching xce_core_decision_cb_t's
|
|
* signature exactly -- forwards each decision to the right publish call:
|
|
*
|
|
* - XCE_DECISION_DISPATCH -> sched_publish_dispatch() called exactly
|
|
* once, with njobs=1 and allocs[0] == the SAME xce_job_alloc_t* the
|
|
* decision carried (not a copy).
|
|
* - XCE_DECISION_PEND -> sched_publish_pend_reason() called exactly
|
|
* once, with nreasons=1 and reasons[0] == the SAME xce_pendreason_t*
|
|
* the decision carried.
|
|
* - XCE_DECISION_PREEMPT -> NEITHER publish function is called (no live
|
|
* allocator path produces this decision type today, and no
|
|
* xcecli_preempt() client API exists to publish it through even if one
|
|
* did -- see driver.c's own header comment on this function for the
|
|
* full reasoning); returns -ENOSYS, never crashes, never silently
|
|
* drops without a trace (verified via the ERROR log call count below).
|
|
* - NULL driver_ctx / NULL decision -> -EINVAL, no publish calls.
|
|
*
|
|
* White-box #include of ../src/core.c, ../src/core_clock.c,
|
|
* ../src/core_events.c, ../src/driver.c, and ../src/decision.c -- same
|
|
* convention (and same file set) as test_driver_translate.c (task 5.1),
|
|
* since driver.c's xce_sched_driver_translate_pending_events() -- unused by
|
|
* this file's tests but part of the same TU -- still references
|
|
* xce_core_push_events()/xce_core_now()/sched_get_job() at link time.
|
|
* decision.c is added (over test_driver_translate.c's set) for the real
|
|
* make_dispatch_decision()/make_pend_decision()/make_preempt_decision()
|
|
* constructors this file's fixtures use to build realistic xce_decision_t
|
|
* values, same convention as test_core_session.c.
|
|
*
|
|
* sched_publish_dispatch()/sched_publish_pend_reason() below are SPY test
|
|
* doubles (not the real scheduler.c/xcecli_ctx_t-backed versions -- this
|
|
* file never links scheduler.c, same rationale as test_driver_translate.c's
|
|
* own stubs) that record every call's arguments so this file can assert on
|
|
* the driver's call CONTRACT (Req 5 AC 9: "publish core-emitted decisions
|
|
* back to the controller via the existing xcecli_ctx_t API") without a real
|
|
* or mocked network connection.
|
|
*/
|
|
#include <errno.h>
|
|
#include <stdint.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include <unity/unity.h>
|
|
|
|
#include "controller.capnp.h"
|
|
#include "kvmgr.h"
|
|
#include "preason.h"
|
|
#include "scheduler.h"
|
|
#include "xstring.h"
|
|
|
|
/* Pull in the real xce_sched_driver_on_decision() (and the rest of
|
|
* driver.c's TU -- see file header comment) under test. */
|
|
#include "../src/core.c"
|
|
#include "../src/core_clock.c"
|
|
#include "../src/core_events.c"
|
|
#include "../src/decision.c"
|
|
#include "../src/driver.c"
|
|
|
|
void kill_sched_job(void* arg) { (void)arg; }
|
|
void kill_sched_queue(void* arg) { (void)arg; }
|
|
void kill_sched_worker(void* arg) { (void)arg; }
|
|
void kill_sched_user(void* arg) { (void)arg; }
|
|
void kill_sched_project(void* arg) { (void)arg; }
|
|
void kill_sched_resval(xce_val_t* val) { (void)val; }
|
|
|
|
/* resource-accounting.md's reconciliation (resource.c, not linked here --
|
|
* this file only #includes core_events.c). core_events.c's
|
|
* XCE_CORE_EVENT_JOB_RUSAGE handler calls this whenever job->accounts !=
|
|
* NULL; link-satisfying stub only, same rationale as every other stub in
|
|
* this block (see test_job_update.c's identical stub). */
|
|
int sched_reconcile_job_resource_usage(xce_sched_core_t* c,
|
|
xce_job_account_t* a,
|
|
xce_job_rusages_t* r)
|
|
{
|
|
(void)c; (void)a; (void)r; return 0;
|
|
}
|
|
|
|
/* dup_val - real reimplementation of snapshot_wire.c's function of the same
|
|
* name (verbatim logic), needed because core_events.c's RESVALS_FULL/
|
|
* RESVALS_DELTA handlers now call it (bug-1585 fix) and this file's
|
|
* white-box #include of ../src/core_events.c does not pull in
|
|
* snapshot_wire.c (same "reimplement the one small function instead of the
|
|
* whole file" rationale this file's own sched_resolve_job_picker copy
|
|
* already documents). This file's own kill_sched_resval is a no-op stub, so
|
|
* the deep copy's contents are never inspected here -- only link-time
|
|
* presence matters for this file's own tests. */
|
|
xce_val_t* dup_val(const xce_val_t* src)
|
|
{
|
|
xce_val_t* v;
|
|
|
|
if (src == NULL) {
|
|
return NULL;
|
|
}
|
|
|
|
v = (xce_val_t*)calloc(1, sizeof(*v));
|
|
if (v == NULL) {
|
|
return NULL;
|
|
}
|
|
|
|
v->which = src->which;
|
|
if (src->which == XCE_VAL_STRING) {
|
|
v->u.svalue
|
|
= (src->u.svalue != NULL) ? makeString(src->u.svalue) : NULL;
|
|
} else {
|
|
v->u = src->u;
|
|
}
|
|
|
|
return v;
|
|
}
|
|
void plugin_unregister(void* arg) { (void)arg; }
|
|
|
|
/* Link-satisfying stubs: as of scheduler-core spec task 5.2,
|
|
* xce_sched_driver_run_normal_mode_round() (driver.c, real, #included
|
|
* above, alongside xce_sched_driver_on_decision() this file DOES exercise)
|
|
* references xce_core_begin_session()/xce_core_signal_go()/
|
|
* xce_core_end_session() (core_session.c) and plugin_begin_session()/
|
|
* plugin_end_session() (plugin.c) / sched_account_jobs() (scheduler.c).
|
|
* This file never exercises xce_sched_driver_run_normal_mode_round() itself
|
|
* (out of THIS file's scope -- see test_driver_session_ordering.c) and
|
|
* never links core_session.c/plugin.c/scheduler.c. Never actually invoked
|
|
* at runtime by any test in this file. */
|
|
int xce_core_begin_session(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int xce_core_signal_go(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int xce_core_end_session(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int plugin_begin_session(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int plugin_end_session(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
int sched_account_jobs(xce_sched_core_t* core)
|
|
{
|
|
(void)core;
|
|
return 0;
|
|
}
|
|
|
|
/* bug-1836 (scheduler-driver-equivalence fix, Recorded-Decision
|
|
* Reconciliation capacity gate): link-satisfying stubs, mirroring this
|
|
* file's existing sched_account_jobs() no-op precedent immediately above --
|
|
* sim_loop.c's sim_reconcile_unaccounted_jobs() now also calls
|
|
* sched_check_host_satisfy()/sched_check_shared_satisfy() (resource.c) to
|
|
* re-validate a recorded decision against live capacity, and
|
|
* sched_charge_account() (resource.c) to keep that check's own view of
|
|
* capacity up to date across multiple candidates in the same round. This
|
|
* file never exercises real resource contention (same rationale as the
|
|
* existing sched_account_jobs()/sched_ctx_get_uncomit_host() no-ops), so
|
|
* the satisfy checks trivially always succeed and the charge call is a
|
|
* no-op. */
|
|
int sched_check_host_satisfy(xce_sched_core_t* core, xce_sched_worker_t* w,
|
|
xvec_t* uncomitted, xvec_idval_t* wanted,
|
|
xvec_idval_t* unmeet)
|
|
{
|
|
(void)core;
|
|
(void)w;
|
|
(void)uncomitted;
|
|
(void)wanted;
|
|
(void)unmeet;
|
|
return 1;
|
|
}
|
|
|
|
int sched_check_shared_satisfy(xce_sched_core_t* core, xvec_t* uncomitted,
|
|
xvec_idval_t* wanted, xvec_idval_t* unmeet)
|
|
{
|
|
(void)core;
|
|
(void)uncomitted;
|
|
(void)wanted;
|
|
(void)unmeet;
|
|
return 1;
|
|
}
|
|
|
|
int sched_charge_account(xce_sched_core_t* core, xce_job_account_t* acct)
|
|
{
|
|
(void)core;
|
|
(void)acct;
|
|
return 0;
|
|
}
|
|
|
|
int sched_uncharge_account(xce_sched_core_t* core, xce_job_account_t* acct)
|
|
{
|
|
(void)core;
|
|
(void)acct;
|
|
return 0;
|
|
}
|
|
|
|
|
|
/* Link-satisfying stubs for recording.c/recording_events.c/
|
|
* recording_session.c's real functions -- as of scheduler-core spec task
|
|
* 9.2, driver.c's (real, #included above)
|
|
* xce_sched_driver_translate_pending_events()/
|
|
* xce_sched_driver_run_normal_mode_round() reference these. Neither is
|
|
* exercised by this file's tests (out of THIS file's scope -- see
|
|
* test_driver_translate.c/test_driver_session_ordering.c), and this file
|
|
* never links recording.c's real implementations (they need
|
|
* xce_sched_core_save_snapshot(), snapshot.c). Never actually invoked at
|
|
* runtime by any test in this file. */
|
|
int sched_record_write_events(xce_scheduler_t* sched, xce_core_event_t* events,
|
|
int n)
|
|
{
|
|
(void)sched;
|
|
(void)events;
|
|
(void)n;
|
|
return 0;
|
|
}
|
|
|
|
/* bug-1852: driver.c now checkpoints separately, after applying a batch. */
|
|
int sched_record_checkpoint_after_apply(xce_scheduler_t* sched)
|
|
{
|
|
(void)sched;
|
|
return 0;
|
|
}
|
|
|
|
/* sched_record_start/sched_record_stop (scheduler-core spec, task 9.4) --
|
|
* driver.c's handle_deferred_admin_record() (new this task) calls both;
|
|
* this file never enqueues an XCE_SCHED_OP_ADMIN_RECORD request, so
|
|
* neither is actually invoked at runtime, same rationale as
|
|
* sched_record_write_events above. */
|
|
int sched_record_start(xce_scheduler_t* sched, const char* record_file)
|
|
{
|
|
(void)sched;
|
|
(void)record_file;
|
|
return 0;
|
|
}
|
|
|
|
int sched_record_stop(xce_scheduler_t* sched)
|
|
{
|
|
(void)sched;
|
|
return 0;
|
|
}
|
|
|
|
sched_record_session_ctx_t* sched_record_session_begin(xce_scheduler_t* sched,
|
|
uint64_t begin_round_ts)
|
|
{
|
|
(void)sched;
|
|
(void)begin_round_ts;
|
|
return NULL;
|
|
}
|
|
|
|
void sched_record_session_add_decision(sched_record_session_ctx_t* ctx,
|
|
uint64_t ts, xce_decision_t* decision)
|
|
{
|
|
(void)ctx;
|
|
(void)ts;
|
|
(void)decision;
|
|
}
|
|
|
|
int sched_record_session_end(xce_scheduler_t* sched,
|
|
sched_record_session_ctx_t* ctx, uint64_t end_ts)
|
|
{
|
|
(void)sched;
|
|
(void)ctx;
|
|
(void)end_ts;
|
|
return 0;
|
|
}
|
|
|
|
/* Real reimplementation of schedobj.c's sched_resolve_job_picker -- same
|
|
* rationale as test_driver_translate.c's own copy (this file never links
|
|
* schedobj.c/scheduler.c). Never actually exercised (no test here pushes
|
|
* core events), present only to satisfy the linker for core_events.c. */
|
|
int sched_resolve_job_picker(xce_sched_core_t* core, xce_sched_plugin_t** out)
|
|
{
|
|
(void)core;
|
|
(void)out;
|
|
return -EINVAL;
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* Spy test doubles for scheduler.c's sched_publish_dispatch()/ */
|
|
/* sched_publish_pend_reason() -- see file header comment. */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static int g_dispatch_calls;
|
|
static xce_job_dispatch_t g_last_dispatch;
|
|
static xce_job_alloc_t* g_last_dispatch_alloc;
|
|
static xce_scheduler_t* g_last_dispatch_sched;
|
|
|
|
static int g_pend_calls;
|
|
static xce_preason_set_t g_last_pend;
|
|
static xce_pendreason_t* g_last_pend_reason;
|
|
static xce_scheduler_t* g_last_pend_sched;
|
|
|
|
static void reset_spies(void)
|
|
{
|
|
g_dispatch_calls = 0;
|
|
g_last_dispatch_alloc = NULL;
|
|
g_last_dispatch_sched = NULL;
|
|
memset(&g_last_dispatch, 0, sizeof(g_last_dispatch));
|
|
|
|
g_pend_calls = 0;
|
|
g_last_pend_reason = NULL;
|
|
g_last_pend_sched = NULL;
|
|
memset(&g_last_pend, 0, sizeof(g_last_pend));
|
|
}
|
|
|
|
int sched_publish_dispatch(xce_scheduler_t* sched, xce_job_dispatch_t* d)
|
|
{
|
|
g_dispatch_calls++;
|
|
g_last_dispatch_sched = sched;
|
|
g_last_dispatch = *d;
|
|
g_last_dispatch_alloc = (d->njobs > 0) ? d->allocs[0] : NULL;
|
|
return 0;
|
|
}
|
|
|
|
int sched_publish_pend_reason(xce_scheduler_t* sched, xce_preason_set_t* r)
|
|
{
|
|
g_pend_calls++;
|
|
g_last_pend_sched = sched;
|
|
g_last_pend = *r;
|
|
g_last_pend_reason = (r->nreasons > 0) ? r->reasons[0] : NULL;
|
|
return 0;
|
|
}
|
|
|
|
void setUp(void) { reset_spies(); }
|
|
void tearDown(void) { return; }
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* XCE_DECISION_DISPATCH */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_dispatch_decision_publishes_via_sched_publish_dispatch(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_job_alloc_t* alloc = (xce_job_alloc_t*)calloc(1, sizeof(*alloc));
|
|
xce_decision_t* d;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
alloc->jobid = 42;
|
|
|
|
d = make_dispatch_decision(alloc);
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(1, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_pend_calls);
|
|
TEST_ASSERT_EQUAL_PTR(&sched, g_last_dispatch_sched);
|
|
TEST_ASSERT_EQUAL_INT(1, g_last_dispatch.njobs);
|
|
TEST_ASSERT_EQUAL_PTR(alloc, g_last_dispatch_alloc);
|
|
TEST_ASSERT_EQUAL_UINT64(42, g_last_dispatch_alloc->jobid);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* XCE_DECISION_PEND */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_pend_decision_publishes_via_sched_publish_pend_reason(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_pendreason_t* reason = make_pendreason(7);
|
|
xce_decision_t* d;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
TEST_ASSERT_NOT_NULL(reason);
|
|
|
|
d = make_pend_decision(reason);
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(0, rc);
|
|
TEST_ASSERT_EQUAL_INT(1, g_pend_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_PTR(&sched, g_last_pend_sched);
|
|
TEST_ASSERT_EQUAL_INT(1, g_last_pend.nreasons);
|
|
TEST_ASSERT_EQUAL_PTR(reason, g_last_pend_reason);
|
|
TEST_ASSERT_EQUAL_UINT64(7, g_last_pend_reason->jobid);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* XCE_DECISION_PREEMPT -- documented gap, must fail loudly, not crash */
|
|
/* or fabricate a call to a nonexistent client API. */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_preempt_decision_fails_loudly_publishes_nothing(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
xce_preempt_t preempt;
|
|
xce_decision_t* d;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
memset(&preempt, 0, sizeof(preempt));
|
|
preempt.jobid = 99;
|
|
preempt.nvictims = 0;
|
|
preempt.victims = NULL;
|
|
preempt.reason = make_pendreason(99);
|
|
|
|
d = make_preempt_decision(&preempt);
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-ENOSYS, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_pend_calls);
|
|
}
|
|
|
|
/* ---------------------------------------------------------------------- */
|
|
/* NULL argument guards */
|
|
/* ---------------------------------------------------------------------- */
|
|
|
|
static void test_null_driver_ctx_returns_einval_no_publish(void)
|
|
{
|
|
xce_job_alloc_t* alloc = (xce_job_alloc_t*)calloc(1, sizeof(*alloc));
|
|
xce_decision_t* d = make_dispatch_decision(alloc);
|
|
int rc;
|
|
|
|
TEST_ASSERT_NOT_NULL(d);
|
|
|
|
rc = xce_sched_driver_on_decision(NULL, d);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
}
|
|
|
|
static void test_null_decision_returns_einval_no_publish(void)
|
|
{
|
|
xce_scheduler_t sched;
|
|
int rc;
|
|
|
|
memset(&sched, 0, sizeof(sched));
|
|
|
|
rc = xce_sched_driver_on_decision(&sched, NULL);
|
|
|
|
TEST_ASSERT_EQUAL_INT(-EINVAL, rc);
|
|
TEST_ASSERT_EQUAL_INT(0, g_dispatch_calls);
|
|
TEST_ASSERT_EQUAL_INT(0, g_pend_calls);
|
|
}
|
|
|
|
int main(void)
|
|
{
|
|
UNITY_BEGIN();
|
|
RUN_TEST(test_dispatch_decision_publishes_via_sched_publish_dispatch);
|
|
RUN_TEST(test_pend_decision_publishes_via_sched_publish_pend_reason);
|
|
RUN_TEST(test_preempt_decision_fails_loudly_publishes_nothing);
|
|
RUN_TEST(test_null_driver_ctx_returns_einval_no_publish);
|
|
RUN_TEST(test_null_decision_returns_einval_no_publish);
|
|
return UNITY_END();
|
|
}
|