myxce/src/scheduler/testsuite/test_sim_alloc_limit_fanout.c

/*
 * test_sim_alloc_limit_fanout.c - scheduler-sim-live-allocation spec, task
 * 4.3 (Req 2.2, boundary: Unified Picker/Allocator Threads, Limit Plugin
 * Core-Safe Matching).
 *
 * ── The gap this closes (design.md's "Known Seam Gap: Host-Scoped Limit
 *    Fan-Out During Live Seqjob/Parjob Dispatch") ─────────────────────────
 * Task 4.1's sim_bootstrap_seqjob_allocator()/sim_bootstrap_parjob_allocator()
 * (sim_alloc.c) registered the seqjob/parjob allocator plugin against a
 * throwaway xce_scheduler_t wrapper's OWN separate core.plugins registry --
 * required only because register_seqjob_allocator()/register_parjob_
 * allocator()'s own init_fn (seqjob_allocator_init()/parjob_allocator_init(),
 * plugins/seqjob_allocator.c/parjob_allocator.c) still read plugin->sched,
 * needing a real xce_scheduler_t*. seqjob_allocator.c's/parjob_allocator.c's
 * own my_alloc() then resolves its plugin-hook fan-out (plugin_prepare_
 * allocation()) via THAT SAME plugin instance's own plugin->core field, not
 * the core parameter threaded down from sched_make_allocation() -- so with
 * plugin->core pointing at the wrapper's own core, the fan-out iterates the
 * wrapper's registry (holding only the allocator plugin itself), never
 * reaching a plugin registered on the REAL simulation core (sim->core) --
 * the limit plugin (sim_bootstrap_limit_plugin()) included. A host-scoped
 * limit's usage instance could therefore never be created via a live
 * seqjob/parjob dispatch attempt.
 *
 * This test proves the fix: a host-scoped limit registered directly against
 * simulation mode's real core (sim_bootstrap_limit_plugin(), unchanged) is
 * matched -- and its usage instance created -- during a GENUINE seqjob
 * dispatch attempt, driven through the real, unmodified sched_job_picker()/
 * sched_allocator() thread pair (sched_election.c) spawned against nothing
 * but &(sim.core), the exact shape sched_sim_driver_run() itself uses.
 * Independent of recorded-decision reconciliation: this fixture never
 * builds a .replay stream or calls sim_reconcile_unaccounted_jobs() at all.
 * Independent of global-limit matching: the one limit configured here is
 * host-scoped (a percent-based limit on a real, non-GLOBAL/non-SHARED
 * resource), never a global limit.
 *
 * ── Link style: real scheduler_srcs, not a white-box #include ────────────
 * Same technique test_sched_election_sim_fixture.c/test_sim_live_
 * allocation.c already establish (see those files' own header comments):
 * links the REAL, unmodified `scheduler_srcs` object list (minus
 * src/main.c) so my_alloc()'s real fan-out through the real plugin.c/
 * resource.c/plugins/limit.c chain is genuinely exercised, not stubbed.
 *
 * ── Fixture shape ──────────────────────────────────────────────────────
 * A bare xce_sched_sim_driver_t, built and bootstrapped by hand (mirroring
 * test_sched_election_sim_fixture.c's own "bare core fixture" technique)
 * rather than driven through a real .replay file: xce_sched_core_init(),
 * then the SAME four bootstrap calls sched_sim_driver_run() (sim_loop.c)
 * itself makes, in the same order, all against this one real sim.core --
 * sim_bootstrap_picker(), sim_bootstrap_limit_plugin() (seeded with one
 * host-scoped limit config), sim_bootstrap_seqjob_allocator(),
 * sim_bootstrap_parjob_allocator(). A real worker (added directly via
 * sched_add_worker(), not a wire WORKERS_PUSH ingest -- this fixture needs
 * no replay/capnp machinery) gives the job's auto-injected "slots" want and
 * this test's own limit resource genuine capacity, so the fixture proves
 * the STRONGEST possible outcome: a real DISPATCH, not merely a PEND that
 * happened to touch the fan-out.
 *
 * `job_wrapper` (a genuine, real, but otherwise-unused xce_scheduler_t)
 * exists SOLELY to satisfy make_sched_job()'s own out-of-boundary
 * precondition (schedobj.c still takes `void* arg` cast to xce_scheduler_t*
 * -- a pre-existing gap task 3.1's own fixture already routes around the
 * same way). Its core.resmgr/workers are ALIASED (not copied) to sim.core's
 * own -- the same "var is a plain assignable reference" technique task
 * 4.1's own sim_alloc.c fix established -- so parse_resreq() resolves
 * resource names against the exact same resmgr this test's real dispatch
 * attempt uses. job_wrapper is never passed to anything else.
 *
 * Same "fixtures here are never torn down, exactly one RUN_TEST(), process
 * exits immediately after" convention every other real-scheduler_srcs
 * bare-core-fixture test in this directory follows (test_sched_election_
 * sim_fixture.c's own file header comment) -- the two spawned, never-
 * joined picker/allocator threads read a stack-local `sim` for the rest of
 * the process's short remaining life, which is safe for the same reason
 * documented there.
 */
#include <errno.h>
#include <pthread.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>

#include <unity/unity.h>

#include "controller.capnp.h"
#include "decision.h"
#include "plugin.h"
#include "preason.h"
#include "resobj.h"
#include "sched/types.h"
#include "scheduler.h"
#include "sim.h"
#include "xcelimit.h"
#include "xstring.h"

/* Consumed the same "extern declared where consumed" way sim_loop.c itself
 * declares these four (sim_loop.c's own file header comment) -- none of
 * them has a shared header declaration. */
extern void sim_bootstrap_picker(xce_sched_sim_driver_t* sim);
extern void sim_bootstrap_limit_plugin(xce_sched_sim_driver_t* sim);
extern void sim_bootstrap_seqjob_allocator(xce_sched_sim_driver_t* sim);
extern void sim_bootstrap_parjob_allocator(xce_sched_sim_driver_t* sim);

#define TEST_JOBID     ((uint64_t)430301ULL)
#define QUEUE_NAME     "normal"
#define TEST_RESREQ    ((char*)"select[type==any] rusage[mem=1]")
#define WORKER_NAME    "sim_alloc_limit_worker"
#define WORKER_SLOTS   4.0
#define WORKER_HOSTRES 1000.0
#define LIMIT_RESNAME  "sim_alloc_host_res"
#define LIMIT_NAME     "sim_alloc_host_limit"

void setUp(void) { return; }
void tearDown(void) { return; }

/* ---------------------------------------------------------------------- */
/* Fixture helpers                                                         */
/* ---------------------------------------------------------------------- */

static xce_resource_t* register_plain_resource(var resmgr, const char* name)
{
    xce_resource_t res;

    memset(&res, 0, sizeof(res));
    res.name = strdup(name);

    return makeResource(resmgr, &res);
}

/* add_test_worker_resval - same field-population step core_build_worker()/
 * make_sched_worker() (core_events.c/schedobj.c) use to fill a worker's
 * resvals, applied by hand since neither of those two helpers is reachable
 * without a real wire xce_worker_t payload (out of this task's boundary to
 * add one). */
static void add_test_worker_resval(xce_sched_worker_t* worker,
                                   xce_sched_core_t*   core,
                                   const char* resname, double value)
{
    xce_resource_t* r   = NULL;
    xce_val_t*      val = NULL;
    int              rc;

    rc = key_find(core->resmgr, (void*)resname, (void**)&r);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_NOT_NULL(r);

    val = (xce_val_t*)calloc(1, sizeof(*val));
    TEST_ASSERT_NOT_NULL(val);
    val->which    = Value_u_fvalue;
    val->u.fvalue = value;

    zarray_put(&(worker->resvals), r->idx, val);
}

static xce_sched_worker_t* make_test_worker(xce_sched_core_t* core,
                                            const char*       name)
{
    xce_sched_worker_t* worker = (xce_sched_worker_t*)calloc(1,
                                                             sizeof(*worker));

    TEST_ASSERT_NOT_NULL(worker);

    worker->name   = strdup(name);
    worker->status = 0;
    worker->idx    = -1;

    zarray_init(&(worker->resvals), 128, (zarray_item_killer_t)kill_sched_resval);
    zarray_init(&(worker->resInUse), 128, (zarray_item_killer_t)kill_sched_resval);

    add_test_worker_resval(worker, core, "slots", WORKER_SLOTS);
    add_test_worker_resval(worker, core, LIMIT_RESNAME, WORKER_HOSTRES);

    return worker;
}

/* ---------------------------------------------------------------------- */
/* The test                                                                */
/* ---------------------------------------------------------------------- */

static void
test_host_scoped_limit_usage_instance_created_via_live_sim_dispatch(void)
{
    xce_sched_sim_driver_t sim;
    xce_scheduler_t         job_wrapper; /* setup-only, see file header */
    xce_sched_plugin_t*    picker       = NULL;
    xce_sched_plugin_t*    limit_plugin = NULL;
    xce_sched_queue_t*     queue;
    xce_sched_worker_t*    worker;
    xce_job_t              j;
    job_submit_data_t      submit;
    xce_sched_job_t*       job;
    xce_limit_val_t        limit_val = { .resname = (char*)LIMIT_RESNAME,
                                         .percent = 1,
                                         .val     = 50.0 };
    xce_limit_config_t     limit_cfg = { .name = (char*)LIMIT_NAME,
                                         .val  = &limit_val };
    xce_limit_config_t*    limits[1] = { &limit_cfg };
    xce_conf4sched_t       config;
    pthread_t              picker_tid;
    pthread_t              allocator_tid;
    xce_decision_t*        decision   = NULL;
    xce_decision_t*        end_marker = (xce_decision_t*)0x1;
    var                    instances_handle;
    iterator_t             it;
    void*                  item = NULL;
    int                    rc;

    memset(&sim, 0, sizeof(sim));
    rc = xce_sched_core_init(&(sim.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    /* Real production resources: "slots" (the auto-injected per-task want
     * every primary resreq gets) and LIMIT_RESNAME (this test's own
     * host-scoped limit's underlying resource -- instantiate_limit()
     * (xcelimit.c) requires it to genuinely exist in core->resmgr, same
     * requirement test_repl_limit_table.c's own host-scoped fixture
     * documents). Neither carries XCE_RESF_GLOBAL/XCE_RESF_SHARED, so
     * is_host_based_limit() resolves LIMIT_NAME as host-based purely from
     * its own percent!=0 shape. */
    TEST_ASSERT_NOT_NULL(register_plain_resource(sim.core.resmgr, "slots"));
    TEST_ASSERT_NOT_NULL(
        register_plain_resource(sim.core.resmgr, LIMIT_RESNAME));

    /* Real production bootstrap sequence -- the SAME four calls
     * sched_sim_driver_run() (sim_loop.c) makes, in the SAME order, all
     * against this one real sim.core. */
    sim_bootstrap_picker(&sim);
    rc = key_find(sim.core.plugins, "builtin_fcfs", (void**)&picker);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_NOT_NULL(picker);

    memset(&config, 0, sizeof(config));
    config.nlimits  = 1;
    config.limits   = limits;
    sim.core.config = &config;

    sim_bootstrap_limit_plugin(&sim);
    rc = key_find(sim.core.plugins, "xce_sched_limit", (void**)&limit_plugin);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_NOT_NULL(limit_plugin);
    TEST_ASSERT_EQUAL_PTR_MESSAGE(
        &(sim.core), limit_plugin->core,
        "sanity: the limit plugin must be registered directly against "
        "sim.core -- the SAME registry this test's seqjob/parjob allocator "
        "plugin must ALSO reach for its own prepare_allocation_hook "
        "fan-out to ever find it");

    /* THE fix under test (task 4.3): sim_bootstrap_seqjob_allocator()/
     * sim_bootstrap_parjob_allocator() must register their plugin
     * instance's own `core` field against this SAME sim.core -- not a
     * throwaway wrapper's own separate registry (task 4.1's own
     * documented gap, design.md's "Known Seam Gap: Host-Scoped Limit
     * Fan-Out During Live Seqjob/Parjob Dispatch"). */
    sim_bootstrap_seqjob_allocator(&sim);
    sim_bootstrap_parjob_allocator(&sim);
    TEST_ASSERT_NOT_NULL_MESSAGE(
        sim.core.seqjob_allocator,
        "sim_bootstrap_seqjob_allocator() must populate core->seqjob_"
        "allocator (task 4.1, unrelated to this task's own fix)");

    /* Real queue registration -- same sequence test_sched_election_sim_
     * fixture.c's own fixture establishes. */
    queue = (xce_sched_queue_t*)calloc(1, sizeof(*queue));
    TEST_ASSERT_NOT_NULL(queue);
    queue->name = makeString(QUEUE_NAME);
    rc          = sched_add_queue(&(sim.core), queue);
    TEST_ASSERT_TRUE(rc >= 0);
    picker->picker_iface->on_queue_update(picker, queue);

    /* A real worker with genuine "slots" AND LIMIT_RESNAME capacity, added
     * directly via the real sched_add_worker() (core.c). */
    worker = make_test_worker(&(sim.core), WORKER_NAME);
    rc     = sched_add_worker(&(sim.core), worker);
    TEST_ASSERT_TRUE(rc >= 0);

    /* Real job construction via make_sched_job() -- see file header for
     * why job_wrapper (a real xce_scheduler_t, its core.resmgr/workers
     * ALIASED to sim.core's own) is used only for this one call. */
    memset(&job_wrapper, 0, sizeof(job_wrapper));
    rc = xce_sched_core_init(&(job_wrapper.core));
    TEST_ASSERT_EQUAL_INT(0, rc);
    job_wrapper.core.resmgr  = sim.core.resmgr;
    job_wrapper.core.workers = sim.core.workers;
    /* Also alias .queues (not just .resmgr/.workers): make_sched_job()'s
     * own internal sched_get_queue(&(job_wrapper.core), ...) call is
     * get-only (never auto-creates, core.c) -- without this alias it
     * would look up "normal" in job_wrapper.core's own separate, empty
     * queue registry and return NULL, leaving job->queue NULL, which
     * silently fails builtin_fcfs_on_job_push()'s own (job->queue == NULL)
     * guard (returns -EINVAL, job never becomes a picker candidate) --
     * caught via gdb during this test's own RED-phase debugging. */
    job_wrapper.core.queues = sim.core.queues;

    memset(&submit, 0, sizeof(submit));
    submit.queue        = (char*)QUEUE_NAME;
    submit.user         = (char*)"sim_alloc_limit_user";
    submit.project_name = (char*)"default";
    submit.resreq       = TEST_RESREQ;
    submit.mintasks     = 1;
    submit.maxtasks     = 1;

    memset(&j, 0, sizeof(j));
    j.jobid  = TEST_JOBID;
    j.state  = JOB_STAT_PEND;
    j.submit = &submit;

    job = make_sched_job(&job_wrapper, &j);
    TEST_ASSERT_NOT_NULL(job);
    TEST_ASSERT_EQUAL_INT(XCE_RESREQ_PRIMARY, job->schedobj->tag);

    /* make_sched_job()'s own parse_resreq() call does not thread
     * submit.mintasks/maxtasks into the parsed primary's own minTasks/
     * maxTasks (task 3.1's own documented gap) -- set directly so this
     * fixture deterministically exercises the seqjob path. */
    job->schedobj->u.primary->resreq->minTasks = 1;
    job->schedobj->u.primary->resreq->maxTasks = 1;

    rc = sched_add_job(&(sim.core), job);
    TEST_ASSERT_TRUE(rc >= 0);
    picker->picker_iface->on_job_push(picker, job);

    /* ── The actual thing under test ─────────────────────────────────
     * the REAL, unmodified sched_job_picker()/sched_allocator() thread
     * pair (sched_election.c), spawned against nothing but &(sim.core) --
     * exactly the shape sched_sim_driver_run() itself uses. */
    rc = pthread_create(&picker_tid, NULL, sched_job_picker, &(sim.core));
    TEST_ASSERT_EQUAL_INT(0, rc);
    rc = pthread_create(&allocator_tid, NULL, sched_allocator, &(sim.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    rc = xce_core_go(&(sim.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    cbuf_get(&(sim.core.decisions), &decision);
    TEST_ASSERT_NOT_NULL_MESSAGE(
        decision, "the real thread pair must carry this job all the way to "
                  "a committed decision");

    cbuf_get(&(sim.core.decisions), &end_marker);
    TEST_ASSERT_NULL_MESSAGE(end_marker,
                             "second decisions-cbuf entry must be the "
                             "end-of-session NULL marker");

    rc = xce_core_end_session(&(sim.core));
    TEST_ASSERT_EQUAL_INT(0, rc);

    TEST_ASSERT_EQUAL_UINT_MESSAGE(
        JOB_STAT_RUN, job->state,
        "the worker's real slots/host-resource capacity must be genuinely "
        "sufficient for a real DISPATCH -- the strongest possible proof "
        "that the per-host prepare_allocation_hook fan-out this test cares "
        "about actually ran, not merely that some decision was reached");

    /* The Observable this task's brief requires: a host-scoped limit's
     * usage instance is visible via the existing usage-instances accessor,
     * created by this LIVE dispatch attempt alone -- reconciliation and
     * global-limit matching were never invoked anywhere in this fixture. */
    instances_handle = limit_get_usage_instances(&(sim.core));
    TEST_ASSERT_NOT_NULL(instances_handle);

    rc = iter_make(&it, instances_handle);
    TEST_ASSERT_TRUE(rc >= 0);

    rc = iter_next(&it, &item);
    TEST_ASSERT_TRUE(rc >= 0);
    TEST_ASSERT_NOT_EQUAL(RTTI_ITER_NO_DATA, rc);
    TEST_ASSERT_NOT_NULL_MESSAGE(
        item,
        "a host-scoped limit registered directly against sim.core must be "
        "matched -- and its usage instance created -- during a genuine "
        "live seqjob/parjob dispatch attempt; before task 4.3's fix the "
        "seqjob/parjob allocator plugin's own plugin->core pointed at a "
        "throwaway wrapper's separate registry, so this fan-out never "
        "reached the limit plugin registered on sim.core");

    iter_destroy(&it);

    printf(
        "test_host_scoped_limit_usage_instance_created_via_live_sim_"
        "dispatch: PASS\n");
}

int main(void)
{
    UNITY_BEGIN();
    RUN_TEST(
        test_host_scoped_limit_usage_instance_created_via_live_sim_dispatch);
    return UNITY_END();
}