From 54f9139e776805a7c0deb221ae1bc015404431bc Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 8 Oct 2026 09:08:23 +0000 Subject: [PATCH] feat(scheduler-sim-decision-trace): capture selection facts, cluster-wide limits and attempt reasons Task 2.1: policy, queue and priority/queue-time ordering keys, each matched cluster-wide limit as (name, scope), and per-code attempt failure reasons with the hosts and resources they name, all copied into the record arena. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduler-sim-decision-trace/tasks.md | 3 +- src/scheduler/include/sched/trace.h | 54 +- src/scheduler/src/sched_trace_capture.c | 298 ++++++++++- src/scheduler/testsuite/meson.build | 16 + src/scheduler/testsuite/test_trace_capture.c | 506 ++++++++++++++++++ 5 files changed, 873 insertions(+), 4 deletions(-) create mode 100644 src/scheduler/testsuite/test_trace_capture.c diff --git a/.kiro/specs/scheduler-sim-decision-trace/tasks.md b/.kiro/specs/scheduler-sim-decision-trace/tasks.md index 1b0dcbb5..9e484358 100644 --- a/.kiro/specs/scheduler-sim-decision-trace/tasks.md +++ b/.kiro/specs/scheduler-sim-decision-trace/tasks.md @@ -83,7 +83,7 @@ - _Requirements: 11.1, 11.2, 11.3_ - [ ] 2. Allocator-side capture of scheduler state -- [ ] 2.1 Capture job selection facts, cluster-wide limits and attempt failure reasons, test first +- [x] 2.1 Capture job selection facts, cluster-wide limits and attempt failure reasons, test first - Selection capture: policy name, the job's queue and queue priority, the job's priority and its queue time. - Cluster-wide limit capture: each matched limit's name and scope instance, read from the job's matched cluster-wide limits. - Attempt failure reasons: summarized per pend-reason code, with the hosts and resources they name, captured before the scheduler replaces the job's stored reason. @@ -346,3 +346,4 @@ - 1.6: 27 tests include a hooked source and now `#include` sched_trace.c and sched_trace_capture.c. test_snapshot_load_allocator_regression.c matches the grep only in a comment and links scheduler_srcs, which already has both. Still open for the hook tasks: `sched_sim_isolation_lib` (src/scheduler/meson.build ~165-173) compiles the hooked `sched_election.c` without the trace sources (1.8/3.1 must add them or a stub). The 19 repl/sim tests matched via repl.c/sim_loop.c belong to 5.1. - 1.7 OPEN for 5.3: the sim decision-free sites at sim_loop.c ~1327 (reconstructed dispatch) and ~1709 (drain loop) don't yet call `sched_trace_rec_free(d->trace)`, and the TRACE op still falls to the sim's switch defaults. 5.3 must add the early TRACE branch before verify/record/breakpoint, or sim_verify reports false mismatches and breakpoints count it as "NONE". decision.h forward-declares `struct xce_sched_trace_rec_s` rather than including sched/trace.h. - 1.8: `core->trace_ctl` and `core->trace_rec` are plain forward-declared pointers (core.h does not include sched/trace.h), and `xce_sched_core_fini()` clears them without freeing because core.c can't link the trace sources. test_trace_hooks_gate.c counts `sched_trace_rec_new` with a link-time wrap. Its "full allocation creates no record" check only becomes meaningful once 3.1 adds the per-job hook. The 11.3 check (non-interactive sim never publishes trace_ctl) is in 5.3. +- 2.1: capture reads struct fields only and calls no scheduler functions, so the white-box tests need no extra sources. Policy name = ctl policy_name, else core->job_pick_plugin; the queue picker override is ignored because election always uses the core picker. Cluster-wide limit scope is stored as "u=..,q=..,p=.." ("" when unscoped); 2.2 should reuse split_vres_name/copy_scope and append "h=". 4.1 must render selection_set/glimits_set false as "unavailable" and an ATTEMPT_REASONS entry with ncodes==0. diff --git a/src/scheduler/include/sched/trace.h b/src/scheduler/include/sched/trace.h index ef06f759..475edf38 100644 --- a/src/scheduler/include/sched/trace.h +++ b/src/scheduler/include/sched/trace.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "resreq.h" #include "sched/types.h" @@ -212,6 +213,15 @@ typedef struct { int count; } sched_trace_limit_tally_t; +/* One cluster-wide limit matched to the job (2.3), from the schedobj's + * glimits. `scope` is the scope instance the limit applies to, as + * comma-separated `u=..,q=..,p=..` pairs; "" when the limit has no + * per-consumer scope. */ +typedef struct { + const char* limit; /* arena copy */ + const char* scope; /* arena copy */ +} sched_trace_limit_ref_t; + /* One requested resource of a single requirement or member (3.2). */ typedef struct { const char* name; /* arena copy */ @@ -357,8 +367,21 @@ struct xce_sched_trace_rec_s { int line_bound; bool incomplete; /* arena growth failed: appending stopped */ - /* selection block */ - const char* policy_name; /* arena copy of ctl->policy_name, or NULL */ + /* selection block (2.2, 2.3), filled by TraceCapture */ + const char* policy_name; /* the job's ordering policy: arena copy of + * the queue's picker override, else of + * ctl->policy_name, else of the core's + * picker; NULL if none is known */ + bool selection_set; /* capture_selection ran: the ordering keys + * below are known (not NOT_FOUND) */ + const char* queue_name; /* arena copy; NULL: the job has no queue */ + int32_t queue_priority; + int32_t job_priority; + time_t queue_time; + bool glimits_set; /* capture_glimits ran: the limits are known + * (false: no schedobj, limits unavailable) */ + int nglimits; + sched_trace_limit_ref_t* glimits; /* in the schedobj's glimits order */ /* current requirement path (pushed and popped by the allocation chain) */ sched_trace_path_t path; @@ -560,4 +583,31 @@ extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err); */ extern void sched_trace_search_end(xce_sched_trace_rec_t* rec, int rc); +/* + * TraceCapture (src/sched_trace_capture.c, design.md "TraceCapture"). Each + * reads scheduler state on the thread that owns it, read-only, and copies + * what it reports into `rec` (never NULL: the SCHED_TRACE gate). + */ + +/* The job's ordering policy and keys at selection (2.2): policy name, queue + * name and priority, job priority, queue time. A NULL job (not found by + * the allocator, 2.4) captures nothing. */ +extern void sched_trace_capture_selection(xce_sched_trace_rec_t* rec, + const xce_sched_core_t* core, + const xce_sched_job_t* job); + +/* The job's matched cluster-wide limits (2.3): each xce_vres_t in + * obj->glimits as (limit name, scope instance). A NULL obj (no schedobj, + * 2.4) leaves the limits unavailable. */ +extern void sched_trace_capture_glimits(xce_sched_trace_rec_t* rec, + const xce_sched_obj_t* obj); + +/* A failed compound attempt's reasons (3.5), from its pend-reason set before + * the scheduler replaces obj->reason: one ATTEMPT_REASONS entry with a + * count per pend-reason code (main reason, host and resource reasons, in + * the order first seen) and the distinct hosts and resources they name. + * A NULL or empty set stores the entry with no reasons. */ +extern void sched_trace_capture_attempt_reasons(xce_sched_trace_rec_t* rec, + const xce_pendreason_t* reason); + #endif diff --git a/src/scheduler/src/sched_trace_capture.c b/src/scheduler/src/sched_trace_capture.c index cfd6aed9..41c90d81 100644 --- a/src/scheduler/src/sched_trace_capture.c +++ b/src/scheduler/src/sched_trace_capture.c @@ -11,6 +11,302 @@ * The helpers are added by the capture tasks; this module is registered in * the scheduler build and in test_core_isolation.c's source scan from the * start, so the no-I/O rule (no stdio, no file I/O, no logging) applies to - * every line that lands here. + * every line that lands here. Capture copies raw strings and numbers; + * formatting them is the renderer's job. + * + * Task 2.1: selection facts (2.2), cluster-wide limits (2.3) and attempt + * failure reasons (3.5). These read struct fields only and call no + * scheduler function, so every white-box test that #includes this file + * links without extra sources. */ +#include +#include + +#include "core.h" #include "sched/trace.h" + +/* ---- limit name and scope instance ------------------------------------ */ + +#define VRES_PREFIX "limit=" +#define VRES_PREFIX_LEN (sizeof(VRES_PREFIX) - 1) + +/* + * Splits a limit vres name, "limit=[/u=..][/q=..][/p=..]" (lib + * xcelimit.c build_vres_name()), into the limit name and the scope part. + * `known` is the limit's own name when the vres has its back-pointer: the + * prefix "limit=" is then stripped exactly, so a '/' inside a limit + * name cannot split it. Without it the name runs to the first '/'. + * Returns false when `vname` is not a limit vres name (*scope is then ""). + */ +static bool split_vres_name(const char* vname, const char* known, + const char** name, size_t* name_len, + const char** scope) +{ + const char* p; + + *name = NULL; + *name_len = 0; + *scope = ""; + if ((vname == NULL) || (strncmp(vname, VRES_PREFIX, VRES_PREFIX_LEN) != 0)) { + return false; + } + p = vname + VRES_PREFIX_LEN; + if ((known != NULL) && (strncmp(p, known, strlen(known)) == 0) + && ((p[strlen(known)] == '\0') || (p[strlen(known)] == '/'))) { + *name_len = strlen(known); + } else { + const char* slash = strchr(p, '/'); + + *name_len = (slash != NULL) ? (size_t)(slash - p) : strlen(p); + } + *name = p; + *scope = p + *name_len; + if (**scope == '/') { + (*scope)++; + } + return true; +} + +/* Arena copy of a vres scope part, "u=a/q=b" -> "u=a,q=b". */ +static char* copy_scope(xce_sched_trace_rec_t* rec, const char* scope) +{ + size_t n = strlen(scope); + char* out; + size_t i; + + out = (char*)sched_trace_arena_alloc(rec, n + 1); + if (out == NULL) { + return NULL; + } + for (i = 0; i < n; i++) { + out[i] = (scope[i] == '/') ? ',' : scope[i]; + } + out[n] = '\0'; + return out; +} + +/* Arena copy of the first `n` bytes of `s`, NUL-terminated. */ +static char* copy_prefix(xce_sched_trace_rec_t* rec, const char* s, size_t n) +{ + char* out = (char*)sched_trace_arena_alloc(rec, n + 1); + + if (out == NULL) { + return NULL; + } + memcpy(out, s, n); + out[n] = '\0'; + return out; +} + +/* + * (limit name, scope instance) of one matched limit. The name comes from + * the limit itself when the vres has its back-pointer, else it is parsed + * from the vres resource name; the scope always comes from the vres name. + * Returns false (nothing stored) when neither names the limit, or on an + * arena failure. + */ +static bool limit_ref(xce_sched_trace_rec_t* rec, const xce_vres_t* vres, + sched_trace_limit_ref_t* out) +{ + const char* known = NULL; + const char* vname = NULL; + const char* name; + size_t name_len; + const char* scope; + + if (vres == NULL) { + return false; + } + if (vres->limit != NULL) { + known = vres->limit->name; + } + if (vres->vres != NULL) { + vname = vres->vres->name; + } + if (!split_vres_name(vname, known, &name, &name_len, &scope) + && (known == NULL)) { + return false; + } + + if (known != NULL) { + out->limit = sched_trace_arena_strdup(rec, known); + } else { + out->limit = copy_prefix(rec, name, name_len); + } + out->scope = copy_scope(rec, scope); + return (out->limit != NULL) && (out->scope != NULL); +} + +/* ---- selection (2.2) and cluster-wide limits (2.3) -------------------- */ + +void sched_trace_capture_selection(xce_sched_trace_rec_t* rec, + const xce_sched_core_t* core, + const xce_sched_job_t* job) +{ + const char* policy = NULL; + + if (job == NULL) { + return; /* not found: the ordering keys are unavailable (2.4) */ + } + + /* the policy in use: the session's (already copied from the ctl), else + * the core's. A queue's picker override is not reported: the picker + * thread orders every job by core->job_pick_plugin (sched_election.c). */ + if ((rec->policy_name == NULL) && (core != NULL)) { + policy = core->job_pick_plugin; + } + if (policy != NULL) { + rec->policy_name = sched_trace_arena_strdup(rec, policy); + } + + if (job->queue != NULL) { + rec->queue_name = sched_trace_arena_strdup(rec, job->queue->name); + rec->queue_priority = job->queue->priority; + } + rec->job_priority = job->priority; + rec->queue_time = job->queue_time; + rec->selection_set = true; +} + +void sched_trace_capture_glimits(xce_sched_trace_rec_t* rec, + const xce_sched_obj_t* obj) +{ + int n; + int i; + + if (obj == NULL) { + return; /* no schedobj: the limits are unavailable (2.4) */ + } + rec->glimits_set = true; + rec->nglimits = 0; + rec->glimits = NULL; + + n = obj->glimits.count; + if ((n <= 0) || (obj->glimits.data == NULL)) { + return; + } + rec->glimits = (sched_trace_limit_ref_t*)sched_trace_arena_alloc( + rec, sizeof(*rec->glimits) * (size_t)n); + if (rec->glimits == NULL) { + return; + } + /* read the list's slots directly: xlist_get() takes a non-const list */ + for (i = 0; i < n; i++) { + const xce_vres_t* vres = (const xce_vres_t*)obj->glimits.data[i]; + + if (limit_ref(rec, vres, &rec->glimits[rec->nglimits])) { + rec->nglimits++; + } + } +} + +/* ---- attempt failure reasons (3.5) ------------------------------------ */ + +static void count_code(sched_trace_reason_count_t* codes, int* ncodes, + int code) +{ + int i; + + for (i = 0; i < *ncodes; i++) { + if (codes[i].code == code) { + codes[i].count++; + return; + } + } + codes[*ncodes].code = code; + codes[*ncodes].count = 1; + (*ncodes)++; +} + +/* Adds `s` to the distinct-name list unless it is NULL, empty (a shared + * resource reason names no host) or already there. */ +static void add_distinct(const char** list, int* n, const char* s) +{ + int i; + + if ((s == NULL) || (s[0] == '\0')) { + return; + } + for (i = 0; i < *n; i++) { + if (strcmp(list[i], s) == 0) { + return; + } + } + list[(*n)++] = s; +} + +void sched_trace_capture_attempt_reasons(xce_sched_trace_rec_t* rec, + const xce_pendreason_t* reason) +{ + const xce_job_preason_t* r = NULL; + sched_trace_reason_count_t* codes; + const char** hosts; + const char** res; + int nh; + int nr; + int ncodes = 0; + int nhosts = 0; + int nres = 0; + int i; + + if (reason != NULL) { + r = reason->reasons; + } + /* nothing to summarize, or the line will not be stored: the model + * still makes (or counts) the one line */ + if ((r == NULL) || rec->incomplete || (rec->nentries >= rec->line_bound)) { + sched_trace_attempt_reasons(rec, NULL, 0, NULL, 0, NULL, 0); + return; + } + + nh = (r->host_based_reasons != NULL) ? r->nhost_reasons : 0; + nr = (r->res_based_reasons != NULL) ? r->nres_reasons : 0; + if (nh < 0) { + nh = 0; + } + if (nr < 0) { + nr = 0; + } + + /* scratch arrays (the model copies what they point to) */ + codes = (sched_trace_reason_count_t*)malloc(sizeof(*codes) + * (size_t)(nh + nr + 1)); + hosts = (const char**)malloc(sizeof(*hosts) * (size_t)(nh + nr + 1)); + res = (const char**)malloc(sizeof(*res) * (size_t)(nr + 1)); + if ((codes == NULL) || (hosts == NULL) || (res == NULL)) { + free(codes); + free((void*)hosts); + free((void*)res); + rec->incomplete = true; /* design "Error Strategy" */ + return; + } + + if (r->main_reason != 0) { + count_code(codes, &ncodes, r->main_reason); + } + for (i = 0; i < nh; i++) { + const xce_host_reason_t* h = r->host_based_reasons[i]; + + if (h == NULL) { + continue; + } + count_code(codes, &ncodes, h->reason); + add_distinct(hosts, &nhosts, h->host); + } + for (i = 0; i < nr; i++) { + const xce_res_reason_t* x = r->res_based_reasons[i]; + + if (x == NULL) { + continue; + } + count_code(codes, &ncodes, x->reason); + add_distinct(hosts, &nhosts, x->host); + add_distinct(res, &nres, x->resource); + } + + sched_trace_attempt_reasons(rec, codes, ncodes, hosts, nhosts, res, nres); + + free(codes); + free((void*)hosts); + free((void*)res); +} diff --git a/src/scheduler/testsuite/meson.build b/src/scheduler/testsuite/meson.build index 03d10402..55571fe5 100644 --- a/src/scheduler/testsuite/meson.build +++ b/src/scheduler/testsuite/meson.build @@ -2546,6 +2546,22 @@ test('scheduler decision trace test', sched_decision_trace_test, suite: 'scheduler', env: ['ASAN_OPTIONS=detect_leaks=0']) +# scheduler-sim-decision-trace task 2.1: capture helpers (design.md +# "TraceCapture") over a fixture core and job -- selection facts, +# cluster-wide limits and attempt failure reasons. Links the record model +# and the capture source; libxce supplies the pend-reason helpers. +sched_trace_capture_test = executable( + 'sched_trace_capture_test', + ['test_trace_capture.c', '../src/sched_trace.c', + '../src/sched_trace_capture.c'], + include_directories:[incdir], + dependencies: [scheduler_test_deps], + install: false) + +test('scheduler trace capture test', + sched_trace_capture_test, suite: 'scheduler', + env: ['ASAN_OPTIONS=detect_leaks=0']) + # bug-1923: shared (sharedInst) and per-worker resource values at sparse # resource indices survive snapshot save and load, in memory and through # the real sched_record_start() -> .replay -> sched_sim_driver_start() seam diff --git a/src/scheduler/testsuite/test_trace_capture.c b/src/scheduler/testsuite/test_trace_capture.c new file mode 100644 index 00000000..276874c0 --- /dev/null +++ b/src/scheduler/testsuite/test_trace_capture.c @@ -0,0 +1,506 @@ +/* + * test_trace_capture.c - unit tests for the decision trace capture helpers + * (scheduler-sim-decision-trace spec, task 2.1; design.md "TraceCapture"). + * + * Task 2.1 covers the selection-time capture, over a fixture core and job: + * - selection facts: the job-ordering policy, the job's queue and queue + * priority, the job's priority and its queue time (Requirement 2.2); + * - the job's matched cluster-wide limits, each as (limit name, scope + * instance), read from the schedobj's glimits (Requirement 2.3); + * - an attempt's failure reasons summarized per pend-reason code, with the + * hosts and resources they name (Requirement 3.5). + * Every captured value is an exact copy: changing or freeing the source + * structures afterwards leaves the record unchanged (design "Invariants": + * a record never references scheduler-owned memory). + * + * Links src/sched_trace.c and src/sched_trace_capture.c, plus libxce for + * the pend-reason helpers. + */ +#include +#include +#include + +#include + +#include "core.h" +#include "preason.h" +#include "sched/trace.h" +#include "xlist.h" + +void setUp(void) { return; } +void tearDown(void) { return; } + +/* ---- fixture --------------------------------------------------------- */ + +typedef struct { + xce_sched_core_t core; + xce_sched_queue_t queue; + xce_sched_user_t user; + xce_sched_project_t project; + xce_sched_job_t job; + xce_sched_obj_t obj; + /* three matched cluster-wide limits */ + xce_limit_t lim_scoped; + xce_limit_t lim_plain; + xce_resource_t vres_res[3]; + xce_resource_t lic; /* the limited resource */ + xce_vres_t vres[3]; +} fixture_t; + +static fixture_t* fx_new(void) +{ + fixture_t* fx = (fixture_t*)calloc(1, sizeof(*fx)); + + TEST_ASSERT_NOT_NULL(fx); + + fx->core.job_pick_plugin = strdup("core-picker"); + + fx->queue.name = strdup("normal"); + fx->queue.priority = 40; + fx->user.name = strdup("alice"); + fx->project.name = strdup("proj1"); + + fx->job.jobid = 0x1234; + fx->job.priority = 7; + fx->job.queue_time = (time_t)1700000123; + fx->job.queue = &fx->queue; + fx->job.user = &fx->user; + fx->job.project = &fx->project; + fx->job.schedobj = &fx->obj; + + fx->obj.user = fx->user.name; + fx->obj.queue = fx->queue.name; + fx->obj.project = fx->project.name; + xlist_init(&fx->obj.glimits, 8); + + fx->lic.name = strdup("lic"); + + /* per-user, per-queue limit */ + fx->lim_scoped.name = strdup("lic_cap"); + fx->vres_res[0].name = strdup("limit=lic_cap/u=alice/q=normal"); + fx->vres_res[0].flags = XCE_RESF_LIMIT; + fx->vres[0].vres = &fx->vres_res[0]; + fx->vres[0].res = &fx->lic; + fx->vres[0].limit = &fx->lim_scoped; + /* limit with no per-consumer scope */ + fx->lim_plain.name = strdup("site_total"); + fx->vres_res[1].name = strdup("limit=site_total"); + fx->vres_res[1].flags = XCE_RESF_LIMIT; + fx->vres[1].vres = &fx->vres_res[1]; + fx->vres[1].res = &fx->lic; + fx->vres[1].limit = &fx->lim_plain; + /* no limit back-pointer: name and scope come from the vres name */ + fx->vres_res[2].name = strdup("limit=orphan/p=proj1"); + fx->vres_res[2].flags = XCE_RESF_LIMIT; + fx->vres[2].vres = &fx->vres_res[2]; + fx->vres[2].res = NULL; + fx->vres[2].limit = NULL; + + return fx; +} + +static void fx_free(fixture_t* fx) +{ + int i; + + free(fx->core.job_pick_plugin); + free(fx->queue.name); + free(fx->queue.job_pick_plugin); + free(fx->user.name); + free(fx->project.name); + free(fx->lic.name); + free(fx->lim_scoped.name); + free(fx->lim_plain.name); + for (i = 0; i < 3; i++) { + free(fx->vres_res[i].name); + } + xlist_fini(&fx->obj.glimits); + free(fx); +} + +static xce_sched_trace_ctl_t make_ctl(const char* policy) +{ + xce_sched_trace_ctl_t ctl; + + memset(&ctl, 0, sizeof(ctl)); + ctl.all = true; + ctl.level = XCE_TRACE_LEVEL_FULL; + ctl.line_bound = XCE_TRACE_LINE_BOUND_DEFAULT; + ctl.session_no = 2; + ctl.policy_name = policy; + ctl.position = 1; + return ctl; +} + +/* True iff p lies inside the record's arena (a copy, not a source + * pointer). */ +static bool in_arena(const xce_sched_trace_rec_t* rec, const void* p) +{ + const sched_trace_chunk_t* c; + + for (c = rec->arena; c != NULL; c = c->next) { + const char* base = (const char*)(c + 1); + + if (((const char*)p >= base) && ((const char*)p < base + c->cap)) { + return true; + } + } + return false; +} + +static void free_pendreason(xce_pendreason_t* r) +{ + if (r == NULL) { + return; + } + erase_job_preason(r->reasons); + free(r->reasons); + free(r); +} + +/* ---- selection facts (2.2) ------------------------------------------- */ + +static void test_selection_copies_ordering_keys(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, fx->job.jobid, + XCE_TRACE_REC_DECISION); + + TEST_ASSERT_NOT_NULL(rec); + TEST_ASSERT_FALSE(rec->selection_set); + + sched_trace_capture_selection(rec, &fx->core, &fx->job); + + TEST_ASSERT_TRUE(rec->selection_set); + TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name); + TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name); + TEST_ASSERT_TRUE(in_arena(rec, rec->queue_name)); + TEST_ASSERT_EQUAL_INT(40, rec->queue_priority); + TEST_ASSERT_EQUAL_INT(7, rec->job_priority); + TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time); + TEST_ASSERT_FALSE(rec->incomplete); + + /* the source changes later: the record does not */ + fx->queue.name[0] = 'X'; + fx->queue.priority = 99; + fx->job.priority = 1; + fx->job.queue_time = 5; + TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name); + TEST_ASSERT_EQUAL_INT(40, rec->queue_priority); + TEST_ASSERT_EQUAL_INT(7, rec->job_priority); + TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time); + + fx_free(fx); + /* and after it is freed, still readable (ASAN catches a dangling + * pointer) */ + TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name); + TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name); + sched_trace_rec_free(rec); +} + +/* The picker thread orders every job by the core's picker + * (sched_election.c), so a queue's picker override is not the policy in + * use (Req 2.2) and must not be reported as one. */ +static void test_selection_policy_ignores_queue_override(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + + fx->queue.job_pick_plugin = strdup("qprio"); + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION); + sched_trace_capture_selection(rec, &fx->core, &fx->job); + + TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name); + TEST_ASSERT_TRUE(in_arena(rec, rec->policy_name)); + fx_free(fx); + TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name); + sched_trace_rec_free(rec); +} + +static void test_selection_policy_falls_back_to_core_picker(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl(NULL); + xce_sched_trace_rec_t* rec; + + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION); + sched_trace_capture_selection(rec, &fx->core, &fx->job); + + TEST_ASSERT_EQUAL_STRING("core-picker", rec->policy_name); + TEST_ASSERT_TRUE(in_arena(rec, rec->policy_name)); + fx_free(fx); + TEST_ASSERT_EQUAL_STRING("core-picker", rec->policy_name); + sched_trace_rec_free(rec); +} + +static void test_selection_job_without_queue(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + + fx->job.queue = NULL; + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION); + sched_trace_capture_selection(rec, &fx->core, &fx->job); + + TEST_ASSERT_TRUE(rec->selection_set); + TEST_ASSERT_NULL(rec->queue_name); + TEST_ASSERT_EQUAL_INT(0, rec->queue_priority); + TEST_ASSERT_EQUAL_INT(7, rec->job_priority); + TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time); + fx_free(fx); + sched_trace_rec_free(rec); +} + +static void test_selection_null_job_leaves_selection_unset(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_NOT_FOUND); + sched_trace_capture_selection(rec, &fx->core, NULL); + TEST_ASSERT_FALSE(rec->selection_set); + TEST_ASSERT_NULL(rec->queue_name); + fx_free(fx); + sched_trace_rec_free(rec); +} + +/* ---- cluster-wide limits (2.3) --------------------------------------- */ + +static void test_glimits_copies_name_and_scope(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + int i; + + for (i = 0; i < 3; i++) { + xlist_store(&fx->obj.glimits, &fx->vres[i]); + } + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION); + TEST_ASSERT_FALSE(rec->glimits_set); + + sched_trace_capture_glimits(rec, &fx->obj); + + TEST_ASSERT_TRUE(rec->glimits_set); + TEST_ASSERT_EQUAL_INT(3, rec->nglimits); + TEST_ASSERT_NOT_NULL(rec->glimits); + TEST_ASSERT_TRUE(in_arena(rec, rec->glimits)); + + TEST_ASSERT_EQUAL_STRING("lic_cap", rec->glimits[0].limit); + TEST_ASSERT_EQUAL_STRING("u=alice,q=normal", rec->glimits[0].scope); + TEST_ASSERT_EQUAL_STRING("site_total", rec->glimits[1].limit); + TEST_ASSERT_EQUAL_STRING("", rec->glimits[1].scope); + /* no back-pointer: parsed from the vres name */ + TEST_ASSERT_EQUAL_STRING("orphan", rec->glimits[2].limit); + TEST_ASSERT_EQUAL_STRING("p=proj1", rec->glimits[2].scope); + for (i = 0; i < 3; i++) { + TEST_ASSERT_TRUE(in_arena(rec, rec->glimits[i].limit)); + TEST_ASSERT_TRUE(in_arena(rec, rec->glimits[i].scope)); + } + + /* the source changes later: the record does not */ + fx->lim_scoped.name[0] = 'Z'; + fx->vres_res[0].name[8] = 'Z'; + xlist_clear(&fx->obj.glimits); + fx_free(fx); + TEST_ASSERT_EQUAL_INT(3, rec->nglimits); + TEST_ASSERT_EQUAL_STRING("lic_cap", rec->glimits[0].limit); + TEST_ASSERT_EQUAL_STRING("u=alice,q=normal", rec->glimits[0].scope); + TEST_ASSERT_EQUAL_STRING("orphan", rec->glimits[2].limit); + TEST_ASSERT_FALSE(rec->incomplete); + sched_trace_rec_free(rec); +} + +static void test_glimits_none_matched_is_known_empty(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION); + sched_trace_capture_glimits(rec, &fx->obj); + TEST_ASSERT_TRUE(rec->glimits_set); + TEST_ASSERT_EQUAL_INT(0, rec->nglimits); + fx_free(fx); + sched_trace_rec_free(rec); +} + +static void test_glimits_null_obj_leaves_limits_unavailable(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_NO_SCHEDOBJ); + sched_trace_capture_glimits(rec, NULL); + TEST_ASSERT_FALSE(rec->glimits_set); + TEST_ASSERT_EQUAL_INT(0, rec->nglimits); + fx_free(fx); + sched_trace_rec_free(rec); +} + +static void test_glimits_skips_null_and_nameless_entries(void) +{ + fixture_t* fx = fx_new(); + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_vres_t bare; + xce_sched_trace_rec_t* rec; + + memset(&bare, 0, sizeof(bare)); /* no vres, no limit: nothing to name */ + xlist_store(&fx->obj.glimits, &bare); + xlist_store(&fx->obj.glimits, &fx->vres[1]); + rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION); + sched_trace_capture_glimits(rec, &fx->obj); + TEST_ASSERT_EQUAL_INT(1, rec->nglimits); + TEST_ASSERT_EQUAL_STRING("site_total", rec->glimits[0].limit); + fx_free(fx); + sched_trace_rec_free(rec); +} + +/* ---- attempt failure reasons (3.5) ----------------------------------- */ + +static sched_trace_entry_t* only_entry(xce_sched_trace_rec_t* rec) +{ + TEST_ASSERT_EQUAL_INT(1, rec->nentries); + TEST_ASSERT_NOT_NULL(rec->entries); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_ATTEMPT_REASONS, rec->entries->type); + return rec->entries; +} + +static void test_attempt_reasons_summarized_per_code(void) +{ + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55, + XCE_TRACE_REC_DECISION); + xce_pendreason_t* r = make_pendreason(0x55); + sched_trace_entry_t* e; + + TEST_ASSERT_NOT_NULL(r); + add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n01", NULL); + add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n02", NULL); + add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n03", NULL); + add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n01", "lic"); + add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n04", "mem"); + add_job_preason(r->reasons, PREASON_NO_RESOURCE, NULL, "lic"); /* shared */ + + sched_trace_capture_attempt_reasons(rec, r); + e = only_entry(rec); + + /* per-code counts, in the order first seen */ + TEST_ASSERT_EQUAL_INT(2, e->u.reasons.ncodes); + TEST_ASSERT_EQUAL_INT(PREASON_MATCH_FAIL, e->u.reasons.codes[0].code); + TEST_ASSERT_EQUAL_INT(2, e->u.reasons.codes[0].count); + TEST_ASSERT_EQUAL_INT(PREASON_NO_RESOURCE, e->u.reasons.codes[1].code); + TEST_ASSERT_EQUAL_INT(4, e->u.reasons.codes[1].count); + + /* distinct named hosts (a shared-resource reason names none) */ + TEST_ASSERT_EQUAL_INT(4, e->u.reasons.nhosts); + TEST_ASSERT_EQUAL_STRING("n01", e->u.reasons.hosts[0]); + TEST_ASSERT_EQUAL_STRING("n02", e->u.reasons.hosts[1]); + TEST_ASSERT_EQUAL_STRING("n03", e->u.reasons.hosts[2]); + TEST_ASSERT_EQUAL_STRING("n04", e->u.reasons.hosts[3]); + + /* distinct named resources */ + TEST_ASSERT_EQUAL_INT(2, e->u.reasons.nres); + TEST_ASSERT_EQUAL_STRING("lic", e->u.reasons.res[0]); + TEST_ASSERT_EQUAL_STRING("mem", e->u.reasons.res[1]); + TEST_ASSERT_TRUE(in_arena(rec, e->u.reasons.hosts[0])); + TEST_ASSERT_TRUE(in_arena(rec, e->u.reasons.res[0])); + + /* the scheduler replaces the stored reason afterwards: the record + * keeps its copy */ + r->reasons->host_based_reasons[0]->host[0] = 'X'; + free_pendreason(r); + TEST_ASSERT_EQUAL_INT(2, e->u.reasons.codes[0].count); + TEST_ASSERT_EQUAL_STRING("n01", e->u.reasons.hosts[0]); + TEST_ASSERT_EQUAL_STRING("mem", e->u.reasons.res[1]); + TEST_ASSERT_FALSE(rec->incomplete); + sched_trace_rec_free(rec); +} + +static void test_attempt_reasons_main_reason_only(void) +{ + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55, + XCE_TRACE_REC_DECISION); + xce_pendreason_t* r = make_pendreason(0x55); + sched_trace_entry_t* e; + + add_job_preason(r->reasons, PREASON_BAD_RESREQ, NULL, NULL); + sched_trace_capture_attempt_reasons(rec, r); + e = only_entry(rec); + TEST_ASSERT_EQUAL_INT(1, e->u.reasons.ncodes); + TEST_ASSERT_EQUAL_INT(PREASON_BAD_RESREQ, e->u.reasons.codes[0].code); + TEST_ASSERT_EQUAL_INT(1, e->u.reasons.codes[0].count); + TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nhosts); + TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nres); + free_pendreason(r); + sched_trace_rec_free(rec); +} + +static void test_attempt_reasons_none_recorded(void) +{ + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55, + XCE_TRACE_REC_DECISION); + xce_pendreason_t* r = make_pendreason(0x55); + sched_trace_entry_t* e; + + /* a failed attempt with no reason set still gets its line */ + sched_trace_capture_attempt_reasons(rec, NULL); + e = only_entry(rec); + TEST_ASSERT_EQUAL_INT(0, e->u.reasons.ncodes); + TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nhosts); + TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nres); + sched_trace_rec_free(rec); + + /* an empty reason set: same */ + rec = sched_trace_rec_new(&ctl, 0x55, XCE_TRACE_REC_DECISION); + sched_trace_capture_attempt_reasons(rec, r); + e = only_entry(rec); + TEST_ASSERT_EQUAL_INT(0, e->u.reasons.ncodes); + free_pendreason(r); + sched_trace_rec_free(rec); +} + +static void test_attempt_reasons_counts_against_the_budget(void) +{ + xce_sched_trace_ctl_t ctl = make_ctl("fcfs"); + xce_sched_trace_rec_t* rec; + xce_pendreason_t* r = make_pendreason(0x55); + int i; + + ctl.line_bound = XCE_TRACE_LINE_BOUND_MIN; + rec = sched_trace_rec_new(&ctl, 0x55, XCE_TRACE_REC_DECISION); + add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n01", NULL); + for (i = 0; i < XCE_TRACE_LINE_BOUND_MIN + 1; i++) { + sched_trace_capture_attempt_reasons(rec, r); + } + TEST_ASSERT_EQUAL_INT(XCE_TRACE_LINE_BOUND_MIN, rec->nentries); + TEST_ASSERT_EQUAL_INT(1, rec->counters.omitted_lines); + free_pendreason(r); + sched_trace_rec_free(rec); +} + +int main(void) +{ + UNITY_BEGIN(); + RUN_TEST(test_selection_copies_ordering_keys); + RUN_TEST(test_selection_policy_ignores_queue_override); + RUN_TEST(test_selection_policy_falls_back_to_core_picker); + RUN_TEST(test_selection_job_without_queue); + RUN_TEST(test_selection_null_job_leaves_selection_unset); + RUN_TEST(test_glimits_copies_name_and_scope); + RUN_TEST(test_glimits_none_matched_is_known_empty); + RUN_TEST(test_glimits_null_obj_leaves_limits_unavailable); + RUN_TEST(test_glimits_skips_null_and_nameless_entries); + RUN_TEST(test_attempt_reasons_summarized_per_code); + RUN_TEST(test_attempt_reasons_main_reason_only); + RUN_TEST(test_attempt_reasons_none_recorded); + RUN_TEST(test_attempt_reasons_counts_against_the_budget); + return UNITY_END(); +}