lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(scheduler-sim-decision-trace): capture selection facts, cluster-wide limits and attempt reasons

Browse source 
Task 2.1: policy, queue and priority/queue-time ordering keys, each
matched cluster-wide limit as (name, scope), and per-code attempt
failure reasons with the hosts and resources they name, all copied into
the record arena.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-08 09:08:23 +00:00
parent a801aff46f
commit 54f9139e77
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
5 changed files with 873 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

3
.kiro/specs/scheduler-sim-decision-trace/tasks.md
View file

@ -83,7 +83,7 @@
- _Requirements: 11.1, 11.2, 11.3_
- [ ] 2. Allocator-side capture of scheduler state
- [ ] 2.1 Capture job selection facts, cluster-wide limits and attempt failure reasons, test first
- [x] 2.1 Capture job selection facts, cluster-wide limits and attempt failure reasons, test first
- Selection capture: policy name, the job's queue and queue priority, the job's priority and its queue time.
- Cluster-wide limit capture: each matched limit's name and scope instance, read from the job's matched cluster-wide limits.
- Attempt failure reasons: summarized per pend-reason code, with the hosts and resources they name, captured before the scheduler replaces the job's stored reason.
@ -346,3 +346,4 @@
- 1.6: 27 tests include a hooked source and now `#include` sched_trace.c and sched_trace_capture.c. test_snapshot_load_allocator_regression.c matches the grep only in a comment and links scheduler_srcs, which already has both. Still open for the hook tasks: `sched_sim_isolation_lib` (src/scheduler/meson.build ~165-173) compiles the hooked `sched_election.c` without the trace sources (1.8/3.1 must add them or a stub). The 19 repl/sim tests matched via repl.c/sim_loop.c belong to 5.1.
- 1.7 OPEN for 5.3: the sim decision-free sites at sim_loop.c ~1327 (reconstructed dispatch) and ~1709 (drain loop) don't yet call `sched_trace_rec_free(d->trace)`, and the TRACE op still falls to the sim's switch defaults. 5.3 must add the early TRACE branch before verify/record/breakpoint, or sim_verify reports false mismatches and breakpoints count it as "NONE". decision.h forward-declares `struct xce_sched_trace_rec_s` rather than including sched/trace.h.
- 1.8: `core->trace_ctl` and `core->trace_rec` are plain forward-declared pointers (core.h does not include sched/trace.h), and `xce_sched_core_fini()` clears them without freeing because core.c can't link the trace sources. test_trace_hooks_gate.c counts `sched_trace_rec_new` with a link-time wrap. Its "full allocation creates no record" check only becomes meaningful once 3.1 adds the per-job hook. The 11.3 check (non-interactive sim never publishes trace_ctl) is in 5.3.
- 2.1: capture reads struct fields only and calls no scheduler functions, so the white-box tests need no extra sources. Policy name = ctl policy_name, else core->job_pick_plugin; the queue picker override is ignored because election always uses the core picker. Cluster-wide limit scope is stored as "u=..,q=..,p=.." ("" when unscoped); 2.2 should reuse split_vres_name/copy_scope and append "h=<host>". 4.1 must render selection_set/glimits_set false as "unavailable" and an ATTEMPT_REASONS entry with ncodes==0.

54
src/scheduler/include/sched/trace.h
View file

@ -25,6 +25,7 @@
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <time.h>
#include "resreq.h"
#include "sched/types.h"
@ -212,6 +213,15 @@ typedef struct {
int count;
} sched_trace_limit_tally_t;
/* One cluster-wide limit matched to the job (2.3), from the schedobj's
* glimits. `scope` is the scope instance the limit applies to, as
* comma-separated `u=..,q=..,p=..` pairs; "" when the limit has no
* per-consumer scope. */
typedef struct {
const char* limit; /* arena copy */
const char* scope; /* arena copy */
} sched_trace_limit_ref_t;
/* One requested resource of a single requirement or member (3.2). */
typedef struct {
const char* name; /* arena copy */
@ -357,8 +367,21 @@ struct xce_sched_trace_rec_s {
int line_bound;
bool incomplete; /* arena growth failed: appending stopped */
/* selection block */
const char* policy_name; /* arena copy of ctl->policy_name, or NULL */
/* selection block (2.2, 2.3), filled by TraceCapture */
const char* policy_name; /* the job's ordering policy: arena copy of
* the queue's picker override, else of
* ctl->policy_name, else of the core's
* picker; NULL if none is known */
bool selection_set; /* capture_selection ran: the ordering keys
* below are known (not NOT_FOUND) */
const char* queue_name; /* arena copy; NULL: the job has no queue */
int32_t queue_priority;
int32_t job_priority;
time_t queue_time;
bool glimits_set; /* capture_glimits ran: the limits are known
* (false: no schedobj, limits unavailable) */
int nglimits;
sched_trace_limit_ref_t* glimits; /* in the schedobj's glimits order */
/* current requirement path (pushed and popped by the allocation chain) */
sched_trace_path_t path;
@ -560,4 +583,31 @@ extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err);
*/
extern void sched_trace_search_end(xce_sched_trace_rec_t* rec, int rc);
/*
* TraceCapture (src/sched_trace_capture.c, design.md "TraceCapture"). Each
* reads scheduler state on the thread that owns it, read-only, and copies
* what it reports into `rec` (never NULL: the SCHED_TRACE gate).
*/
/* The job's ordering policy and keys at selection (2.2): policy name, queue
* name and priority, job priority, queue time. A NULL job (not found by
* the allocator, 2.4) captures nothing. */
extern void sched_trace_capture_selection(xce_sched_trace_rec_t* rec,
const xce_sched_core_t* core,
const xce_sched_job_t* job);
/* The job's matched cluster-wide limits (2.3): each xce_vres_t in
* obj->glimits as (limit name, scope instance). A NULL obj (no schedobj,
* 2.4) leaves the limits unavailable. */
extern void sched_trace_capture_glimits(xce_sched_trace_rec_t* rec,
const xce_sched_obj_t* obj);
/* A failed compound attempt's reasons (3.5), from its pend-reason set before
* the scheduler replaces obj->reason: one ATTEMPT_REASONS entry with a
* count per pend-reason code (main reason, host and resource reasons, in
* the order first seen) and the distinct hosts and resources they name.
* A NULL or empty set stores the entry with no reasons. */
extern void sched_trace_capture_attempt_reasons(xce_sched_trace_rec_t* rec,
const xce_pendreason_t* reason);
#endif

298
src/scheduler/src/sched_trace_capture.c
View file

@ -11,6 +11,302 @@
* The helpers are added by the capture tasks; this module is registered in
* the scheduler build and in test_core_isolation.c's source scan from the
* start, so the no-I/O rule (no stdio, no file I/O, no logging) applies to
* every line that lands here.
* every line that lands here. Capture copies raw strings and numbers;
* formatting them is the renderer's job.
*
* Task 2.1: selection facts (2.2), cluster-wide limits (2.3) and attempt
* failure reasons (3.5). These read struct fields only and call no
* scheduler function, so every white-box test that #includes this file
* links without extra sources.
*/
#include <stdlib.h>
#include <string.h>
#include "core.h"
#include "sched/trace.h"
/* ---- limit name and scope instance ------------------------------------ */
#define VRES_PREFIX "limit="
#define VRES_PREFIX_LEN (sizeof(VRES_PREFIX) - 1)
/*
* Splits a limit vres name, "limit=<name>[/u=..][/q=..][/p=..]" (lib
* xcelimit.c build_vres_name()), into the limit name and the scope part.
* `known` is the limit's own name when the vres has its back-pointer: the
* prefix "limit=<known>" is then stripped exactly, so a '/' inside a limit
* name cannot split it. Without it the name runs to the first '/'.
* Returns false when `vname` is not a limit vres name (*scope is then "").
*/
static bool split_vres_name(const char* vname, const char* known,
const char** name, size_t* name_len,
const char** scope)
{
const char* p;
*name = NULL;
*name_len = 0;
*scope = "";
if ((vname == NULL) || (strncmp(vname, VRES_PREFIX, VRES_PREFIX_LEN) != 0)) {
return false;
}
p = vname + VRES_PREFIX_LEN;
if ((known != NULL) && (strncmp(p, known, strlen(known)) == 0)
&& ((p[strlen(known)] == '\0') || (p[strlen(known)] == '/'))) {
*name_len = strlen(known);
} else {
const char* slash = strchr(p, '/');
*name_len = (slash != NULL) ? (size_t)(slash - p) : strlen(p);
}
*name = p;
*scope = p + *name_len;
if (**scope == '/') {
(*scope)++;
}
return true;
}
/* Arena copy of a vres scope part, "u=a/q=b" -> "u=a,q=b". */
static char* copy_scope(xce_sched_trace_rec_t* rec, const char* scope)
{
size_t n = strlen(scope);
char* out;
size_t i;
out = (char*)sched_trace_arena_alloc(rec, n + 1);
if (out == NULL) {
return NULL;
}
for (i = 0; i < n; i++) {
out[i] = (scope[i] == '/') ? ',' : scope[i];
}
out[n] = '\0';
return out;
}
/* Arena copy of the first `n` bytes of `s`, NUL-terminated. */
static char* copy_prefix(xce_sched_trace_rec_t* rec, const char* s, size_t n)
{
char* out = (char*)sched_trace_arena_alloc(rec, n + 1);
if (out == NULL) {
return NULL;
}
memcpy(out, s, n);
out[n] = '\0';
return out;
}
/*
* (limit name, scope instance) of one matched limit. The name comes from
* the limit itself when the vres has its back-pointer, else it is parsed
* from the vres resource name; the scope always comes from the vres name.
* Returns false (nothing stored) when neither names the limit, or on an
* arena failure.
*/
static bool limit_ref(xce_sched_trace_rec_t* rec, const xce_vres_t* vres,
sched_trace_limit_ref_t* out)
{
const char* known = NULL;
const char* vname = NULL;
const char* name;
size_t name_len;
const char* scope;
if (vres == NULL) {
return false;
}
if (vres->limit != NULL) {
known = vres->limit->name;
}
if (vres->vres != NULL) {
vname = vres->vres->name;
}
if (!split_vres_name(vname, known, &name, &name_len, &scope)
&& (known == NULL)) {
return false;
}
if (known != NULL) {
out->limit = sched_trace_arena_strdup(rec, known);
} else {
out->limit = copy_prefix(rec, name, name_len);
}
out->scope = copy_scope(rec, scope);
return (out->limit != NULL) && (out->scope != NULL);
}
/* ---- selection (2.2) and cluster-wide limits (2.3) -------------------- */
void sched_trace_capture_selection(xce_sched_trace_rec_t* rec,
const xce_sched_core_t* core,
const xce_sched_job_t* job)
{
const char* policy = NULL;
if (job == NULL) {
return; /* not found: the ordering keys are unavailable (2.4) */
}
/* the policy in use: the session's (already copied from the ctl), else
* the core's. A queue's picker override is not reported: the picker
* thread orders every job by core->job_pick_plugin (sched_election.c). */
if ((rec->policy_name == NULL) && (core != NULL)) {
policy = core->job_pick_plugin;
}
if (policy != NULL) {
rec->policy_name = sched_trace_arena_strdup(rec, policy);
}
if (job->queue != NULL) {
rec->queue_name = sched_trace_arena_strdup(rec, job->queue->name);
rec->queue_priority = job->queue->priority;
}
rec->job_priority = job->priority;
rec->queue_time = job->queue_time;
rec->selection_set = true;
}
void sched_trace_capture_glimits(xce_sched_trace_rec_t* rec,
const xce_sched_obj_t* obj)
{
int n;
int i;
if (obj == NULL) {
return; /* no schedobj: the limits are unavailable (2.4) */
}
rec->glimits_set = true;
rec->nglimits = 0;
rec->glimits = NULL;
n = obj->glimits.count;
if ((n <= 0) || (obj->glimits.data == NULL)) {
return;
}
rec->glimits = (sched_trace_limit_ref_t*)sched_trace_arena_alloc(
rec, sizeof(*rec->glimits) * (size_t)n);
if (rec->glimits == NULL) {
return;
}
/* read the list's slots directly: xlist_get() takes a non-const list */
for (i = 0; i < n; i++) {
const xce_vres_t* vres = (const xce_vres_t*)obj->glimits.data[i];
if (limit_ref(rec, vres, &rec->glimits[rec->nglimits])) {
rec->nglimits++;
}
}
}
/* ---- attempt failure reasons (3.5) ------------------------------------ */
static void count_code(sched_trace_reason_count_t* codes, int* ncodes,
int code)
{
int i;
for (i = 0; i < *ncodes; i++) {
if (codes[i].code == code) {
codes[i].count++;
return;
}
}
codes[*ncodes].code = code;
codes[*ncodes].count = 1;
(*ncodes)++;
}
/* Adds `s` to the distinct-name list unless it is NULL, empty (a shared
* resource reason names no host) or already there. */
static void add_distinct(const char** list, int* n, const char* s)
{
int i;
if ((s == NULL) || (s[0] == '\0')) {
return;
}
for (i = 0; i < *n; i++) {
if (strcmp(list[i], s) == 0) {
return;
}
}
list[(*n)++] = s;
}
void sched_trace_capture_attempt_reasons(xce_sched_trace_rec_t* rec,
const xce_pendreason_t* reason)
{
const xce_job_preason_t* r = NULL;
sched_trace_reason_count_t* codes;
const char** hosts;
const char** res;
int nh;
int nr;
int ncodes = 0;
int nhosts = 0;
int nres = 0;
int i;
if (reason != NULL) {
r = reason->reasons;
}
/* nothing to summarize, or the line will not be stored: the model
* still makes (or counts) the one line */
if ((r == NULL) || rec->incomplete || (rec->nentries >= rec->line_bound)) {
sched_trace_attempt_reasons(rec, NULL, 0, NULL, 0, NULL, 0);
return;
}
nh = (r->host_based_reasons != NULL) ? r->nhost_reasons : 0;
nr = (r->res_based_reasons != NULL) ? r->nres_reasons : 0;
if (nh < 0) {
nh = 0;
}
if (nr < 0) {
nr = 0;
}
/* scratch arrays (the model copies what they point to) */
codes = (sched_trace_reason_count_t*)malloc(sizeof(*codes)
* (size_t)(nh + nr + 1));
hosts = (const char**)malloc(sizeof(*hosts) * (size_t)(nh + nr + 1));
res = (const char**)malloc(sizeof(*res) * (size_t)(nr + 1));
if ((codes == NULL) || (hosts == NULL) || (res == NULL)) {
free(codes);
free((void*)hosts);
free((void*)res);
rec->incomplete = true; /* design "Error Strategy" */
return;
}
if (r->main_reason != 0) {
count_code(codes, &ncodes, r->main_reason);
}
for (i = 0; i < nh; i++) {
const xce_host_reason_t* h = r->host_based_reasons[i];
if (h == NULL) {
continue;
}
count_code(codes, &ncodes, h->reason);
add_distinct(hosts, &nhosts, h->host);
}
for (i = 0; i < nr; i++) {
const xce_res_reason_t* x = r->res_based_reasons[i];
if (x == NULL) {
continue;
}
count_code(codes, &ncodes, x->reason);
add_distinct(hosts, &nhosts, x->host);
add_distinct(res, &nres, x->resource);
}
sched_trace_attempt_reasons(rec, codes, ncodes, hosts, nhosts, res, nres);
free(codes);
free((void*)hosts);
free((void*)res);
}

16
src/scheduler/testsuite/meson.build
View file

@ -2546,6 +2546,22 @@ test('scheduler decision trace test',
sched_decision_trace_test, suite: 'scheduler',
env: ['ASAN_OPTIONS=detect_leaks=0'])
# scheduler-sim-decision-trace task 2.1: capture helpers (design.md
# "TraceCapture") over a fixture core and job -- selection facts,
# cluster-wide limits and attempt failure reasons. Links the record model
# and the capture source; libxce supplies the pend-reason helpers.
sched_trace_capture_test = executable(
'sched_trace_capture_test',
['test_trace_capture.c', '../src/sched_trace.c',
'../src/sched_trace_capture.c'],
include_directories:[incdir],
dependencies: [scheduler_test_deps],
install: false)
test('scheduler trace capture test',
sched_trace_capture_test, suite: 'scheduler',
env: ['ASAN_OPTIONS=detect_leaks=0'])
# bug-1923: shared (sharedInst) and per-worker resource values at sparse
# resource indices survive snapshot save and load, in memory and through
# the real sched_record_start() -> .replay -> sched_sim_driver_start() seam

506
src/scheduler/testsuite/test_trace_capture.c Normal file
View file

@ -0,0 +1,506 @@
/*
* test_trace_capture.c - unit tests for the decision trace capture helpers
* (scheduler-sim-decision-trace spec, task 2.1; design.md "TraceCapture").
*
* Task 2.1 covers the selection-time capture, over a fixture core and job:
* - selection facts: the job-ordering policy, the job's queue and queue
* priority, the job's priority and its queue time (Requirement 2.2);
* - the job's matched cluster-wide limits, each as (limit name, scope
* instance), read from the schedobj's glimits (Requirement 2.3);
* - an attempt's failure reasons summarized per pend-reason code, with the
* hosts and resources they name (Requirement 3.5).
* Every captured value is an exact copy: changing or freeing the source
* structures afterwards leaves the record unchanged (design "Invariants":
* a record never references scheduler-owned memory).
*
* Links src/sched_trace.c and src/sched_trace_capture.c, plus libxce for
* the pend-reason helpers.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "core.h"
#include "preason.h"
#include "sched/trace.h"
#include "xlist.h"
void setUp(void) { return; }
void tearDown(void) { return; }
/* ---- fixture --------------------------------------------------------- */
typedef struct {
xce_sched_core_t core;
xce_sched_queue_t queue;
xce_sched_user_t user;
xce_sched_project_t project;
xce_sched_job_t job;
xce_sched_obj_t obj;
/* three matched cluster-wide limits */
xce_limit_t lim_scoped;
xce_limit_t lim_plain;
xce_resource_t vres_res[3];
xce_resource_t lic; /* the limited resource */
xce_vres_t vres[3];
} fixture_t;
static fixture_t* fx_new(void)
{
fixture_t* fx = (fixture_t*)calloc(1, sizeof(*fx));
TEST_ASSERT_NOT_NULL(fx);
fx->core.job_pick_plugin = strdup("core-picker");
fx->queue.name = strdup("normal");
fx->queue.priority = 40;
fx->user.name = strdup("alice");
fx->project.name = strdup("proj1");
fx->job.jobid = 0x1234;
fx->job.priority = 7;
fx->job.queue_time = (time_t)1700000123;
fx->job.queue = &fx->queue;
fx->job.user = &fx->user;
fx->job.project = &fx->project;
fx->job.schedobj = &fx->obj;
fx->obj.user = fx->user.name;
fx->obj.queue = fx->queue.name;
fx->obj.project = fx->project.name;
xlist_init(&fx->obj.glimits, 8);
fx->lic.name = strdup("lic");
/* per-user, per-queue limit */
fx->lim_scoped.name = strdup("lic_cap");
fx->vres_res[0].name = strdup("limit=lic_cap/u=alice/q=normal");
fx->vres_res[0].flags = XCE_RESF_LIMIT;
fx->vres[0].vres = &fx->vres_res[0];
fx->vres[0].res = &fx->lic;
fx->vres[0].limit = &fx->lim_scoped;
/* limit with no per-consumer scope */
fx->lim_plain.name = strdup("site_total");
fx->vres_res[1].name = strdup("limit=site_total");
fx->vres_res[1].flags = XCE_RESF_LIMIT;
fx->vres[1].vres = &fx->vres_res[1];
fx->vres[1].res = &fx->lic;
fx->vres[1].limit = &fx->lim_plain;
/* no limit back-pointer: name and scope come from the vres name */
fx->vres_res[2].name = strdup("limit=orphan/p=proj1");
fx->vres_res[2].flags = XCE_RESF_LIMIT;
fx->vres[2].vres = &fx->vres_res[2];
fx->vres[2].res = NULL;
fx->vres[2].limit = NULL;
return fx;
}
static void fx_free(fixture_t* fx)
{
int i;
free(fx->core.job_pick_plugin);
free(fx->queue.name);
free(fx->queue.job_pick_plugin);
free(fx->user.name);
free(fx->project.name);
free(fx->lic.name);
free(fx->lim_scoped.name);
free(fx->lim_plain.name);
for (i = 0; i < 3; i++) {
free(fx->vres_res[i].name);
}
xlist_fini(&fx->obj.glimits);
free(fx);
}
static xce_sched_trace_ctl_t make_ctl(const char* policy)
{
xce_sched_trace_ctl_t ctl;
memset(&ctl, 0, sizeof(ctl));
ctl.all = true;
ctl.level = XCE_TRACE_LEVEL_FULL;
ctl.line_bound = XCE_TRACE_LINE_BOUND_DEFAULT;
ctl.session_no = 2;
ctl.policy_name = policy;
ctl.position = 1;
return ctl;
}
/* True iff p lies inside the record's arena (a copy, not a source
* pointer). */
static bool in_arena(const xce_sched_trace_rec_t* rec, const void* p)
{
const sched_trace_chunk_t* c;
for (c = rec->arena; c != NULL; c = c->next) {
const char* base = (const char*)(c + 1);
if (((const char*)p >= base) && ((const char*)p < base + c->cap)) {
return true;
}
}
return false;
}
static void free_pendreason(xce_pendreason_t* r)
{
if (r == NULL) {
return;
}
erase_job_preason(r->reasons);
free(r->reasons);
free(r);
}
/* ---- selection facts (2.2) ------------------------------------------- */
static void test_selection_copies_ordering_keys(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, fx->job.jobid,
XCE_TRACE_REC_DECISION);
TEST_ASSERT_NOT_NULL(rec);
TEST_ASSERT_FALSE(rec->selection_set);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_TRUE(rec->selection_set);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name);
TEST_ASSERT_TRUE(in_arena(rec, rec->queue_name));
TEST_ASSERT_EQUAL_INT(40, rec->queue_priority);
TEST_ASSERT_EQUAL_INT(7, rec->job_priority);
TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time);
TEST_ASSERT_FALSE(rec->incomplete);
/* the source changes later: the record does not */
fx->queue.name[0] = 'X';
fx->queue.priority = 99;
fx->job.priority = 1;
fx->job.queue_time = 5;
TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name);
TEST_ASSERT_EQUAL_INT(40, rec->queue_priority);
TEST_ASSERT_EQUAL_INT(7, rec->job_priority);
TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time);
fx_free(fx);
/* and after it is freed, still readable (ASAN catches a dangling
* pointer) */
TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
sched_trace_rec_free(rec);
}
/* The picker thread orders every job by the core's picker
* (sched_election.c), so a queue's picker override is not the policy in
* use (Req 2.2) and must not be reported as one. */
static void test_selection_policy_ignores_queue_override(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
fx->queue.job_pick_plugin = strdup("qprio");
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
TEST_ASSERT_TRUE(in_arena(rec, rec->policy_name));
fx_free(fx);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
sched_trace_rec_free(rec);
}
static void test_selection_policy_falls_back_to_core_picker(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl(NULL);
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_EQUAL_STRING("core-picker", rec->policy_name);
TEST_ASSERT_TRUE(in_arena(rec, rec->policy_name));
fx_free(fx);
TEST_ASSERT_EQUAL_STRING("core-picker", rec->policy_name);
sched_trace_rec_free(rec);
}
static void test_selection_job_without_queue(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
fx->job.queue = NULL;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_TRUE(rec->selection_set);
TEST_ASSERT_NULL(rec->queue_name);
TEST_ASSERT_EQUAL_INT(0, rec->queue_priority);
TEST_ASSERT_EQUAL_INT(7, rec->job_priority);
TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time);
fx_free(fx);
sched_trace_rec_free(rec);
}
static void test_selection_null_job_leaves_selection_unset(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_NOT_FOUND);
sched_trace_capture_selection(rec, &fx->core, NULL);
TEST_ASSERT_FALSE(rec->selection_set);
TEST_ASSERT_NULL(rec->queue_name);
fx_free(fx);
sched_trace_rec_free(rec);
}
/* ---- cluster-wide limits (2.3) --------------------------------------- */
static void test_glimits_copies_name_and_scope(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
int i;
for (i = 0; i < 3; i++) {
xlist_store(&fx->obj.glimits, &fx->vres[i]);
}
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
TEST_ASSERT_FALSE(rec->glimits_set);
sched_trace_capture_glimits(rec, &fx->obj);
TEST_ASSERT_TRUE(rec->glimits_set);
TEST_ASSERT_EQUAL_INT(3, rec->nglimits);
TEST_ASSERT_NOT_NULL(rec->glimits);
TEST_ASSERT_TRUE(in_arena(rec, rec->glimits));
TEST_ASSERT_EQUAL_STRING("lic_cap", rec->glimits[0].limit);
TEST_ASSERT_EQUAL_STRING("u=alice,q=normal", rec->glimits[0].scope);
TEST_ASSERT_EQUAL_STRING("site_total", rec->glimits[1].limit);
TEST_ASSERT_EQUAL_STRING("", rec->glimits[1].scope);
/* no back-pointer: parsed from the vres name */
TEST_ASSERT_EQUAL_STRING("orphan", rec->glimits[2].limit);
TEST_ASSERT_EQUAL_STRING("p=proj1", rec->glimits[2].scope);
for (i = 0; i < 3; i++) {
TEST_ASSERT_TRUE(in_arena(rec, rec->glimits[i].limit));
TEST_ASSERT_TRUE(in_arena(rec, rec->glimits[i].scope));
}
/* the source changes later: the record does not */
fx->lim_scoped.name[0] = 'Z';
fx->vres_res[0].name[8] = 'Z';
xlist_clear(&fx->obj.glimits);
fx_free(fx);
TEST_ASSERT_EQUAL_INT(3, rec->nglimits);
TEST_ASSERT_EQUAL_STRING("lic_cap", rec->glimits[0].limit);
TEST_ASSERT_EQUAL_STRING("u=alice,q=normal", rec->glimits[0].scope);
TEST_ASSERT_EQUAL_STRING("orphan", rec->glimits[2].limit);
TEST_ASSERT_FALSE(rec->incomplete);
sched_trace_rec_free(rec);
}
static void test_glimits_none_matched_is_known_empty(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_glimits(rec, &fx->obj);
TEST_ASSERT_TRUE(rec->glimits_set);
TEST_ASSERT_EQUAL_INT(0, rec->nglimits);
fx_free(fx);
sched_trace_rec_free(rec);
}
static void test_glimits_null_obj_leaves_limits_unavailable(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_NO_SCHEDOBJ);
sched_trace_capture_glimits(rec, NULL);
TEST_ASSERT_FALSE(rec->glimits_set);
TEST_ASSERT_EQUAL_INT(0, rec->nglimits);
fx_free(fx);
sched_trace_rec_free(rec);
}
static void test_glimits_skips_null_and_nameless_entries(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_vres_t bare;
xce_sched_trace_rec_t* rec;
memset(&bare, 0, sizeof(bare)); /* no vres, no limit: nothing to name */
xlist_store(&fx->obj.glimits, &bare);
xlist_store(&fx->obj.glimits, &fx->vres[1]);
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_glimits(rec, &fx->obj);
TEST_ASSERT_EQUAL_INT(1, rec->nglimits);
TEST_ASSERT_EQUAL_STRING("site_total", rec->glimits[0].limit);
fx_free(fx);
sched_trace_rec_free(rec);
}
/* ---- attempt failure reasons (3.5) ----------------------------------- */
static sched_trace_entry_t* only_entry(xce_sched_trace_rec_t* rec)
{
TEST_ASSERT_EQUAL_INT(1, rec->nentries);
TEST_ASSERT_NOT_NULL(rec->entries);
TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_ATTEMPT_REASONS, rec->entries->type);
return rec->entries;
}
static void test_attempt_reasons_summarized_per_code(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55,
XCE_TRACE_REC_DECISION);
xce_pendreason_t* r = make_pendreason(0x55);
sched_trace_entry_t* e;
TEST_ASSERT_NOT_NULL(r);
add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n01", NULL);
add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n02", NULL);
add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n03", NULL);
add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n01", "lic");
add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n04", "mem");
add_job_preason(r->reasons, PREASON_NO_RESOURCE, NULL, "lic"); /* shared */
sched_trace_capture_attempt_reasons(rec, r);
e = only_entry(rec);
/* per-code counts, in the order first seen */
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.ncodes);
TEST_ASSERT_EQUAL_INT(PREASON_MATCH_FAIL, e->u.reasons.codes[0].code);
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.codes[0].count);
TEST_ASSERT_EQUAL_INT(PREASON_NO_RESOURCE, e->u.reasons.codes[1].code);
TEST_ASSERT_EQUAL_INT(4, e->u.reasons.codes[1].count);
/* distinct named hosts (a shared-resource reason names none) */
TEST_ASSERT_EQUAL_INT(4, e->u.reasons.nhosts);
TEST_ASSERT_EQUAL_STRING("n01", e->u.reasons.hosts[0]);
TEST_ASSERT_EQUAL_STRING("n02", e->u.reasons.hosts[1]);
TEST_ASSERT_EQUAL_STRING("n03", e->u.reasons.hosts[2]);
TEST_ASSERT_EQUAL_STRING("n04", e->u.reasons.hosts[3]);
/* distinct named resources */
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.nres);
TEST_ASSERT_EQUAL_STRING("lic", e->u.reasons.res[0]);
TEST_ASSERT_EQUAL_STRING("mem", e->u.reasons.res[1]);
TEST_ASSERT_TRUE(in_arena(rec, e->u.reasons.hosts[0]));
TEST_ASSERT_TRUE(in_arena(rec, e->u.reasons.res[0]));
/* the scheduler replaces the stored reason afterwards: the record
* keeps its copy */
r->reasons->host_based_reasons[0]->host[0] = 'X';
free_pendreason(r);
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.codes[0].count);
TEST_ASSERT_EQUAL_STRING("n01", e->u.reasons.hosts[0]);
TEST_ASSERT_EQUAL_STRING("mem", e->u.reasons.res[1]);
TEST_ASSERT_FALSE(rec->incomplete);
sched_trace_rec_free(rec);
}
static void test_attempt_reasons_main_reason_only(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55,
XCE_TRACE_REC_DECISION);
xce_pendreason_t* r = make_pendreason(0x55);
sched_trace_entry_t* e;
add_job_preason(r->reasons, PREASON_BAD_RESREQ, NULL, NULL);
sched_trace_capture_attempt_reasons(rec, r);
e = only_entry(rec);
TEST_ASSERT_EQUAL_INT(1, e->u.reasons.ncodes);
TEST_ASSERT_EQUAL_INT(PREASON_BAD_RESREQ, e->u.reasons.codes[0].code);
TEST_ASSERT_EQUAL_INT(1, e->u.reasons.codes[0].count);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nhosts);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nres);
free_pendreason(r);
sched_trace_rec_free(rec);
}
static void test_attempt_reasons_none_recorded(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55,
XCE_TRACE_REC_DECISION);
xce_pendreason_t* r = make_pendreason(0x55);
sched_trace_entry_t* e;
/* a failed attempt with no reason set still gets its line */
sched_trace_capture_attempt_reasons(rec, NULL);
e = only_entry(rec);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.ncodes);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nhosts);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nres);
sched_trace_rec_free(rec);
/* an empty reason set: same */
rec = sched_trace_rec_new(&ctl, 0x55, XCE_TRACE_REC_DECISION);
sched_trace_capture_attempt_reasons(rec, r);
e = only_entry(rec);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.ncodes);
free_pendreason(r);
sched_trace_rec_free(rec);
}
static void test_attempt_reasons_counts_against_the_budget(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
xce_pendreason_t* r = make_pendreason(0x55);
int i;
ctl.line_bound = XCE_TRACE_LINE_BOUND_MIN;
rec = sched_trace_rec_new(&ctl, 0x55, XCE_TRACE_REC_DECISION);
add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n01", NULL);
for (i = 0; i < XCE_TRACE_LINE_BOUND_MIN + 1; i++) {
sched_trace_capture_attempt_reasons(rec, r);
}
TEST_ASSERT_EQUAL_INT(XCE_TRACE_LINE_BOUND_MIN, rec->nentries);
TEST_ASSERT_EQUAL_INT(1, rec->counters.omitted_lines);
free_pendreason(r);
sched_trace_rec_free(rec);
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_selection_copies_ordering_keys);
RUN_TEST(test_selection_policy_ignores_queue_override);
RUN_TEST(test_selection_policy_falls_back_to_core_picker);
RUN_TEST(test_selection_job_without_queue);
RUN_TEST(test_selection_null_job_leaves_selection_unset);
RUN_TEST(test_glimits_copies_name_and_scope);
RUN_TEST(test_glimits_none_matched_is_known_empty);
RUN_TEST(test_glimits_null_obj_leaves_limits_unavailable);
RUN_TEST(test_glimits_skips_null_and_nameless_entries);
RUN_TEST(test_attempt_reasons_summarized_per_code);
RUN_TEST(test_attempt_reasons_main_reason_only);
RUN_TEST(test_attempt_reasons_none_recorded);
RUN_TEST(test_attempt_reasons_counts_against_the_budget);
return UNITY_END();
}