feat(scheduler-sim-decision-trace): capture selection facts, cluster-wide limits and attempt reasons

Task 2.1: policy, queue and priority/queue-time ordering keys, each
matched cluster-wide limit as (name, scope), and per-code attempt
failure reasons with the hosts and resources they name, all copied into
the record arena.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Rongsong Shen 2026-10-08 09:08:23 +00:00
parent a801aff46f
commit 54f9139e77
Signed by untrusted user: rshen
GPG key ID: 80533536B0EB651C
5 changed files with 873 additions and 4 deletions

View file

@ -2546,6 +2546,22 @@ test('scheduler decision trace test',
sched_decision_trace_test, suite: 'scheduler',
env: ['ASAN_OPTIONS=detect_leaks=0'])
# scheduler-sim-decision-trace task 2.1: capture helpers (design.md
# "TraceCapture") over a fixture core and job -- selection facts,
# cluster-wide limits and attempt failure reasons. Links the record model
# and the capture source; libxce supplies the pend-reason helpers.
sched_trace_capture_test = executable(
'sched_trace_capture_test',
['test_trace_capture.c', '../src/sched_trace.c',
'../src/sched_trace_capture.c'],
include_directories:[incdir],
dependencies: [scheduler_test_deps],
install: false)
test('scheduler trace capture test',
sched_trace_capture_test, suite: 'scheduler',
env: ['ASAN_OPTIONS=detect_leaks=0'])
# bug-1923: shared (sharedInst) and per-worker resource values at sparse
# resource indices survive snapshot save and load, in memory and through
# the real sched_record_start() -> .replay -> sched_sim_driver_start() seam

View file

@ -0,0 +1,506 @@
/*
* test_trace_capture.c - unit tests for the decision trace capture helpers
* (scheduler-sim-decision-trace spec, task 2.1; design.md "TraceCapture").
*
* Task 2.1 covers the selection-time capture, over a fixture core and job:
* - selection facts: the job-ordering policy, the job's queue and queue
* priority, the job's priority and its queue time (Requirement 2.2);
* - the job's matched cluster-wide limits, each as (limit name, scope
* instance), read from the schedobj's glimits (Requirement 2.3);
* - an attempt's failure reasons summarized per pend-reason code, with the
* hosts and resources they name (Requirement 3.5).
* Every captured value is an exact copy: changing or freeing the source
* structures afterwards leaves the record unchanged (design "Invariants":
* a record never references scheduler-owned memory).
*
* Links src/sched_trace.c and src/sched_trace_capture.c, plus libxce for
* the pend-reason helpers.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unity/unity.h>
#include "core.h"
#include "preason.h"
#include "sched/trace.h"
#include "xlist.h"
void setUp(void) { return; }
void tearDown(void) { return; }
/* ---- fixture --------------------------------------------------------- */
typedef struct {
xce_sched_core_t core;
xce_sched_queue_t queue;
xce_sched_user_t user;
xce_sched_project_t project;
xce_sched_job_t job;
xce_sched_obj_t obj;
/* three matched cluster-wide limits */
xce_limit_t lim_scoped;
xce_limit_t lim_plain;
xce_resource_t vres_res[3];
xce_resource_t lic; /* the limited resource */
xce_vres_t vres[3];
} fixture_t;
static fixture_t* fx_new(void)
{
fixture_t* fx = (fixture_t*)calloc(1, sizeof(*fx));
TEST_ASSERT_NOT_NULL(fx);
fx->core.job_pick_plugin = strdup("core-picker");
fx->queue.name = strdup("normal");
fx->queue.priority = 40;
fx->user.name = strdup("alice");
fx->project.name = strdup("proj1");
fx->job.jobid = 0x1234;
fx->job.priority = 7;
fx->job.queue_time = (time_t)1700000123;
fx->job.queue = &fx->queue;
fx->job.user = &fx->user;
fx->job.project = &fx->project;
fx->job.schedobj = &fx->obj;
fx->obj.user = fx->user.name;
fx->obj.queue = fx->queue.name;
fx->obj.project = fx->project.name;
xlist_init(&fx->obj.glimits, 8);
fx->lic.name = strdup("lic");
/* per-user, per-queue limit */
fx->lim_scoped.name = strdup("lic_cap");
fx->vres_res[0].name = strdup("limit=lic_cap/u=alice/q=normal");
fx->vres_res[0].flags = XCE_RESF_LIMIT;
fx->vres[0].vres = &fx->vres_res[0];
fx->vres[0].res = &fx->lic;
fx->vres[0].limit = &fx->lim_scoped;
/* limit with no per-consumer scope */
fx->lim_plain.name = strdup("site_total");
fx->vres_res[1].name = strdup("limit=site_total");
fx->vres_res[1].flags = XCE_RESF_LIMIT;
fx->vres[1].vres = &fx->vres_res[1];
fx->vres[1].res = &fx->lic;
fx->vres[1].limit = &fx->lim_plain;
/* no limit back-pointer: name and scope come from the vres name */
fx->vres_res[2].name = strdup("limit=orphan/p=proj1");
fx->vres_res[2].flags = XCE_RESF_LIMIT;
fx->vres[2].vres = &fx->vres_res[2];
fx->vres[2].res = NULL;
fx->vres[2].limit = NULL;
return fx;
}
static void fx_free(fixture_t* fx)
{
int i;
free(fx->core.job_pick_plugin);
free(fx->queue.name);
free(fx->queue.job_pick_plugin);
free(fx->user.name);
free(fx->project.name);
free(fx->lic.name);
free(fx->lim_scoped.name);
free(fx->lim_plain.name);
for (i = 0; i < 3; i++) {
free(fx->vres_res[i].name);
}
xlist_fini(&fx->obj.glimits);
free(fx);
}
static xce_sched_trace_ctl_t make_ctl(const char* policy)
{
xce_sched_trace_ctl_t ctl;
memset(&ctl, 0, sizeof(ctl));
ctl.all = true;
ctl.level = XCE_TRACE_LEVEL_FULL;
ctl.line_bound = XCE_TRACE_LINE_BOUND_DEFAULT;
ctl.session_no = 2;
ctl.policy_name = policy;
ctl.position = 1;
return ctl;
}
/* True iff p lies inside the record's arena (a copy, not a source
* pointer). */
static bool in_arena(const xce_sched_trace_rec_t* rec, const void* p)
{
const sched_trace_chunk_t* c;
for (c = rec->arena; c != NULL; c = c->next) {
const char* base = (const char*)(c + 1);
if (((const char*)p >= base) && ((const char*)p < base + c->cap)) {
return true;
}
}
return false;
}
static void free_pendreason(xce_pendreason_t* r)
{
if (r == NULL) {
return;
}
erase_job_preason(r->reasons);
free(r->reasons);
free(r);
}
/* ---- selection facts (2.2) ------------------------------------------- */
static void test_selection_copies_ordering_keys(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, fx->job.jobid,
XCE_TRACE_REC_DECISION);
TEST_ASSERT_NOT_NULL(rec);
TEST_ASSERT_FALSE(rec->selection_set);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_TRUE(rec->selection_set);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name);
TEST_ASSERT_TRUE(in_arena(rec, rec->queue_name));
TEST_ASSERT_EQUAL_INT(40, rec->queue_priority);
TEST_ASSERT_EQUAL_INT(7, rec->job_priority);
TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time);
TEST_ASSERT_FALSE(rec->incomplete);
/* the source changes later: the record does not */
fx->queue.name[0] = 'X';
fx->queue.priority = 99;
fx->job.priority = 1;
fx->job.queue_time = 5;
TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name);
TEST_ASSERT_EQUAL_INT(40, rec->queue_priority);
TEST_ASSERT_EQUAL_INT(7, rec->job_priority);
TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time);
fx_free(fx);
/* and after it is freed, still readable (ASAN catches a dangling
* pointer) */
TEST_ASSERT_EQUAL_STRING("normal", rec->queue_name);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
sched_trace_rec_free(rec);
}
/* The picker thread orders every job by the core's picker
* (sched_election.c), so a queue's picker override is not the policy in
* use (Req 2.2) and must not be reported as one. */
static void test_selection_policy_ignores_queue_override(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
fx->queue.job_pick_plugin = strdup("qprio");
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
TEST_ASSERT_TRUE(in_arena(rec, rec->policy_name));
fx_free(fx);
TEST_ASSERT_EQUAL_STRING("fcfs", rec->policy_name);
sched_trace_rec_free(rec);
}
static void test_selection_policy_falls_back_to_core_picker(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl(NULL);
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_EQUAL_STRING("core-picker", rec->policy_name);
TEST_ASSERT_TRUE(in_arena(rec, rec->policy_name));
fx_free(fx);
TEST_ASSERT_EQUAL_STRING("core-picker", rec->policy_name);
sched_trace_rec_free(rec);
}
static void test_selection_job_without_queue(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
fx->job.queue = NULL;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_selection(rec, &fx->core, &fx->job);
TEST_ASSERT_TRUE(rec->selection_set);
TEST_ASSERT_NULL(rec->queue_name);
TEST_ASSERT_EQUAL_INT(0, rec->queue_priority);
TEST_ASSERT_EQUAL_INT(7, rec->job_priority);
TEST_ASSERT_EQUAL_INT64(1700000123, (int64_t)rec->queue_time);
fx_free(fx);
sched_trace_rec_free(rec);
}
static void test_selection_null_job_leaves_selection_unset(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_NOT_FOUND);
sched_trace_capture_selection(rec, &fx->core, NULL);
TEST_ASSERT_FALSE(rec->selection_set);
TEST_ASSERT_NULL(rec->queue_name);
fx_free(fx);
sched_trace_rec_free(rec);
}
/* ---- cluster-wide limits (2.3) --------------------------------------- */
static void test_glimits_copies_name_and_scope(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
int i;
for (i = 0; i < 3; i++) {
xlist_store(&fx->obj.glimits, &fx->vres[i]);
}
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
TEST_ASSERT_FALSE(rec->glimits_set);
sched_trace_capture_glimits(rec, &fx->obj);
TEST_ASSERT_TRUE(rec->glimits_set);
TEST_ASSERT_EQUAL_INT(3, rec->nglimits);
TEST_ASSERT_NOT_NULL(rec->glimits);
TEST_ASSERT_TRUE(in_arena(rec, rec->glimits));
TEST_ASSERT_EQUAL_STRING("lic_cap", rec->glimits[0].limit);
TEST_ASSERT_EQUAL_STRING("u=alice,q=normal", rec->glimits[0].scope);
TEST_ASSERT_EQUAL_STRING("site_total", rec->glimits[1].limit);
TEST_ASSERT_EQUAL_STRING("", rec->glimits[1].scope);
/* no back-pointer: parsed from the vres name */
TEST_ASSERT_EQUAL_STRING("orphan", rec->glimits[2].limit);
TEST_ASSERT_EQUAL_STRING("p=proj1", rec->glimits[2].scope);
for (i = 0; i < 3; i++) {
TEST_ASSERT_TRUE(in_arena(rec, rec->glimits[i].limit));
TEST_ASSERT_TRUE(in_arena(rec, rec->glimits[i].scope));
}
/* the source changes later: the record does not */
fx->lim_scoped.name[0] = 'Z';
fx->vres_res[0].name[8] = 'Z';
xlist_clear(&fx->obj.glimits);
fx_free(fx);
TEST_ASSERT_EQUAL_INT(3, rec->nglimits);
TEST_ASSERT_EQUAL_STRING("lic_cap", rec->glimits[0].limit);
TEST_ASSERT_EQUAL_STRING("u=alice,q=normal", rec->glimits[0].scope);
TEST_ASSERT_EQUAL_STRING("orphan", rec->glimits[2].limit);
TEST_ASSERT_FALSE(rec->incomplete);
sched_trace_rec_free(rec);
}
static void test_glimits_none_matched_is_known_empty(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_glimits(rec, &fx->obj);
TEST_ASSERT_TRUE(rec->glimits_set);
TEST_ASSERT_EQUAL_INT(0, rec->nglimits);
fx_free(fx);
sched_trace_rec_free(rec);
}
static void test_glimits_null_obj_leaves_limits_unavailable(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_NO_SCHEDOBJ);
sched_trace_capture_glimits(rec, NULL);
TEST_ASSERT_FALSE(rec->glimits_set);
TEST_ASSERT_EQUAL_INT(0, rec->nglimits);
fx_free(fx);
sched_trace_rec_free(rec);
}
static void test_glimits_skips_null_and_nameless_entries(void)
{
fixture_t* fx = fx_new();
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_vres_t bare;
xce_sched_trace_rec_t* rec;
memset(&bare, 0, sizeof(bare)); /* no vres, no limit: nothing to name */
xlist_store(&fx->obj.glimits, &bare);
xlist_store(&fx->obj.glimits, &fx->vres[1]);
rec = sched_trace_rec_new(&ctl, fx->job.jobid, XCE_TRACE_REC_DECISION);
sched_trace_capture_glimits(rec, &fx->obj);
TEST_ASSERT_EQUAL_INT(1, rec->nglimits);
TEST_ASSERT_EQUAL_STRING("site_total", rec->glimits[0].limit);
fx_free(fx);
sched_trace_rec_free(rec);
}
/* ---- attempt failure reasons (3.5) ----------------------------------- */
static sched_trace_entry_t* only_entry(xce_sched_trace_rec_t* rec)
{
TEST_ASSERT_EQUAL_INT(1, rec->nentries);
TEST_ASSERT_NOT_NULL(rec->entries);
TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_ATTEMPT_REASONS, rec->entries->type);
return rec->entries;
}
static void test_attempt_reasons_summarized_per_code(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55,
XCE_TRACE_REC_DECISION);
xce_pendreason_t* r = make_pendreason(0x55);
sched_trace_entry_t* e;
TEST_ASSERT_NOT_NULL(r);
add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n01", NULL);
add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n02", NULL);
add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n03", NULL);
add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n01", "lic");
add_job_preason(r->reasons, PREASON_NO_RESOURCE, "n04", "mem");
add_job_preason(r->reasons, PREASON_NO_RESOURCE, NULL, "lic"); /* shared */
sched_trace_capture_attempt_reasons(rec, r);
e = only_entry(rec);
/* per-code counts, in the order first seen */
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.ncodes);
TEST_ASSERT_EQUAL_INT(PREASON_MATCH_FAIL, e->u.reasons.codes[0].code);
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.codes[0].count);
TEST_ASSERT_EQUAL_INT(PREASON_NO_RESOURCE, e->u.reasons.codes[1].code);
TEST_ASSERT_EQUAL_INT(4, e->u.reasons.codes[1].count);
/* distinct named hosts (a shared-resource reason names none) */
TEST_ASSERT_EQUAL_INT(4, e->u.reasons.nhosts);
TEST_ASSERT_EQUAL_STRING("n01", e->u.reasons.hosts[0]);
TEST_ASSERT_EQUAL_STRING("n02", e->u.reasons.hosts[1]);
TEST_ASSERT_EQUAL_STRING("n03", e->u.reasons.hosts[2]);
TEST_ASSERT_EQUAL_STRING("n04", e->u.reasons.hosts[3]);
/* distinct named resources */
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.nres);
TEST_ASSERT_EQUAL_STRING("lic", e->u.reasons.res[0]);
TEST_ASSERT_EQUAL_STRING("mem", e->u.reasons.res[1]);
TEST_ASSERT_TRUE(in_arena(rec, e->u.reasons.hosts[0]));
TEST_ASSERT_TRUE(in_arena(rec, e->u.reasons.res[0]));
/* the scheduler replaces the stored reason afterwards: the record
* keeps its copy */
r->reasons->host_based_reasons[0]->host[0] = 'X';
free_pendreason(r);
TEST_ASSERT_EQUAL_INT(2, e->u.reasons.codes[0].count);
TEST_ASSERT_EQUAL_STRING("n01", e->u.reasons.hosts[0]);
TEST_ASSERT_EQUAL_STRING("mem", e->u.reasons.res[1]);
TEST_ASSERT_FALSE(rec->incomplete);
sched_trace_rec_free(rec);
}
static void test_attempt_reasons_main_reason_only(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55,
XCE_TRACE_REC_DECISION);
xce_pendreason_t* r = make_pendreason(0x55);
sched_trace_entry_t* e;
add_job_preason(r->reasons, PREASON_BAD_RESREQ, NULL, NULL);
sched_trace_capture_attempt_reasons(rec, r);
e = only_entry(rec);
TEST_ASSERT_EQUAL_INT(1, e->u.reasons.ncodes);
TEST_ASSERT_EQUAL_INT(PREASON_BAD_RESREQ, e->u.reasons.codes[0].code);
TEST_ASSERT_EQUAL_INT(1, e->u.reasons.codes[0].count);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nhosts);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nres);
free_pendreason(r);
sched_trace_rec_free(rec);
}
static void test_attempt_reasons_none_recorded(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec = sched_trace_rec_new(&ctl, 0x55,
XCE_TRACE_REC_DECISION);
xce_pendreason_t* r = make_pendreason(0x55);
sched_trace_entry_t* e;
/* a failed attempt with no reason set still gets its line */
sched_trace_capture_attempt_reasons(rec, NULL);
e = only_entry(rec);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.ncodes);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nhosts);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.nres);
sched_trace_rec_free(rec);
/* an empty reason set: same */
rec = sched_trace_rec_new(&ctl, 0x55, XCE_TRACE_REC_DECISION);
sched_trace_capture_attempt_reasons(rec, r);
e = only_entry(rec);
TEST_ASSERT_EQUAL_INT(0, e->u.reasons.ncodes);
free_pendreason(r);
sched_trace_rec_free(rec);
}
static void test_attempt_reasons_counts_against_the_budget(void)
{
xce_sched_trace_ctl_t ctl = make_ctl("fcfs");
xce_sched_trace_rec_t* rec;
xce_pendreason_t* r = make_pendreason(0x55);
int i;
ctl.line_bound = XCE_TRACE_LINE_BOUND_MIN;
rec = sched_trace_rec_new(&ctl, 0x55, XCE_TRACE_REC_DECISION);
add_job_preason(r->reasons, PREASON_MATCH_FAIL, "n01", NULL);
for (i = 0; i < XCE_TRACE_LINE_BOUND_MIN + 1; i++) {
sched_trace_capture_attempt_reasons(rec, r);
}
TEST_ASSERT_EQUAL_INT(XCE_TRACE_LINE_BOUND_MIN, rec->nentries);
TEST_ASSERT_EQUAL_INT(1, rec->counters.omitted_lines);
free_pendreason(r);
sched_trace_rec_free(rec);
}
int main(void)
{
UNITY_BEGIN();
RUN_TEST(test_selection_copies_ordering_keys);
RUN_TEST(test_selection_policy_ignores_queue_override);
RUN_TEST(test_selection_policy_falls_back_to_core_picker);
RUN_TEST(test_selection_job_without_queue);
RUN_TEST(test_selection_null_job_leaves_selection_unset);
RUN_TEST(test_glimits_copies_name_and_scope);
RUN_TEST(test_glimits_none_matched_is_known_empty);
RUN_TEST(test_glimits_null_obj_leaves_limits_unavailable);
RUN_TEST(test_glimits_skips_null_and_nameless_entries);
RUN_TEST(test_attempt_reasons_summarized_per_code);
RUN_TEST(test_attempt_reasons_main_reason_only);
RUN_TEST(test_attempt_reasons_none_recorded);
RUN_TEST(test_attempt_reasons_counts_against_the_budget);
return UNITY_END();
}