From d277bfb65749bd332f0ae04924dbe537210d272b Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 8 Oct 2026 08:21:57 +0000 Subject: [PATCH] feat(scheduler-sim-decision-trace): plan/verdict counters, search-end, summary tallies, rollback merging Task 1.5: per-requirement plan counts and search-end statements, decision-wide counters exact past truncation, summary-level position tallies (per verdict kind and per limit), merged rollback entries. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../scheduler-sim-decision-trace/tasks.md | 4 +- src/scheduler/include/sched/trace.h | 95 ++- src/scheduler/src/sched_trace.c | 320 +++++++- src/scheduler/testsuite/test_trace_model.c | 755 +++++++++++++++++- 4 files changed, 1159 insertions(+), 15 deletions(-) diff --git a/.kiro/specs/scheduler-sim-decision-trace/tasks.md b/.kiro/specs/scheduler-sim-decision-trace/tasks.md index 272f3cf2..80dde5ae 100644 --- a/.kiro/specs/scheduler-sim-decision-trace/tasks.md +++ b/.kiro/specs/scheduler-sim-decision-trace/tasks.md @@ -49,7 +49,7 @@ - Done when model tests pass under ASAN for nesting and balance, every entry kind, and budget edges (bound-1, bound and bound+1) with exact omitted counts. - _Requirements: 3.3, 3.4, 3.6, 4.3, 4.4, 4.5, 4.7, 4.8, 5.10, 9.1, 9.2_ -- [ ] 1.5 Add plan and verdict counters, search-end statements, summary tallies and rollback merging, test first +- [x] 1.5 Add plan and verdict counters, search-end statements, summary tallies and rollback merging, test first - The model counts generated, tried, discarded and not-tried plans for each requirement scope. - The search-end entry's statement follows the design's conditions: none generated, all discarded, or none placed. The decision-wide counters use the same conditions. - Per-verdict host evaluations and requirement outcomes stay exact after truncation. @@ -341,3 +341,5 @@ - 1.3: design's record header lists a "file tag", but `xce_sched_trace_ctl_t` has no such field; the 4.2/4.6 store/emit tasks must set it or drop it explicitly. - 1.3: trace sources (sched_trace.c, sched_trace_capture.c) may not include (core isolation test); capture copies raw strings/numbers and leaves formatting to the renderer. - 1.4: `sched_trace_host_verdict(rec, 0, err)` means "derive from rows" (rec->host_row_verdicts shortfall/limit, else SELECT, phase per-host); an explicit SELECT is the pre-filter phase. 2.2's row capture must set host_row_verdicts and 3.3/3.4 hooks must follow this. `sched_trace_pos_scan_error` stores no entry; it turns the next UNFILLED position end into SCAN_ERROR. Plan numbers restart per requirement. trace.h forward-declares `struct xce_alloc_plan_s` (tag added in alloc_iter.h) to avoid the core.h include loop. `resreq_begin` may store a NULL req name for a NULL rusage; 4.1's renderer must tolerate it. +- 1.5: the per-limit summary tally goes through `sched_trace_host_limit_hit(rec, limit, scope)` (added; not in the design's interface); 2.2 calls it for each unmet limit row in a host scope and keeps setting `host_row_verdicts |= SHORTFALL` directly. At summary level `pos_end` stores a POS_SUMMARY in place of POS_END, keeping the end state, error and host; 4.1 must render a scan error in that line. The search-end statement is written only when rc==0. +- 1.5 OPEN for 3.4/4.1: a requirement the allocator declines (alloc_runner.c ~474) or whose plan generator fails to start (~427, returns 0) never reaches `sched_trace_search_end` when that call sits only at the last return, so `reqs_none_generated` misses it (Req 4.7, 9.3); calling search_end on that path as well double-reports "no plan generated". Settle it in 3.4, e.g. `sched_trace_no_plan` bumps the counter and search_end skips its statement when a no-plan entry is open. `test_search_end_nonzero_rc_has_no_statement` models a generator failure as rc -12, but the runner returns 0 there. diff --git a/src/scheduler/include/sched/trace.h b/src/scheduler/include/sched/trace.h index 7c04e6b0..689d56a8 100644 --- a/src/scheduler/include/sched/trace.h +++ b/src/scheduler/include/sched/trace.h @@ -141,8 +141,8 @@ typedef struct { * Entry types (design.md "Logical Data Model"). Every stored entry renders * as exactly one line in the budgeted sections (requirement, cluster-wide * check, plan and host), so the number of stored entries is the number of - * budgeted lines. The check rows, host rows, search-end, position-summary - * and rollback entries are added by the tasks that capture them. + * budgeted lines. The check rows and host rows are added by the task that + * captures them (TraceCapture). */ typedef enum { SCHED_TRACE_E_RESREQ_BEGIN = 0, /* requirement named by the path */ @@ -163,6 +163,11 @@ typedef enum { SCHED_TRACE_E_HOST, /* host verdict (5.1-5.7, 5.10) */ SCHED_TRACE_E_COMMIT, /* committed plan (4.5) */ SCHED_TRACE_E_FINALIZE_FAILED, /* allocation build failed (4.5) */ + SCHED_TRACE_E_SEARCH_END, /* plan counts of a requirement, and + * the statement (4.6, 4.7, 4.9) */ + SCHED_TRACE_E_POS_SUMMARY, /* summary level: the position end with + * its verdict counts (5.9) */ + SCHED_TRACE_E_ROLLBACK, /* released hosts, merged (5.8) */ SCHED_TRACE_E_NTYPES } sched_trace_entry_type_t; @@ -174,6 +179,24 @@ typedef enum { SCHED_TRACE_PHASE_PERHOST /* per-host resource and limit check */ } sched_trace_phase_t; +/* The statement a search-end line ends with (design "TraceHooks", search + * end). At most one; none when the search returned rc != 0. */ +typedef enum { + SCHED_TRACE_SEARCH_NO_STMT = 0, + SCHED_TRACE_SEARCH_NONE_GENERATED, /* g == 0 (4.7) */ + SCHED_TRACE_SEARCH_ALL_DISCARDED, /* g > 0 and d == g (4.9) */ + SCHED_TRACE_SEARCH_NONE_PLACED /* t > 0, no plan ended in an + * internal error (4.6) */ +} sched_trace_search_stmt_t; + +/* Hosts of one position that hit one limit (5.9): `count` hosts with a + * limit-exceeded verdict hit the limit `limit` in scope instance `scope`. */ +typedef struct { + const char* limit; /* arena copy */ + const char* scope; /* arena copy */ + int count; +} sched_trace_limit_tally_t; + /* One requested resource of a single requirement or member (3.2). */ typedef struct { const char* name; /* arena copy */ @@ -268,6 +291,30 @@ typedef struct sched_trace_entry_s { int plan_no; /* the last plan begun */ int err; /* FINALIZE_FAILED only */ } commit; + struct { /* SEARCH_END: plans of the requirement's search */ + int generated; /* tried+discarded+not tried */ + int tried; + int discarded; + int not_tried; + int rc; + sched_trace_search_stmt_t stmt; + } search_end; + struct { /* POS_SUMMARY: the POS_END fields plus the host tallies */ + int pos; + xce_sched_trace_pos_end_t how; + const char* host; /* the accepted host, or NULL */ + int err; + int counts[XCE_TRACE_V_NKINDS]; /* hosts + * per verdict kind (bit pos) */ + int nlimits; + sched_trace_limit_tally_t* limits; /* hits per limit, in the + * order first hit */ + } pos_summary; + struct { /* ROLLBACK: consecutive released hosts, in call order */ + int nhosts; + int cap; + const char** hosts; + } rollback; } u; } sched_trace_entry_t; @@ -319,6 +366,25 @@ struct xce_sched_trace_rec_s { bool scan_err_set; unsigned host_row_verdicts; /* SHORTFALL/LIMIT from the current host's * unmet rows (set by the row capture) */ + /* plans of the current requirement's search (reset by resreq_begin + * and after search_end) */ + int scope_tried; + int scope_discarded; + int scope_not_tried; + bool scope_plan_err; /* a tried plan ended in an internal error */ + /* summary level: the current position's host tallies. The limit + * tallies live in builder memory (malloc'd, not the arena) until the + * position ends, so an omitted position allocates nothing in the + * record; they are not counted by sched_trace_rec_bytes. */ + int pos_counts[XCE_TRACE_V_NKINDS]; + struct sched_trace_tally_build_s* pos_limits; + int pos_nlimits; + int pos_limits_cap; + unsigned host_seq; /* numbers host scopes, for one hit per host */ + /* the ROLLBACK entry the last call opened, while no other call has + * been made since (NULL when that entry was omitted) */ + bool rollback_open; + sched_trace_entry_t* rollback_e; /* scope pointers, kept only until the matching scope ends and never * dereferenced after it (TraceCapture's limit lookup) */ const xce_sched_obj_t* scope_obj; @@ -450,8 +516,33 @@ extern void sched_trace_host_begin(xce_sched_trace_rec_t* rec, extern void sched_trace_host_verdict(xce_sched_trace_rec_t* rec, unsigned verdicts, int err); +/* + * The current host's per-host check found limit `limit` exceeded in scope + * instance `scope` (both copied). Sets LIMIT in the host's row verdicts + * and, at summary level, tallies the limit for the position summary once + * per host whose verdict is limit exceeded (5.9). Outside a host scope it + * does nothing. The model-side seam of TraceCapture's limit rows (2.2). + */ +extern void sched_trace_host_limit_hit(xce_sched_trace_rec_t* rec, + const char* limit, const char* scope); + +/* One host whose claim a failed plan released. Consecutive calls merge + * into one ROLLBACK entry, "released hosts: n01, n02" (5.8). */ +extern void sched_trace_rollback_host(xce_sched_trace_rec_t* rec, + const char* host); + /* The plan begun last was committed, or building its allocation failed. */ extern void sched_trace_commit(xce_sched_trace_rec_t* rec); extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err); +/* + * End of a requirement's plan search with its result `rc`: one line with + * the plans generated, tried, discarded and not tried since the + * requirement began, and at most one statement (rc == 0 only): no plan + * generated (g == 0), all discarded (g > 0, d == g), or none placed (t > 0 + * and no tried plan ended in an internal error). The matching decision- + * wide counter is bumped under the same conditions, stored or not (9.3). + */ +extern void sched_trace_search_end(xce_sched_trace_rec_t* rec, int rc); + #endif diff --git a/src/scheduler/src/sched_trace.c b/src/scheduler/src/sched_trace.c index c8ff2697..561cb697 100644 --- a/src/scheduler/src/sched_trace.c +++ b/src/scheduler/src/sched_trace.c @@ -18,6 +18,13 @@ * nesting level, plan numbers, scope pointers) is kept whether or not an * entry is stored, so a truncated record still nests correctly. * + * Counters: the per-requirement plan counts behind the search-end line and + * the decision-wide counters (plans tried and discarded, requirement + * outcomes, host evaluations per verdict kind) are updated on every call, + * stored or not, so they stay exact after truncation (9.3). At summary + * level host verdicts are not stored: they are tallied per position, and + * the position's end becomes one position-summary entry (5.9). + * * No I/O and no logging: test_core_isolation.c scans this file for stdio * and file I/O. */ @@ -50,6 +57,34 @@ static unsigned char* chunk_data(sched_trace_chunk_t* c) return ((unsigned char*)c) + chunk_hdr_size(); } +/* Summary level: one limit hit by hosts of the current position. Builder + * memory, owned by the record until the position ends. */ +struct sched_trace_tally_build_s { + char* limit; + char* scope; + int count; /* hosts with a limit-exceeded verdict */ + unsigned last_host; /* host_seq of the last host that hit it */ +}; + +/* Drops the current position's limit tallies (keeps the array). */ +static void pos_limits_clear(xce_sched_trace_rec_t* rec) +{ + int k; + + for (k = 0; k < rec->pos_nlimits; k++) { + free(rec->pos_limits[k].limit); + free(rec->pos_limits[k].scope); + } + rec->pos_nlimits = 0; +} + +/* Starts the current position's tallies over. */ +static void pos_tally_reset(xce_sched_trace_rec_t* rec) +{ + memset(rec->pos_counts, 0, sizeof(rec->pos_counts)); + pos_limits_clear(rec); +} + bool sched_trace_ctl_wants(const xce_sched_trace_ctl_t* ctl, uint64_t jobid) { int i; @@ -174,6 +209,7 @@ xce_sched_trace_rec_t* sched_trace_rec_new(const xce_sched_trace_ctl_t* ctl, rec->session_no = ctl->session_no; rec->position = ctl->position; rec->line_bound = ctl->line_bound; + rec->host_seq = 1; /* 0: no host yet */ if (ctl->policy_name != NULL) { rec->policy_name = sched_trace_arena_strdup(rec, ctl->policy_name); @@ -196,6 +232,9 @@ void sched_trace_rec_free(xce_sched_trace_rec_t* rec) return; } + pos_limits_clear(rec); + free(rec->pos_limits); + c = rec->arena; while (c != NULL) { sched_trace_chunk_t* next = c->next; @@ -251,6 +290,10 @@ static sched_trace_entry_t* entry_add(xce_sched_trace_rec_t* rec, { sched_trace_entry_t* e; + /* any entry, stored or not, ends a run of released hosts */ + rec->rollback_open = false; + rec->rollback_e = NULL; + if (rec->incomplete) { return NULL; } @@ -278,6 +321,13 @@ static sched_trace_entry_t* entry_add(xce_sched_trace_rec_t* rec, return e; } +/* A call that stores no entry still ends a run of released hosts. */ +static void rollback_run_end(xce_sched_trace_rec_t* rec) +{ + rec->rollback_open = false; + rec->rollback_e = NULL; +} + /* A scope begin is stored at the current level and opens the next one. */ static sched_trace_entry_t* scope_begin(xce_sched_trace_rec_t* rec, sched_trace_entry_type_t type) @@ -332,6 +382,7 @@ void sched_trace_path_push(xce_sched_trace_rec_t* rec, { sched_trace_path_t* p = &rec->path; + rollback_run_end(rec); if (p->depth >= 2) { /* deeper than container > member: not kept, but balanced */ rec->path_overflow++; @@ -345,6 +396,7 @@ void sched_trace_path_push(xce_sched_trace_rec_t* rec, void sched_trace_path_pop(xce_sched_trace_rec_t* rec) { + rollback_run_end(rec); if (rec->path_overflow > 0) { rec->path_overflow--; return; @@ -363,8 +415,12 @@ void sched_trace_resreq_begin(xce_sched_trace_rec_t* rec, int k; rec->resreq_open++; - rec->plan_no = 0; /* plans are numbered per requirement */ + rec->plan_no = 0; /* plans are numbered and counted per requirement */ rec->cur_plan_no = 0; + rec->scope_tried = 0; + rec->scope_discarded = 0; + rec->scope_not_tried = 0; + rec->scope_plan_err = false; e = scope_begin(rec, SCHED_TRACE_E_RESREQ_BEGIN); if (e == NULL) { @@ -488,6 +544,7 @@ void sched_trace_attempt_reasons(xce_sched_trace_rec_t* rec, void sched_trace_check_begin(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj) { + rollback_run_end(rec); rec->scope_obj = obj; rec->scope_w = NULL; } @@ -537,6 +594,8 @@ void sched_trace_plan_begin(xce_sched_trace_rec_t* rec, sched_trace_entry_t* e; rec->cur_plan_no = ++rec->plan_no; + rec->scope_tried++; + rec->counters.plans_tried++; e = scope_begin(rec, SCHED_TRACE_E_PLAN_BEGIN); if (e != NULL) { plan_fill(rec, e, plan, rec->cur_plan_no, 0); @@ -549,6 +608,8 @@ void sched_trace_plan_discarded(xce_sched_trace_rec_t* rec, sched_trace_entry_t* e; rec->plan_no++; + rec->scope_discarded++; + rec->counters.plans_discarded++; e = entry_add(rec, SCHED_TRACE_E_PLAN_DISCARDED); if (e != NULL) { plan_fill(rec, e, plan, rec->plan_no, 0); @@ -561,6 +622,7 @@ void sched_trace_plan_not_tried(xce_sched_trace_rec_t* rec, sched_trace_entry_t* e; rec->plan_no++; + rec->scope_not_tried++; e = entry_add(rec, SCHED_TRACE_E_PLAN_NOT_TRIED); if (e != NULL) { plan_fill(rec, e, plan, rec->plan_no, err); @@ -569,8 +631,13 @@ void sched_trace_plan_not_tried(xce_sched_trace_rec_t* rec, void sched_trace_plan_end(xce_sched_trace_rec_t* rec, int rc, int failed_pos) { - sched_trace_entry_t* e = scope_end(rec, SCHED_TRACE_E_PLAN_END); + sched_trace_entry_t* e; + if (rc < 0) { + rec->scope_plan_err = true; + } + + e = scope_end(rec, SCHED_TRACE_E_PLAN_END); if (e == NULL) { return; } @@ -586,6 +653,7 @@ void sched_trace_pos_begin(xce_sched_trace_rec_t* rec, int pos, int slots) rec->cur_pos = pos; rec->scan_err = 0; rec->scan_err_set = false; + pos_tally_reset(rec); e = scope_begin(rec, SCHED_TRACE_E_POS_BEGIN); if (e == NULL) { @@ -597,10 +665,57 @@ void sched_trace_pos_begin(xce_sched_trace_rec_t* rec, int pos, int slots) void sched_trace_pos_scan_error(xce_sched_trace_rec_t* rec, int err) { + rollback_run_end(rec); rec->scan_err = err; rec->scan_err_set = true; } +/* Fills a POS_SUMMARY entry from the current position's tallies; only + * limits some host was rejected for are kept. */ +static void pos_summary_fill(xce_sched_trace_rec_t* rec, sched_trace_entry_t* e, + xce_sched_trace_pos_end_t how, const char* host, + int err) +{ + int n = 0; + int k; + + e->u.pos_summary.pos = rec->cur_pos; + e->u.pos_summary.how = how; + e->u.pos_summary.err = err; + e->u.pos_summary.host = sched_trace_arena_strdup(rec, host); + memcpy(e->u.pos_summary.counts, rec->pos_counts, + sizeof(e->u.pos_summary.counts)); + + for (k = 0; k < rec->pos_nlimits; k++) { + if (rec->pos_limits[k].count > 0) { + n++; + } + } + if (n == 0) { + return; + } + e->u.pos_summary.limits = (sched_trace_limit_tally_t*) + sched_trace_arena_alloc(rec, sizeof(sched_trace_limit_tally_t) + * (size_t)n); + if (e->u.pos_summary.limits == NULL) { + return; + } + n = 0; + for (k = 0; k < rec->pos_nlimits; k++) { + const struct sched_trace_tally_build_s* b = &rec->pos_limits[k]; + sched_trace_limit_tally_t* t; + + if (b->count <= 0) { + continue; + } + t = &e->u.pos_summary.limits[n++]; + t->limit = sched_trace_arena_strdup(rec, b->limit); + t->scope = sched_trace_arena_strdup(rec, b->scope); + t->count = b->count; + } + e->u.pos_summary.nlimits = n; +} + void sched_trace_pos_end(xce_sched_trace_rec_t* rec, xce_sched_trace_pos_end_t how, const char* host) { @@ -617,6 +732,16 @@ void sched_trace_pos_end(xce_sched_trace_rec_t* rec, rec->scan_err = 0; rec->scan_err_set = false; + if (rec->level == XCE_TRACE_LEVEL_SUMMARY) { + /* the position's end carries its host tallies (5.9) */ + e = scope_end(rec, SCHED_TRACE_E_POS_SUMMARY); + if (e != NULL) { + pos_summary_fill(rec, e, how, host, err); + } + pos_tally_reset(rec); + return; + } + e = scope_end(rec, SCHED_TRACE_E_POS_END); if (e == NULL) { return; @@ -631,11 +756,36 @@ void sched_trace_host_begin(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj, const xce_sched_worker_t* w) { + rollback_run_end(rec); + rec->host_seq++; rec->scope_obj = obj; rec->scope_w = w; rec->host_row_verdicts = 0; } +/* Counts one host evaluation: decision-wide per verdict kind, and at + * summary level per position, with each limit the host hit counted once + * when its verdict is limit exceeded (5.9, 9.3). */ +static void host_count(xce_sched_trace_rec_t* rec, unsigned verdicts) +{ + int k; + + for (k = 0; k < XCE_TRACE_V_NKINDS; k++) { + if ((verdicts & (1u << k)) != 0) { + rec->counters.host_evals[k]++; + rec->pos_counts[k]++; + } + } + if ((verdicts & XCE_TRACE_V_LIMIT) == 0) { + return; + } + for (k = 0; k < rec->pos_nlimits; k++) { + if (rec->pos_limits[k].last_host == rec->host_seq) { + rec->pos_limits[k].count++; + } + } +} + void sched_trace_host_verdict(xce_sched_trace_rec_t* rec, unsigned verdicts, int err) { @@ -659,10 +809,19 @@ void sched_trace_host_verdict(xce_sched_trace_rec_t* rec, unsigned verdicts, phase = SCHED_TRACE_PHASE_PERHOST; } + /* one host evaluation, counted once under each kind it matched (9.3) */ + host_count(rec, verdicts); + /* the host scope ends here */ rec->scope_obj = NULL; rec->scope_w = NULL; rec->host_row_verdicts = 0; + rec->host_seq++; /* later hits belong to no host until the next one */ + rollback_run_end(rec); + + if (rec->level == XCE_TRACE_LEVEL_SUMMARY) { + return; /* tallied for the position summary, no host line */ + } e = entry_add(rec, SCHED_TRACE_E_HOST); if (e == NULL) { @@ -694,3 +853,160 @@ void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err) e->u.commit.plan_no = rec->cur_plan_no; e->u.commit.err = err; } + + +static bool tally_str_eq(const char* a, const char* b) +{ + if ((a == NULL) || (b == NULL)) { + return a == b; + } + return strcmp(a, b) == 0; +} + +/* NULL-preserving heap copy for the builder's tallies; sets *failed when a + * non-NULL string could not be copied. */ +static char* tally_strdup(const char* s, bool* failed) +{ + char* d; + + if (s == NULL) { + return NULL; + } + d = strdup(s); + if (d == NULL) { + *failed = true; + } + return d; +} + +void sched_trace_host_limit_hit(xce_sched_trace_rec_t* rec, const char* limit, + const char* scope) +{ + struct sched_trace_tally_build_s* b; + bool failed = false; + int k; + + rollback_run_end(rec); + if (rec->scope_w == NULL) { + return; /* not a host scope: the cluster-wide check's rows */ + } + rec->host_row_verdicts |= XCE_TRACE_V_LIMIT; + if ((rec->level != XCE_TRACE_LEVEL_SUMMARY) || rec->incomplete) { + return; /* full level shows the rows themselves (2.2) */ + } + + for (k = 0; k < rec->pos_nlimits; k++) { + b = &rec->pos_limits[k]; + if (tally_str_eq(b->limit, limit) && tally_str_eq(b->scope, scope)) { + b->last_host = rec->host_seq; /* once per host, at its verdict */ + return; + } + } + + if (rec->pos_nlimits == rec->pos_limits_cap) { + int cap = (rec->pos_limits_cap > 0) ? rec->pos_limits_cap * 2 : 4; + void* p = realloc(rec->pos_limits, sizeof(*b) * (size_t)cap); + + if (p == NULL) { + rec->incomplete = true; + return; + } + rec->pos_limits = (struct sched_trace_tally_build_s*)p; + rec->pos_limits_cap = cap; + } + b = &rec->pos_limits[rec->pos_nlimits]; + b->limit = tally_strdup(limit, &failed); + b->scope = tally_strdup(scope, &failed); + b->count = 0; + b->last_host = rec->host_seq; + if (failed) { + free(b->limit); + free(b->scope); + rec->incomplete = true; + return; + } + rec->pos_nlimits++; +} + +/* Appends one host to a ROLLBACK entry, growing its array in the arena. */ +static void rollback_append(xce_sched_trace_rec_t* rec, sched_trace_entry_t* e, + const char* host) +{ + if (e->u.rollback.nhosts == e->u.rollback.cap) { + int cap = (e->u.rollback.cap > 0) ? e->u.rollback.cap * 2 : 4; + const char** h = (const char**)sched_trace_arena_alloc( + rec, sizeof(char*) * (size_t)cap); + + if (h == NULL) { + return; + } + if (e->u.rollback.nhosts > 0) { + memcpy(h, e->u.rollback.hosts, + sizeof(char*) * (size_t)e->u.rollback.nhosts); + } + e->u.rollback.hosts = h; + e->u.rollback.cap = cap; + } + e->u.rollback.hosts[e->u.rollback.nhosts++] + = sched_trace_arena_strdup(rec, host); +} + +void sched_trace_rollback_host(xce_sched_trace_rec_t* rec, const char* host) +{ + sched_trace_entry_t* e; + + if (rec->rollback_open) { + /* the same run of released hosts: the same line */ + if ((rec->rollback_e != NULL) && !rec->incomplete) { + rollback_append(rec, rec->rollback_e, host); + } + return; + } + + e = entry_add(rec, SCHED_TRACE_E_ROLLBACK); + rec->rollback_open = true; /* also when omitted: one line either way */ + rec->rollback_e = e; + if (e != NULL) { + rollback_append(rec, e, host); + } +} + +void sched_trace_search_end(xce_sched_trace_rec_t* rec, int rc) +{ + sched_trace_entry_t* e; + sched_trace_search_stmt_t stmt = SCHED_TRACE_SEARCH_NO_STMT; + int t = rec->scope_tried; + int d = rec->scope_discarded; + int x = rec->scope_not_tried; + int g = t + d + x; + + if (rc == 0) { + if (g == 0) { + stmt = SCHED_TRACE_SEARCH_NONE_GENERATED; + rec->counters.reqs_none_generated++; + } else if (d == g) { + stmt = SCHED_TRACE_SEARCH_ALL_DISCARDED; + rec->counters.reqs_all_discarded++; + } else if ((t > 0) && !rec->scope_plan_err) { + stmt = SCHED_TRACE_SEARCH_NONE_PLACED; + rec->counters.reqs_none_placed++; + } + } + + /* the search is over: a later search in this requirement counts anew */ + rec->scope_tried = 0; + rec->scope_discarded = 0; + rec->scope_not_tried = 0; + rec->scope_plan_err = false; + + e = entry_add(rec, SCHED_TRACE_E_SEARCH_END); + if (e == NULL) { + return; + } + e->u.search_end.generated = g; + e->u.search_end.tried = t; + e->u.search_end.discarded = d; + e->u.search_end.not_tried = x; + e->u.search_end.rc = rc; + e->u.search_end.stmt = stmt; +} diff --git a/src/scheduler/testsuite/test_trace_model.c b/src/scheduler/testsuite/test_trace_model.c index fe219fdc..e825a52a 100644 --- a/src/scheduler/testsuite/test_trace_model.c +++ b/src/scheduler/testsuite/test_trace_model.c @@ -21,8 +21,20 @@ * - the line budget at bound-1, bound and bound+1, with the exact * omitted count (9.1, 9.2). * + * Task 1.5 adds the counters and the merged entries: + * - per-requirement plan counts and the search-end statement: none + * generated, all discarded, none placed, including a plan not tried + * after a discarded one (4.6, 4.7, 4.9); + * - decision-wide counters (plans, requirement outcomes, host + * evaluations per verdict kind) stay exact after truncation (9.3); + * - at summary level a position-summary entry replaces the host lines, + * counting a host with several verdict kinds once under each and + * tallying hits per limit (5.9, 5.10); + * - consecutive released-host calls merge into one rollback entry (5.8). + * * Links only src/sched_trace.c (plus libxce for the jobid helpers). */ +#include #include #include @@ -417,15 +429,21 @@ static void test_finish_rejects_non_no_decision_kinds(void) /* ---- entry sequence (task 1.4) ---------------------------------------- */ -static xce_sched_trace_rec_t* new_rec(int line_bound) +static xce_sched_trace_rec_t* new_rec_level(int line_bound, + xce_sched_trace_level_t level) { xce_sched_trace_ctl_t ctl = make_ctl(NULL, 0); - ctl.level = XCE_TRACE_LEVEL_FULL; + ctl.level = level; ctl.line_bound = line_bound; return sched_trace_rec_new(&ctl, PLAIN_ID(5), XCE_TRACE_REC_DECISION); } +static xce_sched_trace_rec_t* new_rec(int line_bound) +{ + return new_rec_level(line_bound, XCE_TRACE_LEVEL_FULL); +} + static int count_entries(const xce_sched_trace_rec_t* rec) { const sched_trace_entry_t* e; @@ -1094,16 +1112,19 @@ static int emit_every_kind(xce_sched_trace_rec_t* rec) sched_trace_host_verdict(rec, XCE_TRACE_V_ACCEPTED, 0); /* 10 */ sched_trace_pos_end(rec, XCE_TRACE_POS_FILLED, hname); /* 11 */ sched_trace_plan_end(rec, 1, 0); /* 12 */ - sched_trace_commit(rec); /* 13 */ - sched_trace_finalize_failed(rec, -5); /* 14 */ - sched_trace_resreq_end(rec, 1, 0); /* 15 */ + sched_trace_rollback_host(rec, hname); /* 13 */ + sched_trace_rollback_host(rec, hname); /* merged */ + sched_trace_commit(rec); /* 14 */ + sched_trace_finalize_failed(rec, -5); /* 15 */ + sched_trace_search_end(rec, 1); /* 16 */ + sched_trace_resreq_end(rec, 1, 0); /* 17 */ sched_trace_path_pop(rec); - sched_trace_member_failed(rec, 1, 1, -12); /* 16 */ - sched_trace_attempt_reasons(rec, &rc, 1, NULL, 0, NULL, 0); /* 17 */ - sched_trace_resreq_end(rec, 0, 0); /* 18 */ + sched_trace_member_failed(rec, 1, 1, -12); /* 18 */ + sched_trace_attempt_reasons(rec, &rc, 1, NULL, 0, NULL, 0); /* 19 */ + sched_trace_resreq_end(rec, 0, 0); /* 20 */ sched_trace_path_pop(rec); - sched_trace_alt_result(rec, 0, 2); /* 19 */ - return 19; + sched_trace_alt_result(rec, 0, 2); /* 21 */ + return 21; } static void test_every_entry_kind_is_one_entry(void) @@ -1128,6 +1149,20 @@ static void test_every_entry_kind_is_one_entry(void) TEST_ASSERT_TRUE(e->type < SCHED_TRACE_E_NTYPES); seen[e->type]++; } + TEST_ASSERT_EQUAL_INT(0, seen[SCHED_TRACE_E_POS_SUMMARY]); + sched_trace_rec_free(rec); + + /* summary level: the position summary replaces the host lines and the + * position end, so the same calls make one entry fewer */ + rec = new_rec_level(XCE_TRACE_LINE_BOUND_DEFAULT, XCE_TRACE_LEVEL_SUMMARY); + TEST_ASSERT_NOT_NULL(rec); + TEST_ASSERT_EQUAL_INT(n, emit_every_kind(rec)); + TEST_ASSERT_EQUAL_INT(n - 1, rec->nentries); + for (e = rec->entries; e != NULL; e = e->next) { + TEST_ASSERT_TRUE(e->type != SCHED_TRACE_E_HOST); + TEST_ASSERT_TRUE(e->type != SCHED_TRACE_E_POS_END); + seen[e->type]++; + } for (t = 0; t < SCHED_TRACE_E_NTYPES; t++) { TEST_ASSERT_TRUE_MESSAGE(seen[t] > 0, "entry kind never stored"); } @@ -1236,6 +1271,692 @@ static void test_budget_cut_inside_a_scope(void) sched_trace_rec_free(rec); } +/* ---- plan counts and search-end statements (4.6, 4.7, 4.9) ------------ */ + +/* Counter slot of a verdict kind (its bit position). */ +static int vi(unsigned verdict) +{ + return __builtin_ctz(verdict); +} + +static void assert_search_end(const sched_trace_entry_t* e, int g, int t, + int d, int x, int rc, + sched_trace_search_stmt_t stmt) +{ + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_SEARCH_END, e->type); + TEST_ASSERT_EQUAL_INT_MESSAGE(g, e->u.search_end.generated, "generated"); + TEST_ASSERT_EQUAL_INT_MESSAGE(t, e->u.search_end.tried, "tried"); + TEST_ASSERT_EQUAL_INT_MESSAGE(d, e->u.search_end.discarded, "discarded"); + TEST_ASSERT_EQUAL_INT_MESSAGE(x, e->u.search_end.not_tried, "not tried"); + TEST_ASSERT_EQUAL_INT(rc, e->u.search_end.rc); + TEST_ASSERT_EQUAL_INT_MESSAGE(stmt, e->u.search_end.stmt, "statement"); +} + +static void assert_req_counters(const xce_sched_trace_rec_t* rec, int gen0, + int alldisc, int noplace) +{ + TEST_ASSERT_EQUAL_INT_MESSAGE(gen0, rec->counters.reqs_none_generated, + "reqs_none_generated"); + TEST_ASSERT_EQUAL_INT_MESSAGE(alldisc, rec->counters.reqs_all_discarded, + "reqs_all_discarded"); + TEST_ASSERT_EQUAL_INT_MESSAGE(noplace, rec->counters.reqs_none_placed, + "reqs_none_placed"); +} + +/* Opens a SINGLE requirement with its cluster-wide check passed. */ +static void open_single(xce_sched_trace_rec_t* rec, fixture_primary_t* fp) +{ + make_primary(fp, 1, 1); + sched_trace_path_push(rec, XCE_TRACE_PATH_SINGLE, 1, 1); + sched_trace_resreq_begin(rec, &fp->p, 0); + sched_trace_check_begin(rec, NULL); + sched_trace_check_end(rec, 1, true); +} + +static void close_single(xce_sched_trace_rec_t* rec, int rc) +{ + sched_trace_resreq_end(rec, rc, 0); + sched_trace_path_pop(rec); +} + +static void test_search_end_none_generated(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + + TEST_ASSERT_NOT_NULL(rec); + open_single(rec, &fp); + sched_trace_no_plan(rec, XCE_TRACE_NOPLAN_NONE_GENERATED); + sched_trace_search_end(rec, 0); + assert_search_end(rec->entries_tail, 0, 0, 0, 0, 0, + SCHED_TRACE_SEARCH_NONE_GENERATED); + TEST_ASSERT_EQUAL_INT(1, rec->entries_tail->depth); /* in the req */ + assert_path1(rec->entries_tail, XCE_TRACE_PATH_SINGLE, 1, 1); + close_single(rec, 0); + assert_req_counters(rec, 1, 0, 0); + TEST_ASSERT_EQUAL_INT(0, rec->counters.plans_tried); + TEST_ASSERT_EQUAL_INT(0, rec->counters.plans_discarded); + sched_trace_rec_free(rec); +} + +static void test_search_end_all_discarded(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + int s[1] = { 1 }; + xce_alloc_plan_t plan = make_plan(s, 1); + + TEST_ASSERT_NOT_NULL(rec); + open_single(rec, &fp); + sched_trace_plan_discarded(rec, &plan); + sched_trace_plan_discarded(rec, &plan); + sched_trace_search_end(rec, 0); + assert_search_end(rec->entries_tail, 2, 0, 2, 0, 0, + SCHED_TRACE_SEARCH_ALL_DISCARDED); + close_single(rec, 0); + assert_req_counters(rec, 0, 1, 0); + TEST_ASSERT_EQUAL_INT(0, rec->counters.plans_tried); + TEST_ASSERT_EQUAL_INT(2, rec->counters.plans_discarded); + sched_trace_rec_free(rec); +} + +static void test_search_end_none_placed(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + int s[1] = { 1 }; + xce_alloc_plan_t plan = make_plan(s, 1); + + TEST_ASSERT_NOT_NULL(rec); + open_single(rec, &fp); + sched_trace_plan_discarded(rec, &plan); + sched_trace_plan_begin(rec, &plan); + sched_trace_pos_begin(rec, 1, 1); + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); + sched_trace_plan_end(rec, 0, 1); + sched_trace_plan_begin(rec, &plan); + sched_trace_plan_end(rec, 0, 1); + sched_trace_search_end(rec, 0); + assert_search_end(rec->entries_tail, 3, 2, 1, 0, 0, + SCHED_TRACE_SEARCH_NONE_PLACED); + close_single(rec, 0); + assert_req_counters(rec, 0, 0, 1); + TEST_ASSERT_EQUAL_INT(2, rec->counters.plans_tried); + TEST_ASSERT_EQUAL_INT(1, rec->counters.plans_discarded); + sched_trace_rec_free(rec); +} + +/* A tried plan that ended in an internal error: no "none placed", whatever + * rc the search returns. */ +static void test_search_end_plan_internal_error_has_no_statement(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + int s[1] = { 1 }; + xce_alloc_plan_t plan = make_plan(s, 1); + + TEST_ASSERT_NOT_NULL(rec); + open_single(rec, &fp); + sched_trace_plan_begin(rec, &plan); + sched_trace_plan_end(rec, 0, 1); + sched_trace_plan_begin(rec, &plan); + sched_trace_plan_end(rec, -5, 1); + sched_trace_search_end(rec, 0); + assert_search_end(rec->entries_tail, 2, 2, 0, 0, 0, + SCHED_TRACE_SEARCH_NO_STMT); + close_single(rec, 0); + + open_single(rec, &fp); + sched_trace_plan_begin(rec, &plan); + sched_trace_plan_end(rec, -5, 1); + sched_trace_search_end(rec, -5); + assert_search_end(rec->entries_tail, 1, 1, 0, 0, -5, + SCHED_TRACE_SEARCH_NO_STMT); + close_single(rec, -5); + + assert_req_counters(rec, 0, 0, 0); + TEST_ASSERT_EQUAL_INT(3, rec->counters.plans_tried); + sched_trace_rec_free(rec); +} + +/* A plan not tried (internal error) after a discarded one: not every + * generated plan was discarded, and none was tried, so no statement. */ +static void test_search_end_not_tried_after_discarded(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + int s[1] = { 1 }; + xce_alloc_plan_t plan = make_plan(s, 1); + + TEST_ASSERT_NOT_NULL(rec); + open_single(rec, &fp); + sched_trace_plan_discarded(rec, &plan); + sched_trace_plan_not_tried(rec, &plan, -12); + sched_trace_search_end(rec, -12); + assert_search_end(rec->entries_tail, 2, 0, 1, 1, -12, + SCHED_TRACE_SEARCH_NO_STMT); + close_single(rec, -12); + + /* the same plans with rc 0 still make no statement: d != g, t == 0 */ + open_single(rec, &fp); + sched_trace_plan_discarded(rec, &plan); + sched_trace_plan_not_tried(rec, &plan, -12); + sched_trace_search_end(rec, 0); + assert_search_end(rec->entries_tail, 2, 0, 1, 1, 0, + SCHED_TRACE_SEARCH_NO_STMT); + close_single(rec, 0); + + assert_req_counters(rec, 0, 0, 0); + TEST_ASSERT_EQUAL_INT(0, rec->counters.plans_tried); + TEST_ASSERT_EQUAL_INT(2, rec->counters.plans_discarded); + sched_trace_rec_free(rec); +} + +/* rc > 0 (committed) and rc < 0 never carry a statement, even when the + * counts alone would match one. */ +static void test_search_end_nonzero_rc_has_no_statement(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + int s[1] = { 1 }; + xce_alloc_plan_t plan = make_plan(s, 1); + + TEST_ASSERT_NOT_NULL(rec); + open_single(rec, &fp); + sched_trace_plan_discarded(rec, &plan); + sched_trace_plan_begin(rec, &plan); + sched_trace_plan_end(rec, 1, 0); + sched_trace_commit(rec); + sched_trace_search_end(rec, 1); + assert_search_end(rec->entries_tail, 2, 1, 1, 0, 1, + SCHED_TRACE_SEARCH_NO_STMT); + close_single(rec, 1); + + open_single(rec, &fp); /* generator failed: no plan, rc < 0 */ + sched_trace_no_plan(rec, XCE_TRACE_NOPLAN_GENERATOR_FAILED); + sched_trace_search_end(rec, -12); + assert_search_end(rec->entries_tail, 0, 0, 0, 0, -12, + SCHED_TRACE_SEARCH_NO_STMT); + close_single(rec, -12); + + assert_req_counters(rec, 0, 0, 0); + sched_trace_rec_free(rec); +} + +/* Counts belong to the requirement the search ran for: they restart with + * each member, while the decision-wide counters add up. */ +static void test_search_end_counts_per_requirement(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + fixture_primary_t fp; + int s[1] = { 1 }; + xce_alloc_plan_t plan = make_plan(s, 1); + + TEST_ASSERT_NOT_NULL(rec); + make_primary(&fp, 1, 1); + sched_trace_path_push(rec, XCE_TRACE_PATH_COMPOUND, 1, 1); + sched_trace_resreq_begin(rec, NULL, 2); + + sched_trace_path_push(rec, XCE_TRACE_PATH_MEMBER, 1, 2); + sched_trace_resreq_begin(rec, &fp.p, 0); + sched_trace_plan_begin(rec, &plan); + sched_trace_plan_end(rec, 1, 0); + sched_trace_commit(rec); + sched_trace_search_end(rec, 1); + assert_search_end(rec->entries_tail, 1, 1, 0, 0, 1, + SCHED_TRACE_SEARCH_NO_STMT); + TEST_ASSERT_EQUAL_INT(2, rec->entries_tail->depth); + TEST_ASSERT_EQUAL_INT(2, rec->entries_tail->path.depth); + TEST_ASSERT_EQUAL_INT(XCE_TRACE_PATH_MEMBER, + rec->entries_tail->path.lvl[1].kind); + sched_trace_resreq_end(rec, 1, 0); + sched_trace_path_pop(rec); + + sched_trace_path_push(rec, XCE_TRACE_PATH_MEMBER, 2, 2); + sched_trace_resreq_begin(rec, &fp.p, 0); + sched_trace_plan_discarded(rec, &plan); + sched_trace_search_end(rec, 0); + assert_search_end(rec->entries_tail, 1, 0, 1, 0, 0, + SCHED_TRACE_SEARCH_ALL_DISCARDED); + sched_trace_resreq_end(rec, 0, 0); + sched_trace_path_pop(rec); + + sched_trace_resreq_end(rec, 0, 0); + sched_trace_path_pop(rec); + + assert_req_counters(rec, 0, 1, 0); + TEST_ASSERT_EQUAL_INT(1, rec->counters.plans_tried); + TEST_ASSERT_EQUAL_INT(1, rec->counters.plans_discarded); + sched_trace_rec_free(rec); +} + +/* ---- counters under truncation (9.3) ---------------------------------- */ + +/* + * One decision with every requirement outcome and every verdict kind: + * requirement 1: two plans discarded (all discarded); + * requirement 2: one plan tried; position 1 sees a host with a shortfall + * and a limit hit, a selection mismatch, an already-claimed host, an + * internal error and an accepted host; position 2 is unfilled, the + * claim is released (none placed); + * requirement 3: no plan generated. + * Returns the lines it makes at full level. + */ +static int drive_mixed(xce_sched_trace_rec_t* rec) +{ + static fixture_primary_t fp; + static char names[5][8] = { "n01", "n02", "n03", "n04", + "n05" }; + xce_sched_worker_t w[5]; + int s[2] = { 1, 1 }; + xce_alloc_plan_t plan = make_plan(s, 2); + int k; + + for (k = 0; k < 5; k++) { + w[k] = make_worker(names[k], k); + } + make_primary(&fp, 2, 2); + + sched_trace_path_push(rec, XCE_TRACE_PATH_SINGLE, 1, 1); + sched_trace_resreq_begin(rec, &fp.p, 0); /* 1 */ + sched_trace_plan_discarded(rec, &plan); /* 2 */ + sched_trace_plan_discarded(rec, &plan); /* 3 */ + sched_trace_search_end(rec, 0); /* 4 */ + sched_trace_resreq_end(rec, 0, 0); /* 5 */ + sched_trace_path_pop(rec); + + sched_trace_path_push(rec, XCE_TRACE_PATH_SINGLE, 1, 1); + sched_trace_resreq_begin(rec, &fp.p, 0); /* 6 */ + sched_trace_plan_begin(rec, &plan); /* 7 */ + sched_trace_pos_begin(rec, 1, 1); /* 8 */ + sched_trace_host_begin(rec, NULL, &w[0]); + sched_trace_host_limit_hit(rec, "lim_host", "h=n01"); + rec->host_row_verdicts |= XCE_TRACE_V_SHORTFALL; + sched_trace_host_verdict(rec, 0, 0); /* 9 */ + sched_trace_host_begin(rec, NULL, &w[1]); + sched_trace_host_verdict(rec, XCE_TRACE_V_SELECT, 0); /* 10 */ + sched_trace_host_begin(rec, NULL, &w[2]); + sched_trace_host_verdict(rec, XCE_TRACE_V_CLAIMED, 0); /* 11 */ + sched_trace_host_begin(rec, NULL, &w[3]); + sched_trace_host_verdict(rec, XCE_TRACE_V_INTERNAL, -5); /* 12 */ + sched_trace_host_begin(rec, NULL, &w[4]); + sched_trace_host_verdict(rec, XCE_TRACE_V_ACCEPTED, 0); /* 13 */ + sched_trace_pos_end(rec, XCE_TRACE_POS_FILLED, "n05"); /* 14 */ + sched_trace_pos_begin(rec, 2, 1); /* 15 */ + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); /* 16 */ + sched_trace_rollback_host(rec, "n05"); /* 17 */ + sched_trace_plan_end(rec, 0, 2); /* 18 */ + sched_trace_search_end(rec, 0); /* 19 */ + sched_trace_resreq_end(rec, 0, 0); /* 20 */ + sched_trace_path_pop(rec); + + sched_trace_path_push(rec, XCE_TRACE_PATH_SINGLE, 1, 1); + sched_trace_resreq_begin(rec, &fp.p, 0); /* 21 */ + sched_trace_no_plan(rec, XCE_TRACE_NOPLAN_NONE_GENERATED); /* 22 */ + sched_trace_search_end(rec, 0); /* 23 */ + sched_trace_resreq_end(rec, 0, 0); /* 24 */ + sched_trace_path_pop(rec); + return 24; +} + +static void assert_mixed_counters(const xce_sched_trace_rec_t* rec) +{ + const sched_trace_counters_t* c = &rec->counters; + + TEST_ASSERT_EQUAL_INT(1, c->plans_tried); + TEST_ASSERT_EQUAL_INT(2, c->plans_discarded); + assert_req_counters(rec, 1, 1, 1); + TEST_ASSERT_EQUAL_INT(1, c->host_evals[vi(XCE_TRACE_V_ACCEPTED)]); + TEST_ASSERT_EQUAL_INT(1, c->host_evals[vi(XCE_TRACE_V_SELECT)]); + TEST_ASSERT_EQUAL_INT(1, c->host_evals[vi(XCE_TRACE_V_SHORTFALL)]); + TEST_ASSERT_EQUAL_INT(1, c->host_evals[vi(XCE_TRACE_V_LIMIT)]); + TEST_ASSERT_EQUAL_INT(1, c->host_evals[vi(XCE_TRACE_V_CLAIMED)]); + TEST_ASSERT_EQUAL_INT(1, c->host_evals[vi(XCE_TRACE_V_INTERNAL)]); +} + +static void test_counters_exact_after_truncation(void) +{ + const int b = XCE_TRACE_LINE_BOUND_MIN; + xce_sched_trace_rec_t* rec; + int n; + + /* untruncated, full level */ + rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + TEST_ASSERT_NOT_NULL(rec); + n = drive_mixed(rec); + TEST_ASSERT_EQUAL_INT(n, rec->nentries); + TEST_ASSERT_EQUAL_INT(0, rec->counters.omitted_lines); + assert_mixed_counters(rec); + sched_trace_rec_free(rec); + + /* truncated before the decision starts: nothing stored, same counts */ + rec = new_rec(b); + TEST_ASSERT_NOT_NULL(rec); + emit_lines(rec, b); + TEST_ASSERT_EQUAL_INT(n, drive_mixed(rec)); + TEST_ASSERT_EQUAL_INT(b, rec->nentries); + TEST_ASSERT_EQUAL_INT(n, rec->counters.omitted_lines); + assert_mixed_counters(rec); + sched_trace_rec_free(rec); + + /* truncated part way through */ + rec = new_rec(b); + TEST_ASSERT_NOT_NULL(rec); + emit_lines(rec, b - 10); + drive_mixed(rec); + TEST_ASSERT_EQUAL_INT(b, rec->nentries); + TEST_ASSERT_EQUAL_INT(n - 10, rec->counters.omitted_lines); + assert_mixed_counters(rec); + sched_trace_rec_free(rec); + + /* summary level, truncated: the five host lines and the position end + * become one summary line per position, and the counts are the same */ + rec = new_rec_level(b, XCE_TRACE_LEVEL_SUMMARY); + TEST_ASSERT_NOT_NULL(rec); + emit_lines(rec, b); + drive_mixed(rec); + TEST_ASSERT_EQUAL_INT(b, rec->nentries); + TEST_ASSERT_EQUAL_INT(n - 5, rec->counters.omitted_lines); + assert_mixed_counters(rec); + sched_trace_rec_free(rec); +} + +/* ---- summary level: position summaries (5.9, 5.10) -------------------- */ + +static const sched_trace_limit_tally_t* +find_tally(const sched_trace_entry_t* e, const char* limit, const char* scope) +{ + int k; + + for (k = 0; k < e->u.pos_summary.nlimits; k++) { + const sched_trace_limit_tally_t* t = &e->u.pos_summary.limits[k]; + + if ((strcmp(t->limit, limit) == 0) && (strcmp(t->scope, scope) == 0)) { + return t; + } + } + return NULL; +} + +static void test_summary_position_tallies(void) +{ + xce_sched_trace_rec_t* rec + = new_rec_level(XCE_TRACE_LINE_BOUND_DEFAULT, XCE_TRACE_LEVEL_SUMMARY); + const sched_trace_entry_t* e; + const sched_trace_limit_tally_t* t; + char names[7][8] + = { "n01", "n02", "n03", "n04", "n05", "n06", "n07" }; + char lim_user[16] = "lim_user"; + char scope_u[16] = "u=alice"; + xce_sched_worker_t w[7]; + int s[2] = { 2, 2 }; + xce_alloc_plan_t plan = make_plan(s, 2); + int k; + + TEST_ASSERT_NOT_NULL(rec); + for (k = 0; k < 7; k++) { + w[k] = make_worker(names[k], k); + } + + sched_trace_plan_begin(rec, &plan); /* 0 */ + sched_trace_pos_begin(rec, 1, 2); /* 1 */ + /* n01: shortfall and a host-limit hit, counted once under each */ + sched_trace_host_begin(rec, NULL, &w[0]); + rec->host_row_verdicts |= XCE_TRACE_V_SHORTFALL; + sched_trace_host_limit_hit(rec, "lim_host", "u=alice,h=n01"); + sched_trace_host_verdict(rec, 0, 0); + /* n02: the same user limit hit on two rows counts once */ + sched_trace_host_begin(rec, NULL, &w[1]); + sched_trace_host_limit_hit(rec, lim_user, scope_u); + sched_trace_host_limit_hit(rec, lim_user, scope_u); + sched_trace_host_verdict(rec, 0, 0); + /* n03: the user limit again; n04: pre-filter; n05 claimed */ + sched_trace_host_begin(rec, NULL, &w[2]); + sched_trace_host_limit_hit(rec, lim_user, scope_u); + sched_trace_host_verdict(rec, 0, 0); + sched_trace_host_begin(rec, NULL, &w[3]); + sched_trace_host_verdict(rec, XCE_TRACE_V_SELECT, 0); + sched_trace_host_begin(rec, NULL, &w[4]); + sched_trace_host_verdict(rec, XCE_TRACE_V_CLAIMED, 0); + sched_trace_host_begin(rec, NULL, &w[5]); + sched_trace_host_verdict(rec, XCE_TRACE_V_ACCEPTED, 0); + sched_trace_pos_end(rec, XCE_TRACE_POS_FILLED, names[5]); /* 2 */ + + sched_trace_pos_begin(rec, 2, 2); /* 3 */ + /* n02 again: a limit hit whose host then ends in an internal error + * is not a limit-exceeded verdict and is not tallied */ + sched_trace_host_begin(rec, NULL, &w[1]); + sched_trace_host_limit_hit(rec, lim_user, scope_u); + sched_trace_host_verdict(rec, XCE_TRACE_V_INTERNAL, -5); + sched_trace_host_begin(rec, NULL, &w[6]); + sched_trace_host_limit_hit(rec, lim_user, scope_u); + sched_trace_host_verdict(rec, 0, 0); + sched_trace_pos_scan_error(rec, -5); + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); /* 4 */ + sched_trace_plan_end(rec, 0, 2); /* 5 */ + + /* no host lines and no position-end lines at summary level */ + TEST_ASSERT_EQUAL_INT(6, rec->nentries); + TEST_ASSERT_EQUAL_INT(0, rec->counters.omitted_lines); + TEST_ASSERT_EQUAL_INT(0, rec->depth); + + e = entry_at(rec, 2); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_POS_SUMMARY, e->type); + TEST_ASSERT_EQUAL_INT(1, e->depth); /* closes the position */ + TEST_ASSERT_EQUAL_INT(1, e->u.pos_summary.pos); + TEST_ASSERT_EQUAL_INT(XCE_TRACE_POS_FILLED, e->u.pos_summary.how); + TEST_ASSERT_EQUAL_STRING("n06", e->u.pos_summary.host); + TEST_ASSERT_TRUE(e->u.pos_summary.host != names[5]); + TEST_ASSERT_EQUAL_INT(1, e->u.pos_summary.counts[vi(XCE_TRACE_V_ACCEPTED)]); + TEST_ASSERT_EQUAL_INT(1, e->u.pos_summary.counts[vi(XCE_TRACE_V_SELECT)]); + TEST_ASSERT_EQUAL_INT(1, + e->u.pos_summary.counts[vi(XCE_TRACE_V_SHORTFALL)]); + TEST_ASSERT_EQUAL_INT(3, e->u.pos_summary.counts[vi(XCE_TRACE_V_LIMIT)]); + TEST_ASSERT_EQUAL_INT(1, e->u.pos_summary.counts[vi(XCE_TRACE_V_CLAIMED)]); + TEST_ASSERT_EQUAL_INT(0, + e->u.pos_summary.counts[vi(XCE_TRACE_V_INTERNAL)]); + TEST_ASSERT_EQUAL_INT(2, e->u.pos_summary.nlimits); + t = find_tally(e, "lim_host", "u=alice,h=n01"); + TEST_ASSERT_NOT_NULL(t); + TEST_ASSERT_EQUAL_INT(1, t->count); + t = find_tally(e, "lim_user", "u=alice"); + TEST_ASSERT_NOT_NULL(t); + TEST_ASSERT_EQUAL_INT(2, t->count); + TEST_ASSERT_TRUE(t->limit != lim_user); + TEST_ASSERT_TRUE(t->scope != scope_u); + + e = entry_at(rec, 4); /* the tallies restart with each position */ + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_POS_SUMMARY, e->type); + TEST_ASSERT_EQUAL_INT(2, e->u.pos_summary.pos); + TEST_ASSERT_EQUAL_INT(XCE_TRACE_POS_SCAN_ERROR, e->u.pos_summary.how); + TEST_ASSERT_EQUAL_INT(-5, e->u.pos_summary.err); + TEST_ASSERT_NULL(e->u.pos_summary.host); + TEST_ASSERT_EQUAL_INT(0, e->u.pos_summary.counts[vi(XCE_TRACE_V_ACCEPTED)]); + TEST_ASSERT_EQUAL_INT(1, e->u.pos_summary.counts[vi(XCE_TRACE_V_LIMIT)]); + TEST_ASSERT_EQUAL_INT(1, + e->u.pos_summary.counts[vi(XCE_TRACE_V_INTERNAL)]); + TEST_ASSERT_EQUAL_INT(1, e->u.pos_summary.nlimits); + t = find_tally(e, "lim_user", "u=alice"); + TEST_ASSERT_NOT_NULL(t); + TEST_ASSERT_EQUAL_INT(1, t->count); + + /* the decision-wide counts agree with the two summaries */ + TEST_ASSERT_EQUAL_INT(4, rec->counters.host_evals[vi(XCE_TRACE_V_LIMIT)]); + TEST_ASSERT_EQUAL_INT( + 1, rec->counters.host_evals[vi(XCE_TRACE_V_SHORTFALL)]); + TEST_ASSERT_EQUAL_INT( + 1, rec->counters.host_evals[vi(XCE_TRACE_V_INTERNAL)]); + + /* every string is a copy */ + strcpy(lim_user, "zz"); + strcpy(scope_u, "zz"); + strcpy(names[5], "zz"); + e = entry_at(rec, 2); + TEST_ASSERT_NOT_NULL(find_tally(e, "lim_user", "u=alice")); + TEST_ASSERT_EQUAL_STRING("n06", e->u.pos_summary.host); + sched_trace_rec_free(rec); +} + +/* A limit hit sets the host's LIMIT row verdict at either level; outside a + * host scope (the cluster-wide check) it is not a host's hit. At full + * level no tallies are kept. */ +static void test_limit_hit_sets_row_verdict(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + char n1[8] = "n01"; + xce_sched_worker_t w1 = make_worker(n1, 0); + + TEST_ASSERT_NOT_NULL(rec); + sched_trace_check_begin(rec, NULL); + sched_trace_host_limit_hit(rec, "lim_user", "u=alice"); + TEST_ASSERT_EQUAL_UINT(0, rec->host_row_verdicts); + sched_trace_check_end(rec, 1, true); + + sched_trace_pos_begin(rec, 1, 1); + sched_trace_host_begin(rec, NULL, &w1); + sched_trace_host_limit_hit(rec, "lim_host", "h=n01"); + TEST_ASSERT_EQUAL_UINT(XCE_TRACE_V_LIMIT, rec->host_row_verdicts); + sched_trace_host_verdict(rec, 0, 0); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_HOST, rec->entries_tail->type); + TEST_ASSERT_EQUAL_UINT(XCE_TRACE_V_LIMIT, + rec->entries_tail->u.host.verdicts); + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_POS_END, rec->entries_tail->type); + sched_trace_rec_free(rec); +} + +/* Summary level past the line bound: the position summary is one omitted + * line and the builder's tallies are released. */ +static void test_summary_truncated(void) +{ + const int b = XCE_TRACE_LINE_BOUND_MIN; + xce_sched_trace_rec_t* rec = new_rec_level(b, XCE_TRACE_LEVEL_SUMMARY); + char n1[8] = "n01"; + xce_sched_worker_t w1 = make_worker(n1, 0); + + TEST_ASSERT_NOT_NULL(rec); + emit_lines(rec, b); + sched_trace_pos_begin(rec, 1, 1); + sched_trace_host_begin(rec, NULL, &w1); + sched_trace_host_limit_hit(rec, "lim_host", "h=n01"); + sched_trace_host_verdict(rec, 0, 0); + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); + TEST_ASSERT_EQUAL_INT(b, rec->nentries); + TEST_ASSERT_EQUAL_INT(2, rec->counters.omitted_lines); + TEST_ASSERT_EQUAL_INT(1, rec->counters.host_evals[vi(XCE_TRACE_V_LIMIT)]); + TEST_ASSERT_EQUAL_INT(0, rec->depth); + sched_trace_rec_free(rec); /* ASAN: no leaked builder tallies */ +} + +/* ---- rollback merging (5.8) ------------------------------------------- */ + +static void test_rollback_merges_consecutive_calls(void) +{ + xce_sched_trace_rec_t* rec = new_rec(XCE_TRACE_LINE_BOUND_DEFAULT); + const sched_trace_entry_t* e; + char h[3][8] = { "n01", "n02", "n03" }; + char many[20][8]; + int s[3] = { 1, 1, 1 }; + xce_alloc_plan_t plan = make_plan(s, 3); + int k; + + TEST_ASSERT_NOT_NULL(rec); + sched_trace_plan_begin(rec, &plan); /* 0 */ + sched_trace_pos_begin(rec, 3, 1); /* 1 */ + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); /* 2 */ + for (k = 0; k < 3; k++) { + sched_trace_rollback_host(rec, h[k]); /* 3 */ + } + sched_trace_plan_end(rec, 0, 3); /* 4 */ + sched_trace_plan_begin(rec, &plan); /* 5 */ + sched_trace_pos_begin(rec, 2, 1); /* 6 */ + sched_trace_pos_end(rec, XCE_TRACE_POS_UNFILLED, NULL); /* 7 */ + sched_trace_rollback_host(rec, h[2]); /* 8 */ + sched_trace_plan_end(rec, 0, 2); /* 9 */ + sched_trace_plan_begin(rec, &plan); /* 10 */ + for (k = 0; k < 20; k++) { + snprintf(many[k], sizeof(many[k]), "m%02d", k); + sched_trace_rollback_host(rec, many[k]); /* 11 */ + } + sched_trace_plan_end(rec, 0, 1); /* 12 */ + + TEST_ASSERT_EQUAL_INT(13, rec->nentries); + + e = entry_at(rec, 3); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_ROLLBACK, e->type); + TEST_ASSERT_EQUAL_INT(1, e->depth); /* inside the plan */ + TEST_ASSERT_EQUAL_INT(3, e->u.rollback.nhosts); + TEST_ASSERT_EQUAL_STRING("n01", e->u.rollback.hosts[0]); + TEST_ASSERT_EQUAL_STRING("n02", e->u.rollback.hosts[1]); + TEST_ASSERT_EQUAL_STRING("n03", e->u.rollback.hosts[2]); + TEST_ASSERT_TRUE(e->u.rollback.hosts[0] != h[0]); + + e = entry_at(rec, 8); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_ROLLBACK, e->type); + TEST_ASSERT_EQUAL_INT(1, e->u.rollback.nhosts); + TEST_ASSERT_EQUAL_STRING("n03", e->u.rollback.hosts[0]); + + e = entry_at(rec, 11); + TEST_ASSERT_EQUAL_INT(SCHED_TRACE_E_ROLLBACK, e->type); + TEST_ASSERT_EQUAL_INT(20, e->u.rollback.nhosts); + for (k = 0; k < 20; k++) { + TEST_ASSERT_EQUAL_STRING(many[k], e->u.rollback.hosts[k]); + } + + strcpy(h[0], "zz"); + TEST_ASSERT_EQUAL_STRING("n01", entry_at(rec, 3)->u.rollback.hosts[0]); + sched_trace_rec_free(rec); +} + +/* Any other model call between two released hosts starts a new entry. */ +static void test_rollback_run_broken_by_other_calls(void) +{ + xce_sched_trace_rec_t* rec = new_rec_level(XCE_TRACE_LINE_BOUND_DEFAULT, + XCE_TRACE_LEVEL_SUMMARY); + char n1[8] = "n01"; + xce_sched_worker_t w1 = make_worker(n1, 0); + + TEST_ASSERT_NOT_NULL(rec); + sched_trace_rollback_host(rec, "n01"); + sched_trace_host_begin(rec, NULL, &w1); /* stores nothing */ + sched_trace_host_verdict(rec, XCE_TRACE_V_SELECT, 0); /* summary: none */ + sched_trace_rollback_host(rec, "n02"); + sched_trace_alt_skipped(rec, 1, 2); + sched_trace_rollback_host(rec, "n03"); + TEST_ASSERT_EQUAL_INT(4, rec->nentries); + TEST_ASSERT_EQUAL_INT(1, entry_at(rec, 0)->u.rollback.nhosts); + TEST_ASSERT_EQUAL_INT(1, entry_at(rec, 1)->u.rollback.nhosts); + TEST_ASSERT_EQUAL_STRING("n02", entry_at(rec, 1)->u.rollback.hosts[0]); + TEST_ASSERT_EQUAL_INT(1, entry_at(rec, 3)->u.rollback.nhosts); + sched_trace_rec_free(rec); +} + +/* Past the bound a merged run is one omitted line, not one per host. */ +static void test_rollback_truncated_is_one_line(void) +{ + const int b = XCE_TRACE_LINE_BOUND_MIN; + xce_sched_trace_rec_t* rec = new_rec(b); + size_t bytes; + + TEST_ASSERT_NOT_NULL(rec); + emit_lines(rec, b - 1); + sched_trace_rollback_host(rec, "n01"); /* the b-th line, stored */ + sched_trace_rollback_host(rec, "n02"); + TEST_ASSERT_EQUAL_INT(b, rec->nentries); + TEST_ASSERT_EQUAL_INT(2, rec->entries_tail->u.rollback.nhosts); + TEST_ASSERT_EQUAL_INT(0, rec->counters.omitted_lines); + + sched_trace_alt_skipped(rec, 1, 1); /* omitted: 1 */ + bytes = sched_trace_rec_bytes(rec); + sched_trace_rollback_host(rec, "n03"); /* omitted: 2 */ + sched_trace_rollback_host(rec, "n04"); + sched_trace_rollback_host(rec, "n05"); + TEST_ASSERT_EQUAL_INT(2, rec->counters.omitted_lines); + TEST_ASSERT_EQUAL_INT(2, rec->entries_tail->u.rollback.nhosts); + TEST_ASSERT_EQUAL_UINT(bytes, sched_trace_rec_bytes(rec)); + sched_trace_rec_free(rec); +} + int main(void) { UNITY_BEGIN(); @@ -1272,5 +1993,19 @@ int main(void) RUN_TEST(test_budget_edges); RUN_TEST(test_budget_counts_every_kind_and_keeps_state); RUN_TEST(test_budget_cut_inside_a_scope); + RUN_TEST(test_search_end_none_generated); + RUN_TEST(test_search_end_all_discarded); + RUN_TEST(test_search_end_none_placed); + RUN_TEST(test_search_end_plan_internal_error_has_no_statement); + RUN_TEST(test_search_end_not_tried_after_discarded); + RUN_TEST(test_search_end_nonzero_rc_has_no_statement); + RUN_TEST(test_search_end_counts_per_requirement); + RUN_TEST(test_counters_exact_after_truncation); + RUN_TEST(test_summary_position_tallies); + RUN_TEST(test_limit_hit_sets_row_verdict); + RUN_TEST(test_summary_truncated); + RUN_TEST(test_rollback_merges_consecutive_calls); + RUN_TEST(test_rollback_run_broken_by_other_calls); + RUN_TEST(test_rollback_truncated_is_one_line); return UNITY_END(); }