#if !defined(_SCHED_TRACE_H_) #define _SCHED_TRACE_H_ 1 /* * trace.h - scheduling decision trace model (scheduler-sim-decision-trace * spec, design.md "TraceModel"). * * A trace record explains one allocator decision for one traced job. It is * built on the allocator thread while the job is allocated, handed to the * simulator REPL on the decision, and rendered there. The record is * self-contained: every string and number it holds is copied into its own * per-record arena, so it never points into scheduler memory and stays * valid after the job, its schedobj or the core state it was captured from * change or go away. * * Only the simulator REPL ever publishes a control block (`trace_ctl`), so * the live scheduler never builds a record (Requirement 11.2). * * The model (src/sched_trace.c, src/sched_trace_scan.c) and the capture * helpers (src/sched_trace_capture.c, src/sched_trace_outcome.c) do no I/O * and no logging; test_core_isolation.c scans these files and their shared * src/sched_trace_internal.h for stdio and file I/O. */ #include #include #include #include #include "resreq.h" #include "sched/types.h" /* Defined in sched/alloc_iter.h, which is not included here: it pulls in * scheduler.h and so core.h. */ typedef struct xce_alloc_plan_s xce_alloc_plan_t; /* Requirement 1.8: two detail levels; summary is the default. */ typedef enum { XCE_TRACE_LEVEL_SUMMARY = 0, XCE_TRACE_LEVEL_FULL } xce_sched_trace_level_t; /* Requirement 5.10 verdict kinds, bit flags (a host may carry several). */ typedef enum { XCE_TRACE_V_ACCEPTED = 1u << 0, XCE_TRACE_V_SELECT = 1u << 1, /* selection mismatch (pre-filter) */ XCE_TRACE_V_SHORTFALL = 1u << 2, /* resource shortfall (per-host check) */ XCE_TRACE_V_LIMIT = 1u << 3, /* limit exceeded (per-host check) */ XCE_TRACE_V_CLAIMED = 1u << 4, /* already claimed by earlier position */ XCE_TRACE_V_INTERNAL = 1u << 5 /* not evaluated: internal error */ } xce_sched_trace_verdict_t; /* Number of verdict kinds above: the size of every per-kind counter array, * indexed by the kind's bit position. */ #define XCE_TRACE_V_NKINDS 6 /* Requirement 2.4 / 2.5 record kinds. */ typedef enum { XCE_TRACE_REC_DECISION = 0, XCE_TRACE_REC_NOT_FOUND, XCE_TRACE_REC_NO_SCHEDOBJ, XCE_TRACE_REC_INTERNAL_ERROR, /* alloc_rc < 0 or decision build failure */ XCE_TRACE_REC_PEND_NO_REASON, XCE_TRACE_REC_RECONSTRUCTED } xce_sched_trace_rec_kind_t; /* One selector from `trace on job ...` (Requirements 1.2, 1.3). */ typedef struct xce_sched_trace_sel_s { uint64_t key; /* plain jobid, array base id, or element id */ bool whole_array; /* true: key is an array base, matches every element */ } xce_sched_trace_sel_t; /* * Settings snapshot published to the core for one session. Immutable during * the session except `position`, which only the allocator thread writes. * REPL-owned; a record copies what it needs and never keeps a pointer here. */ typedef struct xce_sched_trace_ctl_s { bool all; int nsel; xce_sched_trace_sel_t* sel; xce_sched_trace_level_t level; int line_bound; /* 50..100000 */ uint64_t session_no; /* rs->session_count of this session */ const char* policy_name; /* picker name, owned by the ctl */ int position; /* allocator-owned, reset at session begin */ } xce_sched_trace_ctl_t; typedef struct xce_sched_trace_rec_s xce_sched_trace_rec_t; /* * Hook gate (Requirement 11.2). `call` (a statement or expression, side * effects and argument evaluation included) runs only while `core` (an * xce_sched_core_t*) has a record under construction in trace_rec, so a * hook costs one pointer test when nothing is traced, and nothing else. * Only the allocator thread sets trace_rec. A hooked function called from * another thread (the sim thread's reconciliation reaches unsatisfy_) * sees NULL there, because the allocator clears trace_rec before it hands * each decision back. One statement: safe in an unbraced if/else. */ #define SCHED_TRACE(core, call) \ do { \ if ((core)->trace_rec != NULL) { \ call; \ } \ } while (0) #define XCE_TRACE_LINE_BOUND_DEFAULT 1000 /* Requirement 9.1 */ #define XCE_TRACE_LINE_BOUND_MIN 50 #define XCE_TRACE_LINE_BOUND_MAX 100000 #define XCE_TRACE_OUTCOME_HOSTS 50 /* Requirement 9.4 */ typedef enum { XCE_TRACE_PATH_SINGLE, /* PRIMARY job: the single requirement */ XCE_TRACE_PATH_COMPOUND, /* top-level COMPOUND container */ XCE_TRACE_PATH_ALT, /* alternative i/n of an ALTER (a compound) */ XCE_TRACE_PATH_MEMBER /* member j/m of the enclosing compound */ } xce_sched_trace_path_kind_t; /* Kept inside the record, not passed by hooks. */ typedef struct { int depth; /* 0..2: container (COMPOUND|ALT) > MEMBER, or SINGLE */ struct { xce_sched_trace_path_kind_t kind; int i; int n; } lvl[2]; } sched_trace_path_t; /* Requirement 4.7 reasons. */ typedef enum { XCE_TRACE_NOPLAN_DECLINED, /* allocator accepts() declined */ XCE_TRACE_NOPLAN_GENERATOR_FAILED, /* plan_init failed */ XCE_TRACE_NOPLAN_NONE_GENERATED /* generator produced no plan */ } xce_sched_trace_noplan_t; typedef enum { XCE_TRACE_POS_FILLED, XCE_TRACE_POS_UNFILLED, XCE_TRACE_POS_SCAN_ERROR, /* host scan could not start or stopped (4.4) */ XCE_TRACE_POS_INTERNAL /* host accepted but claim snapshot failed */ } xce_sched_trace_pos_end_t; /* * Decision-wide counters (Requirement 9.3). They stay exact when the line * budget truncates the stored entries. */ typedef struct { int plans_tried; int plans_discarded; int reqs_none_generated; int reqs_all_discarded; int reqs_none_placed; int host_evals[XCE_TRACE_V_NKINDS]; /* indexed by verdict bit position */ int omitted_lines; } sched_trace_counters_t; /* * Entry types (design.md "Logical Data Model"). Every stored entry renders * as exactly one line in the budgeted sections (requirement, cluster-wide * check, plan and host), so the number of stored entries is the number of * budgeted lines. */ typedef enum { SCHED_TRACE_E_RESREQ_BEGIN = 0, /* requirement named by the path */ SCHED_TRACE_E_RESREQ_END, /* its outcome; one per begin */ SCHED_TRACE_E_ALT_SKIPPED, /* alternative with no content (3.6) */ SCHED_TRACE_E_ALT_RESULT, /* chosen alternative, or none (3.4) */ SCHED_TRACE_E_MEMBER_FAILED, /* merge failed after member j/m (3.3) */ SCHED_TRACE_E_ATTEMPT_REASONS, /* failure reasons of an attempt (3.5) */ SCHED_TRACE_E_CHECK_END, /* cluster-wide check result (4.8) */ SCHED_TRACE_E_NO_PLAN, /* no plan generated, and why (4.7) */ SCHED_TRACE_E_PLAN_BEGIN, /* plan tried (4.3) */ SCHED_TRACE_E_PLAN_DISCARDED, /* plan generated, discarded (4.9) */ SCHED_TRACE_E_PLAN_NOT_TRIED, /* plan generated, internal error (4.4)*/ SCHED_TRACE_E_PLAN_END, /* plan result (4.4) */ SCHED_TRACE_E_POS_BEGIN, /* plan position (4.4) */ SCHED_TRACE_E_POS_END, /* filled / unfilled / scan error / * internal (4.4) */ SCHED_TRACE_E_HOST, /* host verdict (5.1-5.7, 5.10) */ SCHED_TRACE_E_COMMIT, /* committed plan (4.5) */ SCHED_TRACE_E_FINALIZE_FAILED, /* allocation build failed (4.5) */ SCHED_TRACE_E_SEARCH_END, /* plan counts of a requirement, and * the statement (4.6, 4.7, 4.9) */ SCHED_TRACE_E_POS_SUMMARY, /* summary level: the position end with * its verdict counts (5.9) */ SCHED_TRACE_E_ROLLBACK, /* released hosts, merged (5.8) */ SCHED_TRACE_E_CHECK_ROW, /* cluster-wide check row (4.1, 4.2) */ SCHED_TRACE_E_HOST_ROW, /* unmet row under a host (5.4, 5.5) */ SCHED_TRACE_E_NTYPES } sched_trace_entry_type_t; /* Where a host verdict came from (5.2). Only the selection, shortfall and * limit kinds carry a phase; the other kinds use NONE. */ typedef enum { SCHED_TRACE_PHASE_NONE = 0, SCHED_TRACE_PHASE_PREFILTER, /* host-selection pre-filter */ SCHED_TRACE_PHASE_PERHOST /* per-host resource and limit check */ } sched_trace_phase_t; /* The statement a search-end line ends with (design "TraceHooks", search * end). At most one; none when the search returned rc != 0. */ typedef enum { SCHED_TRACE_SEARCH_NO_STMT = 0, SCHED_TRACE_SEARCH_NONE_GENERATED, /* g == 0 (4.7) */ SCHED_TRACE_SEARCH_ALL_DISCARDED, /* g > 0 and d == g (4.9) */ SCHED_TRACE_SEARCH_NONE_PLACED /* t > 0, no plan ended in an * internal error (4.6) */ } sched_trace_search_stmt_t; /* Hosts of one position that hit one limit (5.9): `count` hosts with a * limit-exceeded verdict hit the limit `limit` in scope instance `scope`. */ typedef struct { const char* limit; /* arena copy */ const char* scope; /* arena copy */ int count; } sched_trace_limit_tally_t; /* One cluster-wide limit matched to the job (2.3), from the schedobj's * glimits. `scope` is the scope instance the limit applies to, as * comma-separated `u=..,q=..,p=..` pairs; "" when the limit has no * per-consumer scope. */ typedef struct { const char* limit; /* arena copy */ const char* scope; /* arena copy */ } sched_trace_limit_ref_t; /* * One row of the satisfaction check (resource.c unsatisfy_()): one * requested resource compared against its total. `met` is the check's own * verdict, !(total < inuse + reserved + requested). A row whose resource * has XCE_RESF_LIMIT is a limit row: `limit`, `scope` and `limited_res` * name the limit, the scope instance it applies to ("u=..,q=..,p=..", with * ",h=" or "h=" appended in a host scope) and the resource it * limits (NULL when no matched limit names it). Every other row is a * resource row, its limit fields NULL. Stored strings are arena copies. */ typedef struct { const char* res; /* resource name; NULL when the index is unknown */ int idx; /* resource index */ int flags; /* XCE_RESF_* */ int valtype; float total; float inuse; float reserved; /* provisionally reserved by earlier parts of * this job's evaluation (uncommitted) */ float requested; bool met; const char* limit; /* limit rows only */ const char* scope; /* limit rows only */ const char* limited_res; /* limit rows only; NULL: unknown */ } sched_trace_row_t; /* One requested resource of a single requirement or member (3.2). */ typedef struct { const char* name; /* arena copy */ float value; int per; /* rusage_per_type_t */ } sched_trace_req_t; /* One pend-reason code of an attempt's failure reasons and how often it * occurred (3.5). */ typedef struct { int code; int count; } sched_trace_reason_count_t; /* * One stored entry. All pointers point into the record's arena. `depth` is * the nesting level the renderer indents by: a requirement, plan or * position begin is stored at the current level and opens the next one; * its end closes it and is stored at the begin's level. `path` is the full * requirement path in effect when the entry was made (3.1). */ typedef struct sched_trace_entry_s { struct sched_trace_entry_s* next; sched_trace_entry_type_t type; int depth; sched_trace_path_t path; union { struct { /* RESREQ_BEGIN */ bool container; /* COMPOUND or ALT: no primary */ int nmembers; /* container only */ int min_tasks; int max_tasks; int nreq; sched_trace_req_t* req; } resreq_begin; struct { /* RESREQ_END */ int rc; /* > 0 satisfied, 0 not, < 0 error */ int not_tried_after; /* members after this one, not tried */ } resreq_end; struct { /* ALT_SKIPPED, ALT_RESULT (i == 0: none chosen) */ int i; int n; } alt; struct { /* MEMBER_FAILED */ int j; int m; int err; } member_failed; struct { /* ATTEMPT_REASONS */ int ncodes; sched_trace_reason_count_t* codes; int nhosts; const char** hosts; int nres; const char** res; } reasons; struct { /* CHECK_END */ int rc; bool plans_follow; } check_end; struct { /* NO_PLAN */ xce_sched_trace_noplan_t why; } no_plan; struct { /* PLAN_BEGIN, PLAN_DISCARDED, PLAN_NOT_TRIED */ int plan_no; /* 1-based, in generation order per requirement */ int nbuckets; /* hosts the plan needs */ int* slots; /* tasks per position, nbuckets of them */ int err; /* PLAN_NOT_TRIED only */ } plan; struct { /* PLAN_END */ int plan_no; int rc; int failed_pos; /* the position that could not be filled */ } plan_end; struct { /* POS_BEGIN */ int pos; int slots; } pos_begin; struct { /* POS_END */ int pos; xce_sched_trace_pos_end_t how; const char* host; /* filled: the accepted host */ int err; /* scan error */ } pos_end; struct { /* HOST */ const char* host; unsigned verdicts; /* xce_sched_trace_verdict_t bits */ sched_trace_phase_t phase; int err; } host; struct { /* COMMIT, FINALIZE_FAILED */ int plan_no; /* the last plan begun */ int err; /* FINALIZE_FAILED only */ } commit; struct { /* SEARCH_END: plans of the requirement's search */ int generated; /* tried+discarded+not tried */ int tried; int discarded; int not_tried; int rc; sched_trace_search_stmt_t stmt; } search_end; struct { /* POS_SUMMARY: the POS_END fields plus the host tallies */ int pos; xce_sched_trace_pos_end_t how; const char* host; /* the accepted host, or NULL */ int err; int counts[XCE_TRACE_V_NKINDS]; /* hosts * per verdict kind (bit pos) */ int nlimits; sched_trace_limit_tally_t* limits; /* hits per limit, in the * order first hit */ } pos_summary; sched_trace_row_t row; /* CHECK_ROW, HOST_ROW */ struct { /* ROLLBACK: consecutive released hosts, in call order */ int nhosts; int cap; const char** hosts; } rollback; } u; } sched_trace_entry_t; /* * Decision outcome (6.1-6.4, 9.4), filled by TraceCapture and rendered * outside the line budget. All pointers point into the record's arena. */ typedef enum { SCHED_TRACE_OUTCOME_NONE = 0, SCHED_TRACE_OUTCOME_DISPATCH, SCHED_TRACE_OUTCOME_PEND } sched_trace_outcome_kind_t; /* One allocated amount: a resource row, or a limit charge (6.3: a charge is * never a resource row). A charge names its limit and scope instance (a * host charge has "h=" appended); `limited_res` is the resource the * limit limits, NULL when no matched limit names the charge, in which case * the amount is a raw number. In an aggregate the scope has no host part. */ typedef struct { const char* res; /* allocated resource name (a charge: the vres name; * NULL in a limit aggregate) */ double value; /* amount, or the total in an aggregate */ const char* limit; /* charges only */ const char* scope; /* charges only */ const char* limited_res; /* charges only; NULL: unknown (raw number) */ } sched_trace_alloc_row_t; /* One allocated host kept in detail (6.1), in allocation order. */ typedef struct { const char* host; int nres; sched_trace_alloc_row_t* res; int ncharges; sched_trace_alloc_row_t* charges; } sched_trace_host_out_t; /* Dispatch outcome: the first XCE_TRACE_OUTCOME_HOSTS hosts in detail, the * job's shared rows and charges (6.2), and the omitted hosts' totals per * resource (limits excluded) and per limit, in the order first seen (9.4). */ typedef struct { int nhosts; /* kept in detail */ sched_trace_host_out_t* hosts; int nshared; sched_trace_alloc_row_t* shared; int nshared_charges; sched_trace_alloc_row_t* shared_charges; int omitted_hosts; int nagg_res; sched_trace_alloc_row_t* agg_res; int nagg_charges; sched_trace_alloc_row_t* agg_charges; } sched_trace_dispatch_out_t; /* Pend outcome (6.4): a copy of the decision's pend-reason set holding the * main reason, every entry that names no host and every entry of the first * XCE_TRACE_OUTCOME_HOSTS distinct hosts, in source order (equal to the * source under job_preason_eq() when no host was omitted). Entries of the * hosts after them are counted per pend-reason code (9.4). */ typedef struct { xce_job_preason_t* reasons; /* NULL: the decision had none */ int omitted_hosts; int ncodes; sched_trace_reason_count_t* omitted_codes; /* first-seen order */ } sched_trace_pend_out_t; /* One arena chunk; the record frees the whole chain at once. */ typedef struct sched_trace_chunk_s { struct sched_trace_chunk_s* next; size_t cap; size_t used; /* followed by `cap` bytes of storage */ } sched_trace_chunk_t; /* * Record layout. Read only by sched_trace*.c and the REPL renderer; every * other file treats the record as opaque. All pointers point into the * record's own arena. */ struct xce_sched_trace_rec_s { /* header */ uint64_t jobid; xce_sched_trace_rec_kind_t kind; int err; /* no-decision kinds: the error, else 0 */ xce_sched_trace_level_t level; uint64_t session_no; int position; int line_bound; bool incomplete; /* arena growth failed: appending stopped */ /* selection block (2.2, 2.3), filled by TraceCapture */ const char* policy_name; /* the job's ordering policy: arena copy of * the queue's picker override, else of * ctl->policy_name, else of the core's * picker; NULL if none is known */ bool selection_set; /* capture_selection ran: the ordering keys * below are known (not NOT_FOUND) */ const char* queue_name; /* arena copy; NULL: the job has no queue */ int32_t queue_priority; int32_t job_priority; time_t queue_time; bool glimits_set; /* capture_glimits ran: the limits are known * (false: no schedobj, limits unavailable) */ int nglimits; sched_trace_limit_ref_t* glimits; /* in the schedobj's glimits order */ /* current requirement path (pushed and popped by the allocation chain) */ sched_trace_path_t path; sched_trace_counters_t counters; /* outcome (6.x, 9.4), outside the line budget */ sched_trace_outcome_kind_t outcome; sched_trace_dispatch_out_t dispatch; /* outcome DISPATCH */ sched_trace_pend_out_t pend; /* outcome PEND */ /* stored entries, in the order made; at most line_bound of them */ sched_trace_entry_t* entries; sched_trace_entry_t* entries_tail; int nentries; /* builder state (allocator thread only, while the record is built) */ int depth; /* nesting level of the next entry */ int path_overflow; /* pushes beyond the two path levels, not kept */ int resreq_open; /* requirement begins without their end yet */ int plan_no; /* plans generated so far in this requirement */ int cur_plan_no; /* the last plan begun */ int cur_pos; /* the position begun last */ int scan_err; /* pos_scan_error() for the current position */ bool scan_err_set; unsigned host_row_verdicts; /* SHORTFALL/LIMIT from the current host's * unmet rows (set by the row capture) */ bool check_open; /* between check_begin and check_end */ /* the current host's unmet rows (full level), held in builder memory * (malloc'd, strings strdup'd) until its verdict line is stored, then * stored after it; dropped when the host scope ends any other way */ sched_trace_row_t* host_rows; int host_nrows; int host_rows_cap; /* plans of the current requirement's search (reset by resreq_begin * and after search_end) */ int scope_tried; int scope_discarded; int scope_not_tried; bool scope_plan_err; /* a tried plan ended in an internal error */ bool scope_no_plan; /* a no-plan reason already counted the search */ /* summary level: the current position's host tallies. The limit * tallies live in builder memory (malloc'd, not the arena) until the * position ends, so an omitted position allocates nothing in the * record; they are not counted by sched_trace_rec_bytes. */ int pos_counts[XCE_TRACE_V_NKINDS]; struct sched_trace_tally_build_s* pos_limits; int pos_nlimits; int pos_limits_cap; unsigned host_seq; /* numbers host scopes, for one hit per host */ /* the ROLLBACK entry the last call opened, while no other call has * been made since (NULL when that entry was omitted) */ bool rollback_open; sched_trace_entry_t* rollback_e; /* scope pointers, kept only until the matching scope ends and never * dereferenced after it (TraceCapture's limit lookup) */ const xce_sched_obj_t* scope_obj; const xce_sched_worker_t* scope_w; /* per-record arena */ sched_trace_chunk_t* arena; size_t arena_bytes; /* sum of chunk sizes, headers included */ }; /* True iff `jobid` is selected by `ctl`: `all`, an exact plain-job or * element selector, or a whole-array selector whose array the element * belongs to (also for elements created after the command). NULL ctl: * false. */ extern bool sched_trace_ctl_wants(const xce_sched_trace_ctl_t* ctl, uint64_t jobid); /* New record for `jobid` with the header copied from `ctl`. NULL when `ctl` * is NULL or allocation fails (the job is then not traced). */ extern xce_sched_trace_rec_t* sched_trace_rec_new(const xce_sched_trace_ctl_t* ctl, uint64_t jobid, xce_sched_trace_rec_kind_t kind); /* Frees the record and its arena. NULL-safe. */ extern void sched_trace_rec_free(xce_sched_trace_rec_t* rec); /* Finish a record as a no-decision kind (NOT_FOUND, NO_SCHEDOBJ, * INTERNAL_ERROR, PEND_NO_REASON) before it is pushed via * make_trace_decision (2.4). Any other kind leaves the record unchanged. */ extern void sched_trace_rec_finish(xce_sched_trace_rec_t* rec, xce_sched_trace_rec_kind_t kind, int err); /* Bytes owned by the record: the record block plus its arena. 0 for NULL. */ extern size_t sched_trace_rec_bytes(const xce_sched_trace_rec_t* rec); /* * Arena copies. Each returns storage owned by the record, freed with it. * On allocation failure they set rec->incomplete and return NULL. A NULL * source (or rec) returns NULL without marking the record incomplete. */ extern void* sched_trace_arena_alloc(xce_sched_trace_rec_t* rec, size_t n); extern void* sched_trace_arena_memdup(xce_sched_trace_rec_t* rec, const void* src, size_t n); extern char* sched_trace_arena_strdup(xce_sched_trace_rec_t* rec, const char* s); /* * Entry sequence (design.md "TraceModel"). `rec` is never NULL (the * SCHED_TRACE gate guarantees it). Calls nest in the order the allocator * makes them: every path_push has one path_pop and every *_begin one * *_end. Entries past rec->line_bound are not stored and each adds one to * counters.omitted_lines (9.1, 9.2); the builder state (path, nesting, * plan numbers, scope pointers) is kept up to date either way. */ /* Requirement path: SINGLE 1/1, COMPOUND 1/1 or ALT i/n as the container * level, MEMBER j/m below it. Not an entry. */ extern void sched_trace_path_push(xce_sched_trace_rec_t* rec, xce_sched_trace_path_kind_t kind, int i, int n); extern void sched_trace_path_pop(xce_sched_trace_rec_t* rec); /* Begin and end of the requirement the current path names. `primary` is * NULL for a container, whose `nmembers` is its member count. */ extern void sched_trace_resreq_begin(xce_sched_trace_rec_t* rec, const xce_primary_resreq_t* primary, int nmembers); extern void sched_trace_resreq_end(xce_sched_trace_rec_t* rec, int rc, int not_tried_after); /* Merge failure after member j of m: members j+1..m not tried (3.3). */ extern void sched_trace_member_failed(xce_sched_trace_rec_t* rec, int j, int m, int err); /* Alternative i/n chosen, later ones not tried (3.4); 0: all failed. */ extern void sched_trace_alt_result(xce_sched_trace_rec_t* rec, int chosen_i, int n); /* Alternative i/n skipped: it has no requirement content (3.6). */ extern void sched_trace_alt_skipped(xce_sched_trace_rec_t* rec, int i, int n); /* Failure reasons of a compound attempt: per-code counts and the hosts and * resources they name, all copied (3.5). TraceCapture builds the arrays * from the attempt's pend-reason set. */ extern void sched_trace_attempt_reasons(xce_sched_trace_rec_t* rec, const sched_trace_reason_count_t* codes, int ncodes, const char* const* hosts, int nhosts, const char* const* res, int nres); /* Cluster-wide check scope. check_begin stores no entry and drops any * host scope left open; check_end stores the result, which on failure * says no placement plans were tried (4.8). */ extern void sched_trace_check_begin(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj); extern void sched_trace_check_end(xce_sched_trace_rec_t* rec, int rc, bool plans_follow); /* No plan generated for the current requirement, and why (4.7). Counts * the requirement under "no plan generated" (9.3), once: a search end * later in the same requirement makes no statement and counts nothing. */ extern void sched_trace_no_plan(xce_sched_trace_rec_t* rec, xce_sched_trace_noplan_t why); /* Plans, numbered in generation order within the current requirement. */ extern void sched_trace_plan_begin(xce_sched_trace_rec_t* rec, const xce_alloc_plan_t* plan); extern void sched_trace_plan_discarded(xce_sched_trace_rec_t* rec, const xce_alloc_plan_t* plan); extern void sched_trace_plan_not_tried(xce_sched_trace_rec_t* rec, const xce_alloc_plan_t* plan, int err); extern void sched_trace_plan_end(xce_sched_trace_rec_t* rec, int rc, int failed_pos); /* Plan positions. A pos_scan_error() stores no entry: it turns the * position's following UNFILLED end into SCAN_ERROR with that error. */ extern void sched_trace_pos_begin(xce_sched_trace_rec_t* rec, int pos, int slots); extern void sched_trace_pos_end(xce_sched_trace_rec_t* rec, xce_sched_trace_pos_end_t how, const char* host); extern void sched_trace_pos_scan_error(xce_sched_trace_rec_t* rec, int err); /* * Per-host scope: host_begin stores no entry; host_verdict stores the * host's verdict and ends the scope. An explicit SELECT verdict comes from * the host-selection pre-filter. `verdicts == 0` means "from the rows": the * SHORTFALL/LIMIT bits the current host's unmet rows set, or SELECT when * there were none (the per-host check's own selection branch); either way * the phase is the per-host check (5.2). */ extern void sched_trace_host_begin(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj, const xce_sched_worker_t* w); extern void sched_trace_host_verdict(xce_sched_trace_rec_t* rec, unsigned verdicts, int err); /* * The current host's per-host check found limit `limit` exceeded in scope * instance `scope` (both copied). Sets LIMIT in the host's row verdicts * and, at summary level, tallies the limit for the position summary once * per host whose verdict is limit exceeded (5.9). Outside a host scope it * does nothing. The model-side seam of TraceCapture's limit rows (2.2). */ extern void sched_trace_host_limit_hit(xce_sched_trace_rec_t* rec, const char* limit, const char* scope); /* * One satisfaction check row (2.2), copied. In a host scope (between * host_begin and host_verdict) it is a HOST_ROW, held until the host's * verdict line and stored right after it, one level deeper; at summary * level it is not stored at all. Otherwise it is a CHECK_ROW, stored now. * Which rows to keep, and the host verdict bits they set, are the caller's * (TraceCapture's) decision. */ extern void sched_trace_row(xce_sched_trace_rec_t* rec, const sched_trace_row_t* row); /* One host whose claim a failed plan released. Consecutive calls merge * into one ROLLBACK entry, "released hosts: n01, n02" (5.8). */ extern void sched_trace_rollback_host(xce_sched_trace_rec_t* rec, const char* host); /* The plan begun last was committed, or building its allocation failed. */ extern void sched_trace_commit(xce_sched_trace_rec_t* rec); extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err); /* * End of a requirement's plan search with its result `rc`: one line with * the plans generated, tried, discarded and not tried since the * requirement began, and at most one statement (rc == 0 only): no plan * generated (g == 0), all discarded (g > 0, d == g), or none placed (t > 0 * and no tried plan ended in an internal error). The matching decision- * wide counter is bumped under the same conditions, stored or not (9.3). * After a no_plan entry in the requirement, there is no statement. */ extern void sched_trace_search_end(xce_sched_trace_rec_t* rec, int rc); /* * TraceCapture (src/sched_trace_capture.c and, for the outcomes, * src/sched_trace_outcome.c; design.md "TraceCapture"). Each * reads scheduler state on the thread that owns it, read-only, and copies * what it reports into `rec` (never NULL: the SCHED_TRACE gate). */ /* The job's ordering policy and keys at selection (2.2): policy name, queue * name and priority, job priority, queue time. A NULL job (not found by * the allocator, 2.4) captures nothing. */ extern void sched_trace_capture_selection(xce_sched_trace_rec_t* rec, const xce_sched_core_t* core, const xce_sched_job_t* job); /* The job's matched cluster-wide limits (2.3): each xce_vres_t in * obj->glimits as (limit name, scope instance). A NULL obj (no schedobj, * 2.4) leaves the limits unavailable. */ extern void sched_trace_capture_glimits(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj); /* * The satisfaction check's rows (4.1, 4.2, 5.4, 5.5), called from * resource.c unsatisfy_() with its inputs: `nres` rows of resource index, * total, in-use and requested, and the dense uncommitted vector keyed by * resource index. Each row's resource is resolved through core->resmgr. * A limit row (XCE_RESF_LIMIT) is resolved to its limit through the * scope's matched limits: obj->glimits in the cluster-wide check scope, * obj->hlimits[w->idx] in a host scope; without a match, the limit name * and scope come from the resource's own name. The cluster-wide scope * keeps every row. A host scope keeps only unmet rows: an unmet resource * row sets the host's SHORTFALL verdict, an unmet limit row its LIMIT * verdict (via sched_trace_host_limit_hit), never SHORTFALL. Outside both * scopes nothing is recorded. `core` is only read. */ extern void sched_trace_check_rows(xce_sched_trace_rec_t* rec, xce_sched_core_t* core, int nres, const int* idxs, const float* total, const float* inuse, const xvec_t* uncomitted, const float* wanted); /* A failed compound attempt's reasons (3.5), from its pend-reason set before * the scheduler replaces obj->reason: one ATTEMPT_REASONS entry with a * count per pend-reason code (main reason, host and resource reasons, in * the order first seen) and the distinct hosts and resources they name. * A NULL or empty set stores the entry with no reasons. */ extern void sched_trace_capture_attempt_reasons(xce_sched_trace_rec_t* rec, const xce_pendreason_t* reason); /* * A dispatch decision's allocation as the record's outcome (6.1-6.3, 9.4). * Each row is a limit charge when its resource, found by name in * core->resmgr, has XCE_RESF_LIMIT, or, when the name is not registered, * when it starts with "limit=". A charge is resolved by vres name through * the job's matched limits: obj->glimits for shared rows, * obj->hlimits[w->idx] for a host's rows (w found by host name in * core->workers); unresolved, its name and scope are parsed from the * resource name and its limited resource is unknown. NULL obj: nothing * resolves. A NULL alloc stores an empty dispatch. `core` is only read. */ extern void sched_trace_capture_dispatch(xce_sched_trace_rec_t* rec, xce_sched_core_t* core, const xce_sched_obj_t* obj, const xce_job_alloc_t* alloc); /* A pend decision's pend-reason set as the record's outcome (6.4, 9.4). */ extern void sched_trace_capture_pend(xce_sched_trace_rec_t* rec, const xce_pendreason_t* reason); /* * A RECONSTRUCTED record (2.5) for a dispatch the simulator rebuilt from a * recorded allocation, built on the sim thread: the header from `ctl` and * `allocs` as its dispatch outcome, captured as above (`obj` is the job's * schedobj, NULL when it has none). NULL when ctl is NULL or the record * cannot be allocated. */ extern xce_sched_trace_rec_t* sched_trace_rec_reconstructed(const xce_sched_trace_ctl_t* ctl, xce_sched_core_t* core, uint64_t jobid, const xce_sched_obj_t* obj, const xce_job_alloc_t* allocs); #endif