#if !defined(_SCHED_TRACE_H_) #define _SCHED_TRACE_H_ 1 /* * trace.h - scheduling decision trace model (scheduler-sim-decision-trace * spec, design.md "TraceModel"). * * A trace record explains one allocator decision for one traced job. It is * built on the allocator thread while the job is allocated, handed to the * simulator REPL on the decision, and rendered there. The record is * self-contained: every string and number it holds is copied into its own * per-record arena, so it never points into scheduler memory and stays * valid after the job, its schedobj or the core state it was captured from * change or go away. * * Only the simulator REPL ever publishes a control block (`trace_ctl`), so * the live scheduler never builds a record (Requirement 11.2). * * The model (src/sched_trace.c) and the capture helpers * (src/sched_trace_capture.c) do no I/O and no logging; test_core_isolation.c * scans both files for stdio and file I/O. */ #include #include #include #include "resreq.h" #include "sched/types.h" /* Defined in sched/alloc_iter.h, which is not included here: it pulls in * scheduler.h and so core.h. */ typedef struct xce_alloc_plan_s xce_alloc_plan_t; /* Requirement 1.8: two detail levels; summary is the default. */ typedef enum { XCE_TRACE_LEVEL_SUMMARY = 0, XCE_TRACE_LEVEL_FULL } xce_sched_trace_level_t; /* Requirement 5.10 verdict kinds, bit flags (a host may carry several). */ typedef enum { XCE_TRACE_V_ACCEPTED = 1u << 0, XCE_TRACE_V_SELECT = 1u << 1, /* selection mismatch (pre-filter) */ XCE_TRACE_V_SHORTFALL = 1u << 2, /* resource shortfall (per-host check) */ XCE_TRACE_V_LIMIT = 1u << 3, /* limit exceeded (per-host check) */ XCE_TRACE_V_CLAIMED = 1u << 4, /* already claimed by earlier position */ XCE_TRACE_V_INTERNAL = 1u << 5 /* not evaluated: internal error */ } xce_sched_trace_verdict_t; /* Number of verdict kinds above: the size of every per-kind counter array, * indexed by the kind's bit position. */ #define XCE_TRACE_V_NKINDS 6 /* Requirement 2.4 / 2.5 record kinds. */ typedef enum { XCE_TRACE_REC_DECISION = 0, XCE_TRACE_REC_NOT_FOUND, XCE_TRACE_REC_NO_SCHEDOBJ, XCE_TRACE_REC_INTERNAL_ERROR, /* alloc_rc < 0 or decision build failure */ XCE_TRACE_REC_PEND_NO_REASON, XCE_TRACE_REC_RECONSTRUCTED } xce_sched_trace_rec_kind_t; /* One selector from `trace on job ...` (Requirements 1.2, 1.3). */ typedef struct xce_sched_trace_sel_s { uint64_t key; /* plain jobid, array base id, or element id */ bool whole_array; /* true: key is an array base, matches every element */ } xce_sched_trace_sel_t; /* * Settings snapshot published to the core for one session. Immutable during * the session except `position`, which only the allocator thread writes. * REPL-owned; a record copies what it needs and never keeps a pointer here. */ typedef struct xce_sched_trace_ctl_s { bool all; int nsel; xce_sched_trace_sel_t* sel; xce_sched_trace_level_t level; int line_bound; /* 50..100000 */ uint64_t session_no; /* rs->session_count of this session */ const char* policy_name; /* picker name, owned by the ctl */ int position; /* allocator-owned, reset at session begin */ } xce_sched_trace_ctl_t; typedef struct xce_sched_trace_rec_s xce_sched_trace_rec_t; #define XCE_TRACE_LINE_BOUND_DEFAULT 1000 /* Requirement 9.1 */ #define XCE_TRACE_LINE_BOUND_MIN 50 #define XCE_TRACE_LINE_BOUND_MAX 100000 #define XCE_TRACE_OUTCOME_HOSTS 50 /* Requirement 9.4 */ typedef enum { XCE_TRACE_PATH_SINGLE, /* PRIMARY job: the single requirement */ XCE_TRACE_PATH_COMPOUND, /* top-level COMPOUND container */ XCE_TRACE_PATH_ALT, /* alternative i/n of an ALTER (a compound) */ XCE_TRACE_PATH_MEMBER /* member j/m of the enclosing compound */ } xce_sched_trace_path_kind_t; /* Kept inside the record, not passed by hooks. */ typedef struct { int depth; /* 0..2: container (COMPOUND|ALT) > MEMBER, or SINGLE */ struct { xce_sched_trace_path_kind_t kind; int i; int n; } lvl[2]; } sched_trace_path_t; /* Requirement 4.7 reasons. */ typedef enum { XCE_TRACE_NOPLAN_DECLINED, /* allocator accepts() declined */ XCE_TRACE_NOPLAN_GENERATOR_FAILED, /* plan_init failed */ XCE_TRACE_NOPLAN_NONE_GENERATED /* generator produced no plan */ } xce_sched_trace_noplan_t; typedef enum { XCE_TRACE_POS_FILLED, XCE_TRACE_POS_UNFILLED, XCE_TRACE_POS_SCAN_ERROR, /* host scan could not start or stopped (4.4) */ XCE_TRACE_POS_INTERNAL /* host accepted but claim snapshot failed */ } xce_sched_trace_pos_end_t; /* * Decision-wide counters (Requirement 9.3). They stay exact when the line * budget truncates the stored entries. */ typedef struct { int plans_tried; int plans_discarded; int reqs_none_generated; int reqs_all_discarded; int reqs_none_placed; int host_evals[XCE_TRACE_V_NKINDS]; /* indexed by verdict bit position */ int omitted_lines; } sched_trace_counters_t; /* * Entry types (design.md "Logical Data Model"). Every stored entry renders * as exactly one line in the budgeted sections (requirement, cluster-wide * check, plan and host), so the number of stored entries is the number of * budgeted lines. The check rows, host rows, search-end, position-summary * and rollback entries are added by the tasks that capture them. */ typedef enum { SCHED_TRACE_E_RESREQ_BEGIN = 0, /* requirement named by the path */ SCHED_TRACE_E_RESREQ_END, /* its outcome; one per begin */ SCHED_TRACE_E_ALT_SKIPPED, /* alternative with no content (3.6) */ SCHED_TRACE_E_ALT_RESULT, /* chosen alternative, or none (3.4) */ SCHED_TRACE_E_MEMBER_FAILED, /* merge failed after member j/m (3.3) */ SCHED_TRACE_E_ATTEMPT_REASONS, /* failure reasons of an attempt (3.5) */ SCHED_TRACE_E_CHECK_END, /* cluster-wide check result (4.8) */ SCHED_TRACE_E_NO_PLAN, /* no plan generated, and why (4.7) */ SCHED_TRACE_E_PLAN_BEGIN, /* plan tried (4.3) */ SCHED_TRACE_E_PLAN_DISCARDED, /* plan generated, discarded (4.9) */ SCHED_TRACE_E_PLAN_NOT_TRIED, /* plan generated, internal error (4.4)*/ SCHED_TRACE_E_PLAN_END, /* plan result (4.4) */ SCHED_TRACE_E_POS_BEGIN, /* plan position (4.4) */ SCHED_TRACE_E_POS_END, /* filled / unfilled / scan error / * internal (4.4) */ SCHED_TRACE_E_HOST, /* host verdict (5.1-5.7, 5.10) */ SCHED_TRACE_E_COMMIT, /* committed plan (4.5) */ SCHED_TRACE_E_FINALIZE_FAILED, /* allocation build failed (4.5) */ SCHED_TRACE_E_NTYPES } sched_trace_entry_type_t; /* Where a host verdict came from (5.2). Only the selection, shortfall and * limit kinds carry a phase; the other kinds use NONE. */ typedef enum { SCHED_TRACE_PHASE_NONE = 0, SCHED_TRACE_PHASE_PREFILTER, /* host-selection pre-filter */ SCHED_TRACE_PHASE_PERHOST /* per-host resource and limit check */ } sched_trace_phase_t; /* One requested resource of a single requirement or member (3.2). */ typedef struct { const char* name; /* arena copy */ float value; int per; /* rusage_per_type_t */ } sched_trace_req_t; /* One pend-reason code of an attempt's failure reasons and how often it * occurred (3.5). */ typedef struct { int code; int count; } sched_trace_reason_count_t; /* * One stored entry. All pointers point into the record's arena. `depth` is * the nesting level the renderer indents by: a requirement, plan or * position begin is stored at the current level and opens the next one; * its end closes it and is stored at the begin's level. `path` is the full * requirement path in effect when the entry was made (3.1). */ typedef struct sched_trace_entry_s { struct sched_trace_entry_s* next; sched_trace_entry_type_t type; int depth; sched_trace_path_t path; union { struct { /* RESREQ_BEGIN */ bool container; /* COMPOUND or ALT: no primary */ int nmembers; /* container only */ int min_tasks; int max_tasks; int nreq; sched_trace_req_t* req; } resreq_begin; struct { /* RESREQ_END */ int rc; /* > 0 satisfied, 0 not, < 0 error */ int not_tried_after; /* members after this one, not tried */ } resreq_end; struct { /* ALT_SKIPPED, ALT_RESULT (i == 0: none chosen) */ int i; int n; } alt; struct { /* MEMBER_FAILED */ int j; int m; int err; } member_failed; struct { /* ATTEMPT_REASONS */ int ncodes; sched_trace_reason_count_t* codes; int nhosts; const char** hosts; int nres; const char** res; } reasons; struct { /* CHECK_END */ int rc; bool plans_follow; } check_end; struct { /* NO_PLAN */ xce_sched_trace_noplan_t why; } no_plan; struct { /* PLAN_BEGIN, PLAN_DISCARDED, PLAN_NOT_TRIED */ int plan_no; /* 1-based, in generation order per requirement */ int nbuckets; /* hosts the plan needs */ int* slots; /* tasks per position, nbuckets of them */ int err; /* PLAN_NOT_TRIED only */ } plan; struct { /* PLAN_END */ int plan_no; int rc; int failed_pos; /* the position that could not be filled */ } plan_end; struct { /* POS_BEGIN */ int pos; int slots; } pos_begin; struct { /* POS_END */ int pos; xce_sched_trace_pos_end_t how; const char* host; /* filled: the accepted host */ int err; /* scan error */ } pos_end; struct { /* HOST */ const char* host; unsigned verdicts; /* xce_sched_trace_verdict_t bits */ sched_trace_phase_t phase; int err; } host; struct { /* COMMIT, FINALIZE_FAILED */ int plan_no; /* the last plan begun */ int err; /* FINALIZE_FAILED only */ } commit; } u; } sched_trace_entry_t; /* One arena chunk; the record frees the whole chain at once. */ typedef struct sched_trace_chunk_s { struct sched_trace_chunk_s* next; size_t cap; size_t used; /* followed by `cap` bytes of storage */ } sched_trace_chunk_t; /* * Record layout. Read only by sched_trace*.c and the REPL renderer; every * other file treats the record as opaque. All pointers point into the * record's own arena. */ struct xce_sched_trace_rec_s { /* header */ uint64_t jobid; xce_sched_trace_rec_kind_t kind; int err; /* no-decision kinds: the error, else 0 */ xce_sched_trace_level_t level; uint64_t session_no; int position; int line_bound; bool incomplete; /* arena growth failed: appending stopped */ /* selection block */ const char* policy_name; /* arena copy of ctl->policy_name, or NULL */ /* current requirement path (pushed and popped by the allocation chain) */ sched_trace_path_t path; sched_trace_counters_t counters; /* stored entries, in the order made; at most line_bound of them */ sched_trace_entry_t* entries; sched_trace_entry_t* entries_tail; int nentries; /* builder state (allocator thread only, while the record is built) */ int depth; /* nesting level of the next entry */ int path_overflow; /* pushes beyond the two path levels, not kept */ int resreq_open; /* requirement begins without their end yet */ int plan_no; /* plans generated so far in this requirement */ int cur_plan_no; /* the last plan begun */ int cur_pos; /* the position begun last */ int scan_err; /* pos_scan_error() for the current position */ bool scan_err_set; unsigned host_row_verdicts; /* SHORTFALL/LIMIT from the current host's * unmet rows (set by the row capture) */ /* scope pointers, kept only until the matching scope ends and never * dereferenced after it (TraceCapture's limit lookup) */ const xce_sched_obj_t* scope_obj; const xce_sched_worker_t* scope_w; /* per-record arena */ sched_trace_chunk_t* arena; size_t arena_bytes; /* sum of chunk sizes, headers included */ }; /* True iff `jobid` is selected by `ctl`: `all`, an exact plain-job or * element selector, or a whole-array selector whose array the element * belongs to (also for elements created after the command). NULL ctl: * false. */ extern bool sched_trace_ctl_wants(const xce_sched_trace_ctl_t* ctl, uint64_t jobid); /* New record for `jobid` with the header copied from `ctl`. NULL when `ctl` * is NULL or allocation fails (the job is then not traced). */ extern xce_sched_trace_rec_t* sched_trace_rec_new(const xce_sched_trace_ctl_t* ctl, uint64_t jobid, xce_sched_trace_rec_kind_t kind); /* Frees the record and its arena. NULL-safe. */ extern void sched_trace_rec_free(xce_sched_trace_rec_t* rec); /* Finish a record as a no-decision kind (NOT_FOUND, NO_SCHEDOBJ, * INTERNAL_ERROR, PEND_NO_REASON) before it is pushed via * make_trace_decision (2.4). Any other kind leaves the record unchanged. */ extern void sched_trace_rec_finish(xce_sched_trace_rec_t* rec, xce_sched_trace_rec_kind_t kind, int err); /* Bytes owned by the record: the record block plus its arena. 0 for NULL. */ extern size_t sched_trace_rec_bytes(const xce_sched_trace_rec_t* rec); /* * Arena copies. Each returns storage owned by the record, freed with it. * On allocation failure they set rec->incomplete and return NULL. A NULL * source (or rec) returns NULL without marking the record incomplete. */ extern void* sched_trace_arena_alloc(xce_sched_trace_rec_t* rec, size_t n); extern void* sched_trace_arena_memdup(xce_sched_trace_rec_t* rec, const void* src, size_t n); extern char* sched_trace_arena_strdup(xce_sched_trace_rec_t* rec, const char* s); /* * Entry sequence (design.md "TraceModel"). `rec` is never NULL (the * SCHED_TRACE gate guarantees it). Calls nest in the order the allocator * makes them: every path_push has one path_pop and every *_begin one * *_end. Entries past rec->line_bound are not stored and each adds one to * counters.omitted_lines (9.1, 9.2); the builder state (path, nesting, * plan numbers, scope pointers) is kept up to date either way. */ /* Requirement path: SINGLE 1/1, COMPOUND 1/1 or ALT i/n as the container * level, MEMBER j/m below it. Not an entry. */ extern void sched_trace_path_push(xce_sched_trace_rec_t* rec, xce_sched_trace_path_kind_t kind, int i, int n); extern void sched_trace_path_pop(xce_sched_trace_rec_t* rec); /* Begin and end of the requirement the current path names. `primary` is * NULL for a container, whose `nmembers` is its member count. */ extern void sched_trace_resreq_begin(xce_sched_trace_rec_t* rec, const xce_primary_resreq_t* primary, int nmembers); extern void sched_trace_resreq_end(xce_sched_trace_rec_t* rec, int rc, int not_tried_after); /* Merge failure after member j of m: members j+1..m not tried (3.3). */ extern void sched_trace_member_failed(xce_sched_trace_rec_t* rec, int j, int m, int err); /* Alternative i/n chosen, later ones not tried (3.4); 0: all failed. */ extern void sched_trace_alt_result(xce_sched_trace_rec_t* rec, int chosen_i, int n); /* Alternative i/n skipped: it has no requirement content (3.6). */ extern void sched_trace_alt_skipped(xce_sched_trace_rec_t* rec, int i, int n); /* Failure reasons of a compound attempt: per-code counts and the hosts and * resources they name, all copied (3.5). TraceCapture builds the arrays * from the attempt's pend-reason set. */ extern void sched_trace_attempt_reasons(xce_sched_trace_rec_t* rec, const sched_trace_reason_count_t* codes, int ncodes, const char* const* hosts, int nhosts, const char* const* res, int nres); /* Cluster-wide check scope. check_begin stores no entry; check_end stores * the result, which on failure says no placement plans were tried (4.8). */ extern void sched_trace_check_begin(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj); extern void sched_trace_check_end(xce_sched_trace_rec_t* rec, int rc, bool plans_follow); extern void sched_trace_no_plan(xce_sched_trace_rec_t* rec, xce_sched_trace_noplan_t why); /* Plans, numbered in generation order within the current requirement. */ extern void sched_trace_plan_begin(xce_sched_trace_rec_t* rec, const xce_alloc_plan_t* plan); extern void sched_trace_plan_discarded(xce_sched_trace_rec_t* rec, const xce_alloc_plan_t* plan); extern void sched_trace_plan_not_tried(xce_sched_trace_rec_t* rec, const xce_alloc_plan_t* plan, int err); extern void sched_trace_plan_end(xce_sched_trace_rec_t* rec, int rc, int failed_pos); /* Plan positions. A pos_scan_error() stores no entry: it turns the * position's following UNFILLED end into SCAN_ERROR with that error. */ extern void sched_trace_pos_begin(xce_sched_trace_rec_t* rec, int pos, int slots); extern void sched_trace_pos_end(xce_sched_trace_rec_t* rec, xce_sched_trace_pos_end_t how, const char* host); extern void sched_trace_pos_scan_error(xce_sched_trace_rec_t* rec, int err); /* * Per-host scope: host_begin stores no entry; host_verdict stores the * host's verdict and ends the scope. An explicit SELECT verdict comes from * the host-selection pre-filter. `verdicts == 0` means "from the rows": the * SHORTFALL/LIMIT bits the current host's unmet rows set, or SELECT when * there were none (the per-host check's own selection branch); either way * the phase is the per-host check (5.2). */ extern void sched_trace_host_begin(xce_sched_trace_rec_t* rec, const xce_sched_obj_t* obj, const xce_sched_worker_t* w); extern void sched_trace_host_verdict(xce_sched_trace_rec_t* rec, unsigned verdicts, int err); /* The plan begun last was committed, or building its allocation failed. */ extern void sched_trace_commit(xce_sched_trace_rec_t* rec); extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err); #endif