lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions
myxce/src/scheduler/include/sched/trace.h
Rongsong Shen 16f40fa9dc
feat(scheduler-sim-decision-trace): hook the allocator runner 
Task 3.4: the runner records no-plan reasons, the cluster-wide check,
plans (tried, discarded, not tried) and their ends, positions, claimed
skips and per-host candidate verdicts, rollbacks, commit or finalize
failure, and search end. A declined or generator-failed requirement now
counts once as 'no plan generated'. Runner results are unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-08 12:04:16 +00:00

790 lines
37 KiB
C
Raw Blame History

#if !defined(_SCHED_TRACE_H_)
#define _SCHED_TRACE_H_ 1
/*
* trace.h - scheduling decision trace model (scheduler-sim-decision-trace
* spec, design.md "TraceModel").
*
* A trace record explains one allocator decision for one traced job. It is
* built on the allocator thread while the job is allocated, handed to the
* simulator REPL on the decision, and rendered there. The record is
* self-contained: every string and number it holds is copied into its own
* per-record arena, so it never points into scheduler memory and stays
* valid after the job, its schedobj or the core state it was captured from
* change or go away.
*
* Only the simulator REPL ever publishes a control block (`trace_ctl`), so
* the live scheduler never builds a record (Requirement 11.2).
*
* The model (src/sched_trace.c, src/sched_trace_scan.c) and the capture
* helpers (src/sched_trace_capture.c, src/sched_trace_outcome.c) do no I/O
* and no logging; test_core_isolation.c scans these files and their shared
* src/sched_trace_internal.h for stdio and file I/O.
*/
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <time.h>
#include "resreq.h"
#include "sched/types.h"
/* Defined in sched/alloc_iter.h, which is not included here: it pulls in
* scheduler.h and so core.h. */
typedef struct xce_alloc_plan_s xce_alloc_plan_t;
/* Requirement 1.8: two detail levels; summary is the default. */
typedef enum {
XCE_TRACE_LEVEL_SUMMARY = 0,
XCE_TRACE_LEVEL_FULL
} xce_sched_trace_level_t;
/* Requirement 5.10 verdict kinds, bit flags (a host may carry several). */
typedef enum {
XCE_TRACE_V_ACCEPTED = 1u << 0,
XCE_TRACE_V_SELECT = 1u << 1, /* selection mismatch (pre-filter) */
XCE_TRACE_V_SHORTFALL = 1u << 2, /* resource shortfall (per-host check) */
XCE_TRACE_V_LIMIT = 1u << 3, /* limit exceeded (per-host check) */
XCE_TRACE_V_CLAIMED = 1u << 4, /* already claimed by earlier position */
XCE_TRACE_V_INTERNAL = 1u << 5 /* not evaluated: internal error */
} xce_sched_trace_verdict_t;
/* Number of verdict kinds above: the size of every per-kind counter array,
* indexed by the kind's bit position. */
#define XCE_TRACE_V_NKINDS 6
/* Requirement 2.4 / 2.5 record kinds. */
typedef enum {
XCE_TRACE_REC_DECISION = 0,
XCE_TRACE_REC_NOT_FOUND,
XCE_TRACE_REC_NO_SCHEDOBJ,
XCE_TRACE_REC_INTERNAL_ERROR, /* alloc_rc < 0 or decision build failure */
XCE_TRACE_REC_PEND_NO_REASON,
XCE_TRACE_REC_RECONSTRUCTED
} xce_sched_trace_rec_kind_t;
/* One selector from `trace on job ...` (Requirements 1.2, 1.3). */
typedef struct xce_sched_trace_sel_s {
uint64_t key; /* plain jobid, array base id, or element id */
bool whole_array; /* true: key is an array base, matches every element */
} xce_sched_trace_sel_t;
/*
* Settings snapshot published to the core for one session. Immutable during
* the session except `position`, which only the allocator thread writes.
* REPL-owned; a record copies what it needs and never keeps a pointer here.
*/
typedef struct xce_sched_trace_ctl_s {
bool all;
int nsel;
xce_sched_trace_sel_t* sel;
xce_sched_trace_level_t level;
int line_bound; /* 50..100000 */
uint64_t session_no; /* rs->session_count of this session */
const char* policy_name; /* picker name, owned by the ctl */
int position; /* allocator-owned, reset at session begin */
} xce_sched_trace_ctl_t;
typedef struct xce_sched_trace_rec_s xce_sched_trace_rec_t;
/*
* Hook gate (Requirement 11.2). `call` (a statement or expression, side
* effects and argument evaluation included) runs only while `core` (an
* xce_sched_core_t*) has a record under construction in trace_rec, so a
* hook costs one pointer test when nothing is traced, and nothing else.
* Only the allocator thread sets trace_rec. A hooked function called from
* another thread (the sim thread's reconciliation reaches unsatisfy_)
* sees NULL there, because the allocator clears trace_rec before it hands
* each decision back. One statement: safe in an unbraced if/else.
*/
#define SCHED_TRACE(core, call) \
do { \
if ((core)->trace_rec != NULL) { \
call; \
} \
} while (0)
#define XCE_TRACE_LINE_BOUND_DEFAULT 1000 /* Requirement 9.1 */
#define XCE_TRACE_LINE_BOUND_MIN 50
#define XCE_TRACE_LINE_BOUND_MAX 100000
#define XCE_TRACE_OUTCOME_HOSTS 50 /* Requirement 9.4 */
typedef enum {
XCE_TRACE_PATH_SINGLE, /* PRIMARY job: the single requirement */
XCE_TRACE_PATH_COMPOUND, /* top-level COMPOUND container */
XCE_TRACE_PATH_ALT, /* alternative i/n of an ALTER (a compound) */
XCE_TRACE_PATH_MEMBER /* member j/m of the enclosing compound */
} xce_sched_trace_path_kind_t;
/* Kept inside the record, not passed by hooks. */
typedef struct {
int depth; /* 0..2: container (COMPOUND|ALT) > MEMBER, or SINGLE */
struct {
xce_sched_trace_path_kind_t kind;
int i;
int n;
} lvl[2];
} sched_trace_path_t;
/* Requirement 4.7 reasons. */
typedef enum {
XCE_TRACE_NOPLAN_DECLINED, /* allocator accepts() declined */
XCE_TRACE_NOPLAN_GENERATOR_FAILED, /* plan_init failed */
XCE_TRACE_NOPLAN_NONE_GENERATED /* generator produced no plan */
} xce_sched_trace_noplan_t;
typedef enum {
XCE_TRACE_POS_FILLED,
XCE_TRACE_POS_UNFILLED,
XCE_TRACE_POS_SCAN_ERROR, /* host scan could not start or stopped (4.4) */
XCE_TRACE_POS_INTERNAL /* host accepted but claim snapshot failed */
} xce_sched_trace_pos_end_t;
/*
* Decision-wide counters (Requirement 9.3). They stay exact when the line
* budget truncates the stored entries.
*/
typedef struct {
int plans_tried;
int plans_discarded;
int reqs_none_generated;
int reqs_all_discarded;
int reqs_none_placed;
int host_evals[XCE_TRACE_V_NKINDS]; /* indexed by verdict bit position */
int omitted_lines;
} sched_trace_counters_t;
/*
* Entry types (design.md "Logical Data Model"). Every stored entry renders
* as exactly one line in the budgeted sections (requirement, cluster-wide
* check, plan and host), so the number of stored entries is the number of
* budgeted lines.
*/
typedef enum {
SCHED_TRACE_E_RESREQ_BEGIN = 0, /* requirement named by the path */
SCHED_TRACE_E_RESREQ_END, /* its outcome; one per begin */
SCHED_TRACE_E_ALT_SKIPPED, /* alternative with no content (3.6) */
SCHED_TRACE_E_ALT_RESULT, /* chosen alternative, or none (3.4) */
SCHED_TRACE_E_MEMBER_FAILED, /* merge failed after member j/m (3.3) */
SCHED_TRACE_E_ATTEMPT_REASONS, /* failure reasons of an attempt (3.5) */
SCHED_TRACE_E_CHECK_END, /* cluster-wide check result (4.8) */
SCHED_TRACE_E_NO_PLAN, /* no plan generated, and why (4.7) */
SCHED_TRACE_E_PLAN_BEGIN, /* plan tried (4.3) */
SCHED_TRACE_E_PLAN_DISCARDED, /* plan generated, discarded (4.9) */
SCHED_TRACE_E_PLAN_NOT_TRIED, /* plan generated, internal error (4.4)*/
SCHED_TRACE_E_PLAN_END, /* plan result (4.4) */
SCHED_TRACE_E_POS_BEGIN, /* plan position (4.4) */
SCHED_TRACE_E_POS_END, /* filled / unfilled / scan error /
* internal (4.4) */
SCHED_TRACE_E_HOST, /* host verdict (5.1-5.7, 5.10) */
SCHED_TRACE_E_COMMIT, /* committed plan (4.5) */
SCHED_TRACE_E_FINALIZE_FAILED, /* allocation build failed (4.5) */
SCHED_TRACE_E_SEARCH_END, /* plan counts of a requirement, and
* the statement (4.6, 4.7, 4.9) */
SCHED_TRACE_E_POS_SUMMARY, /* summary level: the position end with
* its verdict counts (5.9) */
SCHED_TRACE_E_ROLLBACK, /* released hosts, merged (5.8) */
SCHED_TRACE_E_CHECK_ROW, /* cluster-wide check row (4.1, 4.2) */
SCHED_TRACE_E_HOST_ROW, /* unmet row under a host (5.4, 5.5) */
SCHED_TRACE_E_NTYPES
} sched_trace_entry_type_t;
/* Where a host verdict came from (5.2). Only the selection, shortfall and
* limit kinds carry a phase; the other kinds use NONE. */
typedef enum {
SCHED_TRACE_PHASE_NONE = 0,
SCHED_TRACE_PHASE_PREFILTER, /* host-selection pre-filter */
SCHED_TRACE_PHASE_PERHOST /* per-host resource and limit check */
} sched_trace_phase_t;
/* The statement a search-end line ends with (design "TraceHooks", search
* end). At most one; none when the search returned rc != 0. */
typedef enum {
SCHED_TRACE_SEARCH_NO_STMT = 0,
SCHED_TRACE_SEARCH_NONE_GENERATED, /* g == 0 (4.7) */
SCHED_TRACE_SEARCH_ALL_DISCARDED, /* g > 0 and d == g (4.9) */
SCHED_TRACE_SEARCH_NONE_PLACED /* t > 0, no plan ended in an
* internal error (4.6) */
} sched_trace_search_stmt_t;
/* Hosts of one position that hit one limit (5.9): `count` hosts with a
* limit-exceeded verdict hit the limit `limit` in scope instance `scope`. */
typedef struct {
const char* limit; /* arena copy */
const char* scope; /* arena copy */
int count;
} sched_trace_limit_tally_t;
/* One cluster-wide limit matched to the job (2.3), from the schedobj's
* glimits. `scope` is the scope instance the limit applies to, as
* comma-separated `u=..,q=..,p=..` pairs; "" when the limit has no
* per-consumer scope. */
typedef struct {
const char* limit; /* arena copy */
const char* scope; /* arena copy */
} sched_trace_limit_ref_t;
/*
* One row of the satisfaction check (resource.c unsatisfy_()): one
* requested resource compared against its total. `met` is the check's own
* verdict, !(total < inuse + reserved + requested). A row whose resource
* has XCE_RESF_LIMIT is a limit row: `limit`, `scope` and `limited_res`
* name the limit, the scope instance it applies to ("u=..,q=..,p=..", with
* ",h=<host>" or "h=<host>" appended in a host scope) and the resource it
* limits (NULL when no matched limit names it). Every other row is a
* resource row, its limit fields NULL. Stored strings are arena copies.
*/
typedef struct {
const char* res; /* resource name; NULL when the index is unknown */
int idx; /* resource index */
int flags; /* XCE_RESF_* */
int valtype;
float total;
float inuse;
float reserved; /* provisionally reserved by earlier parts of
* this job's evaluation (uncommitted) */
float requested;
bool met;
const char* limit; /* limit rows only */
const char* scope; /* limit rows only */
const char* limited_res; /* limit rows only; NULL: unknown */
} sched_trace_row_t;
/* One requested resource of a single requirement or member (3.2). */
typedef struct {
const char* name; /* arena copy */
float value;
int per; /* rusage_per_type_t */
} sched_trace_req_t;
/* One pend-reason code of an attempt's failure reasons and how often it
* occurred (3.5). */
typedef struct {
int code;
int count;
} sched_trace_reason_count_t;
/*
* One stored entry. All pointers point into the record's arena. `depth` is
* the nesting level the renderer indents by: a requirement, plan or
* position begin is stored at the current level and opens the next one;
* its end closes it and is stored at the begin's level. `path` is the full
* requirement path in effect when the entry was made (3.1).
*/
typedef struct sched_trace_entry_s {
struct sched_trace_entry_s* next;
sched_trace_entry_type_t type;
int depth;
sched_trace_path_t path;
union {
struct { /* RESREQ_BEGIN */
bool container; /* COMPOUND or ALT: no primary */
int nmembers; /* container only */
int min_tasks;
int max_tasks;
int nreq;
sched_trace_req_t* req;
} resreq_begin;
struct { /* RESREQ_END */
int rc; /* > 0 satisfied, 0 not, < 0 error */
int not_tried_after; /* members after this one, not tried */
} resreq_end;
struct { /* ALT_SKIPPED, ALT_RESULT (i == 0: none chosen) */
int i;
int n;
} alt;
struct { /* MEMBER_FAILED */
int j;
int m;
int err;
} member_failed;
struct { /* ATTEMPT_REASONS */
int ncodes;
sched_trace_reason_count_t* codes;
int nhosts;
const char** hosts;
int nres;
const char** res;
} reasons;
struct { /* CHECK_END */
int rc;
bool plans_follow;
} check_end;
struct { /* NO_PLAN */
xce_sched_trace_noplan_t why;
} no_plan;
struct { /* PLAN_BEGIN, PLAN_DISCARDED, PLAN_NOT_TRIED */
int plan_no; /* 1-based, in generation order per requirement */
int nbuckets; /* hosts the plan needs */
int* slots; /* tasks per position, nbuckets of them */
int err; /* PLAN_NOT_TRIED only */
} plan;
struct { /* PLAN_END */
int plan_no;
int rc;
int failed_pos; /* the position that could not be filled */
} plan_end;
struct { /* POS_BEGIN */
int pos;
int slots;
} pos_begin;
struct { /* POS_END */
int pos;
xce_sched_trace_pos_end_t how;
const char* host; /* filled: the accepted host */
int err; /* scan error */
} pos_end;
struct { /* HOST */
const char* host;
unsigned verdicts; /* xce_sched_trace_verdict_t bits */
sched_trace_phase_t phase;
int err;
} host;
struct { /* COMMIT, FINALIZE_FAILED */
int plan_no; /* the last plan begun */
int err; /* FINALIZE_FAILED only */
} commit;
struct { /* SEARCH_END: plans of the requirement's search */
int generated; /* tried+discarded+not tried */
int tried;
int discarded;
int not_tried;
int rc;
sched_trace_search_stmt_t stmt;
} search_end;
struct { /* POS_SUMMARY: the POS_END fields plus the host tallies */
int pos;
xce_sched_trace_pos_end_t how;
const char* host; /* the accepted host, or NULL */
int err;
int counts[XCE_TRACE_V_NKINDS]; /* hosts
* per verdict kind (bit pos) */
int nlimits;
sched_trace_limit_tally_t* limits; /* hits per limit, in the
* order first hit */
} pos_summary;
sched_trace_row_t row; /* CHECK_ROW, HOST_ROW */
struct { /* ROLLBACK: consecutive released hosts, in call order */
int nhosts;
int cap;
const char** hosts;
} rollback;
} u;
} sched_trace_entry_t;
/*
* Decision outcome (6.1-6.4, 9.4), filled by TraceCapture and rendered
* outside the line budget. All pointers point into the record's arena.
*/
typedef enum {
SCHED_TRACE_OUTCOME_NONE = 0,
SCHED_TRACE_OUTCOME_DISPATCH,
SCHED_TRACE_OUTCOME_PEND
} sched_trace_outcome_kind_t;
/* One allocated amount: a resource row, or a limit charge (6.3: a charge is
* never a resource row). A charge names its limit and scope instance (a
* host charge has "h=<host>" appended); `limited_res` is the resource the
* limit limits, NULL when no matched limit names the charge, in which case
* the amount is a raw number. In an aggregate the scope has no host part. */
typedef struct {
const char* res; /* allocated resource name (a charge: the vres name;
* NULL in a limit aggregate) */
double value; /* amount, or the total in an aggregate */
const char* limit; /* charges only */
const char* scope; /* charges only */
const char* limited_res; /* charges only; NULL: unknown (raw number) */
} sched_trace_alloc_row_t;
/* One allocated host kept in detail (6.1), in allocation order. */
typedef struct {
const char* host;
int nres;
sched_trace_alloc_row_t* res;
int ncharges;
sched_trace_alloc_row_t* charges;
} sched_trace_host_out_t;
/* Dispatch outcome: the first XCE_TRACE_OUTCOME_HOSTS hosts in detail, the
* job's shared rows and charges (6.2), and the omitted hosts' totals per
* resource (limits excluded) and per limit, in the order first seen (9.4). */
typedef struct {
int nhosts; /* kept in detail */
sched_trace_host_out_t* hosts;
int nshared;
sched_trace_alloc_row_t* shared;
int nshared_charges;
sched_trace_alloc_row_t* shared_charges;
int omitted_hosts;
int nagg_res;
sched_trace_alloc_row_t* agg_res;
int nagg_charges;
sched_trace_alloc_row_t* agg_charges;
} sched_trace_dispatch_out_t;
/* Pend outcome (6.4): a copy of the decision's pend-reason set holding the
* main reason, every entry that names no host and every entry of the first
* XCE_TRACE_OUTCOME_HOSTS distinct hosts, in source order (equal to the
* source under job_preason_eq() when no host was omitted). Entries of the
* hosts after them are counted per pend-reason code (9.4). */
typedef struct {
xce_job_preason_t* reasons; /* NULL: the decision had none */
int omitted_hosts;
int ncodes;
sched_trace_reason_count_t* omitted_codes; /* first-seen order */
} sched_trace_pend_out_t;
/* One arena chunk; the record frees the whole chain at once. */
typedef struct sched_trace_chunk_s {
struct sched_trace_chunk_s* next;
size_t cap;
size_t used;
/* followed by `cap` bytes of storage */
} sched_trace_chunk_t;
/*
* Record layout. Read only by sched_trace*.c and the REPL renderer; every
* other file treats the record as opaque. All pointers point into the
* record's own arena.
*/
struct xce_sched_trace_rec_s {
/* header */
uint64_t jobid;
xce_sched_trace_rec_kind_t kind;
int err; /* no-decision kinds: the error, else 0 */
xce_sched_trace_level_t level;
uint64_t session_no;
int position;
int line_bound;
bool incomplete; /* arena growth failed: appending stopped */
/* selection block (2.2, 2.3), filled by TraceCapture */
const char* policy_name; /* the job's ordering policy: arena copy of
* the queue's picker override, else of
* ctl->policy_name, else of the core's
* picker; NULL if none is known */
bool selection_set; /* capture_selection ran: the ordering keys
* below are known (not NOT_FOUND) */
const char* queue_name; /* arena copy; NULL: the job has no queue */
int32_t queue_priority;
int32_t job_priority;
time_t queue_time;
bool glimits_set; /* capture_glimits ran: the limits are known
* (false: no schedobj, limits unavailable) */
int nglimits;
sched_trace_limit_ref_t* glimits; /* in the schedobj's glimits order */
/* current requirement path (pushed and popped by the allocation chain) */
sched_trace_path_t path;
sched_trace_counters_t counters;
/* outcome (6.x, 9.4), outside the line budget */
sched_trace_outcome_kind_t outcome;
sched_trace_dispatch_out_t dispatch; /* outcome DISPATCH */
sched_trace_pend_out_t pend; /* outcome PEND */
/* stored entries, in the order made; at most line_bound of them */
sched_trace_entry_t* entries;
sched_trace_entry_t* entries_tail;
int nentries;
/* builder state (allocator thread only, while the record is built) */
int depth; /* nesting level of the next entry */
int path_overflow; /* pushes beyond the two path levels, not kept */
int resreq_open; /* requirement begins without their end yet */
int plan_no; /* plans generated so far in this requirement */
int cur_plan_no; /* the last plan begun */
int cur_pos; /* the position begun last */
int scan_err; /* pos_scan_error() for the current position */
bool scan_err_set;
unsigned host_row_verdicts; /* SHORTFALL/LIMIT from the current host's
* unmet rows (set by the row capture) */
bool check_open; /* between check_begin and check_end */
/* the current host's unmet rows (full level), held in builder memory
* (malloc'd, strings strdup'd) until its verdict line is stored, then
* stored after it; dropped when the host scope ends any other way */
sched_trace_row_t* host_rows;
int host_nrows;
int host_rows_cap;
/* plans of the current requirement's search (reset by resreq_begin
* and after search_end) */
int scope_tried;
int scope_discarded;
int scope_not_tried;
bool scope_plan_err; /* a tried plan ended in an internal error */
bool scope_no_plan; /* a no-plan reason already counted the search */
/* summary level: the current position's host tallies. The limit
* tallies live in builder memory (malloc'd, not the arena) until the
* position ends, so an omitted position allocates nothing in the
* record; they are not counted by sched_trace_rec_bytes. */
int pos_counts[XCE_TRACE_V_NKINDS];
struct sched_trace_tally_build_s* pos_limits;
int pos_nlimits;
int pos_limits_cap;
unsigned host_seq; /* numbers host scopes, for one hit per host */
/* the ROLLBACK entry the last call opened, while no other call has
* been made since (NULL when that entry was omitted) */
bool rollback_open;
sched_trace_entry_t* rollback_e;
/* scope pointers, kept only until the matching scope ends and never
* dereferenced after it (TraceCapture's limit lookup) */
const xce_sched_obj_t* scope_obj;
const xce_sched_worker_t* scope_w;
/* per-record arena */
sched_trace_chunk_t* arena;
size_t arena_bytes; /* sum of chunk sizes, headers included */
};
/* True iff `jobid` is selected by `ctl`: `all`, an exact plain-job or
* element selector, or a whole-array selector whose array the element
* belongs to (also for elements created after the command). NULL ctl:
* false. */
extern bool sched_trace_ctl_wants(const xce_sched_trace_ctl_t* ctl,
uint64_t jobid);
/* New record for `jobid` with the header copied from `ctl`. NULL when `ctl`
* is NULL or allocation fails (the job is then not traced). */
extern xce_sched_trace_rec_t*
sched_trace_rec_new(const xce_sched_trace_ctl_t* ctl, uint64_t jobid,
xce_sched_trace_rec_kind_t kind);
/* Frees the record and its arena. NULL-safe. */
extern void sched_trace_rec_free(xce_sched_trace_rec_t* rec);
/* Finish a record as a no-decision kind (NOT_FOUND, NO_SCHEDOBJ,
* INTERNAL_ERROR, PEND_NO_REASON) before it is pushed via
* make_trace_decision (2.4). Any other kind leaves the record unchanged. */
extern void sched_trace_rec_finish(xce_sched_trace_rec_t* rec,
xce_sched_trace_rec_kind_t kind, int err);
/* Bytes owned by the record: the record block plus its arena. 0 for NULL. */
extern size_t sched_trace_rec_bytes(const xce_sched_trace_rec_t* rec);
/*
* Arena copies. Each returns storage owned by the record, freed with it.
* On allocation failure they set rec->incomplete and return NULL. A NULL
* source (or rec) returns NULL without marking the record incomplete.
*/
extern void* sched_trace_arena_alloc(xce_sched_trace_rec_t* rec, size_t n);
extern void* sched_trace_arena_memdup(xce_sched_trace_rec_t* rec,
const void* src, size_t n);
extern char* sched_trace_arena_strdup(xce_sched_trace_rec_t* rec,
const char* s);
/*
* Entry sequence (design.md "TraceModel"). `rec` is never NULL (the
* SCHED_TRACE gate guarantees it). Calls nest in the order the allocator
* makes them: every path_push has one path_pop and every *_begin one
* *_end. Entries past rec->line_bound are not stored and each adds one to
* counters.omitted_lines (9.1, 9.2); the builder state (path, nesting,
* plan numbers, scope pointers) is kept up to date either way.
*/
/* Requirement path: SINGLE 1/1, COMPOUND 1/1 or ALT i/n as the container
* level, MEMBER j/m below it. Not an entry. */
extern void sched_trace_path_push(xce_sched_trace_rec_t* rec,
xce_sched_trace_path_kind_t kind, int i,
int n);
extern void sched_trace_path_pop(xce_sched_trace_rec_t* rec);
/* Begin and end of the requirement the current path names. `primary` is
* NULL for a container, whose `nmembers` is its member count. */
extern void sched_trace_resreq_begin(xce_sched_trace_rec_t* rec,
const xce_primary_resreq_t* primary,
int nmembers);
extern void sched_trace_resreq_end(xce_sched_trace_rec_t* rec, int rc,
int not_tried_after);
/* Merge failure after member j of m: members j+1..m not tried (3.3). */
extern void sched_trace_member_failed(xce_sched_trace_rec_t* rec, int j,
int m, int err);
/* Alternative i/n chosen, later ones not tried (3.4); 0: all failed. */
extern void sched_trace_alt_result(xce_sched_trace_rec_t* rec, int chosen_i,
int n);
/* Alternative i/n skipped: it has no requirement content (3.6). */
extern void sched_trace_alt_skipped(xce_sched_trace_rec_t* rec, int i,
int n);
/* Failure reasons of a compound attempt: per-code counts and the hosts and
* resources they name, all copied (3.5). TraceCapture builds the arrays
* from the attempt's pend-reason set. */
extern void
sched_trace_attempt_reasons(xce_sched_trace_rec_t* rec,
const sched_trace_reason_count_t* codes,
int ncodes, const char* const* hosts, int nhosts,
const char* const* res, int nres);
/* Cluster-wide check scope. check_begin stores no entry and drops any
* host scope left open; check_end stores the result, which on failure
* says no placement plans were tried (4.8). */
extern void sched_trace_check_begin(xce_sched_trace_rec_t* rec,
const xce_sched_obj_t* obj);
extern void sched_trace_check_end(xce_sched_trace_rec_t* rec, int rc,
bool plans_follow);
/* No plan generated for the current requirement, and why (4.7). Counts
* the requirement under "no plan generated" (9.3), once: a search end
* later in the same requirement makes no statement and counts nothing. */
extern void sched_trace_no_plan(xce_sched_trace_rec_t* rec,
xce_sched_trace_noplan_t why);
/* Plans, numbered in generation order within the current requirement. */
extern void sched_trace_plan_begin(xce_sched_trace_rec_t* rec,
const xce_alloc_plan_t* plan);
extern void sched_trace_plan_discarded(xce_sched_trace_rec_t* rec,
const xce_alloc_plan_t* plan);
extern void sched_trace_plan_not_tried(xce_sched_trace_rec_t* rec,
const xce_alloc_plan_t* plan, int err);
extern void sched_trace_plan_end(xce_sched_trace_rec_t* rec, int rc,
int failed_pos);
/* Plan positions. A pos_scan_error() stores no entry: it turns the
* position's following UNFILLED end into SCAN_ERROR with that error. */
extern void sched_trace_pos_begin(xce_sched_trace_rec_t* rec, int pos,
int slots);
extern void sched_trace_pos_end(xce_sched_trace_rec_t* rec,
xce_sched_trace_pos_end_t how,
const char* host);
extern void sched_trace_pos_scan_error(xce_sched_trace_rec_t* rec, int err);
/*
* Per-host scope: host_begin stores no entry; host_verdict stores the
* host's verdict and ends the scope. An explicit SELECT verdict comes from
* the host-selection pre-filter. `verdicts == 0` means "from the rows": the
* SHORTFALL/LIMIT bits the current host's unmet rows set, or SELECT when
* there were none (the per-host check's own selection branch); either way
* the phase is the per-host check (5.2).
*/
extern void sched_trace_host_begin(xce_sched_trace_rec_t* rec,
const xce_sched_obj_t* obj,
const xce_sched_worker_t* w);
extern void sched_trace_host_verdict(xce_sched_trace_rec_t* rec,
unsigned verdicts, int err);
/*
* The current host's per-host check found limit `limit` exceeded in scope
* instance `scope` (both copied). Sets LIMIT in the host's row verdicts
* and, at summary level, tallies the limit for the position summary once
* per host whose verdict is limit exceeded (5.9). Outside a host scope it
* does nothing. The model-side seam of TraceCapture's limit rows (2.2).
*/
extern void sched_trace_host_limit_hit(xce_sched_trace_rec_t* rec,
const char* limit, const char* scope);
/*
* One satisfaction check row (2.2), copied. In a host scope (between
* host_begin and host_verdict) it is a HOST_ROW, held until the host's
* verdict line and stored right after it, one level deeper; at summary
* level it is not stored at all. Otherwise it is a CHECK_ROW, stored now.
* Which rows to keep, and the host verdict bits they set, are the caller's
* (TraceCapture's) decision.
*/
extern void sched_trace_row(xce_sched_trace_rec_t* rec,
const sched_trace_row_t* row);
/* One host whose claim a failed plan released. Consecutive calls merge
* into one ROLLBACK entry, "released hosts: n01, n02" (5.8). */
extern void sched_trace_rollback_host(xce_sched_trace_rec_t* rec,
const char* host);
/* The plan begun last was committed, or building its allocation failed. */
extern void sched_trace_commit(xce_sched_trace_rec_t* rec);
extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err);
/*
* End of a requirement's plan search with its result `rc`: one line with
* the plans generated, tried, discarded and not tried since the
* requirement began, and at most one statement (rc == 0 only): no plan
* generated (g == 0), all discarded (g > 0, d == g), or none placed (t > 0
* and no tried plan ended in an internal error). The matching decision-
* wide counter is bumped under the same conditions, stored or not (9.3).
* After a no_plan entry in the requirement, there is no statement.
*/
extern void sched_trace_search_end(xce_sched_trace_rec_t* rec, int rc);
/*
* TraceCapture (src/sched_trace_capture.c and, for the outcomes,
* src/sched_trace_outcome.c; design.md "TraceCapture"). Each
* reads scheduler state on the thread that owns it, read-only, and copies
* what it reports into `rec` (never NULL: the SCHED_TRACE gate).
*/
/* The job's ordering policy and keys at selection (2.2): policy name, queue
* name and priority, job priority, queue time. A NULL job (not found by
* the allocator, 2.4) captures nothing. */
extern void sched_trace_capture_selection(xce_sched_trace_rec_t* rec,
const xce_sched_core_t* core,
const xce_sched_job_t* job);
/* The job's matched cluster-wide limits (2.3): each xce_vres_t in
* obj->glimits as (limit name, scope instance). A NULL obj (no schedobj,
* 2.4) leaves the limits unavailable. */
extern void sched_trace_capture_glimits(xce_sched_trace_rec_t* rec,
const xce_sched_obj_t* obj);
/*
* The satisfaction check's rows (4.1, 4.2, 5.4, 5.5), called from
* resource.c unsatisfy_() with its inputs: `nres` rows of resource index,
* total, in-use and requested, and the dense uncommitted vector keyed by
* resource index. Each row's resource is resolved through core->resmgr.
* A limit row (XCE_RESF_LIMIT) is resolved to its limit through the
* scope's matched limits: obj->glimits in the cluster-wide check scope,
* obj->hlimits[w->idx] in a host scope; without a match, the limit name
* and scope come from the resource's own name. The cluster-wide scope
* keeps every row. A host scope keeps only unmet rows: an unmet resource
* row sets the host's SHORTFALL verdict, an unmet limit row its LIMIT
* verdict (via sched_trace_host_limit_hit), never SHORTFALL. Outside both
* scopes nothing is recorded. `core` is only read.
*/
extern void sched_trace_check_rows(xce_sched_trace_rec_t* rec,
xce_sched_core_t* core, int nres,
const int* idxs, const float* total,
const float* inuse,
const xvec_t* uncomitted,
const float* wanted);
/* A failed compound attempt's reasons (3.5), from its pend-reason set before
* the scheduler replaces obj->reason: one ATTEMPT_REASONS entry with a
* count per pend-reason code (main reason, host and resource reasons, in
* the order first seen) and the distinct hosts and resources they name.
* A NULL or empty set stores the entry with no reasons. */
extern void sched_trace_capture_attempt_reasons(xce_sched_trace_rec_t* rec,
const xce_pendreason_t* reason);
/*
* A dispatch decision's allocation as the record's outcome (6.1-6.3, 9.4).
* Each row is a limit charge when its resource, found by name in
* core->resmgr, has XCE_RESF_LIMIT, or, when the name is not registered,
* when it starts with "limit=". A charge is resolved by vres name through
* the job's matched limits: obj->glimits for shared rows,
* obj->hlimits[w->idx] for a host's rows (w found by host name in
* core->workers); unresolved, its name and scope are parsed from the
* resource name and its limited resource is unknown. NULL obj: nothing
* resolves. A NULL alloc stores an empty dispatch. `core` is only read.
*/
extern void sched_trace_capture_dispatch(xce_sched_trace_rec_t* rec,
xce_sched_core_t* core,
const xce_sched_obj_t* obj,
const xce_job_alloc_t* alloc);
/* A pend decision's pend-reason set as the record's outcome (6.4, 9.4). */
extern void sched_trace_capture_pend(xce_sched_trace_rec_t* rec,
const xce_pendreason_t* reason);
/*
* A RECONSTRUCTED record (2.5) for a dispatch the simulator rebuilt from a
* recorded allocation, built on the sim thread: the header from `ctl` and
* `allocs` as its dispatch outcome, captured as above (`obj` is the job's
* schedobj, NULL when it has none). NULL when ctl is NULL or the record
* cannot be allocated.
*/
extern xce_sched_trace_rec_t*
sched_trace_rec_reconstructed(const xce_sched_trace_ctl_t* ctl,
xce_sched_core_t* core, uint64_t jobid,
const xce_sched_obj_t* obj,
const xce_job_alloc_t* allocs);
#endif
Reference in a new issue View git blame Copy permalink