Rongsong Shen
4f951f2b52
Task 1.4: requirement path push/pop with balanced begin/end entries carrying the full nesting path, one entry per traced event kind, and a line budget that stops storing at the bound and counts omitted lines. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
457 lines
20 KiB
C
457 lines
20 KiB
C
#if !defined(_SCHED_TRACE_H_)
|
|
|
|
#define _SCHED_TRACE_H_ 1
|
|
|
|
/*
|
|
* trace.h - scheduling decision trace model (scheduler-sim-decision-trace
|
|
* spec, design.md "TraceModel").
|
|
*
|
|
* A trace record explains one allocator decision for one traced job. It is
|
|
* built on the allocator thread while the job is allocated, handed to the
|
|
* simulator REPL on the decision, and rendered there. The record is
|
|
* self-contained: every string and number it holds is copied into its own
|
|
* per-record arena, so it never points into scheduler memory and stays
|
|
* valid after the job, its schedobj or the core state it was captured from
|
|
* change or go away.
|
|
*
|
|
* Only the simulator REPL ever publishes a control block (`trace_ctl`), so
|
|
* the live scheduler never builds a record (Requirement 11.2).
|
|
*
|
|
* The model (src/sched_trace.c) and the capture helpers
|
|
* (src/sched_trace_capture.c) do no I/O and no logging; test_core_isolation.c
|
|
* scans both files for stdio and file I/O.
|
|
*/
|
|
|
|
#include <stdbool.h>
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "resreq.h"
|
|
#include "sched/types.h"
|
|
|
|
/* Defined in sched/alloc_iter.h, which is not included here: it pulls in
|
|
* scheduler.h and so core.h. */
|
|
typedef struct xce_alloc_plan_s xce_alloc_plan_t;
|
|
|
|
/* Requirement 1.8: two detail levels; summary is the default. */
|
|
typedef enum {
|
|
XCE_TRACE_LEVEL_SUMMARY = 0,
|
|
XCE_TRACE_LEVEL_FULL
|
|
} xce_sched_trace_level_t;
|
|
|
|
/* Requirement 5.10 verdict kinds, bit flags (a host may carry several). */
|
|
typedef enum {
|
|
XCE_TRACE_V_ACCEPTED = 1u << 0,
|
|
XCE_TRACE_V_SELECT = 1u << 1, /* selection mismatch (pre-filter) */
|
|
XCE_TRACE_V_SHORTFALL = 1u << 2, /* resource shortfall (per-host check) */
|
|
XCE_TRACE_V_LIMIT = 1u << 3, /* limit exceeded (per-host check) */
|
|
XCE_TRACE_V_CLAIMED = 1u << 4, /* already claimed by earlier position */
|
|
XCE_TRACE_V_INTERNAL = 1u << 5 /* not evaluated: internal error */
|
|
} xce_sched_trace_verdict_t;
|
|
|
|
/* Number of verdict kinds above: the size of every per-kind counter array,
|
|
* indexed by the kind's bit position. */
|
|
#define XCE_TRACE_V_NKINDS 6
|
|
|
|
/* Requirement 2.4 / 2.5 record kinds. */
|
|
typedef enum {
|
|
XCE_TRACE_REC_DECISION = 0,
|
|
XCE_TRACE_REC_NOT_FOUND,
|
|
XCE_TRACE_REC_NO_SCHEDOBJ,
|
|
XCE_TRACE_REC_INTERNAL_ERROR, /* alloc_rc < 0 or decision build failure */
|
|
XCE_TRACE_REC_PEND_NO_REASON,
|
|
XCE_TRACE_REC_RECONSTRUCTED
|
|
} xce_sched_trace_rec_kind_t;
|
|
|
|
/* One selector from `trace on job ...` (Requirements 1.2, 1.3). */
|
|
typedef struct xce_sched_trace_sel_s {
|
|
uint64_t key; /* plain jobid, array base id, or element id */
|
|
bool whole_array; /* true: key is an array base, matches every element */
|
|
} xce_sched_trace_sel_t;
|
|
|
|
/*
|
|
* Settings snapshot published to the core for one session. Immutable during
|
|
* the session except `position`, which only the allocator thread writes.
|
|
* REPL-owned; a record copies what it needs and never keeps a pointer here.
|
|
*/
|
|
typedef struct xce_sched_trace_ctl_s {
|
|
bool all;
|
|
int nsel;
|
|
xce_sched_trace_sel_t* sel;
|
|
xce_sched_trace_level_t level;
|
|
int line_bound; /* 50..100000 */
|
|
uint64_t session_no; /* rs->session_count of this session */
|
|
const char* policy_name; /* picker name, owned by the ctl */
|
|
int position; /* allocator-owned, reset at session begin */
|
|
} xce_sched_trace_ctl_t;
|
|
|
|
typedef struct xce_sched_trace_rec_s xce_sched_trace_rec_t;
|
|
|
|
#define XCE_TRACE_LINE_BOUND_DEFAULT 1000 /* Requirement 9.1 */
|
|
#define XCE_TRACE_LINE_BOUND_MIN 50
|
|
#define XCE_TRACE_LINE_BOUND_MAX 100000
|
|
#define XCE_TRACE_OUTCOME_HOSTS 50 /* Requirement 9.4 */
|
|
|
|
typedef enum {
|
|
XCE_TRACE_PATH_SINGLE, /* PRIMARY job: the single requirement */
|
|
XCE_TRACE_PATH_COMPOUND, /* top-level COMPOUND container */
|
|
XCE_TRACE_PATH_ALT, /* alternative i/n of an ALTER (a compound) */
|
|
XCE_TRACE_PATH_MEMBER /* member j/m of the enclosing compound */
|
|
} xce_sched_trace_path_kind_t;
|
|
|
|
/* Kept inside the record, not passed by hooks. */
|
|
typedef struct {
|
|
int depth; /* 0..2: container (COMPOUND|ALT) > MEMBER, or SINGLE */
|
|
struct {
|
|
xce_sched_trace_path_kind_t kind;
|
|
int i;
|
|
int n;
|
|
} lvl[2];
|
|
} sched_trace_path_t;
|
|
|
|
/* Requirement 4.7 reasons. */
|
|
typedef enum {
|
|
XCE_TRACE_NOPLAN_DECLINED, /* allocator accepts() declined */
|
|
XCE_TRACE_NOPLAN_GENERATOR_FAILED, /* plan_init failed */
|
|
XCE_TRACE_NOPLAN_NONE_GENERATED /* generator produced no plan */
|
|
} xce_sched_trace_noplan_t;
|
|
|
|
typedef enum {
|
|
XCE_TRACE_POS_FILLED,
|
|
XCE_TRACE_POS_UNFILLED,
|
|
XCE_TRACE_POS_SCAN_ERROR, /* host scan could not start or stopped (4.4) */
|
|
XCE_TRACE_POS_INTERNAL /* host accepted but claim snapshot failed */
|
|
} xce_sched_trace_pos_end_t;
|
|
|
|
/*
|
|
* Decision-wide counters (Requirement 9.3). They stay exact when the line
|
|
* budget truncates the stored entries.
|
|
*/
|
|
typedef struct {
|
|
int plans_tried;
|
|
int plans_discarded;
|
|
int reqs_none_generated;
|
|
int reqs_all_discarded;
|
|
int reqs_none_placed;
|
|
int host_evals[XCE_TRACE_V_NKINDS]; /* indexed by verdict bit position */
|
|
int omitted_lines;
|
|
} sched_trace_counters_t;
|
|
|
|
/*
|
|
* Entry types (design.md "Logical Data Model"). Every stored entry renders
|
|
* as exactly one line in the budgeted sections (requirement, cluster-wide
|
|
* check, plan and host), so the number of stored entries is the number of
|
|
* budgeted lines. The check rows, host rows, search-end, position-summary
|
|
* and rollback entries are added by the tasks that capture them.
|
|
*/
|
|
typedef enum {
|
|
SCHED_TRACE_E_RESREQ_BEGIN = 0, /* requirement named by the path */
|
|
SCHED_TRACE_E_RESREQ_END, /* its outcome; one per begin */
|
|
SCHED_TRACE_E_ALT_SKIPPED, /* alternative with no content (3.6) */
|
|
SCHED_TRACE_E_ALT_RESULT, /* chosen alternative, or none (3.4) */
|
|
SCHED_TRACE_E_MEMBER_FAILED, /* merge failed after member j/m (3.3) */
|
|
SCHED_TRACE_E_ATTEMPT_REASONS, /* failure reasons of an attempt (3.5) */
|
|
SCHED_TRACE_E_CHECK_END, /* cluster-wide check result (4.8) */
|
|
SCHED_TRACE_E_NO_PLAN, /* no plan generated, and why (4.7) */
|
|
SCHED_TRACE_E_PLAN_BEGIN, /* plan tried (4.3) */
|
|
SCHED_TRACE_E_PLAN_DISCARDED, /* plan generated, discarded (4.9) */
|
|
SCHED_TRACE_E_PLAN_NOT_TRIED, /* plan generated, internal error (4.4)*/
|
|
SCHED_TRACE_E_PLAN_END, /* plan result (4.4) */
|
|
SCHED_TRACE_E_POS_BEGIN, /* plan position (4.4) */
|
|
SCHED_TRACE_E_POS_END, /* filled / unfilled / scan error /
|
|
* internal (4.4) */
|
|
SCHED_TRACE_E_HOST, /* host verdict (5.1-5.7, 5.10) */
|
|
SCHED_TRACE_E_COMMIT, /* committed plan (4.5) */
|
|
SCHED_TRACE_E_FINALIZE_FAILED, /* allocation build failed (4.5) */
|
|
SCHED_TRACE_E_NTYPES
|
|
} sched_trace_entry_type_t;
|
|
|
|
/* Where a host verdict came from (5.2). Only the selection, shortfall and
|
|
* limit kinds carry a phase; the other kinds use NONE. */
|
|
typedef enum {
|
|
SCHED_TRACE_PHASE_NONE = 0,
|
|
SCHED_TRACE_PHASE_PREFILTER, /* host-selection pre-filter */
|
|
SCHED_TRACE_PHASE_PERHOST /* per-host resource and limit check */
|
|
} sched_trace_phase_t;
|
|
|
|
/* One requested resource of a single requirement or member (3.2). */
|
|
typedef struct {
|
|
const char* name; /* arena copy */
|
|
float value;
|
|
int per; /* rusage_per_type_t */
|
|
} sched_trace_req_t;
|
|
|
|
/* One pend-reason code of an attempt's failure reasons and how often it
|
|
* occurred (3.5). */
|
|
typedef struct {
|
|
int code;
|
|
int count;
|
|
} sched_trace_reason_count_t;
|
|
|
|
/*
|
|
* One stored entry. All pointers point into the record's arena. `depth` is
|
|
* the nesting level the renderer indents by: a requirement, plan or
|
|
* position begin is stored at the current level and opens the next one;
|
|
* its end closes it and is stored at the begin's level. `path` is the full
|
|
* requirement path in effect when the entry was made (3.1).
|
|
*/
|
|
typedef struct sched_trace_entry_s {
|
|
struct sched_trace_entry_s* next;
|
|
sched_trace_entry_type_t type;
|
|
int depth;
|
|
sched_trace_path_t path;
|
|
union {
|
|
struct { /* RESREQ_BEGIN */
|
|
bool container; /* COMPOUND or ALT: no primary */
|
|
int nmembers; /* container only */
|
|
int min_tasks;
|
|
int max_tasks;
|
|
int nreq;
|
|
sched_trace_req_t* req;
|
|
} resreq_begin;
|
|
struct { /* RESREQ_END */
|
|
int rc; /* > 0 satisfied, 0 not, < 0 error */
|
|
int not_tried_after; /* members after this one, not tried */
|
|
} resreq_end;
|
|
struct { /* ALT_SKIPPED, ALT_RESULT (i == 0: none chosen) */
|
|
int i;
|
|
int n;
|
|
} alt;
|
|
struct { /* MEMBER_FAILED */
|
|
int j;
|
|
int m;
|
|
int err;
|
|
} member_failed;
|
|
struct { /* ATTEMPT_REASONS */
|
|
int ncodes;
|
|
sched_trace_reason_count_t* codes;
|
|
int nhosts;
|
|
const char** hosts;
|
|
int nres;
|
|
const char** res;
|
|
} reasons;
|
|
struct { /* CHECK_END */
|
|
int rc;
|
|
bool plans_follow;
|
|
} check_end;
|
|
struct { /* NO_PLAN */
|
|
xce_sched_trace_noplan_t why;
|
|
} no_plan;
|
|
struct { /* PLAN_BEGIN, PLAN_DISCARDED, PLAN_NOT_TRIED */
|
|
int plan_no; /* 1-based, in generation order per requirement */
|
|
int nbuckets; /* hosts the plan needs */
|
|
int* slots; /* tasks per position, nbuckets of them */
|
|
int err; /* PLAN_NOT_TRIED only */
|
|
} plan;
|
|
struct { /* PLAN_END */
|
|
int plan_no;
|
|
int rc;
|
|
int failed_pos; /* the position that could not be filled */
|
|
} plan_end;
|
|
struct { /* POS_BEGIN */
|
|
int pos;
|
|
int slots;
|
|
} pos_begin;
|
|
struct { /* POS_END */
|
|
int pos;
|
|
xce_sched_trace_pos_end_t how;
|
|
const char* host; /* filled: the accepted host */
|
|
int err; /* scan error */
|
|
} pos_end;
|
|
struct { /* HOST */
|
|
const char* host;
|
|
unsigned verdicts; /* xce_sched_trace_verdict_t bits */
|
|
sched_trace_phase_t phase;
|
|
int err;
|
|
} host;
|
|
struct { /* COMMIT, FINALIZE_FAILED */
|
|
int plan_no; /* the last plan begun */
|
|
int err; /* FINALIZE_FAILED only */
|
|
} commit;
|
|
} u;
|
|
} sched_trace_entry_t;
|
|
|
|
/* One arena chunk; the record frees the whole chain at once. */
|
|
typedef struct sched_trace_chunk_s {
|
|
struct sched_trace_chunk_s* next;
|
|
size_t cap;
|
|
size_t used;
|
|
/* followed by `cap` bytes of storage */
|
|
} sched_trace_chunk_t;
|
|
|
|
/*
|
|
* Record layout. Read only by sched_trace*.c and the REPL renderer; every
|
|
* other file treats the record as opaque. All pointers point into the
|
|
* record's own arena.
|
|
*/
|
|
struct xce_sched_trace_rec_s {
|
|
/* header */
|
|
uint64_t jobid;
|
|
xce_sched_trace_rec_kind_t kind;
|
|
int err; /* no-decision kinds: the error, else 0 */
|
|
xce_sched_trace_level_t level;
|
|
uint64_t session_no;
|
|
int position;
|
|
int line_bound;
|
|
bool incomplete; /* arena growth failed: appending stopped */
|
|
|
|
/* selection block */
|
|
const char* policy_name; /* arena copy of ctl->policy_name, or NULL */
|
|
|
|
/* current requirement path (pushed and popped by the allocation chain) */
|
|
sched_trace_path_t path;
|
|
|
|
sched_trace_counters_t counters;
|
|
|
|
/* stored entries, in the order made; at most line_bound of them */
|
|
sched_trace_entry_t* entries;
|
|
sched_trace_entry_t* entries_tail;
|
|
int nentries;
|
|
|
|
/* builder state (allocator thread only, while the record is built) */
|
|
int depth; /* nesting level of the next entry */
|
|
int path_overflow; /* pushes beyond the two path levels, not kept */
|
|
int resreq_open; /* requirement begins without their end yet */
|
|
int plan_no; /* plans generated so far in this requirement */
|
|
int cur_plan_no; /* the last plan begun */
|
|
int cur_pos; /* the position begun last */
|
|
int scan_err; /* pos_scan_error() for the current position */
|
|
bool scan_err_set;
|
|
unsigned host_row_verdicts; /* SHORTFALL/LIMIT from the current host's
|
|
* unmet rows (set by the row capture) */
|
|
/* scope pointers, kept only until the matching scope ends and never
|
|
* dereferenced after it (TraceCapture's limit lookup) */
|
|
const xce_sched_obj_t* scope_obj;
|
|
const xce_sched_worker_t* scope_w;
|
|
|
|
/* per-record arena */
|
|
sched_trace_chunk_t* arena;
|
|
size_t arena_bytes; /* sum of chunk sizes, headers included */
|
|
};
|
|
|
|
/* True iff `jobid` is selected by `ctl`: `all`, an exact plain-job or
|
|
* element selector, or a whole-array selector whose array the element
|
|
* belongs to (also for elements created after the command). NULL ctl:
|
|
* false. */
|
|
extern bool sched_trace_ctl_wants(const xce_sched_trace_ctl_t* ctl,
|
|
uint64_t jobid);
|
|
|
|
/* New record for `jobid` with the header copied from `ctl`. NULL when `ctl`
|
|
* is NULL or allocation fails (the job is then not traced). */
|
|
extern xce_sched_trace_rec_t*
|
|
sched_trace_rec_new(const xce_sched_trace_ctl_t* ctl, uint64_t jobid,
|
|
xce_sched_trace_rec_kind_t kind);
|
|
|
|
/* Frees the record and its arena. NULL-safe. */
|
|
extern void sched_trace_rec_free(xce_sched_trace_rec_t* rec);
|
|
|
|
/* Finish a record as a no-decision kind (NOT_FOUND, NO_SCHEDOBJ,
|
|
* INTERNAL_ERROR, PEND_NO_REASON) before it is pushed via
|
|
* make_trace_decision (2.4). Any other kind leaves the record unchanged. */
|
|
extern void sched_trace_rec_finish(xce_sched_trace_rec_t* rec,
|
|
xce_sched_trace_rec_kind_t kind, int err);
|
|
|
|
/* Bytes owned by the record: the record block plus its arena. 0 for NULL. */
|
|
extern size_t sched_trace_rec_bytes(const xce_sched_trace_rec_t* rec);
|
|
|
|
/*
|
|
* Arena copies. Each returns storage owned by the record, freed with it.
|
|
* On allocation failure they set rec->incomplete and return NULL. A NULL
|
|
* source (or rec) returns NULL without marking the record incomplete.
|
|
*/
|
|
extern void* sched_trace_arena_alloc(xce_sched_trace_rec_t* rec, size_t n);
|
|
extern void* sched_trace_arena_memdup(xce_sched_trace_rec_t* rec,
|
|
const void* src, size_t n);
|
|
extern char* sched_trace_arena_strdup(xce_sched_trace_rec_t* rec,
|
|
const char* s);
|
|
|
|
/*
|
|
* Entry sequence (design.md "TraceModel"). `rec` is never NULL (the
|
|
* SCHED_TRACE gate guarantees it). Calls nest in the order the allocator
|
|
* makes them: every path_push has one path_pop and every *_begin one
|
|
* *_end. Entries past rec->line_bound are not stored and each adds one to
|
|
* counters.omitted_lines (9.1, 9.2); the builder state (path, nesting,
|
|
* plan numbers, scope pointers) is kept up to date either way.
|
|
*/
|
|
|
|
/* Requirement path: SINGLE 1/1, COMPOUND 1/1 or ALT i/n as the container
|
|
* level, MEMBER j/m below it. Not an entry. */
|
|
extern void sched_trace_path_push(xce_sched_trace_rec_t* rec,
|
|
xce_sched_trace_path_kind_t kind, int i,
|
|
int n);
|
|
extern void sched_trace_path_pop(xce_sched_trace_rec_t* rec);
|
|
|
|
/* Begin and end of the requirement the current path names. `primary` is
|
|
* NULL for a container, whose `nmembers` is its member count. */
|
|
extern void sched_trace_resreq_begin(xce_sched_trace_rec_t* rec,
|
|
const xce_primary_resreq_t* primary,
|
|
int nmembers);
|
|
extern void sched_trace_resreq_end(xce_sched_trace_rec_t* rec, int rc,
|
|
int not_tried_after);
|
|
|
|
/* Merge failure after member j of m: members j+1..m not tried (3.3). */
|
|
extern void sched_trace_member_failed(xce_sched_trace_rec_t* rec, int j,
|
|
int m, int err);
|
|
/* Alternative i/n chosen, later ones not tried (3.4); 0: all failed. */
|
|
extern void sched_trace_alt_result(xce_sched_trace_rec_t* rec, int chosen_i,
|
|
int n);
|
|
/* Alternative i/n skipped: it has no requirement content (3.6). */
|
|
extern void sched_trace_alt_skipped(xce_sched_trace_rec_t* rec, int i,
|
|
int n);
|
|
/* Failure reasons of a compound attempt: per-code counts and the hosts and
|
|
* resources they name, all copied (3.5). TraceCapture builds the arrays
|
|
* from the attempt's pend-reason set. */
|
|
extern void
|
|
sched_trace_attempt_reasons(xce_sched_trace_rec_t* rec,
|
|
const sched_trace_reason_count_t* codes,
|
|
int ncodes, const char* const* hosts, int nhosts,
|
|
const char* const* res, int nres);
|
|
|
|
/* Cluster-wide check scope. check_begin stores no entry; check_end stores
|
|
* the result, which on failure says no placement plans were tried (4.8). */
|
|
extern void sched_trace_check_begin(xce_sched_trace_rec_t* rec,
|
|
const xce_sched_obj_t* obj);
|
|
extern void sched_trace_check_end(xce_sched_trace_rec_t* rec, int rc,
|
|
bool plans_follow);
|
|
|
|
extern void sched_trace_no_plan(xce_sched_trace_rec_t* rec,
|
|
xce_sched_trace_noplan_t why);
|
|
|
|
/* Plans, numbered in generation order within the current requirement. */
|
|
extern void sched_trace_plan_begin(xce_sched_trace_rec_t* rec,
|
|
const xce_alloc_plan_t* plan);
|
|
extern void sched_trace_plan_discarded(xce_sched_trace_rec_t* rec,
|
|
const xce_alloc_plan_t* plan);
|
|
extern void sched_trace_plan_not_tried(xce_sched_trace_rec_t* rec,
|
|
const xce_alloc_plan_t* plan, int err);
|
|
extern void sched_trace_plan_end(xce_sched_trace_rec_t* rec, int rc,
|
|
int failed_pos);
|
|
|
|
/* Plan positions. A pos_scan_error() stores no entry: it turns the
|
|
* position's following UNFILLED end into SCAN_ERROR with that error. */
|
|
extern void sched_trace_pos_begin(xce_sched_trace_rec_t* rec, int pos,
|
|
int slots);
|
|
extern void sched_trace_pos_end(xce_sched_trace_rec_t* rec,
|
|
xce_sched_trace_pos_end_t how,
|
|
const char* host);
|
|
extern void sched_trace_pos_scan_error(xce_sched_trace_rec_t* rec, int err);
|
|
|
|
/*
|
|
* Per-host scope: host_begin stores no entry; host_verdict stores the
|
|
* host's verdict and ends the scope. An explicit SELECT verdict comes from
|
|
* the host-selection pre-filter. `verdicts == 0` means "from the rows": the
|
|
* SHORTFALL/LIMIT bits the current host's unmet rows set, or SELECT when
|
|
* there were none (the per-host check's own selection branch); either way
|
|
* the phase is the per-host check (5.2).
|
|
*/
|
|
extern void sched_trace_host_begin(xce_sched_trace_rec_t* rec,
|
|
const xce_sched_obj_t* obj,
|
|
const xce_sched_worker_t* w);
|
|
extern void sched_trace_host_verdict(xce_sched_trace_rec_t* rec,
|
|
unsigned verdicts, int err);
|
|
|
|
/* The plan begun last was committed, or building its allocation failed. */
|
|
extern void sched_trace_commit(xce_sched_trace_rec_t* rec);
|
|
extern void sched_trace_finalize_failed(xce_sched_trace_rec_t* rec, int err);
|
|
|
|
#endif
|