docs: incorporate review feedback for resreq-ast-cse-design
- Rename grammar rules: or_expr→select_or, and_expr→select_and, not_expr→select_not, atom_expr→select_primary - Eliminate SEL_BOOL_LITERAL: parse true/false directly as SEL_TRUE/SEL_FALSE; boolean resources used standalone (gpu, not gpu==true) - Remove 'true'|'false' from value production in grammar - Use LSF-style unquoted strings in examples (type==linux, not type=="linux") - Document n-ary vs binary tree decision in §6.2 - Add §8.5 Evaluation Memoization: per-host select_eval_ctx_t with node_id-indexed bool_bits bitmap, num_vals, str_vals arrays; memoized eval_select with cache check; session lifecycle (pre-session capacity, per-host alloc/discard, post-session GC + node ID recompaction) - Add node_id field to xce_select_node_t for eval context indexing - Clarify §9 two-level sharing: select CSE is global, resreq sharing is scoped to (user, queue, project) - Clarify §10 select vs rusage boundary: select evaluates against host snapshot, rusage checks host.available (total - booked)
This commit is contained in:
parent
19898a198a
commit
c2ad96a5e0
1 changed files with 382 additions and 86 deletions
|
|
@ -1,7 +1,7 @@
|
|||
# XCE Resource Requirement AST & CSE Design
|
||||
|
||||
> **Status**: Draft
|
||||
> **Last Updated**: 2026-04-19
|
||||
> **Last Updated**: 2026-04-21
|
||||
> **Target**: `src/lib/` (C shared library), `src/include/resreq.h`
|
||||
|
||||
---
|
||||
|
|
@ -49,7 +49,7 @@ This document describes the design to:
|
|||
- Apply **Common Subexpression Elimination (CSE)** at the AST node level so
|
||||
identical sub-expressions across all jobs share a single allocation.
|
||||
In production HPC workloads, many jobs share identical or overlapping
|
||||
select expressions (e.g., `type=="x86_64" && gpu==true`). CSE at the
|
||||
select expressions (e.g., `type==x86_64 && gpu`). CSE at the
|
||||
AST node level allows these jobs to share interned subtrees, yielding
|
||||
significant memory savings and reducing allocation pressure during
|
||||
scheduling — validated by experience with IBM LSF's select clause
|
||||
|
|
@ -98,19 +98,18 @@ primary_resreq ::= select[...] rusage[...] span[...] same[...] affinity[...]
|
|||
### 3.1 Grammar (new `lib/xce_select.peg`)
|
||||
|
||||
```
|
||||
select_expr ::= or_expr EOF
|
||||
or_expr ::= and_expr ( '||' and_expr )*
|
||||
and_expr ::= not_expr ( '&&' not_expr )*
|
||||
not_expr ::= '!' atom_expr | atom_expr
|
||||
atom_expr ::= '(' or_expr ')' | comparison | resname
|
||||
comparison ::= resname cmp_op value
|
||||
cmp_op ::= '>=' | '<=' | '!=' | '==' | '>' | '<'
|
||||
resname ::= [a-zA-Z_][a-zA-Z_0-9]*
|
||||
value ::= string_literal | number | boolean_literal | ANY
|
||||
boolean_literal ::= 'true' | 'false'
|
||||
ANY ::= 'any'
|
||||
number ::= [0-9]+ ('.' [0-9]*)?
|
||||
string_literal ::= [a-zA-Z_][a-zA-Z_0-9]* (unquoted word, excluding 'any', 'true', 'false')
|
||||
select_expr ::= select_or EOF
|
||||
select_or ::= select_and ( '||' select_and )*
|
||||
select_and ::= select_not ( '&&' select_not )*
|
||||
select_not ::= '!' select_primary | select_primary
|
||||
select_primary ::= '(' select_or ')' | comparison | resname
|
||||
comparison ::= resname cmp_op value
|
||||
cmp_op ::= '>=' | '<=' | '!=' | '==' | '>' | '<'
|
||||
resname ::= [a-zA-Z_][a-zA-Z_0-9]*
|
||||
value ::= string_literal | number | ANY
|
||||
ANY ::= 'any'
|
||||
number ::= [0-9]+ ('.' [0-9]*)?
|
||||
string_literal ::= [a-zA-Z_][a-zA-Z_0-9]* (unquoted word, excluding 'any')
|
||||
```
|
||||
|
||||
**LSF-compatible disambiguation:** `resname` and `string_literal` share
|
||||
|
|
@ -118,8 +117,9 @@ the same lexical pattern. The parser resolves the ambiguity using the
|
|||
resource manager: if a bare word matches a known resource name, it is
|
||||
parsed as `SEL_RESNAME`; otherwise it is treated as `SEL_STR` (a string
|
||||
literal). This is compatible with IBM LSF's select clause semantics.
|
||||
For example, `type=="linux"` — `type` resolves to `SEL_RESNAME` (known
|
||||
For example, `type==linux` — `type` resolves to `SEL_RESNAME` (known
|
||||
resource), `linux` resolves to `SEL_STR` (not a known resource name).
|
||||
String values in LSF select expressions are unquoted words.
|
||||
|
||||
**`any` keyword:** The special keyword `any` matches any value. It is
|
||||
only valid as the right-hand side of `==` or `!=` (e.g., `type == any`,
|
||||
|
|
@ -136,13 +136,29 @@ form implicitly. If this inconsistency proves confusing in practice,
|
|||
a future change can allow `resname != any` at parse time and const-fold
|
||||
it to `SEL_FALSE`.
|
||||
|
||||
**Boolean literals:** `true` and `false` are parsed as `SEL_BOOL_LITERAL`
|
||||
nodes. The value is stored as `int bval` (1 for true, 0 for false) in
|
||||
the node's union. Type validation ensures that boolean literals only
|
||||
appear in `==`/`!=` comparisons with `RES_TYPE_BOOL` resources, or as
|
||||
standalone boolean operands (where they const-fold to `SEL_TRUE` or
|
||||
`SEL_FALSE`). `true` and `false` are reserved keywords excluded from
|
||||
the `string_literal` pattern.
|
||||
**Boolean keywords `true` and `false`:** These are parsed directly as
|
||||
`SEL_TRUE` and `SEL_FALSE` singleton nodes — there is no intermediate
|
||||
node type. They are not part of the `value` production (the RHS of
|
||||
comparisons) because boolean resources should be used as standalone
|
||||
operands, not compared to `true`/`false`. The idiomatic form is:
|
||||
|
||||
- `gpu` (standalone boolean resource — the resource's value IS the boolean)
|
||||
- **Not** `gpu == true` — though the parser accepts this, const-folding
|
||||
reduces `bool_res == SEL_TRUE` → `SEL_RESNAME(bool_res)` and
|
||||
`bool_res == SEL_FALSE` → `SEL_NOT(SEL_RESNAME(bool_res))`
|
||||
|
||||
Since const-folding always eliminates these comparisons, a dedicated
|
||||
`SEL_BOOL_LITERAL` type would never survive to runtime. Parsing
|
||||
`true` → `SEL_TRUE` and `false` → `SEL_FALSE` directly avoids an
|
||||
intermediate type. `true` and `false` are reserved keywords excluded
|
||||
from the `string_literal` pattern.
|
||||
|
||||
**String literals in comparisons:** Unquoted words that are not known
|
||||
resource names and not reserved keywords (`any`, `true`, `false`) are
|
||||
treated as string literals. For example, `type == x86_64` — `type`
|
||||
resolves to `SEL_RESNAME` (known resource), `x86_64` resolves to
|
||||
`SEL_STR` (not a known resource name). String values in comparisons
|
||||
do not need quotes — LSF select expressions use unquoted words.
|
||||
|
||||
### 3.2 Node Types
|
||||
|
||||
|
|
@ -160,10 +176,9 @@ typedef enum {
|
|||
SEL_RESNAME, /* leaf: resource variable */
|
||||
SEL_NUM, /* leaf: numeric literal */
|
||||
SEL_STR, /* leaf: string literal */
|
||||
SEL_BOOL_LITERAL, /* leaf: boolean literal (true/false) */
|
||||
SEL_ANY, /* leaf: any keyword (const-folded to SEL_TRUE) */
|
||||
SEL_TRUE, /* leaf: constant true (from const fold) */
|
||||
SEL_FALSE, /* leaf: constant false (from const fold) */
|
||||
SEL_TRUE, /* leaf: constant true — also produced by keyword `true` */
|
||||
SEL_FALSE, /* leaf: constant false — also produced by keyword `false` */
|
||||
} sel_node_type_t;
|
||||
```
|
||||
|
||||
|
|
@ -173,7 +188,8 @@ typedef enum {
|
|||
struct xce_select_node_ {
|
||||
sel_node_type_t type;
|
||||
uint32_t hashcode; /* cached at construction */
|
||||
unsigned gc_mark; /* set during mark-and-sweep GC (§5.6) */
|
||||
uint32_t node_id; /* dense index into eval context arrays (§8.5) */
|
||||
unsigned gc_mark; /* set during mark-and-sweep GC (§5.5) */
|
||||
union {
|
||||
struct {
|
||||
xce_select_node_t *left, *right;
|
||||
|
|
@ -185,7 +201,6 @@ struct xce_select_node_ {
|
|||
} resname;
|
||||
double numval; /* SEL_NUM */
|
||||
char* strval; /* SEL_STR — interned via makeString() */
|
||||
int bval; /* SEL_BOOL_LITERAL — 1 for true, 0 for false */
|
||||
} u;
|
||||
};
|
||||
```
|
||||
|
|
@ -303,7 +318,6 @@ Key encoding per node type:
|
|||
| `SEL_RESNAME` | `(uintptr_t)res_idx` | `0` |
|
||||
| `SEL_NUM` | bit-cast of `double` | `0` |
|
||||
| `SEL_STR` | `(uintptr_t)xstring_ptr` | `0` |
|
||||
| `SEL_BOOL_LITERAL` | `(uintptr_t)bval` | `0` |
|
||||
| `SEL_ANY` | `0` | `0` |
|
||||
| `SEL_TRUE` | `0` | `0` |
|
||||
| `SEL_FALSE` | `0` | `0` |
|
||||
|
|
@ -362,6 +376,7 @@ xce_select_node_t* makeSelectNode(sel_node_type_t type,
|
|||
if (node == NULL) { errno = ENOMEM; return NULL; }
|
||||
node->type = type;
|
||||
node->hashcode = sel_key_hash(&key);
|
||||
node->node_id = sel_node_next_id++; /* dense index for eval context */
|
||||
/* fill u.* from op1/op2 by type … */
|
||||
|
||||
/* 5. insert — re-check under lock for TOCTOU safety */
|
||||
|
|
@ -449,6 +464,10 @@ entry is marked for removal; the select root pointer is unlinked and
|
|||
the job struct freed at the session boundary, before GC runs. This
|
||||
eliminates the risk of shared sub-nodes becoming unreachable mid-session.
|
||||
|
||||
After GC, node IDs are re-compacted to `0..N-1` (§8.5.4) so that the
|
||||
evaluation context arrays remain dense. This is done by a linear scan
|
||||
of the intern table that reassigns `node_id` sequentially.
|
||||
|
||||
The function is not thread-safe with respect to concurrent
|
||||
`eval_select` calls, so the scheduler must ensure no evaluations are
|
||||
in progress when GC runs.
|
||||
|
|
@ -504,6 +523,19 @@ After flattening, `(A && B) && C` and `A && (B && C)` both produce the
|
|||
same left-associative tree, maximizing CSE sharing. The same transform
|
||||
applies to `SEL_OR`.
|
||||
|
||||
**Why binary trees, not n-ary nodes?** An alternative design uses n-ary
|
||||
nodes like `(AND, A, B, C)` instead of binary `(AND, (AND, A, B), C)`.
|
||||
N-ary nodes make operand reordering trivial (sort the children array) and
|
||||
eliminate the need for flattening. However, they break the fixed-size
|
||||
`sel_node_key_t` — the CSE key becomes variable-length, requiring a
|
||||
hash over N sorted child pointers per lookup. Binary trees preserve the
|
||||
O(1) fixed-size key: `(SEL_AND, left_ptr, right_ptr)`. For typical HPC
|
||||
workloads where `&&`/`||` chains are 2–5 operands, binary trees with
|
||||
flattening + commutativity sort give canonical form with zero per-lookup
|
||||
cost beyond two pointer compares. N-ary representation is noted as a
|
||||
potential future enhancement if operand reordering within long chains
|
||||
provides measurable CSE gains beyond what binary canonicalization achieves.
|
||||
|
||||
### 6.3 Const-Fold Transform
|
||||
|
||||
A **const-fold pass** simplifies the AST by evaluating expressions that
|
||||
|
|
@ -516,6 +548,10 @@ are statically determined at parse time:
|
|||
- Identity laws:`SEL_TRUE && X` → `X`, `X && SEL_TRUE` → `X`; `SEL_FALSE || X` → `X`, `X || SEL_FALSE` → `X`
|
||||
- Annihilation laws: `SEL_FALSE && X` → `SEL_FALSE`, `X && SEL_FALSE` → `SEL_FALSE`; `SEL_TRUE || X` → `SEL_TRUE`, `X || SEL_TRUE` → `SEL_TRUE`
|
||||
- Complement laws: `!SEL_TRUE` → `SEL_FALSE`, `!SEL_FALSE` → `SEL_TRUE`
|
||||
- Boolean equality: `bool_res == SEL_TRUE` → `bool_res` (since `x == true` is just `x`);
|
||||
`bool_res == SEL_FALSE` → `SEL_NOT(bool_res)` (since `x == false` is `!x`);
|
||||
`bool_res != SEL_TRUE` → `SEL_NOT(bool_res)`; `bool_res != SEL_FALSE` → `bool_res`.
|
||||
These apply only when `bool_res` is a `SEL_RESNAME` with `res_type == RES_TYPE_BOOL`.
|
||||
|
||||
The const-fold pass runs **after** construction and flattening, replacing
|
||||
subtrees with constant leaf nodes. Since all nodes are interned,
|
||||
|
|
@ -584,15 +620,16 @@ if (!operand_is_numeric(lnode) || !operand_is_numeric(rnode)) {
|
|||
compatible. When a resource name (SEL_RESNAME) is compared to a literal,
|
||||
the literal is coerced to the resource's type:
|
||||
|
||||
- `type == "linux"` — `type` is `RES_TYPE_STRING`, `"linux"` is already
|
||||
`SEL_STR` — types match.
|
||||
- `type == linux` — `type` is `SEL_RESNAME(RES_TYPE_STRING)`, `linux` is
|
||||
`SEL_STR` (not a known resource name) — types match.
|
||||
- `type == 5` — `type` is `RES_TYPE_STRING`, `5` is `SEL_NUM` — the
|
||||
numeric literal is converted to `SEL_STR` (`"5"`) at parse time so that
|
||||
eval sees two string operands.
|
||||
- `mem > 100` — `mem` is `RES_TYPE_NUMERIC`, `100` is `SEL_NUM` — types
|
||||
match.
|
||||
- `gpu == true` — `gpu` is `RES_TYPE_BOOL`, `true` remains as a boolean
|
||||
value.
|
||||
- `gpu` — `gpu` is `RES_TYPE_BOOL`, used as a standalone boolean operand
|
||||
(idiomatic LSF form). `gpu == true` is also accepted but const-folds
|
||||
to just `gpu`; `gpu == false` const-folds to `!gpu`.
|
||||
|
||||
This coercion guarantees that at eval time, both operands of `EQ`/`NE`
|
||||
always have matching types — the `switch (lv.type)` dispatch in
|
||||
|
|
@ -659,11 +696,11 @@ typedef struct {
|
|||
|
||||
```c
|
||||
static sel_val_t get_leaf_value(const xce_select_node_t* node,
|
||||
const host_eval_ctx_t* host)
|
||||
select_eval_ctx_t* ctx)
|
||||
{
|
||||
switch (node->type) {
|
||||
case SEL_RESNAME:
|
||||
return host_resval_typed(host,
|
||||
return host_resval_typed(ctx->host,
|
||||
node->u.resname.res_idx,
|
||||
node->u.resname.res_type);
|
||||
case SEL_NUM:
|
||||
|
|
@ -678,53 +715,49 @@ static sel_val_t get_leaf_value(const xce_select_node_t* node,
|
|||
|
||||
### 8.3 `eval_select`
|
||||
|
||||
`eval_select` is a post-order traversal of the AST tree. If anything
|
||||
goes wrong during the visit, `-1` is returned immediately and propagated
|
||||
up — `&&`, `||`, and `!` never silently swallow errors.
|
||||
`eval_select` is a post-order traversal of the AST tree with memoization
|
||||
via a per-host evaluation context (§8.5). The outer `eval_select`
|
||||
checks the cache before recursing; `eval_select_inner` contains the
|
||||
actual recursive logic. If anything goes wrong during the visit, `-1` is
|
||||
returned immediately and propagated up — `&&`, `||`, and `!` never
|
||||
silently swallow errors.
|
||||
|
||||
**Host evaluation context:** `xvec_idval_t` stores per-job resource
|
||||
allocation data, not host resource values. The select evaluator needs a
|
||||
host evaluation context that provides resource values by `res_idx`. A
|
||||
`host_eval_ctx_t` struct bundles the host's numeric resource vector and
|
||||
a string resource lookup table:
|
||||
allocation data, not host resource values. The select evaluator uses a
|
||||
per-host `select_eval_ctx_t` (§8.5) that provides resource values by
|
||||
`res_idx` through its `numvals` and `strvals` members, and caches
|
||||
evaluation results by `node_id` so each node is evaluated at most once
|
||||
per host across all jobs.
|
||||
|
||||
```c
|
||||
typedef struct {
|
||||
const xvec_idval_t* numvals; /* numeric resource values by res_idx */
|
||||
const char** strvals; /* string resource values by res_idx */
|
||||
} host_eval_ctx_t;
|
||||
```
|
||||
|
||||
The Go CGO bridge populates `host_eval_ctx_t` from the host's live
|
||||
resource data before each evaluation call. The `strvals` array is
|
||||
indexed by `res_idx`; entries for non-string resources are NULL.
|
||||
|
||||
```c
|
||||
/* Returns: 1 = host matches, 0 = host does not match, -1 = error */
|
||||
int eval_select(const xce_select_node_t* node, const host_eval_ctx_t* host)
|
||||
/* Inner eval — called by memoized eval_select (§8.5.2) on cache miss.
|
||||
* Returns: 1 = host matches, 0 = host does not match, -1 = error */
|
||||
static int eval_select_inner(const xce_select_node_t* node,
|
||||
select_eval_ctx_t* ctx)
|
||||
{
|
||||
int lv, rv;
|
||||
const host_eval_ctx_t* host = ctx->host; /* host snapshot in ctx */
|
||||
|
||||
switch (node->type) {
|
||||
/* logical — explicit error propagation, not C boolean operators */
|
||||
case SEL_AND:
|
||||
lv = eval_select(node->u.binop.left, host);
|
||||
lv = eval_select(node->u.binop.left, ctx);
|
||||
if (lv < 0) return -1;
|
||||
if (lv == 0) return 0;
|
||||
rv = eval_select(node->u.binop.right, host);
|
||||
rv = eval_select(node->u.binop.right, ctx);
|
||||
if (rv < 0) return -1;
|
||||
return rv;
|
||||
|
||||
case SEL_OR:
|
||||
lv = eval_select(node->u.binop.left, host);
|
||||
lv = eval_select(node->u.binop.left, ctx);
|
||||
if (lv < 0) return -1;
|
||||
if (lv == 1) return 1;
|
||||
rv = eval_select(node->u.binop.right, host);
|
||||
rv = eval_select(node->u.binop.right, ctx);
|
||||
if (rv < 0) return -1;
|
||||
return rv;
|
||||
|
||||
case SEL_NOT:
|
||||
lv = eval_select(node->u.operand, host);
|
||||
lv = eval_select(node->u.operand, ctx);
|
||||
if (lv < 0) return -1;
|
||||
return !lv;
|
||||
|
||||
|
|
@ -746,8 +779,8 @@ int eval_select(const xce_select_node_t* node, const host_eval_ctx_t* host)
|
|||
|
||||
/* numeric comparisons — operand types validated at parse time */
|
||||
case SEL_GT: case SEL_LT: case SEL_GE: case SEL_LE: {
|
||||
double l = get_leaf_value(node->u.binop.left, host).u.num;
|
||||
double r = get_leaf_value(node->u.binop.right, host).u.num;
|
||||
double l = get_leaf_value(node->u.binop.left, ctx).u.num;
|
||||
double r = get_leaf_value(node->u.binop.right, ctx).u.num;
|
||||
if (node->type == SEL_GT) return l > r;
|
||||
if (node->type == SEL_LT) return l < r;
|
||||
if (node->type == SEL_GE) return l >= r;
|
||||
|
|
@ -756,8 +789,8 @@ int eval_select(const xce_select_node_t* node, const host_eval_ctx_t* host)
|
|||
|
||||
/* type-dispatched equality — type guaranteed compatible at parse time */
|
||||
case SEL_EQ: case SEL_NE: {
|
||||
sel_val_t l = get_leaf_value(node->u.binop.left, host);
|
||||
sel_val_t r = get_leaf_value(node->u.binop.right, host);
|
||||
sel_val_t l = get_leaf_value(node->u.binop.left, ctx);
|
||||
sel_val_t r = get_leaf_value(node->u.binop.right, ctx);
|
||||
int eq;
|
||||
switch (l.type) {
|
||||
case RES_TYPE_NUMERIC: eq = l.u.num == r.u.num; break;
|
||||
|
|
@ -815,13 +848,215 @@ int eval_select(const xce_select_node_t* node, const host_eval_ctx_t* host)
|
|||
an eval-time reference count (increment before eval, decrement after)
|
||||
must be added to the C API.
|
||||
|
||||
### 8.5 Evaluation Memoization
|
||||
|
||||
Many jobs share sub-expressions via CSE (§5). Without memoization,
|
||||
evaluating `type==linux` for job1 and job2 against the same host
|
||||
computes the same node twice. A per-host evaluation context stores
|
||||
cached results indexed by `node_id`, so each node is evaluated at most
|
||||
once per host across all jobs.
|
||||
|
||||
**Design principle:** The interned AST nodes are **read-only** — they
|
||||
carry no mutable eval state. All ephemeral evaluation state lives in a
|
||||
separate `select_eval_ctx_t` that is allocated per host, used across all
|
||||
jobs for that host within a session, and discarded when scheduling for
|
||||
that host completes.
|
||||
|
||||
#### 8.5.1 Evaluation Context
|
||||
|
||||
```c
|
||||
typedef struct {
|
||||
const host_eval_ctx_t* host; /* host resource snapshot for this evaluation */
|
||||
|
||||
/* Boolean result: 2 bits per node
|
||||
* 00 = not evaluated
|
||||
* 01 = match (true)
|
||||
* 10 = no-match (false)
|
||||
* 11 = error
|
||||
*/
|
||||
uint8_t* bool_bits; /* bitmap, 2 bits per node */
|
||||
uint32_t bool_bits_cap; /* capacity in nodes (not bytes) */
|
||||
|
||||
/* Numeric leaf cache: only populated for SEL_RESNAME(numeric) and SEL_NUM.
|
||||
* Indexed by node_id. Unused entries are 0.0 (which is a valid numeric
|
||||
* value, but leaf nodes are always evaluated before use, so the slot
|
||||
* is always written before it is read). */
|
||||
double* num_vals;
|
||||
uint32_t num_vals_cap;
|
||||
|
||||
/* String leaf cache: only populated for SEL_RESNAME(string) and SEL_STR.
|
||||
* Indexed by node_id. Unused entries are NULL. */
|
||||
const char** str_vals;
|
||||
uint32_t str_vals_cap;
|
||||
} select_eval_ctx_t;
|
||||
```
|
||||
|
||||
Each interned node carries a `node_id` — a dense 0-based index assigned
|
||||
at intern time. The evaluation context arrays are sized to
|
||||
`max(node_id) + 1`.
|
||||
|
||||
```c
|
||||
#define EVAL_NONE 0 /* not yet evaluated */
|
||||
#define EVAL_TRUE 1 /* node evaluates to true */
|
||||
#define EVAL_FALSE 2 /* node evaluates to false */
|
||||
#define EVAL_ERROR 3 /* node evaluates to error */
|
||||
|
||||
static inline int eval_get_bool(const select_eval_ctx_t* ctx, uint32_t node_id)
|
||||
{
|
||||
uint32_t bit_off = node_id * 2;
|
||||
return (ctx->bool_bits[bit_off / 8] >> (bit_off % 8)) & 0x3;
|
||||
}
|
||||
|
||||
static inline void eval_set_bool(select_eval_ctx_t* ctx, uint32_t node_id, int val)
|
||||
{
|
||||
uint32_t bit_off = node_id * 2;
|
||||
uint32_t byte = bit_off / 8;
|
||||
uint32_t shift = bit_off % 8;
|
||||
ctx->bool_bits[byte] = (ctx->bool_bits[byte] & ~(0x3 << shift))
|
||||
| ((val & 0x3) << shift);
|
||||
}
|
||||
```
|
||||
|
||||
#### 8.5.2 Memoized `eval_select`
|
||||
|
||||
```c
|
||||
int eval_select(const xce_select_node_t* node, select_eval_ctx_t* ctx)
|
||||
{
|
||||
/* check cache — O(1) bitmap lookup */
|
||||
int cached = eval_get_bool(ctx, node->node_id);
|
||||
if (cached != EVAL_NONE)
|
||||
return (cached == EVAL_TRUE) ? 1 :
|
||||
(cached == EVAL_FALSE) ? 0 : -1;
|
||||
|
||||
int result = eval_select_inner(node, ctx); /* recursive eval */
|
||||
|
||||
/* write cache */
|
||||
eval_set_bool(ctx, node->node_id,
|
||||
result == 1 ? EVAL_TRUE :
|
||||
result == 0 ? EVAL_FALSE : EVAL_ERROR);
|
||||
|
||||
return result;
|
||||
}
|
||||
```
|
||||
|
||||
`eval_select_inner` contains the recursive evaluation logic (§8.3) but
|
||||
without the cache check — it is called only on cache miss. Leaf nodes
|
||||
(`SEL_RESNAME`, `SEL_NUM`, `SEL_STR`, `SEL_TRUE`, `SEL_FALSE`) write
|
||||
their results to the `num_vals` and `str_vals` arrays as well, so
|
||||
comparison operators can read cached leaf values without re-reading host
|
||||
data.
|
||||
|
||||
#### 8.5.3 Session Lifecycle
|
||||
|
||||
```
|
||||
┌─ Pre-session ──────────────────────────────────────────────────┐
|
||||
│ 1. Check max_node_id against eval context capacity │
|
||||
│ 2. If needed: grow bool_bits / num_vals / str_vals arrays │
|
||||
│ 3. If many nodes GC'd and eval context is oversized by 2x: │
|
||||
│ shrink arrays (optional, avoids unbounded growth) │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─ Scheduling session ─────────────────────────────────────────────┐
|
||||
│ for each host: │
|
||||
│ ctx = select_eval_ctx_for_host(host) /* allocate or reuse */ │
|
||||
│ memset(ctx->bool_bits, 0, bitmap_bytes(max_node_id + 1)) │
|
||||
│ /* num_vals and str_vals: no need to clear — │
|
||||
│ leaf resolution overwrites per-host before comparators │
|
||||
│ read them. Only bool_bits needs clearing because its │
|
||||
│ "not evaluated" state (EVAL_NONE) is represented by 0. */ │
|
||||
│ for each pending_job: │
|
||||
│ rc = eval_select(job->select_root, ctx) │
|
||||
│ if rc < 0: log error, skip host │
|
||||
│ if rc == 0: skip host (select filter failed) │
|
||||
│ if rc == 1: proceed to rusage check │
|
||||
│ select_eval_ctx_discard(ctx) /* free per-host context */ │
|
||||
└──────────────────────────────────────────────────────────────────┘
|
||||
↓
|
||||
┌─ Post-session ──────────────────────────────────────────────────┐
|
||||
│ 1. Unlink completed jobs (primary->select->root = NULL) │
|
||||
│ 2. Run gc_sweep_select_nodes() │
|
||||
│ 3. Re-allocate node IDs: linear scan of intern table, │
|
||||
│ assign 0..N-1 (dense after GC) │
|
||||
│ 4. Adjust eval context for next session capacity check │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
**Per-host allocation and discard:** Each host gets its own
|
||||
`select_eval_ctx_t` that is allocated when the scheduler begins
|
||||
evaluating jobs against that host, shared across all jobs for that host
|
||||
within the session, and freed when the host's evaluation is complete.
|
||||
This means the context does **not** persist between hosts — the next
|
||||
host gets a fresh (or reused) context with `bool_bits` cleared.
|
||||
|
||||
#### 8.5.4 Node ID Re-compaction After GC
|
||||
|
||||
After GC removes unreachable nodes, the remaining nodes have sparse
|
||||
IDs. Compacting to `0..N-1` keeps the eval context arrays dense:
|
||||
|
||||
```c
|
||||
/* Run after gc_sweep_select_nodes() */
|
||||
static void recompact_node_ids(void)
|
||||
{
|
||||
uint32_t next_id = 0;
|
||||
kvtab_iter_t* iter = iter_make(sel_node_table.table);
|
||||
for (void* val; iter_next(iter, NULL, &val); ) {
|
||||
xce_select_node_t* node = (xce_select_node_t*)val;
|
||||
node->node_id = next_id++;
|
||||
}
|
||||
iter_free(iter);
|
||||
sel_node_next_id = next_id; /* reset counter for new allocations */
|
||||
}
|
||||
```
|
||||
|
||||
#### 8.5.5 Memory Cost
|
||||
|
||||
For a cluster with 10,000 unique select nodes:
|
||||
|
||||
| Field | Per-node cost | Total 10K nodes |
|
||||
|-------|:---:|:---:|
|
||||
| `bool_bits` (2 bits/node) | 0.25 B | 2.5 KB |
|
||||
| `num_vals` (float64) | 8 B | 80 KB |
|
||||
| `str_vals` (char*) | 8 B | 80 KB |
|
||||
| **Total per host** | | **~163 KB** |
|
||||
|
||||
By separating the typed arrays instead of using a `sel_val_t` union
|
||||
(24 bytes/node) for every node, the per-host cost drops from ~240 KB to
|
||||
~163 KB — a 32% reduction. The savings is larger in practice because
|
||||
80%+ of nodes are boolean-type (AND, OR, NOT, comparisons), which use
|
||||
only 2 bits instead of 24 bytes.
|
||||
|
||||
#### 8.5.6 Thread Safety for Memoization
|
||||
|
||||
The `select_eval_ctx_t` is per-host and per-session. It is not shared
|
||||
between hosts or between sessions. Since the scheduler goroutine
|
||||
processes one host at a time (§8.4), no concurrent access to the same
|
||||
context occurs. The `bool_bits` bitmap is cleared per host with `memset`,
|
||||
which is safe because no other goroutine accesses it concurrently.
|
||||
|
||||
---
|
||||
|
||||
## 9. Job-Level Sharing
|
||||
|
||||
There are two levels of sharing in the resreq system:
|
||||
|
||||
### 9.1 Select Sharing is Global (Node-Level CSE)
|
||||
|
||||
Select expressions are purely host-filtering — `type==linux` means the
|
||||
same thing regardless of which user, queue, or project submitted the job.
|
||||
Node-level CSE (§5) makes this sharing **global**: all jobs that reference
|
||||
`type==linux` share the same interned `SEL_EQ` subtree, regardless of
|
||||
their user, queue, or project. This is one of the key benefits of CSE at
|
||||
the AST node level rather than string-level interning.
|
||||
|
||||
### 9.2 Resreq Sharing is Scoped to (user, queue, project)
|
||||
|
||||
Two jobs share the same `xce_resreq_t*` pointer if and only if they were
|
||||
submitted with the same user, same queue, same project, and the same resreq
|
||||
string. The sharing mechanism is a second intern table at the resreq level,
|
||||
string. The scoping is necessary because `rusage` clauses involve per-user
|
||||
and per-queue allocation constraints — the same rusage string may result in
|
||||
different allocation behavior for different users or queues.
|
||||
|
||||
The sharing mechanism is a second intern table at the resreq level,
|
||||
modelled on `xstring.c`:
|
||||
|
||||
```
|
||||
|
|
@ -831,10 +1066,18 @@ makeResreq(canonical_resreq_str)
|
|||
→ cache miss: parse_resreq() → intern each select node (CSE) → insert → return
|
||||
```
|
||||
|
||||
### 9.3 Composability of the Two Levels
|
||||
|
||||
Because `makeSelectNode` interns individual AST nodes, jobs that share only a
|
||||
**subset** of their resreq (e.g., same `select[...]` but different `rusage[...]`)
|
||||
still share the common select subtree. The sharing is structural, not just
|
||||
per-string.
|
||||
per-string. The two levels compose:
|
||||
|
||||
- **Node-level CSE** (§5): global — `type==linux` is one interned subtree
|
||||
used by all jobs.
|
||||
- **Resreq-level sharing** (§9.2): scoped to `(user, queue, project, resreq_string)`
|
||||
— because rusage clauses involve allocation constraints that differ per
|
||||
user/queue/project.
|
||||
|
||||
The resreq intern table key is the **canonical resreq string** (raw, after
|
||||
whitespace normalization). A future enhancement could normalize clause order
|
||||
|
|
@ -847,16 +1090,19 @@ whitespace normalization). A future enhancement could normalize clause order
|
|||
Within a single scheduling session:
|
||||
|
||||
```
|
||||
for each pending_job:
|
||||
for each candidate_host:
|
||||
1. rc = eval_select(job.resreq.primary.select, &host.eval_ctx)
|
||||
for each candidate_host:
|
||||
ctx = select_eval_ctx_for_host(host) /* allocate per-host eval context */
|
||||
memset(ctx->bool_bits, 0, ...) /* clear memoization cache */
|
||||
for each pending_job:
|
||||
1. rc = eval_select(job.resreq.primary.select, ctx)
|
||||
→ 1 (true): proceed to rusage check
|
||||
→ 0 (false): skip host (select filter failed)
|
||||
→ -1 (error): log error, skip host — do NOT silently treat as false
|
||||
2. can_satisfy_rusage(job.resreq.primary.rusages, host.available)
|
||||
→ false: skip host (insufficient resources)
|
||||
3. allocate(job, host) → mark job as dispatched, update host.available
|
||||
if no_host_matched:
|
||||
select_eval_ctx_discard(ctx) /* free per-host context */
|
||||
if no_host_matched_for_job:
|
||||
job remains unsatisfied, re-evaluated next session
|
||||
```
|
||||
|
||||
|
|
@ -869,14 +1115,30 @@ silently coerced to truthy.
|
|||
|
||||
Key properties:
|
||||
|
||||
- **select is stateless**: host properties (type, architecture, boolean
|
||||
features) do not change during a session. `eval_select` on the same
|
||||
`(select_node, host)` pair always returns the same result within a session.
|
||||
- **select evaluates against a host snapshot**: `eval_select` uses
|
||||
`host_eval_ctx_t`, which contains the host's static and semi-static
|
||||
properties (type, architecture, boolean features, current load). These
|
||||
do not change during a session, so `eval_select` on the same
|
||||
`(select_node, host)` pair always returns the same result within a
|
||||
session. The `host_eval_ctx_t` does **not** contain allocation or
|
||||
booking data — it is a pure resource snapshot.
|
||||
- **rusage allocation uses snapshot + booked**: `can_satisfy_rusage`
|
||||
checks `host.available = host.total - host.booked`, where `host.booked`
|
||||
accumulates as jobs are dispatched within the session. This is a
|
||||
separate mechanism from select evaluation — the select filter runs
|
||||
first, and only hosts that pass the select filter are considered for
|
||||
rusage allocation.
|
||||
- **rusage is stateful**: as jobs are dispatched, `host.available` shrinks.
|
||||
Later jobs in the same session see reduced available resources.
|
||||
- **unsatisfied jobs persist**: a job that cannot be placed stays in the
|
||||
queue and is re-evaluated in the next scheduling session.
|
||||
|
||||
The separation of concerns is deliberate: select filtering is a pure
|
||||
boolean check against host properties (no side effects, fully
|
||||
memoizable per host), while rusage allocation is a resource consumption
|
||||
check that mutates the host's available resources as dispatch decisions
|
||||
are made.
|
||||
|
||||
---
|
||||
|
||||
## 11. Implementation Plan
|
||||
|
|
@ -904,9 +1166,10 @@ signatures and assertions are defined before implementation code.*
|
|||
- Add `res_type_t`, `sel_node_type_t`, `xce_select_node_t` typedef +
|
||||
struct.
|
||||
- Fill `struct xce_resreq_select_` with `xce_select_node_t* root`.
|
||||
- Declare `makeSelectNode`, `eval_select`,
|
||||
- Declare `makeSelectNode`, `eval_select`, `eval_select_inner`,
|
||||
`select_eval_ctx_for_host`, `select_eval_ctx_discard`,
|
||||
`sel_node_table_init`, `sel_node_table_fini`, `gc_sweep_select_nodes`,
|
||||
`gc_mark_tree`.
|
||||
`gc_mark_tree`, `recompact_node_ids`.
|
||||
- Add `res_type_to_valtype` / `valtype_to_res_type` mapping helpers.
|
||||
- Add `parse_resreq_primary_with_resmgr(primary, resreqString, errmsg, resmgr)`
|
||||
as a new entry point. The existing `parse_resreq_primary` calls it
|
||||
|
|
@ -933,10 +1196,16 @@ signatures and assertions are defined before implementation code.*
|
|||
- `makeSelectNode` (§5.4) — with key_add return code check for TOCTOU
|
||||
safety.
|
||||
- `sel_key_of`, `sel_key_hash` helpers.
|
||||
- `eval_select` (§8.3) — explicit error propagation (no C boolean ops).
|
||||
- `eval_select` / `eval_select_inner` (§8.3/§8.5.2) — memoized evaluation
|
||||
with per-host `select_eval_ctx_t`, explicit error propagation.
|
||||
- `select_eval_ctx_for_host`, `select_eval_ctx_discard` — per-host context
|
||||
allocation/discard (§8.5.3).
|
||||
- `host_resval_typed`, `host_get_bool`, `host_get_float`,
|
||||
`host_get_string` accessors.
|
||||
- Mark-and-sweep GC: `gc_sweep_select_nodes()`, `gc_mark_tree()`.
|
||||
- Node ID assignment: `node_id` field assigned by `makeSelectNode` from
|
||||
a monotonically increasing counter (`sel_node_next_id++`).
|
||||
- `recompact_node_ids()` — reassign dense 0..N-1 IDs after GC (§8.5.4).
|
||||
- AST transform pipeline: flatten, const-fold (§6.2–6.4).
|
||||
- `xce_str_match` wrapper (initially `strcmp` semantics).
|
||||
- Update `kill_primary_resreq` to unlink select root (`primary->select->root = NULL`)
|
||||
|
|
@ -955,16 +1224,23 @@ signatures and assertions are defined before implementation code.*
|
|||
- `backend/clib.go` — replace `EvalSelectForHost` stub with real CGO call.
|
||||
Must marshal a `host_eval_ctx_t` from Go to C, populating both the
|
||||
numeric resource vector (`numvals`) and the string resource table
|
||||
(`strvals`, indexed by `res_idx`). The `strvals` array maps `res_idx`
|
||||
to interned string pointers obtained via `makeString()` for CSE
|
||||
compatibility.
|
||||
(`strvals`, indexed by `res_idx`). The Go bridge then calls
|
||||
`select_eval_ctx_for_host()` to create the per-host memoization context,
|
||||
evaluates each job against it via `eval_select()`, and calls
|
||||
`select_eval_ctx_discard()` when the host's evaluation is complete.
|
||||
The `strvals` array maps `res_idx` to interned string pointers obtained
|
||||
via `makeString()` for CSE compatibility.
|
||||
- `controllers/hosts.go` — wire `FilterHostByResReqSelect` into
|
||||
`preCheckHostRequest`.
|
||||
- **GC trigger:** Add a CGO call `C.gc_sweep_select_nodes()` invoked by
|
||||
the scheduler goroutine at the end of each scheduling session, after
|
||||
all dispatch/defer decisions and job cleanup (unlinking select roots)
|
||||
are complete. This must be a well-defined session boundary — no
|
||||
`eval_select` calls may be in progress during the sweep.
|
||||
- **GC trigger:** Add CGO calls invoked by the scheduler goroutine at the
|
||||
end of each scheduling session, after all dispatch/defer decisions and
|
||||
job cleanup (unlinking select roots) are complete:
|
||||
1. `C.gc_sweep_select_nodes()` — reclaim unreachable nodes.
|
||||
2. `C.recompact_node_ids()` — reassign dense 0..N-1 IDs after GC (§8.5.4).
|
||||
This must be a well-defined session boundary — no `eval_select` calls
|
||||
may be in progress during the sweep or recompaction.
|
||||
Before the next session, check whether the eval context arrays need to
|
||||
grow or shrink to accommodate the new `max_node_id` (§8.5.3).
|
||||
- **CGO concurrency:** `FilterHostByResReqSelect` must be called from a
|
||||
single, pinned scheduler goroutine (§8.4). This is a hard invariant:
|
||||
`eval_select` accesses interned nodes without holding references, so
|
||||
|
|
@ -987,7 +1263,7 @@ typedef enum { RES_TYPE_NUMERIC = 0, RES_TYPE_STRING, RES_TYPE_BOOL } res_type_t
|
|||
typedef enum {
|
||||
SEL_AND, SEL_OR, SEL_NOT,
|
||||
SEL_GT, SEL_LT, SEL_GE, SEL_LE, SEL_EQ, SEL_NE,
|
||||
SEL_RESNAME, SEL_NUM, SEL_STR, SEL_BOOL_LITERAL, SEL_ANY, SEL_TRUE, SEL_FALSE,
|
||||
SEL_RESNAME, SEL_NUM, SEL_STR, SEL_ANY, SEL_TRUE, SEL_FALSE,
|
||||
} sel_node_type_t;
|
||||
|
||||
typedef struct xce_select_node_ xce_select_node_t;
|
||||
|
|
@ -995,6 +1271,7 @@ typedef struct xce_select_node_ xce_select_node_t;
|
|||
struct xce_select_node_ {
|
||||
sel_node_type_t type;
|
||||
uint32_t hashcode;
|
||||
uint32_t node_id; /* dense index into eval context arrays (§8.5) */
|
||||
unsigned gc_mark; /* set during mark-and-sweep GC (§5.5) */
|
||||
union {
|
||||
struct { xce_select_node_t *left, *right; } binop;
|
||||
|
|
@ -1002,7 +1279,6 @@ struct xce_select_node_ {
|
|||
struct { int res_idx; res_type_t res_type; } resname;
|
||||
double numval;
|
||||
char* strval;
|
||||
int bval; /* SEL_BOOL_LITERAL — 1 for true, 0 for false */
|
||||
} u;
|
||||
};
|
||||
```
|
||||
|
|
@ -1035,6 +1311,26 @@ typedef struct {
|
|||
} host_eval_ctx_t;
|
||||
```
|
||||
|
||||
### Select Evaluation Context (per-host, §8.5)
|
||||
|
||||
```c
|
||||
typedef struct {
|
||||
const host_eval_ctx_t* host; /* host resource snapshot */
|
||||
uint8_t* bool_bits; /* bitmap, 2 bits per node (eval result cache) */
|
||||
uint32_t bool_bits_cap; /* capacity in nodes */
|
||||
double* num_vals; /* numeric leaf cache by node_id */
|
||||
uint32_t num_vals_cap;
|
||||
const char** str_vals; /* string leaf cache by node_id */
|
||||
uint32_t str_vals_cap;
|
||||
} select_eval_ctx_t;
|
||||
|
||||
/* Allocate eval context for a host (shared across all jobs for that host) */
|
||||
select_eval_ctx_t* select_eval_ctx_for_host(const host_eval_ctx_t* host);
|
||||
|
||||
/* Free per-host eval context after host scheduling is complete */
|
||||
void select_eval_ctx_discard(select_eval_ctx_t* ctx);
|
||||
```
|
||||
|
||||
### Select Parse Context
|
||||
|
||||
```c
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue