lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

limit audit: correct PEND reason semantics + percent-map + project normalization

Browse source 
Limit-subsystem audit fixes (#1, #2, #4b, #5, #7; #3/#6/#8 deferred):

#1 PEND reason semantics inversion (the real fix):
- plugins/limit.c: remove the bogus rc<=0 PREASON_LIMIT_GLOBAL/HOST attach
  blocks from match_job_global_limit_()/match_job_host_limit_(). match_limit()
  only knows scope membership (1 = in scope, 0 = not), never usage-vs-cap --
  the attach fired "quota exhausted" for every job OUTSIDE the limit's scope
  and never for a real exhaustion.
- resource.c sched_try_alloc_on_host_() + alloc_runner.c
  sched_alloc_prepare_shared_(): attach the real verdict in the generic
  satisfy check's unmet loops -- an unmet resource with XCE_RESF_LIMIT now
  reports PREASON_LIMIT_HOST (per-host loop, host named) / PREASON_LIMIT_GLOBAL
  (shared loop, host-less) instead of the generic NO_RESOURCE; the vres name
  ("limit=<name>[/u=..][/q=..][/p=..]") rides in the reason's resource field,
  reverse-parseable to the bare limit name.

#2: rtr_reason_name() now covers codes 4-9 (aligned with pendReasonName() in
the apiserver and reasonScenario() in the MCP server); preason.h's
main-reason comment states the real producer contract.

#4b: make_sched_obj() normalizes obj->project ""/NULL -> "default", matching
sched_get_project()'s default -- a limit scoped to projects.members:[default]
now matches obj->project the same as job->project (the wire decodes an absent
project as "", not NULL).

#5: percent_capacity_resource_map: "swap" -> "swp" (the live load index name
workers register; "swap" matched no live resource, so a percent limit on it
fell back to the live-value base bug-1834 exists to avoid) + add tmp ->
maxtmp. Fixture and lib tests follow the live names.

#7 (coverage): test_alloc_runner.c asserts an unmet shared limit vres gets
PREASON_LIMIT_GLOBAL; test_alloc_runner_search.c asserts an unmet per-host
limit vres gets PREASON_LIMIT_HOST with the host named;
test_repl_trace_e2e.c / test_trace_render.c and the repl_trace_baseline
fixtures (XCE_REPL_BASELINE_REGEN=1) follow the corrected reason rendering.

Full tree: 134/134 tests green (scheduler 100, libxce, harness, controller,
worker, autoscaler, apiserver).

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-10-22 13:11:01 +00:00
parent 993daa8f95
commit c194973d42
17 changed files with 295 additions and 64 deletions
Download patch file Download diff file Expand all files Collapse all files

8
src/include/preason.h
View file

@ -8,8 +8,12 @@
#define PREASON_NO_RESOURCE (2) #define PREASON_NO_RESOURCE (2)
#define PREASON_MATCH_FAIL (3) #define PREASON_MATCH_FAIL (3)
/* LSF-style PEND reason classes beyond plain capacity / host-match failure. /* LSF-style PEND reason classes beyond plain capacity / host-match failure.
* Codes 4+ feed both main_reason and the host/res detail rows; the API layer * add_job_preason() only ever writes main_reason when called with BOTH host
* maps each to a human-readable name (pendReasonName in apiserver). */ * and res NULL (which erases the detail lists); every current scheduler
* producer of codes 4+ attaches them as host/res DETAIL rows -- the API
* layer maps each code to a human-readable name (pendReasonName in
* apiserver) wherever it appears, and promotes the first detail row to the
* API-visible main_reason_name (pend-reason P0). */
#define PREASON_LIMIT_GLOBAL (4) /* global quota hit (ug1_slots, ug2_slots_per_user, ...) */ #define PREASON_LIMIT_GLOBAL (4) /* global quota hit (ug1_slots, ug2_slots_per_user, ...) */
#define PREASON_LIMIT_HOST (5) /* host-based quota hit (hg1_slots_per_host) */ #define PREASON_LIMIT_HOST (5) /* host-based quota hit (hg1_slots_per_host) */
#define PREASON_QUEUE_CLOSED (6) /* queue closed/disabled, not accepting jobs */ #define PREASON_QUEUE_CLOSED (6) /* queue closed/disabled, not accepting jobs */

74
src/lib/testsuite/test_limit.c
View file

@ -731,7 +731,7 @@ test_instantiate_limit_percent_mem_limit_resolves_percent_res_to_maxmem(
} }
static void static void
test_instantiate_limit_percent_swap_limit_resolves_percent_res_to_maxswap( test_instantiate_limit_percent_swp_limit_resolves_percent_res_to_maxswap(
void) void)
{ {
var umgr; var umgr;
@ -743,9 +743,9 @@ test_instantiate_limit_percent_swap_limit_resolves_percent_res_to_maxswap(
xce_vres_t* vres = NULL; xce_vres_t* vres = NULL;
int rc; int rc;
xce_limit_config_t limit_config xce_limit_config_t limit_config
= { .name = "pct_swap_limit", = { .name = "pct_swp_limit",
.val .val
= &(xce_limit_val_t) { .resname = "swap", .percent = 1, = &(xce_limit_val_t) { .resname = "swp", .percent = 1,
.val = 0.5 } }; .val = 0.5 } };
groupmgr_init(&umgr, 4, 4); groupmgr_init(&umgr, 4, 4);
@ -761,10 +761,15 @@ test_instantiate_limit_percent_swap_limit_resolves_percent_res_to_maxswap(
rc = instantiate_limit(&vres, resmgr, limit, "", "", ""); rc = instantiate_limit(&vres, resmgr, limit, "", "", "");
TEST_ASSERT_TRUE(rc >= 0); TEST_ASSERT_TRUE(rc >= 0);
TEST_ASSERT_NOT_NULL(vres); TEST_ASSERT_NOT_NULL(vres);
/* "swp", not "swap": workers register the swap load index as "swp"
* (resource_sync.c init_builtin_load_indices()); the percent map must
* key the LIVE name or a percent limit on it falls back to the
* live-value base bug-1834 exists to avoid. */
TEST_ASSERT_NOT_NULL_MESSAGE( TEST_ASSERT_NOT_NULL_MESSAGE(
vres->percent_res, vres->percent_res,
"bug-1834: a percent-based 'swap' limit must resolve a static " "bug-1834 follow-up: a percent-based 'swp' limit must resolve a "
"capacity-source resource ('maxswap'), same pairing as mem/maxmem"); "static capacity-source resource ('maxswap'), same pairing as "
"mem/maxmem");
TEST_ASSERT_EQUAL_STRING("maxswap", vres->percent_res->name); TEST_ASSERT_EQUAL_STRING("maxswap", vres->percent_res->name);
TEST_ASSERT_EQUAL_INT(TEST_RES_MAXSWAP_IDX, vres->percent_res->idx); TEST_ASSERT_EQUAL_INT(TEST_RES_MAXSWAP_IDX, vres->percent_res->idx);
@ -776,10 +781,63 @@ test_instantiate_limit_percent_swap_limit_resolves_percent_res_to_maxswap(
destroy(qmgr); destroy(qmgr);
destroy(umgr); destroy(umgr);
printf("test_instantiate_limit_percent_swap_limit_resolves_percent_res_" printf("test_instantiate_limit_percent_swp_limit_resolves_percent_res_"
"to_maxswap: PASS\n"); "to_maxswap: PASS\n");
} }
static void
test_instantiate_limit_percent_tmp_limit_resolves_percent_res_to_maxtmp(
void)
{
var umgr;
var qmgr;
var hmgr;
var pmgr;
var resmgr;
xce_limit_t* limit;
xce_vres_t* vres = NULL;
int rc;
xce_limit_config_t limit_config
= { .name = "pct_tmp_limit",
.val
= &(xce_limit_val_t) { .resname = "tmp", .percent = 1,
.val = 0.5 } };
groupmgr_init(&umgr, 4, 4);
groupmgr_init(&qmgr, 4, 4);
groupmgr_init(&hmgr, 4, 4);
groupmgr_init(&pmgr, 4, 4);
resmgr = make_test_resmgr();
TEST_ASSERT_NOT_NULL(resmgr);
rc = build_limit(&limit, umgr, qmgr, hmgr, pmgr, &limit_config);
TEST_ASSERT_TRUE(rc == 1);
rc = instantiate_limit(&vres, resmgr, limit, "", "", "");
TEST_ASSERT_TRUE(rc >= 0);
TEST_ASSERT_NOT_NULL(vres);
/* tmp/maxtmp: the table's own documented future one-liner, now real --
* same dynamic/static pairing as mem/maxmem. */
TEST_ASSERT_NOT_NULL_MESSAGE(
vres->percent_res,
"bug-1834 follow-up: a percent-based 'tmp' limit must resolve a "
"static capacity-source resource ('maxtmp'), same pairing as "
"mem/maxmem");
TEST_ASSERT_EQUAL_STRING("maxtmp", vres->percent_res->name);
TEST_ASSERT_EQUAL_INT(TEST_RES_MAXTMP_IDX, vres->percent_res->idx);
free_instantiated_vres(resmgr, vres);
kill_limit(limit);
free_test_resmgr(resmgr);
destroy(pmgr);
destroy(hmgr);
destroy(qmgr);
destroy(umgr);
printf("test_instantiate_limit_percent_tmp_limit_resolves_percent_res_"
"to_maxtmp: PASS\n");
}
static void static void
test_instantiate_limit_absolute_value_mem_limit_leaves_percent_res_null( test_instantiate_limit_absolute_value_mem_limit_leaves_percent_res_null(
void) void)
@ -903,7 +961,9 @@ void test_limit(void)
RUN_TEST( RUN_TEST(
test_instantiate_limit_percent_mem_limit_resolves_percent_res_to_maxmem); test_instantiate_limit_percent_mem_limit_resolves_percent_res_to_maxmem);
RUN_TEST( RUN_TEST(
test_instantiate_limit_percent_swap_limit_resolves_percent_res_to_maxswap); test_instantiate_limit_percent_swp_limit_resolves_percent_res_to_maxswap);
RUN_TEST(
test_instantiate_limit_percent_tmp_limit_resolves_percent_res_to_maxtmp);
RUN_TEST( RUN_TEST(
test_instantiate_limit_absolute_value_mem_limit_leaves_percent_res_null); test_instantiate_limit_absolute_value_mem_limit_leaves_percent_res_null);
RUN_TEST( RUN_TEST(

9
src/lib/testsuite/test_resmgr_fixture.c
View file

@ -35,8 +35,10 @@ var make_test_resmgr(void)
add_resource(resmgr, "ncpus", TEST_RES_NCPUS_IDX, XCE_VAL_NUMERIC); add_resource(resmgr, "ncpus", TEST_RES_NCPUS_IDX, XCE_VAL_NUMERIC);
add_resource(resmgr, "arch", TEST_RES_ARCH_IDX, XCE_VAL_STRING); add_resource(resmgr, "arch", TEST_RES_ARCH_IDX, XCE_VAL_STRING);
add_resource(resmgr, "maxmem", TEST_RES_MAXMEM_IDX, XCE_VAL_NUMERIC); add_resource(resmgr, "maxmem", TEST_RES_MAXMEM_IDX, XCE_VAL_NUMERIC);
add_resource(resmgr, "swap", TEST_RES_SWAP_IDX, XCE_VAL_NUMERIC); add_resource(resmgr, "swp", TEST_RES_SWAP_IDX, XCE_VAL_NUMERIC);
add_resource(resmgr, "maxswap", TEST_RES_MAXSWAP_IDX, XCE_VAL_NUMERIC); add_resource(resmgr, "maxswap", TEST_RES_MAXSWAP_IDX, XCE_VAL_NUMERIC);
add_resource(resmgr, "tmp", TEST_RES_TMP_IDX, XCE_VAL_NUMERIC);
add_resource(resmgr, "maxtmp", TEST_RES_MAXTMP_IDX, XCE_VAL_NUMERIC);
return resmgr; return resmgr;
} }
@ -47,8 +49,9 @@ void free_test_resmgr(var resmgr)
/* Free each xce_resource_t we allocated (datakiller=NULL means kvtab /* Free each xce_resource_t we allocated (datakiller=NULL means kvtab
* won't free them, so we must do it here). */ * won't free them, so we must do it here). */
const char* names[] = { "type", "mem", "gpu", "ncpus", const char* names[]
"arch", "maxmem", "swap", "maxswap", NULL }; = { "type", "mem", "gpu", "ncpus", "arch", "maxmem",
"swp", "maxswap", "tmp", "maxtmp", NULL };
for (int i = 0; names[i] != NULL; i++) { for (int i = 0; names[i] != NULL; i++) {
xce_resource_t* r = NULL; xce_resource_t* r = NULL;
int rc = key_find(resmgr, (void*)names[i], (void**)&r); int rc = key_find(resmgr, (void*)names[i], (void**)&r);

20
src/lib/testsuite/test_resmgr_fixture.h
View file

@ -4,16 +4,18 @@
#include <rtti.h> #include <rtti.h>
/* Test resource indices — must match make_test_resmgr() registration order */ /* Test resource indices — must match make_test_resmgr() registration order */
#define TEST_RES_TYPE_IDX 0 /* "type" — RES_TYPE_STRING */ #define TEST_RES_TYPE_IDX 0 /* "type" — RES_TYPE_STRING */
#define TEST_RES_MEM_IDX 1 /* "mem" — RES_TYPE_NUMERIC */ #define TEST_RES_MEM_IDX 1 /* "mem" — RES_TYPE_NUMERIC */
#define TEST_RES_GPU_IDX 2 /* "gpu" — RES_TYPE_BOOL */ #define TEST_RES_GPU_IDX 2 /* "gpu" — RES_TYPE_BOOL */
#define TEST_RES_NCPUS_IDX 3 /* "ncpus" — RES_TYPE_NUMERIC */ #define TEST_RES_NCPUS_IDX 3 /* "ncpus" — RES_TYPE_NUMERIC */
#define TEST_RES_ARCH_IDX 4 /* "arch" — RES_TYPE_STRING */ #define TEST_RES_ARCH_IDX 4 /* "arch" — RES_TYPE_STRING */
#define TEST_RES_MAXMEM_IDX 5 /* "maxmem" — RES_TYPE_NUMERIC (bug-1834) */ #define TEST_RES_MAXMEM_IDX 5 /* "maxmem" — RES_TYPE_NUMERIC (bug-1834) */
#define TEST_RES_SWAP_IDX 6 /* "swap" — RES_TYPE_NUMERIC (bug-1834) */ #define TEST_RES_SWAP_IDX 6 /* "swp" — RES_TYPE_NUMERIC (live swap load-index name, bug-1834 follow-up) */
#define TEST_RES_MAXSWAP_IDX 7 /* "maxswap" — RES_TYPE_NUMERIC (bug-1834) */ #define TEST_RES_MAXSWAP_IDX 7 /* "maxswap" — RES_TYPE_NUMERIC (bug-1834) */
#define TEST_RES_TMP_IDX 8 /* "tmp" — RES_TYPE_NUMERIC (bug-1834 follow-up) */
#define TEST_RES_MAXTMP_IDX 9 /* "maxtmp" — RES_TYPE_NUMERIC (bug-1834 follow-up) */
/* Create a test resmgr with 8 pre-registered resources. /* Create a test resmgr with 10 pre-registered resources.
* Returns a var (kvtab) that can be passed to parse_select / parse_resreq_primary. * Returns a var (kvtab) that can be passed to parse_select / parse_resreq_primary.
* Caller must call free_test_resmgr() when done. */ * Caller must call free_test_resmgr() when done. */
extern var make_test_resmgr(void); extern var make_test_resmgr(void);

23
src/lib/xcelimit.c
View file

@ -477,19 +477,28 @@ int is_host_based_limit(var resmgr, xce_limit_t* limit)
} }
/* bug-1834: a percent-based limit's capacity must come from a resource's /* bug-1834: a percent-based limit's capacity must come from a resource's
* static total, not its live/fluctuating value -- "mem"/"swap" are both * static total, not its live/fluctuating value -- "mem"/"swp"/"tmp" are all
* XCE_RESF_DYNAMIC load indices (resource-accounting.md), so their own * XCE_RESF_DYNAMIC load indices (resource-accounting.md), so their own
* live value is the wrong base for a stable percentage-of-capacity cap. * live value is the wrong base for a stable percentage-of-capacity cap.
* "maxmem"/"maxswap" are the static, self-reported-once counterparts * "maxmem"/"maxswap"/"maxtmp" are the static, self-reported-once
* (resources.yaml) that should be read instead. Table-driven so a future * counterparts (resources.yaml; registered per-worker by
* dynamic/static pair (e.g. tmp/maxtmp) is a one-line addition, not a new * controller/worker.c add_hostattrs_resvals()) that should be read instead.
* branch. */ * Table-driven: a new dynamic/static pair is a one-line addition.
*
* Key names are the LIVE resource names, not display aliases: workers
* register the swap load index as "swp" (resource_sync.c
* init_builtin_load_indices()), never "swap" -- the "swap" key this table
* previously carried matched no live resource, so a percent limit on it
* fell back to the live-value base bug-1834 exists to avoid (a percent
* limit on a "swap" name still can't match a live resource today, but the
* table now names the pair that actually exists). */
static const struct { static const struct {
const char* resname; const char* resname;
const char* percent_resname; const char* percent_resname;
} percent_capacity_resource_map[] = { } percent_capacity_resource_map[] = {
{ "mem", "maxmem" }, { "mem", "maxmem" },
{ "swap", "maxswap" }, { "swp", "maxswap" },
{ "tmp", "maxtmp" },
}; };
static const char* percent_capacity_resname(const char* resname) static const char* percent_capacity_resname(const char* resname)

16
src/scheduler/src/alloc_runner.c
View file

@ -100,10 +100,20 @@ static int sched_alloc_prepare_shared_(xce_sched_core_t* core,
continue; continue;
} }
/* bug-1927: a COMPOUND's later primary short on the same /* A limit's virtual resource (XCE_RESF_LIMIT, added by
* pool does not repeat the entry */ * instantiate_limit()) surfaces here exactly when the GLOBAL
* limit's cap is exhausted cluster-wide -- the real quota
* verdict, replacing the removed plugins/limit.c rc<=0 attach
* (see match_job_global_limit_()'s comment there). r->name is
* the vres name "limit=<name>[/u=..][/q=..][/p=..]",
* reverse-parseable to the bare limit name (split_vres_name()
* convention). Host-less row: a global limit is not a
* per-host fact. */
add_job_preason_once(ctx->reason->reasons, &(ctx->reason_index), add_job_preason_once(ctx->reason->reasons, &(ctx->reason_index),
PREASON_NO_RESOURCE, NULL, r->name); (r->flags & XCE_RESF_LIMIT)
? PREASON_LIMIT_GLOBAL
: PREASON_NO_RESOURCE,
NULL, r->name);
} }
return 0; return 0;

49
src/scheduler/src/plugins/limit.c
View file

@ -157,17 +157,19 @@ static int match_job_global_limit_(xce_sched_plugin_t* plugin,
TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__, TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
limit->name, obj->user, obj->queue, obj->project, rc); limit->name, obj->user, obj->queue, obj->project, rc);
if (rc <= 0) { if (rc <= 0) {
/* global quota hit — report the specific limit (PEND-reason P1, /* rc == 0: the job is NOT in this limit's scope (match_limit()'s
* restored from 4e2ac86). The election return value is ignored by * "no consumer rejects it" verdict) -- nothing to enforce, nothing
* the main loop, so the reason is carried on ctx->reason and moved * to report. rc < 0 (-EINVAL): a wiring error, surfaced by the
* to obj->reason by limit_job_election. */ * caller's rc<0 break.
if (ctx->reason == NULL) { *
ctx->reason = make_pendreason(0); * This is deliberately NOT a PREASON_LIMIT_GLOBAL site, despite
} * 4e2ac86 P0's original comment: match_limit() knows only scope
if (ctx->reason != NULL) { * membership, never usage-vs-cap -- the quota verdict happens
add_job_preason(ctx->reason->reasons, PREASON_LIMIT_GLOBAL, NULL, * later, in the generic satisfy check, where the limit's vres is
limit->name); * an unmet entry like any resource (the real attach lives in
} * resource.c's unmet loops). Attaching "quota exhausted" here
* fired for every job OUTSIDE the limit's scope, and never for an
* actual exhausted quota. */
return rc; return rc;
} }
@ -285,10 +287,11 @@ static int limit_job_election(void* arg, xce_sched_job_t* job)
iter_destroy(&it); iter_destroy(&it);
/* Carry any global-quota reason (PREASON_LIMIT_GLOBAL) attached by /* match_job_global_limit_() can no longer attach a reason (scope
* match_job_global_limit_ onto the job so it survives to the PEND * membership is not a quota verdict -- see its rc<=0 comment), so
* decision. The election's rc itself is ignored by the main loop. * nothing ever reaches here and the election's rc is ignored by the
* (PEND-reason P1, restored from 4e2ac86.) */ * main loop as before. Kept as a cheap NULL check to preserve the
* hook's historical "carry anything the election built" shape. */
if (ctx.reason != NULL) { if (ctx.reason != NULL) {
MOVE_PTR(obj->reason, ctx.reason); MOVE_PTR(obj->reason, ctx.reason);
} }
@ -380,16 +383,12 @@ static int match_job_host_limit_(xce_sched_plugin_t* plugin,
TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__, TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
limit->name, obj->user, obj->queue, obj->project, rc); limit->name, obj->user, obj->queue, obj->project, rc);
if (rc <= 0) { if (rc <= 0) {
/* host-based quota hit — report the specific limit (PEND-reason P1, /* rc == 0: the job (or this host) is NOT in the limit's scope --
* restored from 4e2ac86; previously the block was silent and the job * nothing to enforce. See match_job_global_limit_()'s identical
* just stayed PEND). */ * comment above: the real quota-verdict reason attach lives in
if (ctx->reason == NULL) { * resource.c's unmet loops, keyed on the limit's vres being an
ctx->reason = make_pendreason(0); * unmet entry -- NOT here, where match_limit() only knows scope
} * membership. */
if (ctx->reason != NULL) {
add_job_preason(ctx->reason->reasons, PREASON_LIMIT_HOST,
worker->name, limit->name);
}
return rc; return rc;
} }

15
src/scheduler/src/repl_trace_render_outcome.c
View file

@ -70,6 +70,21 @@ const char* rtr_reason_name(int code, char* buf, size_t bufsz)
return "NO_RESOURCE"; return "NO_RESOURCE";
case PREASON_MATCH_FAIL: case PREASON_MATCH_FAIL:
return "MATCH_FAIL"; return "MATCH_FAIL";
/* codes 4-9: keep the names aligned with pendReasonName() in the
* apiserver (convert_pure.go) and reasonScenario() in the MCP server
* (pendreason.go) -- all three render the same PREASON_* space. */
case PREASON_LIMIT_GLOBAL:
return "LIMIT_GLOBAL";
case PREASON_LIMIT_HOST:
return "LIMIT_HOST";
case PREASON_QUEUE_CLOSED:
return "QUEUE_CLOSED";
case PREASON_QUEUE_DRAINING:
return "QUEUE_DRAINING";
case PREASON_TIME_WINDOW:
return "TIME_WINDOW";
case PREASON_HOST_UNAVAIL:
return "HOST_UNAVAIL";
default: default:
snprintf(buf, bufsz, "code %d", code); snprintf(buf, bufsz, "code %d", code);
return buf; return buf;

15
src/scheduler/src/resource.c
View file

@ -615,11 +615,20 @@ int sched_try_alloc_on_host_(xce_sched_worker_t* w, xce_sched_core_t* core,
continue; continue;
} }
/* bug-1927: every plan re-checks this host; record each /* A limit's virtual resource (XCE_RESF_LIMIT, added by
* (host, resource, code) once */ * instantiate_limit()) surfaces here exactly when its cap
* is exhausted for THIS host -- this is the real quota
* verdict the removed plugins/limit.c rc<=0 attach only
* pretended to make (it fired "quota hit" for jobs OUTSIDE
* a limit's scope). r->name is the vres name
* "limit=<name>[/u=..][/q=..][/p=..]", reverse-parseable to
* the bare limit name (split_vres_name() convention). */
add_job_preason_once(ctx->reason->reasons, add_job_preason_once(ctx->reason->reasons,
&(ctx->reason_index), &(ctx->reason_index),
PREASON_NO_RESOURCE, w->name, r->name); (r->flags & XCE_RESF_LIMIT)
? PREASON_LIMIT_HOST
: PREASON_NO_RESOURCE,
w->name, r->name);
} }
return 0; return 0;
} }

14
src/scheduler/src/schedobj.c
View file

@ -119,7 +119,19 @@ static int make_sched_obj(xce_sched_obj_t* obj, xce_scheduler_t* sched,
obj->user = makeString(j->submit->user); obj->user = makeString(j->submit->user);
obj->queue = makeString(j->submit->queue); obj->queue = makeString(j->submit->queue);
obj->project = makeString(j->submit->project_name); /* project: the live submit wire never carries NULL (an absent project
* decodes as "", STRING_DUP("") in the generated decode), so normalize
* ""/NULL to "default" here -- the SAME default sched_get_project()
* applies when building job->project (core.c). The two views of a
* job's project (obj->project, what limit matching sees;
* job->project->name, what everything else sees) must agree, or a
* limit scoped to projects.members: [default] would match
* job->project but never obj->project. */
if (NULLSTR(j->submit->project_name)) {
obj->project = makeString("default");
} else {
obj->project = makeString(j->submit->project_name);
}
xlist_init(&(obj->glimits), 8); xlist_init(&(obj->glimits), 8);
zarray_init(&(obj->hlimits), 32, kill_hlimits_list); zarray_init(&(obj->hlimits), 32, kill_hlimits_list);

BIN
src/scheduler/testsuite/fixtures/repl_trace_baseline/contention.replay
View file

Binary file not shown.

BIN
src/scheduler/testsuite/fixtures/repl_trace_baseline/set_output.replay
View file

Binary file not shown.

BIN
src/scheduler/testsuite/fixtures/repl_trace_baseline/sim_output.replay
View file

Binary file not shown.

51
src/scheduler/testsuite/test_alloc_runner.c
View file

@ -527,6 +527,56 @@ static void test_shared_met_counted_and_kept_when_no_plan(void)
rt_fixture_fini(&f); rt_fixture_fini(&f);
} }
/* An unmet entry whose resource is a limit's virtual resource
* (XCE_RESF_LIMIT) reports the quota verdict PREASON_LIMIT_GLOBAL, not the
* generic NO_RESOURCE the plain shared shortfalls get (limit audit #1: the
* vres' unmet state IS the quota exhaustion). */
static void test_shared_short_limit_vres_reasons_limit_global(void)
{
rt_fixture_t f;
xce_resource_t vres_in;
xce_resource_t* vres;
xce_res_reason_t* rr;
rt_fixture_init(&f);
memset(&vres_in, 0, sizeof(vres_in));
vres_in.name = (char*)"limit=fake_limit";
vres_in.flags = XCE_RESF_LIMIT;
vres = makeResource(f.sched.core.resmgr, &vres_in);
TEST_ASSERT_NOT_NULL(vres);
{
int idx[1] = { vres->idx };
float val[1] = { 2.0f };
TEST_ASSERT_TRUE(put_resvals(&(f.sched.core.sharedInst), 1, idx, val)
>= 0);
}
/* lic_a met (3 of 10); the limit vres short (3 of 2). Staged after the
* rerun: sched_ctx_set_primary() -> plugin_prepare_allocation()
* resets alloc_req.shared. */
rt_want_shared(&f, 3.0f, 0.0f);
TEST_ASSERT_TRUE(
xvec_idval_store(&(f.ctx.alloc_req.shared), vres->idx, 3.0f) > 0);
TEST_ASSERT_EQUAL_INT(0, rt_run(&f));
TEST_ASSERT_NULL(f.ctx.alloc);
TEST_ASSERT_NOT_NULL(f.ctx.reason);
/* Only the UNMET entry gets a row: lic_a (3 of 10) is met and stays
* silent; the vres (3 of 2) is the one shortfall. */
TEST_ASSERT_EQUAL_INT(1, f.ctx.reason->reasons->nres_reasons);
rr = f.ctx.reason->reasons->res_based_reasons[0];
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_STRING("", rr->host);
TEST_ASSERT_EQUAL_STRING("limit=fake_limit", rr->resource);
TEST_ASSERT_EQUAL_INT_MESSAGE(PREASON_LIMIT_GLOBAL, rr->reason,
"an exhausted limit's vres must report "
"LIMIT_GLOBAL, not NO_RESOURCE");
rt_fixture_fini(&f);
}
/* The helpers directly: a shared request of nothing is met and counts /* The helpers directly: a shared request of nothing is met and counts
* nothing; the reason helper keeps an existing record. */ * nothing; the reason helper keeps an existing record. */
static void test_helpers_empty_shared_is_met(void) static void test_helpers_empty_shared_is_met(void)
@ -554,6 +604,7 @@ int main(void)
RUN_TEST(test_existing_reason_is_reused); RUN_TEST(test_existing_reason_is_reused);
RUN_TEST(test_reason_oom_returns_enomem_without_checks); RUN_TEST(test_reason_oom_returns_enomem_without_checks);
RUN_TEST(test_shared_short_one_reason_per_unmet_entry); RUN_TEST(test_shared_short_one_reason_per_unmet_entry);
RUN_TEST(test_shared_short_limit_vres_reasons_limit_global);
RUN_TEST(test_shared_met_counted_and_kept_when_plan_init_fails); RUN_TEST(test_shared_met_counted_and_kept_when_plan_init_fails);
RUN_TEST(test_shared_met_counted_and_kept_when_no_plan); RUN_TEST(test_shared_met_counted_and_kept_when_no_plan);
RUN_TEST(test_helpers_empty_shared_is_met); RUN_TEST(test_helpers_empty_shared_is_met);

54
src/scheduler/testsuite/test_alloc_runner_search.c
View file

@ -138,6 +138,9 @@ static int g_nlog = 0;
static int g_slots_idx = -1; static int g_slots_idx = -1;
static int g_lic_idx = -1; static int g_lic_idx = -1;
static float g_shared_want = 0.0f; /* lic per job; 0 = none */ static float g_shared_want = 0.0f; /* lic per job; 0 = none */
/* per-host limit vres staged by the limit plugin when set (test-only) */
static int g_host_vres_idx = -1;
static float g_host_vres_want = 0.0f;
int plugin_prepare_allocation(xce_sched_core_t* sched, xce_sched_ctx_t* ctx) int plugin_prepare_allocation(xce_sched_core_t* sched, xce_sched_ctx_t* ctx)
{ {
@ -167,6 +170,11 @@ int plugin_prepare_allocation(xce_sched_core_t* sched, xce_sched_ctx_t* ctx)
TEST_ASSERT_TRUE(xvec_idval_store(&(ctx->alloc_req.host), g_slots_idx, TEST_ASSERT_TRUE(xvec_idval_store(&(ctx->alloc_req.host), g_slots_idx,
(float)ctx->nslots) (float)ctx->nslots)
> 0); > 0);
if (g_host_vres_idx >= 0) {
TEST_ASSERT_TRUE(xvec_idval_store(&(ctx->alloc_req.host),
g_host_vres_idx, g_host_vres_want)
> 0);
}
return 0; return 0;
} }
@ -243,6 +251,8 @@ void setUp(void)
g_dup_fail_on = 0; g_dup_fail_on = 0;
g_nlog = 0; g_nlog = 0;
g_shared_want = 0.0f; g_shared_want = 0.0f;
g_host_vres_idx = -1;
g_host_vres_want = 0.0f;
g_nplans = 0; g_nplans = 0;
g_init_result = 0; g_init_result = 0;
g_init_calls = 0; g_init_calls = 0;
@ -635,6 +645,49 @@ static void test_short_host_reason_recorded_once_across_plans(void)
sr_fixture_fini(&f); sr_fixture_fini(&f);
} }
/* An unmet entry whose resource is a host-scoped limit's virtual resource
* (XCE_RESF_LIMIT) reports the quota verdict PREASON_LIMIT_HOST with the
* host named, not the generic NO_RESOURCE (limit audit #1: the vres' unmet
* state IS the quota exhaustion). */
static void test_short_host_limit_vres_reasons_limit_host(void)
{
sr_fixture_t f;
xce_sched_worker_t* a;
xce_resource_t* vres;
xce_res_reason_t* rr;
sr_fixture_init(&f);
a = sr_add_host(&f, "sr-a", 4.0f);
vres = makeResource(
f.sched.core.resmgr,
&(xce_resource_t) { .name = (char*)"limit=fake_host_limit/u=hdedup_user",
.flags = XCE_RESF_LIMIT });
TEST_ASSERT_NOT_NULL(vres);
{
int idx[1] = { vres->idx };
float val[1] = { 3.0f }; /* capacity 3 on this host */
TEST_ASSERT_TRUE(put_resvals(&(a->resvals), 1, idx, val) >= 0);
}
g_host_vres_idx = vres->idx;
g_host_vres_want = 4.0f; /* 4 requested against 3: short */
sr_ctx_begin(&f, 2);
script_plan(1, 2, 0, 0);
TEST_ASSERT_EQUAL_INT(0, sr_run(&f));
TEST_ASSERT_NOT_NULL(f.ctx.reason);
TEST_ASSERT_EQUAL_INT(1, f.ctx.reason->reasons->nres_reasons);
rr = f.ctx.reason->reasons->res_based_reasons[0];
TEST_ASSERT_NOT_NULL(rr);
TEST_ASSERT_EQUAL_STRING("sr-a", rr->host);
TEST_ASSERT_EQUAL_STRING("limit=fake_host_limit/u=hdedup_user",
rr->resource);
TEST_ASSERT_EQUAL_INT_MESSAGE(PREASON_LIMIT_HOST, rr->reason,
"an exhausted host limit's vres must report "
"LIMIT_HOST, not NO_RESOURCE");
sr_fixture_fini(&f);
}
/* ---------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- */
/* Malformed plans and the claims buffer (Req 3.8, 5.7) */ /* Malformed plans and the claims buffer (Req 3.8, 5.7) */
/* ---------------------------------------------------------------------- */ /* ---------------------------------------------------------------------- */
@ -795,6 +848,7 @@ int main(void)
RUN_TEST(test_rollback_then_later_plan_with_bucket_counts); RUN_TEST(test_rollback_then_later_plan_with_bucket_counts);
RUN_TEST(test_every_short_host_gets_reason); RUN_TEST(test_every_short_host_gets_reason);
RUN_TEST(test_short_host_reason_recorded_once_across_plans); RUN_TEST(test_short_host_reason_recorded_once_across_plans);
RUN_TEST(test_short_host_limit_vres_reasons_limit_host);
RUN_TEST(test_malformed_plans_skipped); RUN_TEST(test_malformed_plans_skipped);
RUN_TEST(test_claims_buffer_grows_to_larger_plan); RUN_TEST(test_claims_buffer_grows_to_larger_plan);
RUN_TEST(test_claims_buffer_growth_failure); RUN_TEST(test_claims_buffer_growth_failure);

9
src/scheduler/testsuite/test_repl_trace_e2e.c
View file

@ -354,7 +354,10 @@ static void test_step_mode_full_all_jobs_and_file_copy(void)
"\n outcome: PEND\n", "\n outcome: PEND\n",
"\n main reason: none\n", "\n main reason: none\n",
"\n resource license: NO_RESOURCE\n", "\n resource license: NO_RESOURCE\n",
"\n resource limit=cluster_slots: NO_RESOURCE\n"); /* The limit's vres being unmet carries the real quota verdict
* (PREASON_LIMIT_GLOBAL), not the generic NO_RESOURCE it used
* to share with the license shortfall. */
"\n resource limit=cluster_slots: LIMIT_GLOBAL\n");
TEST_ASSERT_NULL(strstr(b, "plan 1")); TEST_ASSERT_NULL(strstr(b, "plan 1"));
free(b); free(b);
/* 4.2: the cluster-wide limit is never shown as a resource row. */ /* 4.2: the cluster-wide limit is never shown as a resource row. */
@ -368,11 +371,11 @@ static void test_step_mode_full_all_jobs_and_file_copy(void)
"\n requirement alternative 1/2 > member 1/1 tasks 4..4 " "\n requirement alternative 1/2 > member 1/1 tasks 4..4 "
"requests mem=12288.000000 (12.00 KB)\n", "requests mem=12288.000000 (12.00 KB)\n",
"\n member 1/1 not satisfied\n", "\n member 1/1 not satisfied\n",
"\n failure reasons: NO_RESOURCE x1 (resources: " "\n failure reasons: LIMIT_GLOBAL x1 (resources: "
"limit=cluster_slots)\n", "limit=cluster_slots)\n",
"\n alternative 1/2 not satisfied\n", "\n alternative 1/2 not satisfied\n",
"\n requirement alternative 2/2 (compound, 1 member)\n", "\n requirement alternative 2/2 (compound, 1 member)\n",
"\n failure reasons: NO_RESOURCE x1 (resources: " "\n failure reasons: LIMIT_GLOBAL x1 (resources: "
"limit=cluster_slots)\n", "limit=cluster_slots)\n",
"\n alternative 2/2 not satisfied\n", "\n alternative 2/2 not satisfied\n",
"\n no alternative chosen: none of the 2 alternatives was " "\n no alternative chosen: none of the 2 alternatives was "

2
src/scheduler/testsuite/test_trace_render.c
View file

@ -815,7 +815,7 @@ static void test_pend_outcome(void)
" resource license: NO_RESOURCE\n" " resource license: NO_RESOURCE\n"
" host n01: MATCH_FAIL, resource mem: NO_RESOURCE\n" " host n01: MATCH_FAIL, resource mem: NO_RESOURCE\n"
" host n02: MATCH_FAIL\n" " host n02: MATCH_FAIL\n"
" omitted hosts' pend reasons: MATCH_FAIL x2, code 9 x1\n" " omitted hosts' pend reasons: MATCH_FAIL x2, HOST_UNAVAIL x1\n"
" ... 2 hosts omitted\n", " ... 2 hosts omitted\n",
out); out);
free(out); free(out);