lei.chen/myxce
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(scheduler): PEND reason classification — main_reason + limit/queue/worker reasons (POC "why is my job waiting")

Browse source 
P0 (done, cluster-verified):
- main_reason was always 0: add_job_preason only set it when BOTH host and res
  were NULL, but the allocator always attaches host/res detail rows. Now the
  first detail add wins as main_reason, so the API's main_reason_name is
  meaningful (was "unknown(0)"). Verified live: PEND jobs now report
  main_reason=3/match_fail + per-host reason_name.
- limit plugin attach: host-based quota hit (hg1_slots_per_host) -> LIMIT_HOST
  + limit name; global quota hit (ug1_slots, ...) -> LIMIT_GLOBAL + limit name.
  Previously both blocks were silent (job just stayed PEND).
- Extended the PREASON_* enum (was 3 codes) with LSF-style classes:
  LIMIT_GLOBAL/LIMIT_HOST/QUEUE_CLOSED/QUEUE_DRAINING/TIME_WINDOW/HOST_UNAVAIL.
  apiserver pendReasonName maps them; host/res detail rows now carry reason_name.

P1 (implemented):
- queue status: sched_queue_is_ready no longer a stub — closed/inactivated
  queues are not electable; the fcfs election attaches QUEUE_CLOSED /
  QUEUE_DRAINING to every PEND job in a skipped queue.
- worker filter: sched_try_alloc_on_host_ skips UNAVAIL/CLOSED workers and
  attaches HOST_UNAVAIL instead of trying to allocate on them.

Unit tests: preason suite updated for the main_reason change + 3 new tests
(limit host/global, first-add-wins); 295 lib tests pass under ASAN.
apiserver backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen 2026-08-14 09:43:21 +00:00
parent 62778930e2
commit 4e2ac861f4
10 changed files with 211 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/frontend/apiserver/backend/convert.go
View file

@ -133,7 +133,8 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
hr := C.job_host_reason(pr, C.int(i))
if hr != nil && hr.host != nil {
r.pendReason.HostReasons = append(r.pendReason.HostReasons,
rawHostReason{Host: goString(hr.host), Reason: int(hr.reason)})
rawHostReason{Host: goString(hr.host), Reason: int(hr.reason),
ReasonName: pendReasonName(int(hr.reason))})
}
}
for i := 0; i < int(pr.nres_reasons); i++ {
@ -141,7 +142,8 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
if rr != nil && rr.resource != nil {
r.pendReason.ResReasons = append(r.pendReason.ResReasons,
rawResReason{Resource: goString(rr.resource),
Host: goString(rr.host), Reason: int(rr.reason)})
Host: goString(rr.host), Reason: int(rr.reason),
ReasonName: pendReasonName(int(rr.reason))})
}
}
}

20
src/frontend/apiserver/backend/convert_pure.go
View file

@ -163,6 +163,7 @@ type rawSubmitData struct {
type rawHostReason struct {
Host string `json:"host"`
Reason int `json:"reason"`
ReasonName string `json:"reason_name"`
}
// rawResReason mirrors the C xce_res_reason_t (one resource-level PEND reason).
@ -170,6 +171,7 @@ type rawResReason struct {
Resource string `json:"resource"`
Host string `json:"host"`
Reason int `json:"reason"`
ReasonName string `json:"reason_name"`
}
// rawPendReason mirrors the C xce_job_preason_t attached to a PEND job by the
@ -184,9 +186,9 @@ type rawPendReason struct {
}
// pendReasonName maps a PREASON_* main_reason code to a readable name. The
// codes are the PEND reason bitfield of xce_job_preason_t (PREASON_BAD_RESREQ/
// NO_RESOURCE/MATCH_FAIL, src/include/preason.h). Unknown codes are reported
// verbatim so a new scheduler reason never silently disappears.
// codes are the PEND reason bitfield of xce_job_preason_t (src/include/
// preason.h). Unknown codes are reported verbatim so a new scheduler reason
// never silently disappears.
func pendReasonName(code int) string {
switch code {
case 1: // PREASON_BAD_RESREQ
@ -195,6 +197,18 @@ func pendReasonName(code int) string {
return "no_resource"
case 3: // PREASON_MATCH_FAIL
return "match_fail"
case 4: // PREASON_LIMIT_GLOBAL
return "limit_global"
case 5: // PREASON_LIMIT_HOST
return "limit_host"
case 6: // PREASON_QUEUE_CLOSED
return "queue_closed"
case 7: // PREASON_QUEUE_DRAINING
return "queue_draining"
case 8: // PREASON_TIME_WINDOW
return "time_window"
case 9: // PREASON_HOST_UNAVAIL
return "host_unavail"
default:
return fmt.Sprintf("unknown(%d)", code)
}

9
src/include/preason.h
View file

@ -7,6 +7,15 @@
#define PREASON_BAD_RESREQ (1)
#define PREASON_NO_RESOURCE (2)
#define PREASON_MATCH_FAIL (3)
/* LSF-style PEND reason classes beyond plain capacity / host-match failure.
* Codes 4+ feed both main_reason and the host/res detail rows; the API layer
* maps each to a human-readable name (pendReasonName in apiserver). */
#define PREASON_LIMIT_GLOBAL (4) /* global quota hit (ug1_slots, ug2_slots_per_user, ...) */
#define PREASON_LIMIT_HOST (5) /* host-based quota hit (hg1_slots_per_host) */
#define PREASON_QUEUE_CLOSED (6) /* queue closed/disabled, not accepting jobs */
#define PREASON_QUEUE_DRAINING (7) /* queue draining (no new jobs until empty) */
#define PREASON_TIME_WINDOW (8) /* outside the queue's dispatch_window */
#define PREASON_HOST_UNAVAIL (9) /* host down/unavailable, skipped for allocation */
extern void erase_job_preason(xce_job_preason_t*);
extern int add_job_preason(xce_job_preason_t*, int, char*, char*);

12
src/lib/preason.c
View file

@ -141,6 +141,10 @@ int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res)
{
int rc = 0;
if (reasons == NULL) {
return -EINVAL;
}
if ((host == NULL) && (res == NULL)) {
/* set main reason */
erase_job_preason(reasons);
@ -153,6 +157,14 @@ int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res)
rc = add_res_preason(reasons, code, host, res);
}
/* The allocator always attaches host/res DETAIL rows (MATCH_FAIL per host,
* NO_RESOURCE per resource) and never calls with both host and res NULL —
* without this main_reason stays 0 and the API can only surface the detail
* rows. First successful detail add wins as the overall reason. */
if ((rc > 0) && (reasons->main_reason == 0)) {
reasons->main_reason = code;
}
return rc;
}

68
src/lib/testsuite/test_preason.c
View file

@ -26,7 +26,7 @@ static void test_add_host_reason(void)
memset(&reason, 0x0, sizeof(reason));
add_job_preason(&reason, 1, "h1", NULL);
TEST_ASSERT_TRUE(reason.main_reason == 0);
TEST_ASSERT_TRUE(reason.main_reason == 1);
TEST_ASSERT_TRUE((reason.nhost_reasons == 1)
&& (reason.host_based_reasons != NULL)
&& (reason.host_based_reasons[0] != NULL));
@ -45,7 +45,7 @@ static void test_add_res_reason(void)
memset(&reason, 0x0, sizeof(reason));
add_job_preason(&reason, 1, "h1", "res1");
TEST_ASSERT_TRUE(reason.main_reason == 0);
TEST_ASSERT_TRUE(reason.main_reason == 1);
TEST_ASSERT_TRUE((reason.nres_reasons == 1)
&& (reason.res_based_reasons != NULL)
&& (reason.res_based_reasons[0] != NULL));
@ -65,7 +65,7 @@ static void test_add_shared_res_reason(void)
memset(&reason, 0x0, sizeof(reason));
add_job_preason(&reason, 1, NULL, "res1");
TEST_ASSERT_TRUE(reason.main_reason == 0);
TEST_ASSERT_TRUE(reason.main_reason == 1);
TEST_ASSERT_TRUE((reason.nres_reasons == 1)
&& (reason.res_based_reasons != NULL)
&& (reason.res_based_reasons[0] != NULL));
@ -88,7 +88,7 @@ static void test_add_res_host_reason(void)
add_job_preason(&reason, 1, "h1", "res1");
add_job_preason(&reason, 2, "h1", NULL);
TEST_ASSERT_TRUE(reason.main_reason == 0);
TEST_ASSERT_TRUE(reason.main_reason == 1);
TEST_ASSERT_TRUE((reason.nres_reasons == 1)
&& (reason.res_based_reasons != NULL)
&& (reason.res_based_reasons[0] != NULL));
@ -132,7 +132,7 @@ static void test_add_more_host_reason(void)
add_job_preason(&reason, 1, "h1", NULL);
add_job_preason(&reason, 2, "h2", NULL);
TEST_ASSERT_TRUE(reason.main_reason == 0);
TEST_ASSERT_TRUE(reason.main_reason == 1);
TEST_ASSERT_TRUE((reason.nhost_reasons == 2)
&& (reason.host_based_reasons != NULL)
&& (reason.host_based_reasons[0] != NULL)
@ -156,7 +156,7 @@ static void test_add_more_res_reason(void)
add_job_preason(&reason, 1, "h1", "res1");
add_job_preason(&reason, 2, "h2", "res2");
TEST_ASSERT_TRUE(reason.main_reason == 0);
TEST_ASSERT_TRUE(reason.main_reason == 1);
TEST_ASSERT_TRUE((reason.nres_reasons == 2)
&& (reason.res_based_reasons != NULL)
&& (reason.res_based_reasons[0] != NULL)
@ -175,6 +175,59 @@ static void test_add_more_res_reason(void)
erase_job_preason(&reason);
}
/* LSF-style reason classes: a limit/queue/time/worker reason attached as a
* host/res detail row must surface both in main_reason AND in the detail row's
* own code (bug PEND-reason: POC "why is my job waiting"). */
static void test_limit_host_reason(void)
{
xce_job_preason_t reason;
memset(&reason, 0x0, sizeof(reason));
add_job_preason(&reason, PREASON_LIMIT_HOST, "t1", "hg1_slots_per_host");
TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_HOST);
TEST_ASSERT_TRUE(reason.nres_reasons == 1);
TEST_ASSERT_TRUE(reason.res_based_reasons[0]->reason == PREASON_LIMIT_HOST);
TEST_ASSERT_TRUE(
strcmp(reason.res_based_reasons[0]->host, "t1") == 0);
TEST_ASSERT_TRUE(
strcmp(reason.res_based_reasons[0]->resource, "hg1_slots_per_host")
== 0);
erase_job_preason(&reason);
}
static void test_limit_global_reason(void)
{
xce_job_preason_t reason;
memset(&reason, 0x0, sizeof(reason));
add_job_preason(&reason, PREASON_LIMIT_GLOBAL, NULL, "ug1_slots");
TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_GLOBAL);
TEST_ASSERT_TRUE(reason.nres_reasons == 1);
TEST_ASSERT_TRUE(
reason.res_based_reasons[0]->reason == PREASON_LIMIT_GLOBAL);
TEST_ASSERT_TRUE(
strcmp(reason.res_based_reasons[0]->host, "") == 0);
TEST_ASSERT_TRUE(strcmp(reason.res_based_reasons[0]->resource, "ug1_slots")
== 0);
erase_job_preason(&reason);
}
static void test_second_reason_keeps_first_main(void)
{
xce_job_preason_t reason;
memset(&reason, 0x0, sizeof(reason));
/* First detail add wins as main_reason; a later add does not overwrite. */
add_job_preason(&reason, PREASON_LIMIT_HOST, "t1", "hg1");
add_job_preason(&reason, PREASON_NO_RESOURCE, "t2", "slots");
TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_HOST);
TEST_ASSERT_TRUE(reason.nres_reasons == 2);
erase_job_preason(&reason);
}
void test_preason(void)
{
printf("\n>> preason test\n\n");
@ -187,4 +240,7 @@ void test_preason(void)
RUN_TEST(test_add_main_reset_other);
RUN_TEST(test_add_more_host_reason);
RUN_TEST(test_add_more_res_reason);
RUN_TEST(test_limit_host_reason);
RUN_TEST(test_limit_global_reason);
RUN_TEST(test_second_reason_keeps_first_main);
}

1
src/scheduler/include/scheduler.h
View file

@ -148,6 +148,7 @@ extern int sched_add_worker(xce_scheduler_t*, xce_sched_worker_t*);
extern int sched_add_project(xce_scheduler_t*, xce_sched_project_t*);
extern int sched_job_is_ready(xce_sched_job_t*);
extern int sched_queue_is_ready(xce_sched_queue_t*);
extern int sched_queue_status_reason(uint32_t status);
extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);

29
src/scheduler/src/plugins/fcfs.c
View file

@ -230,11 +230,36 @@ static void* fcfs_elect_job(xce_sched_plugin_t* plugin)
pthread_testcancel();
pendJobs = &(q->pendJobs);
if (!sched_queue_is_ready(q->queue)) {
/* Attach the queue-state reason (closed/draining/windowed-out)
* to every PEND job in this queue so a skipped queue explains
* itself (PEND-reason P1). */
int qreason = sched_queue_status_reason(q->queue->status);
if (qreason != 0) {
ordq_item_t* it3;
for (it3 = pendJobs->head; it3 != NULL; it3 = it3->next) {
fcfs_job_t* j = (fcfs_job_t*)it3->data;
xce_sched_obj_t* obj;
if (j == NULL || j->job == NULL) {
continue;
}
obj = j->job->schedobj;
if (obj == NULL) {
continue;
}
if (obj->reason == NULL) {
obj->reason = make_pendreason(0);
}
if (obj->reason != NULL) {
add_job_preason(obj->reason->reasons, qreason,
q->queue->name, NULL);
}
}
}
continue;
}
pendJobs = &(q->pendJobs);
for (it2 = pendJobs->head; it2 != NULL; it2 = it2->next) {
fcfs_job_t* j = (fcfs_job_t*)it2->data;

27
src/scheduler/src/plugins/limit.c
View file

@ -6,6 +6,7 @@
#include "kvmgr.h"
#include "plugin.h"
#include "preason.h"
#include "sched/types.h"
#include "scheduler.h"
#include "utils.h"
@ -184,6 +185,16 @@ static int match_job_global_limit_(xce_sched_plugin_t* plugin,
TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
limit->name, obj->user, obj->queue, obj->project, rc);
if (rc <= 0) {
/* global quota hit — report the specific limit. The election return
* value is ignored by the main loop, so the reason is carried on
* ctx->reason and moved to obj->reason by limit_job_election. */
if (ctx->reason == NULL) {
ctx->reason = make_pendreason(0);
}
if (ctx->reason != NULL) {
add_job_preason(ctx->reason->reasons, PREASON_LIMIT_GLOBAL, NULL,
limit->name);
}
return rc;
}
@ -275,6 +286,13 @@ static int limit_job_election(void* arg, xce_sched_job_t* job)
iter_destroy(&it);
/* Carry any global-quota reason (PREASON_LIMIT_GLOBAL) attached by
* match_job_global_limit_ onto the job so it survives to the PEND
* decision. The election's rc itself is ignored by the main loop. */
if (ctx.reason != NULL) {
MOVE_PTR(obj->reason, ctx.reason);
}
obj->configver = sched->configver;
return rc;
@ -334,6 +352,15 @@ static int match_job_host_limit_(xce_sched_plugin_t* plugin,
TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
limit->name, obj->user, obj->queue, obj->project, rc);
if (rc <= 0) {
/* host-based quota hit — report the specific limit (bug PEND-reason:
* previously the block was silent, job just stayed PEND). */
if (ctx->reason == NULL) {
ctx->reason = make_pendreason(0);
}
if (ctx->reason != NULL) {
add_job_preason(ctx->reason->reasons, PREASON_LIMIT_HOST,
worker->name, limit->name);
}
return rc;
}

14
src/scheduler/src/resource.c
View file

@ -469,6 +469,20 @@ static int sched_try_alloc_on_host_(xce_sched_worker_t* w, va_list ap)
int rc;
xvec_t* uncomit = NULL;
/* Worker down/closed — never eligible. Attach HOST_UNAVAIL instead of
* trying to allocate on it (PEND-reason P1, E17). */
if (w->status
& (XCE_WORKER_UNAVAIL | XCE_WORKER_CLOSED | XCE_WORKER_CLOSED_BY_ADMIN)) {
if (ctx->reason == NULL) {
ctx->reason = make_pendreason(0);
}
if (ctx->reason != NULL) {
add_job_preason(ctx->reason->reasons, PREASON_HOST_UNAVAIL,
w->name, NULL);
}
return 0;
}
rc = sched_meet_host_selection(sched, w, primary->select);
if (rc == 0) {
if (ctx->reason == NULL) {

32
src/scheduler/src/scheduler.c
View file

@ -1449,8 +1449,36 @@ int sched_job_is_ready(xce_sched_job_t* j)
return 1;
}
/* FIXME this is just stub, we need to add more */
int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
/* A queue is electable only while it accepts new jobs. The controller
* maintains q->status (queue.c: closed / inactivated-by-admin); the dispatch
* window state (QUEUE_STAT_INACT_WINDOW) is not yet computed anywhere, so it
* is not checked here (PEND-reason P1 note). */
int sched_queue_is_ready(xce_sched_queue_t* q)
{
if (q == NULL) {
return 0;
}
if (q->status
& (QUEUE_STAT_CLOSED | QUEUE_STAT_INACT_ADMIN | QUEUE_STAT_INACT_WINDOW)) {
return 0;
}
return 1;
}
/* Map a queue status bitmask to the PEND reason it should report. */
int sched_queue_status_reason(uint32_t status)
{
if (status & QUEUE_STAT_CLOSED) {
return PREASON_QUEUE_CLOSED;
}
if (status & QUEUE_STAT_INACT_ADMIN) {
return PREASON_QUEUE_DRAINING;
}
if (status & QUEUE_STAT_INACT_WINDOW) {
return PREASON_TIME_WINDOW;
}
return 0;
}
int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
{