diff --git a/src/frontend/apiserver/backend/convert.go b/src/frontend/apiserver/backend/convert.go index ddf459fb..1304ac3d 100644 --- a/src/frontend/apiserver/backend/convert.go +++ b/src/frontend/apiserver/backend/convert.go @@ -133,7 +133,8 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo { hr := C.job_host_reason(pr, C.int(i)) if hr != nil && hr.host != nil { r.pendReason.HostReasons = append(r.pendReason.HostReasons, - rawHostReason{Host: goString(hr.host), Reason: int(hr.reason)}) + rawHostReason{Host: goString(hr.host), Reason: int(hr.reason), + ReasonName: pendReasonName(int(hr.reason))}) } } for i := 0; i < int(pr.nres_reasons); i++ { @@ -141,7 +142,8 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo { if rr != nil && rr.resource != nil { r.pendReason.ResReasons = append(r.pendReason.ResReasons, rawResReason{Resource: goString(rr.resource), - Host: goString(rr.host), Reason: int(rr.reason)}) + Host: goString(rr.host), Reason: int(rr.reason), + ReasonName: pendReasonName(int(rr.reason))}) } } } diff --git a/src/frontend/apiserver/backend/convert_pure.go b/src/frontend/apiserver/backend/convert_pure.go index ce6cb0f4..aa77b372 100644 --- a/src/frontend/apiserver/backend/convert_pure.go +++ b/src/frontend/apiserver/backend/convert_pure.go @@ -161,15 +161,17 @@ type rawSubmitData struct { // rawHostReason mirrors the C xce_host_reason_t (one host-level PEND reason). type rawHostReason struct { - Host string `json:"host"` - Reason int `json:"reason"` + Host string `json:"host"` + Reason int `json:"reason"` + ReasonName string `json:"reason_name"` } // rawResReason mirrors the C xce_res_reason_t (one resource-level PEND reason). type rawResReason struct { - Resource string `json:"resource"` - Host string `json:"host"` - Reason int `json:"reason"` + Resource string `json:"resource"` + Host string `json:"host"` + Reason int `json:"reason"` + ReasonName string `json:"reason_name"` } // rawPendReason mirrors the C xce_job_preason_t attached to a PEND job by the @@ -184,9 +186,9 @@ type rawPendReason struct { } // pendReasonName maps a PREASON_* main_reason code to a readable name. The -// codes are the PEND reason bitfield of xce_job_preason_t (PREASON_BAD_RESREQ/ -// NO_RESOURCE/MATCH_FAIL, src/include/preason.h). Unknown codes are reported -// verbatim so a new scheduler reason never silently disappears. +// codes are the PEND reason bitfield of xce_job_preason_t (src/include/ +// preason.h). Unknown codes are reported verbatim so a new scheduler reason +// never silently disappears. func pendReasonName(code int) string { switch code { case 1: // PREASON_BAD_RESREQ @@ -195,6 +197,18 @@ func pendReasonName(code int) string { return "no_resource" case 3: // PREASON_MATCH_FAIL return "match_fail" + case 4: // PREASON_LIMIT_GLOBAL + return "limit_global" + case 5: // PREASON_LIMIT_HOST + return "limit_host" + case 6: // PREASON_QUEUE_CLOSED + return "queue_closed" + case 7: // PREASON_QUEUE_DRAINING + return "queue_draining" + case 8: // PREASON_TIME_WINDOW + return "time_window" + case 9: // PREASON_HOST_UNAVAIL + return "host_unavail" default: return fmt.Sprintf("unknown(%d)", code) } diff --git a/src/include/preason.h b/src/include/preason.h index 470451a7..a7b5343a 100644 --- a/src/include/preason.h +++ b/src/include/preason.h @@ -4,9 +4,18 @@ #include "xce/types.h" -#define PREASON_BAD_RESREQ (1) -#define PREASON_NO_RESOURCE (2) -#define PREASON_MATCH_FAIL (3) +#define PREASON_BAD_RESREQ (1) +#define PREASON_NO_RESOURCE (2) +#define PREASON_MATCH_FAIL (3) +/* LSF-style PEND reason classes beyond plain capacity / host-match failure. + * Codes 4+ feed both main_reason and the host/res detail rows; the API layer + * maps each to a human-readable name (pendReasonName in apiserver). */ +#define PREASON_LIMIT_GLOBAL (4) /* global quota hit (ug1_slots, ug2_slots_per_user, ...) */ +#define PREASON_LIMIT_HOST (5) /* host-based quota hit (hg1_slots_per_host) */ +#define PREASON_QUEUE_CLOSED (6) /* queue closed/disabled, not accepting jobs */ +#define PREASON_QUEUE_DRAINING (7) /* queue draining (no new jobs until empty) */ +#define PREASON_TIME_WINDOW (8) /* outside the queue's dispatch_window */ +#define PREASON_HOST_UNAVAIL (9) /* host down/unavailable, skipped for allocation */ extern void erase_job_preason(xce_job_preason_t*); extern int add_job_preason(xce_job_preason_t*, int, char*, char*); diff --git a/src/lib/preason.c b/src/lib/preason.c index 6baefd27..6b9a3731 100644 --- a/src/lib/preason.c +++ b/src/lib/preason.c @@ -141,6 +141,10 @@ int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res) { int rc = 0; + if (reasons == NULL) { + return -EINVAL; + } + if ((host == NULL) && (res == NULL)) { /* set main reason */ erase_job_preason(reasons); @@ -153,6 +157,14 @@ int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res) rc = add_res_preason(reasons, code, host, res); } + /* The allocator always attaches host/res DETAIL rows (MATCH_FAIL per host, + * NO_RESOURCE per resource) and never calls with both host and res NULL — + * without this main_reason stays 0 and the API can only surface the detail + * rows. First successful detail add wins as the overall reason. */ + if ((rc > 0) && (reasons->main_reason == 0)) { + reasons->main_reason = code; + } + return rc; } diff --git a/src/lib/testsuite/test_preason.c b/src/lib/testsuite/test_preason.c index 99a7fa6a..856f4783 100644 --- a/src/lib/testsuite/test_preason.c +++ b/src/lib/testsuite/test_preason.c @@ -26,7 +26,7 @@ static void test_add_host_reason(void) memset(&reason, 0x0, sizeof(reason)); add_job_preason(&reason, 1, "h1", NULL); - TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE(reason.main_reason == 1); TEST_ASSERT_TRUE((reason.nhost_reasons == 1) && (reason.host_based_reasons != NULL) && (reason.host_based_reasons[0] != NULL)); @@ -45,7 +45,7 @@ static void test_add_res_reason(void) memset(&reason, 0x0, sizeof(reason)); add_job_preason(&reason, 1, "h1", "res1"); - TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE(reason.main_reason == 1); TEST_ASSERT_TRUE((reason.nres_reasons == 1) && (reason.res_based_reasons != NULL) && (reason.res_based_reasons[0] != NULL)); @@ -65,7 +65,7 @@ static void test_add_shared_res_reason(void) memset(&reason, 0x0, sizeof(reason)); add_job_preason(&reason, 1, NULL, "res1"); - TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE(reason.main_reason == 1); TEST_ASSERT_TRUE((reason.nres_reasons == 1) && (reason.res_based_reasons != NULL) && (reason.res_based_reasons[0] != NULL)); @@ -88,7 +88,7 @@ static void test_add_res_host_reason(void) add_job_preason(&reason, 1, "h1", "res1"); add_job_preason(&reason, 2, "h1", NULL); - TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE(reason.main_reason == 1); TEST_ASSERT_TRUE((reason.nres_reasons == 1) && (reason.res_based_reasons != NULL) && (reason.res_based_reasons[0] != NULL)); @@ -132,7 +132,7 @@ static void test_add_more_host_reason(void) add_job_preason(&reason, 1, "h1", NULL); add_job_preason(&reason, 2, "h2", NULL); - TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE(reason.main_reason == 1); TEST_ASSERT_TRUE((reason.nhost_reasons == 2) && (reason.host_based_reasons != NULL) && (reason.host_based_reasons[0] != NULL) @@ -156,7 +156,7 @@ static void test_add_more_res_reason(void) add_job_preason(&reason, 1, "h1", "res1"); add_job_preason(&reason, 2, "h2", "res2"); - TEST_ASSERT_TRUE(reason.main_reason == 0); + TEST_ASSERT_TRUE(reason.main_reason == 1); TEST_ASSERT_TRUE((reason.nres_reasons == 2) && (reason.res_based_reasons != NULL) && (reason.res_based_reasons[0] != NULL) @@ -175,6 +175,59 @@ static void test_add_more_res_reason(void) erase_job_preason(&reason); } +/* LSF-style reason classes: a limit/queue/time/worker reason attached as a + * host/res detail row must surface both in main_reason AND in the detail row's + * own code (bug PEND-reason: POC "why is my job waiting"). */ +static void test_limit_host_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, PREASON_LIMIT_HOST, "t1", "hg1_slots_per_host"); + TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_HOST); + TEST_ASSERT_TRUE(reason.nres_reasons == 1); + TEST_ASSERT_TRUE(reason.res_based_reasons[0]->reason == PREASON_LIMIT_HOST); + TEST_ASSERT_TRUE( + strcmp(reason.res_based_reasons[0]->host, "t1") == 0); + TEST_ASSERT_TRUE( + strcmp(reason.res_based_reasons[0]->resource, "hg1_slots_per_host") + == 0); + erase_job_preason(&reason); +} + +static void test_limit_global_reason(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + add_job_preason(&reason, PREASON_LIMIT_GLOBAL, NULL, "ug1_slots"); + TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_GLOBAL); + TEST_ASSERT_TRUE(reason.nres_reasons == 1); + TEST_ASSERT_TRUE( + reason.res_based_reasons[0]->reason == PREASON_LIMIT_GLOBAL); + TEST_ASSERT_TRUE( + strcmp(reason.res_based_reasons[0]->host, "") == 0); + TEST_ASSERT_TRUE(strcmp(reason.res_based_reasons[0]->resource, "ug1_slots") + == 0); + erase_job_preason(&reason); +} + +static void test_second_reason_keeps_first_main(void) +{ + xce_job_preason_t reason; + + memset(&reason, 0x0, sizeof(reason)); + + /* First detail add wins as main_reason; a later add does not overwrite. */ + add_job_preason(&reason, PREASON_LIMIT_HOST, "t1", "hg1"); + add_job_preason(&reason, PREASON_NO_RESOURCE, "t2", "slots"); + TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_HOST); + TEST_ASSERT_TRUE(reason.nres_reasons == 2); + erase_job_preason(&reason); +} + void test_preason(void) { printf("\n>> preason test\n\n"); @@ -187,4 +240,7 @@ void test_preason(void) RUN_TEST(test_add_main_reset_other); RUN_TEST(test_add_more_host_reason); RUN_TEST(test_add_more_res_reason); + RUN_TEST(test_limit_host_reason); + RUN_TEST(test_limit_global_reason); + RUN_TEST(test_second_reason_keeps_first_main); } diff --git a/src/scheduler/include/scheduler.h b/src/scheduler/include/scheduler.h index 064f70e0..3d9c8a9b 100644 --- a/src/scheduler/include/scheduler.h +++ b/src/scheduler/include/scheduler.h @@ -148,6 +148,7 @@ extern int sched_add_worker(xce_scheduler_t*, xce_sched_worker_t*); extern int sched_add_project(xce_scheduler_t*, xce_sched_project_t*); extern int sched_job_is_ready(xce_sched_job_t*); extern int sched_queue_is_ready(xce_sched_queue_t*); +extern int sched_queue_status_reason(uint32_t status); extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*); extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*); extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*); diff --git a/src/scheduler/src/plugins/fcfs.c b/src/scheduler/src/plugins/fcfs.c index 10de80ba..14cfcbc4 100644 --- a/src/scheduler/src/plugins/fcfs.c +++ b/src/scheduler/src/plugins/fcfs.c @@ -230,11 +230,36 @@ static void* fcfs_elect_job(xce_sched_plugin_t* plugin) pthread_testcancel(); + pendJobs = &(q->pendJobs); + if (!sched_queue_is_ready(q->queue)) { + /* Attach the queue-state reason (closed/draining/windowed-out) + * to every PEND job in this queue so a skipped queue explains + * itself (PEND-reason P1). */ + int qreason = sched_queue_status_reason(q->queue->status); + if (qreason != 0) { + ordq_item_t* it3; + for (it3 = pendJobs->head; it3 != NULL; it3 = it3->next) { + fcfs_job_t* j = (fcfs_job_t*)it3->data; + xce_sched_obj_t* obj; + if (j == NULL || j->job == NULL) { + continue; + } + obj = j->job->schedobj; + if (obj == NULL) { + continue; + } + if (obj->reason == NULL) { + obj->reason = make_pendreason(0); + } + if (obj->reason != NULL) { + add_job_preason(obj->reason->reasons, qreason, + q->queue->name, NULL); + } + } + } continue; } - - pendJobs = &(q->pendJobs); for (it2 = pendJobs->head; it2 != NULL; it2 = it2->next) { fcfs_job_t* j = (fcfs_job_t*)it2->data; diff --git a/src/scheduler/src/plugins/limit.c b/src/scheduler/src/plugins/limit.c index 168ee028..b90acb3a 100644 --- a/src/scheduler/src/plugins/limit.c +++ b/src/scheduler/src/plugins/limit.c @@ -6,6 +6,7 @@ #include "kvmgr.h" #include "plugin.h" +#include "preason.h" #include "sched/types.h" #include "scheduler.h" #include "utils.h" @@ -184,6 +185,16 @@ static int match_job_global_limit_(xce_sched_plugin_t* plugin, TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__, limit->name, obj->user, obj->queue, obj->project, rc); if (rc <= 0) { + /* global quota hit — report the specific limit. The election return + * value is ignored by the main loop, so the reason is carried on + * ctx->reason and moved to obj->reason by limit_job_election. */ + if (ctx->reason == NULL) { + ctx->reason = make_pendreason(0); + } + if (ctx->reason != NULL) { + add_job_preason(ctx->reason->reasons, PREASON_LIMIT_GLOBAL, NULL, + limit->name); + } return rc; } @@ -275,6 +286,13 @@ static int limit_job_election(void* arg, xce_sched_job_t* job) iter_destroy(&it); + /* Carry any global-quota reason (PREASON_LIMIT_GLOBAL) attached by + * match_job_global_limit_ onto the job so it survives to the PEND + * decision. The election's rc itself is ignored by the main loop. */ + if (ctx.reason != NULL) { + MOVE_PTR(obj->reason, ctx.reason); + } + obj->configver = sched->configver; return rc; @@ -334,6 +352,15 @@ static int match_job_host_limit_(xce_sched_plugin_t* plugin, TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__, limit->name, obj->user, obj->queue, obj->project, rc); if (rc <= 0) { + /* host-based quota hit — report the specific limit (bug PEND-reason: + * previously the block was silent, job just stayed PEND). */ + if (ctx->reason == NULL) { + ctx->reason = make_pendreason(0); + } + if (ctx->reason != NULL) { + add_job_preason(ctx->reason->reasons, PREASON_LIMIT_HOST, + worker->name, limit->name); + } return rc; } diff --git a/src/scheduler/src/resource.c b/src/scheduler/src/resource.c index af0314e7..1a88638c 100644 --- a/src/scheduler/src/resource.c +++ b/src/scheduler/src/resource.c @@ -469,6 +469,20 @@ static int sched_try_alloc_on_host_(xce_sched_worker_t* w, va_list ap) int rc; xvec_t* uncomit = NULL; + /* Worker down/closed — never eligible. Attach HOST_UNAVAIL instead of + * trying to allocate on it (PEND-reason P1, E17). */ + if (w->status + & (XCE_WORKER_UNAVAIL | XCE_WORKER_CLOSED | XCE_WORKER_CLOSED_BY_ADMIN)) { + if (ctx->reason == NULL) { + ctx->reason = make_pendreason(0); + } + if (ctx->reason != NULL) { + add_job_preason(ctx->reason->reasons, PREASON_HOST_UNAVAIL, + w->name, NULL); + } + return 0; + } + rc = sched_meet_host_selection(sched, w, primary->select); if (rc == 0) { if (ctx->reason == NULL) { diff --git a/src/scheduler/src/scheduler.c b/src/scheduler/src/scheduler.c index 7ccbf4cc..d2448146 100644 --- a/src/scheduler/src/scheduler.c +++ b/src/scheduler/src/scheduler.c @@ -1449,8 +1449,36 @@ int sched_job_is_ready(xce_sched_job_t* j) return 1; } -/* FIXME this is just stub, we need to add more */ -int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; } +/* A queue is electable only while it accepts new jobs. The controller + * maintains q->status (queue.c: closed / inactivated-by-admin); the dispatch + * window state (QUEUE_STAT_INACT_WINDOW) is not yet computed anywhere, so it + * is not checked here (PEND-reason P1 note). */ +int sched_queue_is_ready(xce_sched_queue_t* q) +{ + if (q == NULL) { + return 0; + } + if (q->status + & (QUEUE_STAT_CLOSED | QUEUE_STAT_INACT_ADMIN | QUEUE_STAT_INACT_WINDOW)) { + return 0; + } + return 1; +} + +/* Map a queue status bitmask to the PEND reason it should report. */ +int sched_queue_status_reason(uint32_t status) +{ + if (status & QUEUE_STAT_CLOSED) { + return PREASON_QUEUE_CLOSED; + } + if (status & QUEUE_STAT_INACT_ADMIN) { + return PREASON_QUEUE_DRAINING; + } + if (status & QUEUE_STAT_INACT_WINDOW) { + return PREASON_TIME_WINDOW; + } + return 0; +} int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision) {