feat(scheduler): PEND reason classification — main_reason + limit/queue/worker reasons (POC "why is my job waiting")

P0 (done, cluster-verified):
- main_reason was always 0: add_job_preason only set it when BOTH host and res
  were NULL, but the allocator always attaches host/res detail rows. Now the
  first detail add wins as main_reason, so the API's main_reason_name is
  meaningful (was "unknown(0)"). Verified live: PEND jobs now report
  main_reason=3/match_fail + per-host reason_name.
- limit plugin attach: host-based quota hit (hg1_slots_per_host) -> LIMIT_HOST
  + limit name; global quota hit (ug1_slots, ...) -> LIMIT_GLOBAL + limit name.
  Previously both blocks were silent (job just stayed PEND).
- Extended the PREASON_* enum (was 3 codes) with LSF-style classes:
  LIMIT_GLOBAL/LIMIT_HOST/QUEUE_CLOSED/QUEUE_DRAINING/TIME_WINDOW/HOST_UNAVAIL.
  apiserver pendReasonName maps them; host/res detail rows now carry reason_name.

P1 (implemented):
- queue status: sched_queue_is_ready no longer a stub — closed/inactivated
  queues are not electable; the fcfs election attaches QUEUE_CLOSED /
  QUEUE_DRAINING to every PEND job in a skipped queue.
- worker filter: sched_try_alloc_on_host_ skips UNAVAIL/CLOSED workers and
  attaches HOST_UNAVAIL instead of trying to allocate on them.

Unit tests: preason suite updated for the main_reason change + 3 new tests
(limit host/global, first-add-wins); 295 lib tests pass under ASAN.
apiserver backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>

src/frontend/apiserver/backend/convert.go

@@ -133,7 +133,8 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
 			hr := C.job_host_reason(pr, C.int(i))
 			if hr != nil && hr.host != nil {
 				r.pendReason.HostReasons = append(r.pendReason.HostReasons,
-					rawHostReason{Host: goString(hr.host), Reason: int(hr.reason)})
+					rawHostReason{Host: goString(hr.host), Reason: int(hr.reason),
+						ReasonName: pendReasonName(int(hr.reason))})
 			}
 		}
 		for i := 0; i < int(pr.nres_reasons); i++ {
@@ -141,7 +142,8 @@ func cJobToModel(job *C.xce_job_t) *models.JobInfo {
 			if rr != nil && rr.resource != nil {
 				r.pendReason.ResReasons = append(r.pendReason.ResReasons,
 					rawResReason{Resource: goString(rr.resource),
-						Host: goString(rr.host), Reason: int(rr.reason)})
+						Host: goString(rr.host), Reason: int(rr.reason),
+						ReasonName: pendReasonName(int(rr.reason))})
 			}
 		}
 	}

src/frontend/apiserver/backend/convert_pure.go

@@ -163,6 +163,7 @@ type rawSubmitData struct {
 type rawHostReason struct {
 	Host       string `json:"host"`
 	Reason     int    `json:"reason"`
+	ReasonName string `json:"reason_name"`
 }
 
 // rawResReason mirrors the C xce_res_reason_t (one resource-level PEND reason).
@@ -170,6 +171,7 @@ type rawResReason struct {
 	Resource   string `json:"resource"`
 	Host       string `json:"host"`
 	Reason     int    `json:"reason"`
+	ReasonName string `json:"reason_name"`
 }
 
 // rawPendReason mirrors the C xce_job_preason_t attached to a PEND job by the
@@ -184,9 +186,9 @@ type rawPendReason struct {
 }
 
 // pendReasonName maps a PREASON_* main_reason code to a readable name. The
-// codes are the PEND reason bitfield of xce_job_preason_t (PREASON_BAD_RESREQ/
+// codes are the PEND reason bitfield of xce_job_preason_t (src/include/
-// NO_RESOURCE/MATCH_FAIL, src/include/preason.h). Unknown codes are reported
+// preason.h). Unknown codes are reported verbatim so a new scheduler reason
-// verbatim so a new scheduler reason never silently disappears.
+// never silently disappears.
 func pendReasonName(code int) string {
 	switch code {
 	case 1: // PREASON_BAD_RESREQ
@@ -195,6 +197,18 @@ func pendReasonName(code int) string {
 		return "no_resource"
 	case 3: // PREASON_MATCH_FAIL
 		return "match_fail"
+	case 4: // PREASON_LIMIT_GLOBAL
+		return "limit_global"
+	case 5: // PREASON_LIMIT_HOST
+		return "limit_host"
+	case 6: // PREASON_QUEUE_CLOSED
+		return "queue_closed"
+	case 7: // PREASON_QUEUE_DRAINING
+		return "queue_draining"
+	case 8: // PREASON_TIME_WINDOW
+		return "time_window"
+	case 9: // PREASON_HOST_UNAVAIL
+		return "host_unavail"
 	default:
 		return fmt.Sprintf("unknown(%d)", code)
 	}

src/include/preason.h

@@ -7,6 +7,15 @@
 #define PREASON_BAD_RESREQ    (1)
 #define PREASON_NO_RESOURCE   (2)
 #define PREASON_MATCH_FAIL    (3)
+/* LSF-style PEND reason classes beyond plain capacity / host-match failure.
+ * Codes 4+ feed both main_reason and the host/res detail rows; the API layer
+ * maps each to a human-readable name (pendReasonName in apiserver). */
+#define PREASON_LIMIT_GLOBAL  (4) /* global quota hit (ug1_slots, ug2_slots_per_user, ...) */
+#define PREASON_LIMIT_HOST    (5) /* host-based quota hit (hg1_slots_per_host) */
+#define PREASON_QUEUE_CLOSED  (6) /* queue closed/disabled, not accepting jobs */
+#define PREASON_QUEUE_DRAINING (7) /* queue draining (no new jobs until empty) */
+#define PREASON_TIME_WINDOW   (8) /* outside the queue's dispatch_window */
+#define PREASON_HOST_UNAVAIL  (9) /* host down/unavailable, skipped for allocation */
 
 extern void              erase_job_preason(xce_job_preason_t*);
 extern int               add_job_preason(xce_job_preason_t*, int, char*, char*);

src/lib/preason.c

@@ -141,6 +141,10 @@ int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res)
 {
     int rc = 0;
 
+    if (reasons == NULL) {
+        return -EINVAL;
+    }
+
     if ((host == NULL) && (res == NULL)) {
         /* set main reason */
         erase_job_preason(reasons);
@@ -153,6 +157,14 @@ int add_job_preason(xce_job_preason_t* reasons, int code, char* host, char* res)
         rc = add_res_preason(reasons, code, host, res);
     }
 
+    /* The allocator always attaches host/res DETAIL rows (MATCH_FAIL per host,
+     * NO_RESOURCE per resource) and never calls with both host and res NULL —
+     * without this main_reason stays 0 and the API can only surface the detail
+     * rows. First successful detail add wins as the overall reason. */
+    if ((rc > 0) && (reasons->main_reason == 0)) {
+        reasons->main_reason = code;
+    }
+
     return rc;
 }
 

src/lib/testsuite/test_preason.c

@@ -26,7 +26,7 @@ static void test_add_host_reason(void)
     memset(&reason, 0x0, sizeof(reason));
 
     add_job_preason(&reason, 1, "h1", NULL);
-    TEST_ASSERT_TRUE(reason.main_reason == 0);
+    TEST_ASSERT_TRUE(reason.main_reason == 1);
     TEST_ASSERT_TRUE((reason.nhost_reasons == 1)
                      && (reason.host_based_reasons != NULL)
                      && (reason.host_based_reasons[0] != NULL));
@@ -45,7 +45,7 @@ static void test_add_res_reason(void)
     memset(&reason, 0x0, sizeof(reason));
 
     add_job_preason(&reason, 1, "h1", "res1");
-    TEST_ASSERT_TRUE(reason.main_reason == 0);
+    TEST_ASSERT_TRUE(reason.main_reason == 1);
     TEST_ASSERT_TRUE((reason.nres_reasons == 1)
                      && (reason.res_based_reasons != NULL)
                      && (reason.res_based_reasons[0] != NULL));
@@ -65,7 +65,7 @@ static void test_add_shared_res_reason(void)
     memset(&reason, 0x0, sizeof(reason));
 
     add_job_preason(&reason, 1, NULL, "res1");
-    TEST_ASSERT_TRUE(reason.main_reason == 0);
+    TEST_ASSERT_TRUE(reason.main_reason == 1);
     TEST_ASSERT_TRUE((reason.nres_reasons == 1)
                      && (reason.res_based_reasons != NULL)
                      && (reason.res_based_reasons[0] != NULL));
@@ -88,7 +88,7 @@ static void test_add_res_host_reason(void)
     add_job_preason(&reason, 1, "h1", "res1");
     add_job_preason(&reason, 2, "h1", NULL);
 
-    TEST_ASSERT_TRUE(reason.main_reason == 0);
+    TEST_ASSERT_TRUE(reason.main_reason == 1);
     TEST_ASSERT_TRUE((reason.nres_reasons == 1)
                      && (reason.res_based_reasons != NULL)
                      && (reason.res_based_reasons[0] != NULL));
@@ -132,7 +132,7 @@ static void test_add_more_host_reason(void)
     add_job_preason(&reason, 1, "h1", NULL);
     add_job_preason(&reason, 2, "h2", NULL);
 
-    TEST_ASSERT_TRUE(reason.main_reason == 0);
+    TEST_ASSERT_TRUE(reason.main_reason == 1);
     TEST_ASSERT_TRUE((reason.nhost_reasons == 2)
                      && (reason.host_based_reasons != NULL)
                      && (reason.host_based_reasons[0] != NULL)
@@ -156,7 +156,7 @@ static void test_add_more_res_reason(void)
     add_job_preason(&reason, 1, "h1", "res1");
     add_job_preason(&reason, 2, "h2", "res2");
 
-    TEST_ASSERT_TRUE(reason.main_reason == 0);
+    TEST_ASSERT_TRUE(reason.main_reason == 1);
     TEST_ASSERT_TRUE((reason.nres_reasons == 2)
                      && (reason.res_based_reasons != NULL)
                      && (reason.res_based_reasons[0] != NULL)
@@ -175,6 +175,59 @@ static void test_add_more_res_reason(void)
     erase_job_preason(&reason);
 }
 
+/* LSF-style reason classes: a limit/queue/time/worker reason attached as a
+ * host/res detail row must surface both in main_reason AND in the detail row's
+ * own code (bug PEND-reason: POC "why is my job waiting"). */
+static void test_limit_host_reason(void)
+{
+    xce_job_preason_t reason;
+
+    memset(&reason, 0x0, sizeof(reason));
+
+    add_job_preason(&reason, PREASON_LIMIT_HOST, "t1", "hg1_slots_per_host");
+    TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_HOST);
+    TEST_ASSERT_TRUE(reason.nres_reasons == 1);
+    TEST_ASSERT_TRUE(reason.res_based_reasons[0]->reason == PREASON_LIMIT_HOST);
+    TEST_ASSERT_TRUE(
+        strcmp(reason.res_based_reasons[0]->host, "t1") == 0);
+    TEST_ASSERT_TRUE(
+        strcmp(reason.res_based_reasons[0]->resource, "hg1_slots_per_host")
+        == 0);
+    erase_job_preason(&reason);
+}
+
+static void test_limit_global_reason(void)
+{
+    xce_job_preason_t reason;
+
+    memset(&reason, 0x0, sizeof(reason));
+
+    add_job_preason(&reason, PREASON_LIMIT_GLOBAL, NULL, "ug1_slots");
+    TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_GLOBAL);
+    TEST_ASSERT_TRUE(reason.nres_reasons == 1);
+    TEST_ASSERT_TRUE(
+        reason.res_based_reasons[0]->reason == PREASON_LIMIT_GLOBAL);
+    TEST_ASSERT_TRUE(
+        strcmp(reason.res_based_reasons[0]->host, "") == 0);
+    TEST_ASSERT_TRUE(strcmp(reason.res_based_reasons[0]->resource, "ug1_slots")
+                     == 0);
+    erase_job_preason(&reason);
+}
+
+static void test_second_reason_keeps_first_main(void)
+{
+    xce_job_preason_t reason;
+
+    memset(&reason, 0x0, sizeof(reason));
+
+    /* First detail add wins as main_reason; a later add does not overwrite. */
+    add_job_preason(&reason, PREASON_LIMIT_HOST, "t1", "hg1");
+    add_job_preason(&reason, PREASON_NO_RESOURCE, "t2", "slots");
+    TEST_ASSERT_TRUE(reason.main_reason == PREASON_LIMIT_HOST);
+    TEST_ASSERT_TRUE(reason.nres_reasons == 2);
+    erase_job_preason(&reason);
+}
+
 void test_preason(void)
 {
     printf("\n>> preason test\n\n");
@@ -187,4 +240,7 @@ void test_preason(void)
     RUN_TEST(test_add_main_reset_other);
     RUN_TEST(test_add_more_host_reason);
     RUN_TEST(test_add_more_res_reason);
+    RUN_TEST(test_limit_host_reason);
+    RUN_TEST(test_limit_global_reason);
+    RUN_TEST(test_second_reason_keeps_first_main);
 }

src/scheduler/include/scheduler.h

@@ -148,6 +148,7 @@ extern int sched_add_worker(xce_scheduler_t*, xce_sched_worker_t*);
 extern int sched_add_project(xce_scheduler_t*, xce_sched_project_t*);
 extern int sched_job_is_ready(xce_sched_job_t*);
 extern int sched_queue_is_ready(xce_sched_queue_t*);
+extern int sched_queue_status_reason(uint32_t status);
 extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
 extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
 extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);

src/scheduler/src/plugins/fcfs.c

@@ -230,11 +230,36 @@ static void* fcfs_elect_job(xce_sched_plugin_t* plugin)
 
             pthread_testcancel();
 
+            pendJobs = &(q->pendJobs);
+
             if (!sched_queue_is_ready(q->queue)) {
+                /* Attach the queue-state reason (closed/draining/windowed-out)
+                 * to every PEND job in this queue so a skipped queue explains
+                 * itself (PEND-reason P1). */
+                int qreason = sched_queue_status_reason(q->queue->status);
+                if (qreason != 0) {
+                    ordq_item_t* it3;
+                    for (it3 = pendJobs->head; it3 != NULL; it3 = it3->next) {
+                        fcfs_job_t* j   = (fcfs_job_t*)it3->data;
+                        xce_sched_obj_t* obj;
+                        if (j == NULL || j->job == NULL) {
+                            continue;
+                        }
+                        obj = j->job->schedobj;
+                        if (obj == NULL) {
+                            continue;
+                        }
+                        if (obj->reason == NULL) {
+                            obj->reason = make_pendreason(0);
+                        }
+                        if (obj->reason != NULL) {
+                            add_job_preason(obj->reason->reasons, qreason,
+                                            q->queue->name, NULL);
+                        }
+                    }
+                }
                 continue;
             }
-
-            pendJobs = &(q->pendJobs);
             for (it2 = pendJobs->head; it2 != NULL; it2 = it2->next) {
                 fcfs_job_t* j = (fcfs_job_t*)it2->data;
 

src/scheduler/src/plugins/limit.c

@@ -6,6 +6,7 @@
 
 #include "kvmgr.h"
 #include "plugin.h"
+#include "preason.h"
 #include "sched/types.h"
 #include "scheduler.h"
 #include "utils.h"
@@ -184,6 +185,16 @@ static int match_job_global_limit_(xce_sched_plugin_t* plugin,
     TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
           limit->name, obj->user, obj->queue, obj->project, rc);
     if (rc <= 0) {
+        /* global quota hit — report the specific limit. The election return
+         * value is ignored by the main loop, so the reason is carried on
+         * ctx->reason and moved to obj->reason by limit_job_election. */
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_LIMIT_GLOBAL, NULL,
+                            limit->name);
+        }
         return rc;
     }
 
@@ -275,6 +286,13 @@ static int limit_job_election(void* arg, xce_sched_job_t* job)
 
     iter_destroy(&it);
 
+    /* Carry any global-quota reason (PREASON_LIMIT_GLOBAL) attached by
+     * match_job_global_limit_ onto the job so it survives to the PEND
+     * decision. The election's rc itself is ignored by the main loop. */
+    if (ctx.reason != NULL) {
+        MOVE_PTR(obj->reason, ctx.reason);
+    }
+
     obj->configver = sched->configver;
 
     return rc;
@@ -334,6 +352,15 @@ static int match_job_host_limit_(xce_sched_plugin_t* plugin,
     TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
           limit->name, obj->user, obj->queue, obj->project, rc);
     if (rc <= 0) {
+        /* host-based quota hit — report the specific limit (bug PEND-reason:
+         * previously the block was silent, job just stayed PEND). */
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_LIMIT_HOST,
+                            worker->name, limit->name);
+        }
         return rc;
     }
 

src/scheduler/src/resource.c

@@ -469,6 +469,20 @@ static int sched_try_alloc_on_host_(xce_sched_worker_t* w, va_list ap)
     int                   rc;
     xvec_t*               uncomit = NULL;
 
+    /* Worker down/closed — never eligible. Attach HOST_UNAVAIL instead of
+     * trying to allocate on it (PEND-reason P1, E17). */
+    if (w->status
+        & (XCE_WORKER_UNAVAIL | XCE_WORKER_CLOSED | XCE_WORKER_CLOSED_BY_ADMIN)) {
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_HOST_UNAVAIL,
+                            w->name, NULL);
+        }
+        return 0;
+    }
+
     rc = sched_meet_host_selection(sched, w, primary->select);
     if (rc == 0) {
         if (ctx->reason == NULL) {

src/scheduler/src/scheduler.c

@@ -1449,8 +1449,36 @@ int sched_job_is_ready(xce_sched_job_t* j)
     return 1;
 }
 
-/* FIXME this is just stub, we need to add more */
+/* A queue is electable only while it accepts new jobs. The controller
-int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
+ * maintains q->status (queue.c: closed / inactivated-by-admin); the dispatch
+ * window state (QUEUE_STAT_INACT_WINDOW) is not yet computed anywhere, so it
+ * is not checked here (PEND-reason P1 note). */
+int sched_queue_is_ready(xce_sched_queue_t* q)
+{
+    if (q == NULL) {
+        return 0;
+    }
+    if (q->status
+        & (QUEUE_STAT_CLOSED | QUEUE_STAT_INACT_ADMIN | QUEUE_STAT_INACT_WINDOW)) {
+        return 0;
+    }
+    return 1;
+}
+
+/* Map a queue status bitmask to the PEND reason it should report. */
+int sched_queue_status_reason(uint32_t status)
+{
+    if (status & QUEUE_STAT_CLOSED) {
+        return PREASON_QUEUE_CLOSED;
+    }
+    if (status & QUEUE_STAT_INACT_ADMIN) {
+        return PREASON_QUEUE_DRAINING;
+    }
+    if (status & QUEUE_STAT_INACT_WINDOW) {
+        return PREASON_TIME_WINDOW;
+    }
+    return 0;
+}
 
 int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
 {