feat(scheduler): PEND reason classification — main_reason + limit/queue/worker reasons (POC "why is my job waiting")

P0 (done, cluster-verified):
- main_reason was always 0: add_job_preason only set it when BOTH host and res
  were NULL, but the allocator always attaches host/res detail rows. Now the
  first detail add wins as main_reason, so the API's main_reason_name is
  meaningful (was "unknown(0)"). Verified live: PEND jobs now report
  main_reason=3/match_fail + per-host reason_name.
- limit plugin attach: host-based quota hit (hg1_slots_per_host) -> LIMIT_HOST
  + limit name; global quota hit (ug1_slots, ...) -> LIMIT_GLOBAL + limit name.
  Previously both blocks were silent (job just stayed PEND).
- Extended the PREASON_* enum (was 3 codes) with LSF-style classes:
  LIMIT_GLOBAL/LIMIT_HOST/QUEUE_CLOSED/QUEUE_DRAINING/TIME_WINDOW/HOST_UNAVAIL.
  apiserver pendReasonName maps them; host/res detail rows now carry reason_name.

P1 (implemented):
- queue status: sched_queue_is_ready no longer a stub — closed/inactivated
  queues are not electable; the fcfs election attaches QUEUE_CLOSED /
  QUEUE_DRAINING to every PEND job in a skipped queue.
- worker filter: sched_try_alloc_on_host_ skips UNAVAIL/CLOSED workers and
  attaches HOST_UNAVAIL instead of trying to allocate on them.

Unit tests: preason suite updated for the main_reason change + 3 new tests
(limit host/global, first-add-wins); 295 lib tests pass under ASAN.
apiserver backend tests pass.

Co-Authored-By: Claude <noreply@anthropic.com>

src/scheduler/include/scheduler.h

@@ -148,6 +148,7 @@ extern int sched_add_worker(xce_scheduler_t*, xce_sched_worker_t*);
 extern int sched_add_project(xce_scheduler_t*, xce_sched_project_t*);
 extern int sched_job_is_ready(xce_sched_job_t*);
 extern int sched_queue_is_ready(xce_sched_queue_t*);
+extern int sched_queue_status_reason(uint32_t status);
 extern int sched_commit_decision(xce_scheduler_t*, xce_decision_t*);
 extern int sched_publish_pend_reason(xce_scheduler_t*, xce_preason_set_t*);
 extern int sched_publish_dispatch(xce_scheduler_t*, xce_job_dispatch_t*);

src/scheduler/src/plugins/fcfs.c

@@ -230,11 +230,36 @@ static void* fcfs_elect_job(xce_sched_plugin_t* plugin)
 
             pthread_testcancel();
 
+            pendJobs = &(q->pendJobs);
+
             if (!sched_queue_is_ready(q->queue)) {
+                /* Attach the queue-state reason (closed/draining/windowed-out)
+                 * to every PEND job in this queue so a skipped queue explains
+                 * itself (PEND-reason P1). */
+                int qreason = sched_queue_status_reason(q->queue->status);
+                if (qreason != 0) {
+                    ordq_item_t* it3;
+                    for (it3 = pendJobs->head; it3 != NULL; it3 = it3->next) {
+                        fcfs_job_t* j   = (fcfs_job_t*)it3->data;
+                        xce_sched_obj_t* obj;
+                        if (j == NULL || j->job == NULL) {
+                            continue;
+                        }
+                        obj = j->job->schedobj;
+                        if (obj == NULL) {
+                            continue;
+                        }
+                        if (obj->reason == NULL) {
+                            obj->reason = make_pendreason(0);
+                        }
+                        if (obj->reason != NULL) {
+                            add_job_preason(obj->reason->reasons, qreason,
+                                            q->queue->name, NULL);
+                        }
+                    }
+                }
                 continue;
             }
-
-            pendJobs = &(q->pendJobs);
             for (it2 = pendJobs->head; it2 != NULL; it2 = it2->next) {
                 fcfs_job_t* j = (fcfs_job_t*)it2->data;
 

src/scheduler/src/plugins/limit.c

@@ -6,6 +6,7 @@
 
 #include "kvmgr.h"
 #include "plugin.h"
+#include "preason.h"
 #include "sched/types.h"
 #include "scheduler.h"
 #include "utils.h"
@@ -184,6 +185,16 @@ static int match_job_global_limit_(xce_sched_plugin_t* plugin,
     TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
           limit->name, obj->user, obj->queue, obj->project, rc);
     if (rc <= 0) {
+        /* global quota hit — report the specific limit. The election return
+         * value is ignored by the main loop, so the reason is carried on
+         * ctx->reason and moved to obj->reason by limit_job_election. */
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_LIMIT_GLOBAL, NULL,
+                            limit->name);
+        }
         return rc;
     }
 
@@ -275,6 +286,13 @@ static int limit_job_election(void* arg, xce_sched_job_t* job)
 
     iter_destroy(&it);
 
+    /* Carry any global-quota reason (PREASON_LIMIT_GLOBAL) attached by
+     * match_job_global_limit_ onto the job so it survives to the PEND
+     * decision. The election's rc itself is ignored by the main loop. */
+    if (ctx.reason != NULL) {
+        MOVE_PTR(obj->reason, ctx.reason);
+    }
+
     obj->configver = sched->configver;
 
     return rc;
@@ -334,6 +352,15 @@ static int match_job_host_limit_(xce_sched_plugin_t* plugin,
     TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
           limit->name, obj->user, obj->queue, obj->project, rc);
     if (rc <= 0) {
+        /* host-based quota hit — report the specific limit (bug PEND-reason:
+         * previously the block was silent, job just stayed PEND). */
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_LIMIT_HOST,
+                            worker->name, limit->name);
+        }
         return rc;
     }
 

src/scheduler/src/resource.c

@@ -469,6 +469,20 @@ static int sched_try_alloc_on_host_(xce_sched_worker_t* w, va_list ap)
     int                   rc;
     xvec_t*               uncomit = NULL;
 
+    /* Worker down/closed — never eligible. Attach HOST_UNAVAIL instead of
+     * trying to allocate on it (PEND-reason P1, E17). */
+    if (w->status
+        & (XCE_WORKER_UNAVAIL | XCE_WORKER_CLOSED | XCE_WORKER_CLOSED_BY_ADMIN)) {
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_HOST_UNAVAIL,
+                            w->name, NULL);
+        }
+        return 0;
+    }
+
     rc = sched_meet_host_selection(sched, w, primary->select);
     if (rc == 0) {
         if (ctx->reason == NULL) {

src/scheduler/src/scheduler.c

@@ -1449,8 +1449,36 @@ int sched_job_is_ready(xce_sched_job_t* j)
     return 1;
 }
 
-/* FIXME this is just stub, we need to add more */
-int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
+/* A queue is electable only while it accepts new jobs. The controller
+ * maintains q->status (queue.c: closed / inactivated-by-admin); the dispatch
+ * window state (QUEUE_STAT_INACT_WINDOW) is not yet computed anywhere, so it
+ * is not checked here (PEND-reason P1 note). */
+int sched_queue_is_ready(xce_sched_queue_t* q)
+{
+    if (q == NULL) {
+        return 0;
+    }
+    if (q->status
+        & (QUEUE_STAT_CLOSED | QUEUE_STAT_INACT_ADMIN | QUEUE_STAT_INACT_WINDOW)) {
+        return 0;
+    }
+    return 1;
+}
+
+/* Map a queue status bitmask to the PEND reason it should report. */
+int sched_queue_status_reason(uint32_t status)
+{
+    if (status & QUEUE_STAT_CLOSED) {
+        return PREASON_QUEUE_CLOSED;
+    }
+    if (status & QUEUE_STAT_INACT_ADMIN) {
+        return PREASON_QUEUE_DRAINING;
+    }
+    if (status & QUEUE_STAT_INACT_WINDOW) {
+        return PREASON_TIME_WINDOW;
+    }
+    return 0;
+}
 
 int sched_commit_decision(xce_scheduler_t* sched, xce_decision_t* decision)
 {