feat(scheduler): restore PEND-reason P1 + fix dispatchwindow merge regressions

The scheduler-framework merge (2026-08-25) silently regressed the PEND
reason P1 work (4e2ac86/ee79179): the old fcfs.c became dead-on-default
(builtin_fcfs replaced it), resource.c/limit.c conflict resolution took
Rongsong's side, and the merge-fix only re-added the dispatchwindow FIELD
without the logic. After the merge, sched_queue_is_ready/sched_job_is_ready
were `return 1` stubs and only MATCH_FAIL/NO_RESOURCE were ever emitted.

Restore P1 on the active (framework) path, keeping every reason attach in
the allocator thread (bug-1521: the picker thread must not write
job->schedobj->reason — it races the allocator's MOVE_PTR):

- core.c: sched_job_is_ready (PEND & not suspended) and sched_queue_is_ready
  (QUEUE_STAT_CLOSED/INACT_ADMIN/INACT_WINDOW gate) real implementations.
- scheduler.c: allocator loop injects sched_queue_status_reason() — queue
  status first, then the dispatch window evaluated against the core's
  virtual clock (xce_core_now/1e9) so live and sim modes agree; attaches
  QUEUE_CLOSED/DRAINING/TIME_WINDOW and pends the job. Adds <time.h>.
- resource.c: skip UNAVAIL/CLOSED/CLOSED_BY_ADMIN workers + HOST_UNAVAIL
  (restored from 4e2ac86).
- limit.c: attach LIMIT_GLOBAL/LIMIT_HOST + carry reason in
  limit_job_election (restored from 4e2ac86).

Two further merge regressions found during cluster verification (the
framework core path never carried the queue dispatch window):

- core_events.c core_build_queue: duplicate q->dispatchwindow_config onto
  xce_sched_queue_t.dispatchwindow (make_sched_queue did this; the core path
  dropped it, so TIME_WINDOW could never fire on the default path).
- core_events.c core_upsert_queue update branch: refresh dispatchwindow on
  QUEUE_UPDATE.

Regression test: test_core_ready_gates.c (white-box #include of core.c,
7 cases covering both gates), registered in scheduler testsuite meson.build.

Cluster-verified on t1/t2/t3 (nix xcecore closure + AppImage standard
deployment): night queue (window closed) -> main_reason=8 time_window;
all workers down -> main_reason=9 host_unavail with per-host detail;
closed queue -> job stays PEND; normal queue baseline -> DONE. Scheduler
suite 52/53 (1 pre-existing admin-request symlink-escape failure, unrelated).

Co-Authored-By: Claude Code <noreply@anthropic.com>

src/scheduler/src/core.c

@@ -432,8 +432,41 @@ xce_sched_project_t* sched_get_project(xce_sched_core_t* core, char* name)
     return make_sched_project(core, name);
 }
 
-/* FIXME this is just stub, we need to add more */
-int sched_job_is_ready(xce_sched_job_t* j) { return 1; }
+/* A job is electable only while it is PENDING and not suspended. Once
+ * dispatched (RUN), finished (EXIT/DONE), or suspended (PSUSP/USUSP/SSUSP)
+ * it must never be re-elected — bug-1498 (restored from the pre-framework
+ * scheduler.c after the scheduler-framework merge dropped it). */
+int sched_job_is_ready(xce_sched_job_t* j)
+{
+    if (j == NULL) {
+        return 0;
+    }
+    if ((j->state & JOB_STAT_PEND) == 0) {
+        return 0; /* already dispatched/running or terminal */
+    }
+    if (j->state & (JOB_STAT_PSUSP | JOB_STAT_USUSP | JOB_STAT_SSUSP)) {
+        return 0; /* suspended — not electable */
+    }
+    return 1;
+}
 
-/* FIXME this is just stub, we need to add more */
-int sched_queue_is_ready(xce_sched_queue_t* q) { return 1; }
+/* A queue is electable only while it accepts new jobs. The controller
+ * maintains q->status (closed / inactivated-by-admin / inactive-by-window).
+ * The dispatch window (q->dispatchwindow, from queues.yaml) is NOT evaluated
+ * here: core.c is isolated from wall-clock time (test_core_isolation.c) and
+ * the window can only be judged against the core's virtual clock
+ * (xce_core_now), which this signature has no access to. Windowed-out queues
+ * therefore pass this gate and are re-pended with PREASON_TIME_WINDOW by the
+ * allocator (scheduler.c, where xce_core_now() is reachable) — same net
+ * effect, with a safe (allocator-thread) reason attach. PEND-reason P1. */
+int sched_queue_is_ready(xce_sched_queue_t* q)
+{
+    if (q == NULL) {
+        return 0;
+    }
+    if (q->status
+        & (QUEUE_STAT_CLOSED | QUEUE_STAT_INACT_ADMIN | QUEUE_STAT_INACT_WINDOW)) {
+        return 0;
+    }
+    return 1;
+}

src/scheduler/src/core_events.c

@@ -284,6 +284,15 @@ static xce_sched_queue_t* core_build_queue(xce_queue_t* q)
     queue->priority        = q->priority;
     queue->accept_interval = q->accept_interval;
     queue->sched_delay     = q->sched_delay;
+    /* PEND-reason P1 fix (2026-08-30): the scheduler-framework merge
+     * dropped the dispatch-window config here — make_sched_queue (schedobj.c)
+     * duplicates q->dispatchwindow_config, but this framework core path did
+     * not, so the TIME_WINDOW PEND reason could never fire on the live
+     * default path. Mirror make_sched_queue: dup the window (freed by
+     * kill_sched_queue via core->queues' datakiller). */
+    if (q->dispatchwindow_config != NULL) {
+        queue->dispatchwindow = dup_timewindow_config(q->dispatchwindow_config);
+    }
     /* Same scope note as make_sched_queue's own (schedobj.c): project_name/
      * job_pick_plugin have no wire-config source yet, and (matching
      * existing behavior) resreq is not populated here either -- wiring
@@ -336,6 +345,15 @@ static void core_upsert_queue(xce_sched_core_t* core, xce_sched_plugin_t* picker
             FREEUP(q->resreq);
             q->resreq = makeString(updated->resreq);
         }
+
+        /* PEND-reason P1 fix (2026-08-30): same merge regression as
+         * core_build_queue above — the update branch never carried the
+         * dispatch window. Refresh it so a QUEUE_UPDATE that changes the
+         * window (or a re-push after the initial build) takes effect. */
+        if (updated->dispatchwindow_config != NULL) {
+            free_timewindow_config(q->dispatchwindow);
+            q->dispatchwindow = dup_timewindow_config(updated->dispatchwindow_config);
+        }
     }
 
     if (picker != NULL) {

src/scheduler/src/plugins/limit.c

@@ -6,6 +6,7 @@
 
 #include "kvmgr.h"
 #include "plugin.h"
+#include "preason.h"
 #include "sched/types.h"
 #include "scheduler.h"
 #include "utils.h"
@@ -176,6 +177,17 @@ static int match_job_global_limit_(xce_sched_plugin_t* plugin,
     TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
           limit->name, obj->user, obj->queue, obj->project, rc);
     if (rc <= 0) {
+        /* global quota hit — report the specific limit (PEND-reason P1,
+         * restored from 4e2ac86). The election return value is ignored by
+         * the main loop, so the reason is carried on ctx->reason and moved
+         * to obj->reason by limit_job_election. */
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_LIMIT_GLOBAL, NULL,
+                            limit->name);
+        }
         return rc;
     }
 
@@ -267,6 +279,14 @@ static int limit_job_election(void* arg, xce_sched_job_t* job)
 
     iter_destroy(&it);
 
+    /* Carry any global-quota reason (PREASON_LIMIT_GLOBAL) attached by
+     * match_job_global_limit_ onto the job so it survives to the PEND
+     * decision. The election's rc itself is ignored by the main loop.
+     * (PEND-reason P1, restored from 4e2ac86.) */
+    if (ctx.reason != NULL) {
+        MOVE_PTR(obj->reason, ctx.reason);
+    }
+
     obj->configver = sched->core.configver;
 
     return rc;
@@ -326,6 +346,16 @@ static int match_job_host_limit_(xce_sched_plugin_t* plugin,
     TRACE("%s match_limit limit %s user %s queue %s project %s rc %d", __func__,
           limit->name, obj->user, obj->queue, obj->project, rc);
     if (rc <= 0) {
+        /* host-based quota hit — report the specific limit (PEND-reason P1,
+         * restored from 4e2ac86; previously the block was silent and the job
+         * just stayed PEND). */
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_LIMIT_HOST,
+                            worker->name, limit->name);
+        }
         return rc;
     }
 

src/scheduler/src/resource.c

@@ -484,6 +484,23 @@ static int sched_try_alloc_on_host_(xce_sched_worker_t* w, va_list ap)
     int                   rc;
     xvec_t*               uncomit = NULL;
 
+    /* Worker down/closed — never eligible. Attach HOST_UNAVAIL instead of
+     * trying to allocate on it (PEND-reason P1, E17 — restored from
+     * 4e2ac86 after the scheduler-framework merge dropped it). Runs in the
+     * allocator thread (sched_try_alloc_on_hosts), so ctx->reason is safe
+     * to touch here. */
+    if (w->status
+        & (XCE_WORKER_UNAVAIL | XCE_WORKER_CLOSED | XCE_WORKER_CLOSED_BY_ADMIN)) {
+        if (ctx->reason == NULL) {
+            ctx->reason = make_pendreason(0);
+        }
+        if (ctx->reason != NULL) {
+            add_job_preason(ctx->reason->reasons, PREASON_HOST_UNAVAIL,
+                            w->name, NULL);
+        }
+        return 0;
+    }
+
     rc = sched_meet_host_selection(sched, w, primary->select);
     if (rc == 0) {
         if (ctx->reason == NULL) {

src/scheduler/src/scheduler.c

@@ -5,6 +5,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <time.h>
 
 #include "client.h"
 #include "controller.capnp.h"
@@ -266,6 +267,39 @@ static int sched_make_alter_allocation(xce_scheduler_t* sched,
     return 0;
 }
 
+/* Map a queue to the PEND reason it should report (PEND-reason P1 restore,
+ * resurrected from the pre-framework scheduler.c). Status flags take
+ * precedence, then the dispatch window. The window is evaluated against the
+ * core's virtual clock (xce_core_now, /1e9 → seconds) so live and sim modes
+ * agree; a NULL/empty window is always open. Runs only from the allocator
+ * thread (see the gate at the top of the allocator loop), so its reason
+ * attachment cannot race MOVE_PTR(obj->reason, ...) (bug-1521). */
+static int sched_queue_status_reason(xce_scheduler_t* sched,
+                                     xce_sched_queue_t* q)
+{
+    if (q == NULL) {
+        return 0;
+    }
+    if (q->status & QUEUE_STAT_CLOSED) {
+        return PREASON_QUEUE_CLOSED;
+    }
+    if (q->status & QUEUE_STAT_INACT_ADMIN) {
+        return PREASON_QUEUE_DRAINING;
+    }
+    if (q->status & QUEUE_STAT_INACT_WINDOW) {
+        return PREASON_TIME_WINDOW;
+    }
+    if (q->dispatchwindow != NULL) {
+        uint64_t now_ns = xce_core_now(&(sched->core));
+
+        if (!xce_timewindow_is_open(q->dispatchwindow,
+                                    (time_t)(now_ns / 1000000000ULL))) {
+            return PREASON_TIME_WINDOW;
+        }
+    }
+    return 0;
+}
+
 static int sched_make_allocation(xce_scheduler_t* sched, xce_sched_obj_t* obj)
 {
     int rc = 0;
@@ -326,6 +360,7 @@ static void* sched_allocator(void* arg)
         xce_decision_t*  decision = NULL;
         xce_sched_job_t* job      = NULL;
         int              alloc_rc = 0;
+        int              qr       = 0;
 
         pthread_testcancel();
 
@@ -346,6 +381,35 @@ static void* sched_allocator(void* arg)
 
         plugin_job_election(sched, job);
 
+        /* PEND-reason P1 restore (bug-1521-safe): a job whose queue is not
+         * ready (closed / drained / outside its dispatch window) must not be
+         * dispatched. The gate and reason attach live HERE (allocator thread)
+         * rather than in the picker: the picker thread writing
+         * job->schedobj->reason races this thread's MOVE_PTR(obj->reason, ...)
+         * (bug-1521 heap corruption). The picker still skips persistently-
+         * closed queues per its spec contract, so this path fires for queues
+         * that close or window-out between pick and allocation, and for
+         * windowed-out queues that pass the picker's status-only gate. */
+        qr = sched_queue_status_reason(sched, job->queue);
+        if (qr != 0) {
+            xce_pendreason_t* qreason = make_pendreason(0);
+
+            job_picker->picker_iface->on_pend(job_picker, jobid);
+
+            if (qreason != NULL) {
+                qreason->jobid = job->jobid;
+                add_job_preason(qreason->reasons, qr,
+                                job->queue != NULL ? job->queue->name : NULL,
+                                NULL);
+                decision = make_pend_decision(qreason);
+            }
+            if (decision != NULL) {
+                commit_decision(sched, decision);
+                cbuf_put(&(sched->core.decisions), &decision);
+            }
+            continue;
+        }
+
         /* calculate decision for jobid */
         alloc_rc = sched_make_allocation(sched, job->schedobj);
         if (alloc_rc < 0) {

src/scheduler/testsuite/meson.build

@@ -264,6 +264,19 @@ sched_core_jbuf_capacity_pin_test = executable(
              + '"'],
     install: false)
 
+# Regression guard for the PEND-reason P1 readiness gates restored after
+# the scheduler-framework merge dropped them (sched_job_is_ready /
+# sched_queue_is_ready were `return 1` stubs in core.c). Same white-box
+# #include pattern as test_core_jbuf_capacity_pin.c (pulls the real
+# ../src/core.c into this TU); exercises the REAL gate functions, not a
+# re-implementation.
+sched_core_ready_gates_test = executable(
+    'sched_core_ready_gates_test',
+    ['test_core_ready_gates.c'],
+    include_directories:[incdir],
+    dependencies: [scheduler_test_deps],
+    install: false)
+
 # Contract test for the core event interface and batch event push
 # (scheduler-core spec, task 4.3, Req 5 AC 4 / AC 8): xce_core_push_events()
 # fires the resolved picker's corresponding state-change hook exactly once
@@ -1166,6 +1179,9 @@ test('scheduler no bare printf test',
 test('scheduler core jbuf capacity pin test',
      sched_core_jbuf_capacity_pin_test, suite: 'scheduler',
      env: ['ASAN_OPTIONS=detect_leaks=0'])
+test('scheduler core ready gates test',
+     sched_core_ready_gates_test, suite: 'scheduler',
+     env: ['ASAN_OPTIONS=detect_leaks=0'])
 test('scheduler core events test',
      sched_core_events_test, suite: 'scheduler',
      env: ['ASAN_OPTIONS=detect_leaks=0'])

src/scheduler/testsuite/test_core_ready_gates.c

@@ -0,0 +1,121 @@
+/*
+ * test_core_ready_gates.c - contract test for the PEND-reason P1 readiness
+ * gates restored after the scheduler-framework merge dropped them
+ * (sched_job_is_ready / sched_queue_is_ready were `return 1` stubs in
+ * core.c; the pre-framework scheduler.c implementations were lost).
+ *
+ * Observable under test:
+ *   - sched_job_is_ready(): a job is electable only while PENDING and not
+ *     suspended. NULL / RUN / EXIT / DONE / PSUSP / USUSP / SSUSP -> 0.
+ *     PEND -> 1. (bug-1498: a running job lingering in the pendJobs queue
+ *     must never be re-elected and re-dispatched.)
+ *   - sched_queue_is_ready(): a queue is electable only while it accepts
+ *     new jobs. NULL / QUEUE_STAT_CLOSED / QUEUE_STAT_INACT_ADMIN /
+ *     QUEUE_STAT_INACT_WINDOW -> 0. Status 0 -> 1. The dispatch-window
+ *     half of the gate intentionally lives in the allocator (scheduler.c,
+ *     xce_core_now) not here: core.c is isolated from wall-clock time, and
+ *     the queue has no back-pointer to the core (see the function's own
+ *     comment in core.c).
+ *
+ * White-box #include of ../src/core.c (same convention as
+ * test_core_jbuf_capacity_pin.c): these tests exercise the REAL core.c
+ * functions, not a re-implementation. Stub surface is exactly the seven
+ * kvmgr datakillers/plugin_unregister core.c references as addresses in
+ * xce_sched_core_init() (never actually invoked -- fixtures aren't torn
+ * down, same convention as the sibling core tests).
+ */
+#include <unity/unity.h>
+
+#include "sched/types.h"
+#include "xce/types.h"
+
+#include "../src/core.c"
+
+/* Linker stubs for core.c's extern datakillers (never invoked). */
+void kill_sched_job(void* arg) { (void)arg; }
+void kill_sched_queue(void* arg) { (void)arg; }
+void kill_sched_worker(void* arg) { (void)arg; }
+void kill_sched_user(void* arg) { (void)arg; }
+void kill_sched_project(void* arg) { (void)arg; }
+void kill_sched_resval(xce_val_t* val) { (void)val; }
+void plugin_unregister(void* arg) { (void)arg; }
+
+void setUp(void) { return; }
+void tearDown(void) { return; }
+
+static void test_job_is_ready_rejects_null(void)
+{
+    TEST_ASSERT_EQUAL_INT(0, sched_job_is_ready(NULL));
+}
+
+static void test_job_is_ready_accepts_pend(void)
+{
+    xce_sched_job_t j = { .state = JOB_STAT_PEND };
+
+    TEST_ASSERT_EQUAL_INT(1, sched_job_is_ready(&j));
+}
+
+static void test_job_is_ready_rejects_non_pend(void)
+{
+    const uint32_t states[] = { 0, JOB_STAT_RUN, JOB_STAT_DONE, JOB_STAT_EXIT };
+    size_t         i;
+
+    for (i = 0; i < sizeof(states) / sizeof(states[0]); i++) {
+        xce_sched_job_t j = { .state = states[i] };
+
+        TEST_ASSERT_EQUAL_INT_MESSAGE(0, sched_job_is_ready(&j),
+                                      "non-PEND job must not be electable");
+    }
+}
+
+static void test_job_is_ready_rejects_suspended(void)
+{
+    const uint32_t susp[] = { JOB_STAT_PSUSP, JOB_STAT_USUSP, JOB_STAT_SSUSP };
+    size_t         i;
+
+    for (i = 0; i < sizeof(susp) / sizeof(susp[0]); i++) {
+        xce_sched_job_t j = { .state = JOB_STAT_PEND | susp[i] };
+
+        TEST_ASSERT_EQUAL_INT_MESSAGE(0, sched_job_is_ready(&j),
+                                      "suspended job must not be electable");
+    }
+}
+
+static void test_queue_is_ready_rejects_null(void)
+{
+    TEST_ASSERT_EQUAL_INT(0, sched_queue_is_ready(NULL));
+}
+
+static void test_queue_is_ready_accepts_open(void)
+{
+    xce_sched_queue_t q = { .status = 0 };
+
+    TEST_ASSERT_EQUAL_INT(1, sched_queue_is_ready(&q));
+}
+
+static void test_queue_is_ready_rejects_closed_or_inactive(void)
+{
+    const uint32_t statuses[] = { QUEUE_STAT_CLOSED, QUEUE_STAT_INACT_ADMIN,
+                                  QUEUE_STAT_INACT_WINDOW };
+    size_t         i;
+
+    for (i = 0; i < sizeof(statuses) / sizeof(statuses[0]); i++) {
+        xce_sched_queue_t q = { .status = statuses[i] };
+
+        TEST_ASSERT_EQUAL_INT_MESSAGE(0, sched_queue_is_ready(&q),
+                                      "closed/inactive queue must not be electable");
+    }
+}
+
+int main(void)
+{
+    UNITY_BEGIN();
+    RUN_TEST(test_job_is_ready_rejects_null);
+    RUN_TEST(test_job_is_ready_accepts_pend);
+    RUN_TEST(test_job_is_ready_rejects_non_pend);
+    RUN_TEST(test_job_is_ready_rejects_suspended);
+    RUN_TEST(test_queue_is_ready_rejects_null);
+    RUN_TEST(test_queue_is_ready_accepts_open);
+    RUN_TEST(test_queue_is_ready_rejects_closed_or_inactive);
+    return UNITY_END();
+}