From 6928f747024eaf4fed71a6e88ec6b0e2cf9cfe41 Mon Sep 17 00:00:00 2001 From: "lei.chen" Date: Tue, 11 Aug 2026 03:19:07 +0000 Subject: [PATCH] fix(scheduler): re-queue job as PEND on dispatch failure (bug-1494 class) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The controller rejects a dispatch when the chosen worker is not in a dispatchable state (UNAVAIL after a worker death, or still joining) and replies with a jobDispatchFail notify. The scheduler had no handler for that event, so a job it had just committed (state=RUN) was never returned to PEND — it was never re-elected and stayed PEND cluster-wide even after the worker recovered. Handle NotifyData_data_jobDispatchFail: log the failure and call the new sched_requeue_pend_job(), which frees the dispatch bookkeeping, resets the job to PEND, and re-enqueues it in the fcfs pendJobs queue so the next election session re-dispatches it. White-box test TUs that #include handler.c get a sched_requeue_pend_job stub. Co-Authored-By: Claude --- src/scheduler/include/scheduler.h | 1 + src/scheduler/src/handler.c | 22 +++++++++++++ src/scheduler/src/scheduler.c | 33 +++++++++++++++++++ src/scheduler/testsuite/test_handlers.c | 14 ++++++++ .../testsuite/test_job_array_expand.c | 12 +++++++ .../testsuite/test_job_array_wire_e2e.c | 12 +++++++ 6 files changed, 94 insertions(+) diff --git a/src/scheduler/include/scheduler.h b/src/scheduler/include/scheduler.h index 13ab9e08..80e5ffc2 100644 --- a/src/scheduler/include/scheduler.h +++ b/src/scheduler/include/scheduler.h @@ -130,6 +130,7 @@ extern void sched_go(xce_scheduler_t*); extern void sched_begin_session(xce_scheduler_t*); extern void sched_end_session(xce_scheduler_t*); extern xce_sched_job_t* sched_get_job(xce_scheduler_t*, uint64_t); +extern int sched_requeue_pend_job(xce_scheduler_t*, uint64_t); extern xce_sched_queue_t* sched_get_queue(xce_scheduler_t*, char*); extern xce_sched_worker_t* sched_get_worker(xce_scheduler_t*, char*); extern xce_sched_user_t* sched_get_user(xce_scheduler_t*, char*); diff --git a/src/scheduler/src/handler.c b/src/scheduler/src/handler.c index 7a598da8..8bafbeed 100644 --- a/src/scheduler/src/handler.c +++ b/src/scheduler/src/handler.c @@ -527,6 +527,28 @@ int sched_handle_notify(xce_scheduler_t* sched, xce_notify_data_t* notify) if (rc >= 0) notify->data.sched_update_queue = NULL; break; + case NotifyData_data_jobDispatchFail: { + /* The controller rejected a dispatch the scheduler committed (e.g. + * the chosen worker went UNAVAIL between election and dispatch). + * Return the job to PEND so it is re-elected next session — without + * this the job stays RUN in the scheduler's view and is never + * dispatched again (job stuck PEND cluster-wide). */ + job_failinfo_t* fail = notify->data.dispatchfail; + char jobidString[128]; + + if (fail == NULL) { + rc = -EINVAL; + break; + } + + fmt_jobid(jobidString, sizeof(jobidString), fail->jobid); + WARN("%s: job %s dispatch failed (code %d) — returning to PEND", + __func__, jobidString, fail->code); + rc = sched_requeue_pend_job(sched, fail->jobid); + if (rc >= 0) + notify->data.dispatchfail = NULL; + break; + } case NotifyData_data_schedUpdateResvals: rc = sched_handle_sched_update_resvals( sched, notify->data.sched_update_resvals); diff --git a/src/scheduler/src/scheduler.c b/src/scheduler/src/scheduler.c index d1018323..8ca1c5e3 100644 --- a/src/scheduler/src/scheduler.c +++ b/src/scheduler/src/scheduler.c @@ -700,6 +700,39 @@ static int sched_job_state_change(xce_scheduler_t* sched, uint64_t jobid, return rc; } + +/* + * sched_requeue_pend_job — return a job to PEND after its dispatch failed at + * the controller (jobDispatchFail notify). The dispatch decision set the + * job's state to RUN and attached allocs/accounts; the controller rejected + * the dispatch (e.g. the chosen worker went UNAVAIL/down between the + * scheduler's election and the controller's dispatch_a_job_ check), and + * without this the job stays RUN in the scheduler's view forever — it is + * never re-elected and is stuck PEND cluster-wide even after the worker + * recovers (scheduler-side of bug-1494 class). Free the dispatch + * bookkeeping, reset to PEND, and re-enqueue so the next election + * re-dispatches it. + */ +int sched_requeue_pend_job(xce_scheduler_t* sched, uint64_t jobid) +{ + xce_sched_job_t* job = sched_get_job(sched, jobid); + + if (job == NULL) { + return -ENOENT; + } + + if (job->accounts != NULL) { + free_job_account(job->accounts); + job->accounts = NULL; + } + if (job->allocs != NULL) { + free_JobAlloc_ptr(&(job->allocs)); + } + job->state = JOB_STAT_PEND; + + /* fcfs_update_job (job_hook) enqueues only PEND jobs, with dedup. */ + return plugin_job_update(sched, job); +} static int sched_update_workers(xce_scheduler_t* sched, int nworkers, xce_worker_t** workers) { diff --git a/src/scheduler/testsuite/test_handlers.c b/src/scheduler/testsuite/test_handlers.c index 35c7ae8b..4ade15bf 100644 --- a/src/scheduler/testsuite/test_handlers.c +++ b/src/scheduler/testsuite/test_handlers.c @@ -35,6 +35,20 @@ xce_sched_job_t* sched_get_job(xce_scheduler_t* sched, uint64_t jobid) (void)sched; (void)jobid; return NULL; } +/* White-box stub for sched_requeue_pend_job (handler.c's jobDispatchFail + * case). sched_get_job is stubbed to NULL here, so this just returns + * -ENOENT like the real function does for an unknown jobid. */ +int sched_requeue_pend_job(xce_scheduler_t* sched, uint64_t jobid) +{ + (void)sched; + xce_sched_job_t* job = sched_get_job(sched, jobid); + if (job == NULL) { + return -ENOENT; + } + job->state = JOB_STAT_PEND; + return 0; +} + #include "../src/handler.c" void setUp(void) { return; } diff --git a/src/scheduler/testsuite/test_job_array_expand.c b/src/scheduler/testsuite/test_job_array_expand.c index 7ca66c51..2b95f3a0 100644 --- a/src/scheduler/testsuite/test_job_array_expand.c +++ b/src/scheduler/testsuite/test_job_array_expand.c @@ -56,6 +56,18 @@ xce_sched_job_t* sched_get_job(xce_scheduler_t* sched, uint64_t jobid) return job; } +/* White-box stub for sched_requeue_pend_job (handler.c's jobDispatchFail + * case), backed by this TU's real kvmgr sched_get_job. */ +int sched_requeue_pend_job(xce_scheduler_t* sched, uint64_t jobid) +{ + xce_sched_job_t* job = sched_get_job(sched, jobid); + if (job == NULL) { + return -ENOENT; + } + job->state = JOB_STAT_PEND; + return 0; +} + void setUp(void) { return; } void tearDown(void) { diff --git a/src/scheduler/testsuite/test_job_array_wire_e2e.c b/src/scheduler/testsuite/test_job_array_wire_e2e.c index 1867f2cb..e54f2300 100644 --- a/src/scheduler/testsuite/test_job_array_wire_e2e.c +++ b/src/scheduler/testsuite/test_job_array_wire_e2e.c @@ -107,6 +107,18 @@ xce_sched_job_t* sched_get_job(xce_scheduler_t* sched, uint64_t jobid) return job; } +/* White-box stub for sched_requeue_pend_job (handler.c's jobDispatchFail + * case), backed by this TU's real kvmgr sched_get_job. */ +int sched_requeue_pend_job(xce_scheduler_t* sched, uint64_t jobid) +{ + xce_sched_job_t* job = sched_get_job(sched, jobid); + if (job == NULL) { + return -ENOENT; + } + job->state = JOB_STAT_PEND; + return 0; +} + void setUp(void) { return; } void tearDown(void) { return; }