From 1a2bd3f6a8a947f2408fb51229464fb08d20e838 Mon Sep 17 00:00:00 2001 From: "lei.chen" Date: Tue, 11 Aug 2026 06:14:01 +0000 Subject: [PATCH] fix(scheduler): update existing worker in place on re-push (bug-1494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a worker is re-pushed (e.g. it rejoined with fresh resvals after the controller/worker had been down), sched_update_workers built a new worker and sched_add_worker's key_add returned EEXIST — the fresh worker was dropped and the scheduler kept the stale state (slots=0 from the dead/absent period). PEND jobs never dispatched even after the worker returned with full capacity (bug-1494, controller-restart self-heal). sched_update_workers now updates an already-known worker in place: swap the fresh resvals/resInUse/status into the existing worker (the stale arrays are freed when the discarded object is torn down) and adopt its host_eval_ctx. Also NULL-guard make_sched_worker (it can return NULL). Verified on the t1 cluster: scheduler started while the worker was dead (slots=0), worker rejoined (slots=8), previously-PEND jobs dispatched and completed, no ASAN. Co-Authored-By: Claude --- src/scheduler/src/scheduler.c | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/scheduler/src/scheduler.c b/src/scheduler/src/scheduler.c index 8ca1c5e3..4a38978d 100644 --- a/src/scheduler/src/scheduler.c +++ b/src/scheduler/src/scheduler.c @@ -740,8 +740,48 @@ static int sched_update_workers(xce_scheduler_t* sched, int nworkers, for (i = 0; i < nworkers; i++) { xce_sched_worker_t* w = make_sched_worker(sched, workers[i]); + if (w == NULL) { + continue; + } plugin_worker_update(sched, w); + + /* bug-1494 (controller-restart self-heal): a re-push of an already + * known worker (e.g. after it rejoined with fresh resvals) must + * UPDATE it in place. key_add() would EEXIST-drop the fresh worker + * and leave the stale resvals in place — the scheduler keeps the + * dead/absent state (slots=0) and PEND jobs never dispatch even after + * the worker comes back with full capacity. Swap the fresh + * resvals/status into the existing worker, then discard the new + * object (which now holds the stale arrays). */ + xce_sched_worker_t* existing = sched_get_worker(sched, w->name); + if (existing != NULL) { + zarray_t tmp; + + tmp = existing->resvals; + existing->resvals = w->resvals; + w->resvals = tmp; + + tmp = existing->resInUse; + existing->resInUse = w->resInUse; + w->resInUse = tmp; + + existing->status = w->status; + + if (existing->host_eval_ctx != NULL) { + sched_worker_release_host_ctx(existing); + } + existing->host_eval_ctx = w->host_eval_ctx; + w->host_eval_ctx = NULL; + + zarray_fini(&(w->resvals)); + zarray_fini(&(w->resInUse)); + /* w->name is an interned string (makeString) owned by the + * xstring table — never free() it (bad-free). */ + free(w); + continue; + } + sched_add_worker(sched, w); }