From c091a8dae046e2b6add9d35367e4709fa013d54e Mon Sep 17 00:00:00 2001 From: "lei.chen" Date: Wed, 5 Aug 2026 03:40:09 +0000 Subject: [PATCH] fix(controller,worker): sched worker resval UAF + execuser fallback to nobody MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bug-1506 (controller): sched_worker_change pointed w->host.resvals_to_sched at resvalist.data, then xlist_fini freed it — a concurrent reader (the WORKER_JOIN hook encoding the worker's resvals) hit an ASAN heap-use-after-free that crashed the controller and failed every job. Take ownership of the resval data via MOVE_PTR (mirrors collect_resvals_workers); the prior array is intentionally leaked (freeing it races the same readers). bug-1507 (worker): the apiserver's bug-1500 execuser fix sent the submit user ("admin") as execuser, but that user does not exist in the worker host's /etc/passwd. systemd failed the worker unit with EXIT_USER (217), so every job died immediately after submit. job_start_local_task now checks the execuser with getpwnam() and falls back to "nobody" when the user does not exist. Verified with a full job lifecycle (submit -> RUN as nobody -> DONE) on the clean build: all 4 daemons healthy, zero ASAN. Co-Authored-By: Claude --- .wolf/buglog.json | 59 +++++++++++++++++++++++++++ .wolf/memory.md | 2 + src/controller/sched.c | 19 ++++++++- src/worker/src/controller_connector.c | 18 +++++++- 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index ec54f4b..c4c9fb2 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -14816,5 +14816,64 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-08-05" + }, + { + "id": "bug-1505", + "timestamp": "2026-08-05", + "error_message": "build failed: controller_connector.c:186:12: error: 'xcecd_ctx' undeclared (first use in this function); did you mean 'xcecli_ctx'?", + "file": "src/worker/src/controller_connector.c", + "root_cause": "checking_connection() was changed to loop indefinitely on xcecd_ctx.running && cc_ctx->running, but xcecd_ctx is a global owned by xcecd.c (xcecd_context_t xcecd_ctx) and the extern declaration lives in include/xcecd.h, which controller_connector.c did not include -> undeclared identifier.", + "fix": "added #include \"xcecd.h\" to controller_connector.c (guarded header already declares extern xcecd_context_t xcecd_ctx;). Rebuilt worker/xcecd successfully and deployed.", + "tags": [ + "worker", + "build-failure", + "missing-include", + "controller-connector", + "xcecd_ctx", + "deploy" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-05" + }, + { + "id": "bug-1506", + "timestamp": "2026-08-05", + "error_message": "controller ASAN heap-use-after-free: READ of size 8 in encode_NameValue_list (T6 join thread) on memory freed by xlist_fini in sched_worker_change (T2 hook thread). Crashes the controller, causing all jobs to fail.", + "file": "src/controller/sched.c (sched_worker_change)", + "root_cause": "sched_worker_change pointed w->host.resvals_to_sched at resvalist.data, then xlist_fini(&resvalist) freed that array \u2014 leaving the worker's resvals_to_sched dangling. A concurrent reader (the WORKER_JOIN hook encoding the worker's resvals via encode_NameValue_list) read the freed memory -> UAF. The old #if 0 free_name_val_list block was disabled for the same race.", + "fix": "Take ownership of resvalist.data via MOVE_PTR before xlist_fini (mirrors collect_resvals_workers). The prior resvals_to_sched is intentionally leaked (freeing it would race the same readers). Added #include \"utils.h\".", + "tags": [ + "asan", + "heap-use-after-free", + "controller", + "sched", + "resvals", + "race" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-05" + }, + { + "id": "bug-1507", + "timestamp": "2026-08-05", + "error_message": "\u4f5c\u4e1a\u63d0\u4ea4\u540e\u7acb\u5373 EXIT: systemd \u5355\u5143 xcecd-worker-.service \u5931\u8d25, status=217/USER (\u670d\u52a1\u7528\u6237\u4e0d\u5b58\u5728). \u7528\u6237 1-0-1785897786-0 \u7b49\u5168\u90e8\u5931\u8d25.", + "file": "src/worker/src/controller_connector.c (job_start_local_task)", + "root_cause": "apiserver bug-1500 \u4fee\u590d\u8bbe\u7f6e si.execuser=resolveExecUser(job)=\"admin\" (\u63d0\u4ea4\u7528\u6237). \u4f46 admin \u5728 worker \u4e3b\u673a /etc/passwd \u4e0d\u5b58\u5728. worker \u628a limits->user_name \u8bbe\u4e3a execuser, start_systemd_job \u751f\u6210 User=admin \u5355\u5143 -> systemd EXIT_USER 217 -> \u4f5c\u4e1a\u5931\u8d25. \u4e4b\u524d execuser \u4e3a\u7a7a\u65f6\u9ed8\u8ba4 nobody(\u5b58\u5728) \u80fd\u8dd1, bug-1500 \u90e8\u7f72\u540e\u53d8\u6210 admin \u53cd\u800c\u5168\u5931\u8d25.", + "fix": "job_start_local_task \u7528 getpwnam() \u68c0\u67e5 execuser \u662f\u5426\u5b58\u5728; \u4e0d\u5b58\u5728\u5219\u56de\u9000 nobody. \u4f5c\u4e1a\u4ee5 nobody \u8fd0\u884c(\u7528\u6237\u5df2\u786e\u8ba4\u73b0\u9636\u6bb5\u53ef\u4ee5).", + "tags": [ + "worker", + "execuser", + "systemd", + "217", + "nobody", + "regression" + ], + "related_bugs": [ + "bug-1500" + ], + "occurrences": 1, + "last_seen": "2026-08-05" } ] \ No newline at end of file diff --git a/.wolf/memory.md b/.wolf/memory.md index 5867910..96cbb75 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -7371,3 +7371,5 @@ | 09:39 | 修复 controller 反复崩溃 bug-1502: persist_job_hook_fun 把 RUSAGE_UPDATE hook 的第2参 (xce_job_rusages_t*) 当 job_t* 用, 读 offset 56 越界 56字节结构 -> ASAN heap-buffer-overflow. 加专用 case 从 jobmgr 取当前 state | src/controller/persist.c, testsuite/test_persist_hooks.c | RED(旧代码 stack-buffer-overflow)/GREEN(修复后 PASS), 全套458测试通过 | ~350 | | 01:08 | 验证完成: bug-1502(controller ASAN崩溃)修复生产生效(sleep 666跑满666秒, controller 0崩溃); 作业PENDING根因=配置正确但scheduler先于worker join拿不到slots(操作时序, bug-1503); 干净controller部署, 集群全链路(apiserver提交->scheduler->controller->worker)冒烟通过 | persist.c, sched.c, worker.c | 验证通过 | ~300 | | 01:42 | 修复 bug-1504: xcecli jobs list -a 返回空, 因 FilterRows 把 job_status notnull 路由到 MatchesBitmaskCondition(仅实现OpEQ/OpNE) -> 全过滤. 加 OpISNULL/OpNOTNULL 分支 + 3回归测试, 部署后 -a 正常显示全部作业 | src/frontend/apiserver/store/query.go + query_test.go | RED->GREEN, 生产验证 -a 列出全部 | ~250 | +| 02:40 | 增量重建并部署 xcecd (ASAN): bug-1505修复(worker join无限重试, checking_connection 不再5次放弃+删handle_disconnect). 首次编译失败: controller_connector.c 引用 xcecd_ctx 未包含 xcecd.h -> 加 #include "xcecd.h" 修复. ninja worker/xcecd 成功, mtime 2026-08-05 02:28:50, md5 36b166545caf266e30e194e626494c6d 源/部署一致 | src/worker/src/controller_connector.c, xcecd.h | 部署完成(未重启守护进程) | ~200 | +| 03:39 | 完整验证通过: 干净 controller(bug-1506 UAF修复)+worker(bug-1507 execuser回退nobody) 下 final-verify sleep 12 完整跑完=DONE, 4 daemon健康, ASAN 0. bug-1506(sched_worker_change MOVE_PTR所有权)+bug-1507(getpwnam回退nobody) 修复, 按新准则验证后提交 | sched.c, controller_connector.c, buglog | 验证通过, 准备提交 | ~200 | diff --git a/src/controller/sched.c b/src/controller/sched.c index 7458b5d..13f50ac 100644 --- a/src/controller/sched.c +++ b/src/controller/sched.c @@ -4,6 +4,7 @@ #include "cluster.h" #include "controller.capnp.h" #include "request.h" +#include "utils.h" /* MOVE_PTR (bug-1506) */ #include "worker.h" #include "xce/cluster_def.h" #include "xce/types.h" @@ -500,8 +501,22 @@ static int sched_worker_change(int type, int situation, void* data, xlist_init(&resvalist, 256); collect_resval(c->resmgr, &(w->host.resvals), &resvalist); - w->host.nresvals_to_sched = resvalist.count; - w->host.resvals_to_sched = (xce_name_val_t**)resvalist.data; + { + /* bug-1506: take OWNERSHIP of resvalist.data so the xlist_fini + * below does not free the array w->host.resvals_to_sched aliases. + * The old code pointed resvals_to_sched at resvalist.data, then + * xlist_fini(&resvalist) freed it — a concurrent reader (the + * WORKER_JOIN hook encoding the worker's resvals, via + * encode_NameValue_list) read the dangling pointer → ASAN + * heap-use-after-free that crashed the controller. The prior + * resvals_to_sched (if any) is intentionally left allocated: freeing + * it here would race the same readers (the old #if 0 + * free_name_val_list block was disabled for this reason). */ + void* p = NULL; + MOVE_PTR(p, resvalist.data); + w->host.nresvals_to_sched = resvalist.count; + w->host.resvals_to_sched = (xce_name_val_t**)p; + } mylock_lock(&(c->sched.lock)); if (c->sched.ep != NULL) { diff --git a/src/worker/src/controller_connector.c b/src/worker/src/controller_connector.c index 96a606b..028751c 100755 --- a/src/worker/src/controller_connector.c +++ b/src/worker/src/controller_connector.c @@ -8,6 +8,7 @@ #include "xcelog.h" #include #include +#include #include #include #include @@ -799,7 +800,22 @@ static int job_start_local_task(xce_request_t* request) } if (job_ctx->startinfo->execuser != NULL) { - job_ctx->limits->user_name = strdup(job_ctx->startinfo->execuser); + /* bug-1507: only use the exec user if it actually exists on this + * host. The API-layer submit user (e.g. "admin") is not + * necessarily a real /etc/passwd account, and systemd then fails + * the worker unit with EXIT_USER (217), killing every job. Fall + * back to nobody so the job still runs. */ + struct passwd* pw = getpwnam(job_ctx->startinfo->execuser); + if (pw != NULL) { + job_ctx->limits->user_name + = strdup(job_ctx->startinfo->execuser); + } else { + job_ctx->limits->user_name = strdup("nobody"); + XCECD_LOG(XCE_LOG_WARN, + "exec user '%s' does not exist on this host, " + "running job as nobody", + job_ctx->startinfo->execuser); + } } else { job_ctx->limits->user_name = strdup("nobody"); XCECD_LOG(XCE_LOG_WARN,