forked from lei.chen/myxce
fix(controller,worker): sched worker resval UAF + execuser fallback to nobody
bug-1506 (controller): sched_worker_change pointed w->host.resvals_to_sched
at resvalist.data, then xlist_fini freed it — a concurrent reader (the
WORKER_JOIN hook encoding the worker's resvals) hit an ASAN
heap-use-after-free that crashed the controller and failed every job. Take
ownership of the resval data via MOVE_PTR (mirrors collect_resvals_workers);
the prior array is intentionally leaked (freeing it races the same readers).
bug-1507 (worker): the apiserver's bug-1500 execuser fix sent the submit
user ("admin") as execuser, but that user does not exist in the worker
host's /etc/passwd. systemd failed the worker unit with EXIT_USER (217),
so every job died immediately after submit. job_start_local_task now
checks the execuser with getpwnam() and falls back to "nobody" when the
user does not exist.
Verified with a full job lifecycle (submit -> RUN as nobody -> DONE) on
the clean build: all 4 daemons healthy, zero ASAN.
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen
parent
c630809116
commit
c091a8dae0
4 changed files with 95 additions and 3 deletions
|
|
@ -14816,5 +14816,64 @@
|
|||
"related_bugs": [],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-08-05"
|
||||
},
|
||||
{
|
||||
"id": "bug-1505",
|
||||
"timestamp": "2026-08-05",
|
||||
"error_message": "build failed: controller_connector.c:186:12: error: 'xcecd_ctx' undeclared (first use in this function); did you mean 'xcecli_ctx'?",
|
||||
"file": "src/worker/src/controller_connector.c",
|
||||
"root_cause": "checking_connection() was changed to loop indefinitely on xcecd_ctx.running && cc_ctx->running, but xcecd_ctx is a global owned by xcecd.c (xcecd_context_t xcecd_ctx) and the extern declaration lives in include/xcecd.h, which controller_connector.c did not include -> undeclared identifier.",
|
||||
"fix": "added #include \"xcecd.h\" to controller_connector.c (guarded header already declares extern xcecd_context_t xcecd_ctx;). Rebuilt worker/xcecd successfully and deployed.",
|
||||
"tags": [
|
||||
"worker",
|
||||
"build-failure",
|
||||
"missing-include",
|
||||
"controller-connector",
|
||||
"xcecd_ctx",
|
||||
"deploy"
|
||||
],
|
||||
"related_bugs": [],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-08-05"
|
||||
},
|
||||
{
|
||||
"id": "bug-1506",
|
||||
"timestamp": "2026-08-05",
|
||||
"error_message": "controller ASAN heap-use-after-free: READ of size 8 in encode_NameValue_list (T6 join thread) on memory freed by xlist_fini in sched_worker_change (T2 hook thread). Crashes the controller, causing all jobs to fail.",
|
||||
"file": "src/controller/sched.c (sched_worker_change)",
|
||||
"root_cause": "sched_worker_change pointed w->host.resvals_to_sched at resvalist.data, then xlist_fini(&resvalist) freed that array \u2014 leaving the worker's resvals_to_sched dangling. A concurrent reader (the WORKER_JOIN hook encoding the worker's resvals via encode_NameValue_list) read the freed memory -> UAF. The old #if 0 free_name_val_list block was disabled for the same race.",
|
||||
"fix": "Take ownership of resvalist.data via MOVE_PTR before xlist_fini (mirrors collect_resvals_workers). The prior resvals_to_sched is intentionally leaked (freeing it would race the same readers). Added #include \"utils.h\".",
|
||||
"tags": [
|
||||
"asan",
|
||||
"heap-use-after-free",
|
||||
"controller",
|
||||
"sched",
|
||||
"resvals",
|
||||
"race"
|
||||
],
|
||||
"related_bugs": [],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-08-05"
|
||||
},
|
||||
{
|
||||
"id": "bug-1507",
|
||||
"timestamp": "2026-08-05",
|
||||
"error_message": "\u4f5c\u4e1a\u63d0\u4ea4\u540e\u7acb\u5373 EXIT: systemd \u5355\u5143 xcecd-worker-<jobid>.service \u5931\u8d25, status=217/USER (\u670d\u52a1\u7528\u6237\u4e0d\u5b58\u5728). \u7528\u6237 1-0-1785897786-0 \u7b49\u5168\u90e8\u5931\u8d25.",
|
||||
"file": "src/worker/src/controller_connector.c (job_start_local_task)",
|
||||
"root_cause": "apiserver bug-1500 \u4fee\u590d\u8bbe\u7f6e si.execuser=resolveExecUser(job)=\"admin\" (\u63d0\u4ea4\u7528\u6237). \u4f46 admin \u5728 worker \u4e3b\u673a /etc/passwd \u4e0d\u5b58\u5728. worker \u628a limits->user_name \u8bbe\u4e3a execuser, start_systemd_job \u751f\u6210 User=admin \u5355\u5143 -> systemd EXIT_USER 217 -> \u4f5c\u4e1a\u5931\u8d25. \u4e4b\u524d execuser \u4e3a\u7a7a\u65f6\u9ed8\u8ba4 nobody(\u5b58\u5728) \u80fd\u8dd1, bug-1500 \u90e8\u7f72\u540e\u53d8\u6210 admin \u53cd\u800c\u5168\u5931\u8d25.",
|
||||
"fix": "job_start_local_task \u7528 getpwnam() \u68c0\u67e5 execuser \u662f\u5426\u5b58\u5728; \u4e0d\u5b58\u5728\u5219\u56de\u9000 nobody. \u4f5c\u4e1a\u4ee5 nobody \u8fd0\u884c(\u7528\u6237\u5df2\u786e\u8ba4\u73b0\u9636\u6bb5\u53ef\u4ee5).",
|
||||
"tags": [
|
||||
"worker",
|
||||
"execuser",
|
||||
"systemd",
|
||||
"217",
|
||||
"nobody",
|
||||
"regression"
|
||||
],
|
||||
"related_bugs": [
|
||||
"bug-1500"
|
||||
],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-08-05"
|
||||
}
|
||||
]
|
||||
|
|
@ -7371,3 +7371,5 @@
|
|||
| 09:39 | 修复 controller 反复崩溃 bug-1502: persist_job_hook_fun 把 RUSAGE_UPDATE hook 的第2参 (xce_job_rusages_t*) 当 job_t* 用, 读 offset 56 越界 56字节结构 -> ASAN heap-buffer-overflow. 加专用 case 从 jobmgr 取当前 state | src/controller/persist.c, testsuite/test_persist_hooks.c | RED(旧代码 stack-buffer-overflow)/GREEN(修复后 PASS), 全套458测试通过 | ~350 |
|
||||
| 01:08 | 验证完成: bug-1502(controller ASAN崩溃)修复生产生效(sleep 666跑满666秒, controller 0崩溃); 作业PENDING根因=配置正确但scheduler先于worker join拿不到slots(操作时序, bug-1503); 干净controller部署, 集群全链路(apiserver提交->scheduler->controller->worker)冒烟通过 | persist.c, sched.c, worker.c | 验证通过 | ~300 |
|
||||
| 01:42 | 修复 bug-1504: xcecli jobs list -a 返回空, 因 FilterRows 把 job_status notnull 路由到 MatchesBitmaskCondition(仅实现OpEQ/OpNE) -> 全过滤. 加 OpISNULL/OpNOTNULL 分支 + 3回归测试, 部署后 -a 正常显示全部作业 | src/frontend/apiserver/store/query.go + query_test.go | RED->GREEN, 生产验证 -a 列出全部 | ~250 |
|
||||
| 02:40 | 增量重建并部署 xcecd (ASAN): bug-1505修复(worker join无限重试, checking_connection 不再5次放弃+删handle_disconnect). 首次编译失败: controller_connector.c 引用 xcecd_ctx 未包含 xcecd.h -> 加 #include "xcecd.h" 修复. ninja worker/xcecd 成功, mtime 2026-08-05 02:28:50, md5 36b166545caf266e30e194e626494c6d 源/部署一致 | src/worker/src/controller_connector.c, xcecd.h | 部署完成(未重启守护进程) | ~200 |
|
||||
| 03:39 | 完整验证通过: 干净 controller(bug-1506 UAF修复)+worker(bug-1507 execuser回退nobody) 下 final-verify sleep 12 完整跑完=DONE, 4 daemon健康, ASAN 0. bug-1506(sched_worker_change MOVE_PTR所有权)+bug-1507(getpwnam回退nobody) 修复, 按新准则验证后提交 | sched.c, controller_connector.c, buglog | 验证通过, 准备提交 | ~200 |
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@
|
|||
#include "cluster.h"
|
||||
#include "controller.capnp.h"
|
||||
#include "request.h"
|
||||
#include "utils.h" /* MOVE_PTR (bug-1506) */
|
||||
#include "worker.h"
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xce/types.h"
|
||||
|
|
@ -500,8 +501,22 @@ static int sched_worker_change(int type, int situation, void* data,
|
|||
|
||||
xlist_init(&resvalist, 256);
|
||||
collect_resval(c->resmgr, &(w->host.resvals), &resvalist);
|
||||
{
|
||||
/* bug-1506: take OWNERSHIP of resvalist.data so the xlist_fini
|
||||
* below does not free the array w->host.resvals_to_sched aliases.
|
||||
* The old code pointed resvals_to_sched at resvalist.data, then
|
||||
* xlist_fini(&resvalist) freed it — a concurrent reader (the
|
||||
* WORKER_JOIN hook encoding the worker's resvals, via
|
||||
* encode_NameValue_list) read the dangling pointer → ASAN
|
||||
* heap-use-after-free that crashed the controller. The prior
|
||||
* resvals_to_sched (if any) is intentionally left allocated: freeing
|
||||
* it here would race the same readers (the old #if 0
|
||||
* free_name_val_list block was disabled for this reason). */
|
||||
void* p = NULL;
|
||||
MOVE_PTR(p, resvalist.data);
|
||||
w->host.nresvals_to_sched = resvalist.count;
|
||||
w->host.resvals_to_sched = (xce_name_val_t**)resvalist.data;
|
||||
w->host.resvals_to_sched = (xce_name_val_t**)p;
|
||||
}
|
||||
|
||||
mylock_lock(&(c->sched.lock));
|
||||
if (c->sched.ep != NULL) {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@
|
|||
#include "xcelog.h"
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
#include <pwd.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
|
@ -799,7 +800,22 @@ static int job_start_local_task(xce_request_t* request)
|
|||
}
|
||||
|
||||
if (job_ctx->startinfo->execuser != NULL) {
|
||||
job_ctx->limits->user_name = strdup(job_ctx->startinfo->execuser);
|
||||
/* bug-1507: only use the exec user if it actually exists on this
|
||||
* host. The API-layer submit user (e.g. "admin") is not
|
||||
* necessarily a real /etc/passwd account, and systemd then fails
|
||||
* the worker unit with EXIT_USER (217), killing every job. Fall
|
||||
* back to nobody so the job still runs. */
|
||||
struct passwd* pw = getpwnam(job_ctx->startinfo->execuser);
|
||||
if (pw != NULL) {
|
||||
job_ctx->limits->user_name
|
||||
= strdup(job_ctx->startinfo->execuser);
|
||||
} else {
|
||||
job_ctx->limits->user_name = strdup("nobody");
|
||||
XCECD_LOG(XCE_LOG_WARN,
|
||||
"exec user '%s' does not exist on this host, "
|
||||
"running job as nobody",
|
||||
job_ctx->startinfo->execuser);
|
||||
}
|
||||
} else {
|
||||
job_ctx->limits->user_name = strdup("nobody");
|
||||
XCECD_LOG(XCE_LOG_WARN,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue