From 2fa9bedcd3095cb5cab2838134bb438a7a23d351 Mon Sep 17 00:00:00 2001 From: "lei.chen" Date: Tue, 4 Aug 2026 09:39:57 +0000 Subject: [PATCH] fix(controller): persist_job_hook_fun OOB read on RUSAGE_UPDATE hook (bug-1502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The RUSAGE_UPDATE job hook fires with (cluster_t*, xce_job_rusages_t*) as its varargs — the apiserver and scheduler subscribers parse exactly that contract — but persist_job_hook_fun's default case treated the second arg as a job_t* and read j->job.state at offset 56. A xce_job_rusages_t is 56 bytes and shares only jobid@0 with xce_job_t (which is why the jobid read coincidentally worked), so the state read landed one byte past the struct: ASAN heap-buffer-overflow (READ of size 4) in thread T7 whenever a job runs and the worker pushes rusage updates. This was the recurring controller crash (term 3/4/5) blocking job-PENDING verification. Add a dedicated HOOK_DETAIL_JOB_RUSAGE_UPDATE case that reads jobid from the xce_job_rusages_t* and bakes the job's current state from a jobmgr key_find on the vararg cluster_t* (race-free: the sole caller update_job_rusages() holds that job's write lock while firing the AFTER_EVENT hook; re-locking the job inside the hook would deadlock). Regression test: test_job_hook_rusage_writes_wal_entry now passes a real 56-byte xce_job_rusages_t instead of a full job_t (which masked the overflow) — ASAN stack-buffer-overflow under the old code, PASS under the fix. Full controller unit suite: 458 tests, 0 failures. Co-Authored-By: Claude --- .wolf/buglog.json | 20 +++++++++++++ .wolf/memory.md | 1 + src/controller/persist.c | 29 ++++++++++++++++++- src/controller/testsuite/test_persist_hooks.c | 13 ++++++++- 4 files changed, 61 insertions(+), 2 deletions(-) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 63d9a03..75b2d8b 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -14756,5 +14756,25 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-08-04" + }, + { + "id": "bug-1502", + "timestamp": "2026-08-04", + "error_message": "controller ASAN heap-buffer-overflow (READ of size 4, thread T7): 56-byte xce_job_rusages_t buffer from decode_JobRusages_ptr, read at offset 56 (1 byte past end). Recurring crash that kills the controller whenever a job runs and the worker pushes rusage updates (term=3/4/5 all crashed).", + "file": "src/controller/persist.c (persist_job_hook_fun)", + "root_cause": "RUSAGE_UPDATE job hook arg mismatch: update_job_rusages() fires HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) as its 2nd va_arg (apiserver/sched subscribers parse this exact contract), but persist_job_hook_fun default case treated it as job_t*. xce_job_rusages_t is 56 bytes and has jobid@0 (coincidentally correct) but NO state field; j->job.state lives at offset 56 of xce_job_t, so the read went 4 bytes past the rusage struct -> heap-buffer-overflow. The unit test masked it by passing a full job_t.", + "fix": "Added case HOOK_DETAIL_JOB_RUSAGE_UPDATE in persist_job_hook_fun: read jobid from xce_job_rusages_t*, bake the job current state via a jobmgr key_find on the vararg cluster_t* (safe: sole caller update_job_rusages holds that job write lock while firing). Regression test passes a real 56-byte xce_job_rusages_t (stack-buffer-overflow under old code, PASS under fix).", + "tags": [ + "asan", + "heap-buffer-overflow", + "controller", + "persist", + "hook", + "job-rusage", + "crash" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-08-04" } ] \ No newline at end of file diff --git a/.wolf/memory.md b/.wolf/memory.md index 87d491b..62261e6 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -7368,3 +7368,4 @@ | 05:49 | Created ../../../../tmp/frpc-watchdog.sh | — | ~338 | | 13:50 | 看门狗改为纯日志模式(不自动重启): 移到 /home/dankeer/bin/frpc-watchdog.sh + crontab @reboot 开机自启; 只在隧道状态变化时写日志(DOWN抓快照/恢复记录/持续DOWN每5min) | /home/dankeer/bin/frpc-watchdog.sh | 部署完成, 用户人工监视 | ~200 | | 05:50 | Session end: 9 writes across 5 files (frpcdbg_mon.sh, frpc_prod.toml, frpc_mon.sh, frpc_watchdog.sh, frpc-watchdog.sh) | 2 reads | ~2219 tok | +| 09:39 | 修复 controller 反复崩溃 bug-1502: persist_job_hook_fun 把 RUSAGE_UPDATE hook 的第2参 (xce_job_rusages_t*) 当 job_t* 用, 读 offset 56 越界 56字节结构 -> ASAN heap-buffer-overflow. 加专用 case 从 jobmgr 取当前 state | src/controller/persist.c, testsuite/test_persist_hooks.c | RED(旧代码 stack-buffer-overflow)/GREEN(修复后 PASS), 全套458测试通过 | ~350 | diff --git a/src/controller/persist.c b/src/controller/persist.c index 112e1fb..d3e85de 100644 --- a/src/controller/persist.c +++ b/src/controller/persist.c @@ -6573,8 +6573,35 @@ int persist_job_hook_fun(int type, int situation, int detail, js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; break; } + case HOOK_DETAIL_JOB_RUSAGE_UPDATE: { + /* args: cluster_t*, xce_job_rusages_t* — NOT job_t* (bug-1502). + * The RUSAGE_UPDATE hook carries the rusage struct, which has no job + * state (apiserver_job_rusage_update_fun / sched_job_rusage_update_fun + * parse this exact contract). The old default-case path treated the + * 2nd arg as a job_t* and read j->job.state at offset 56 — one byte + * past a 56-byte xce_job_rusages_t → ASAN heap-buffer-overflow (the + * only field the two structs share is jobid@0, so that read + * coincidentally worked). Bake the job's CURRENT state from the + * jobmgr: the sole caller update_job_rusages() fires this AFTER_EVENT + * while holding the job's write lock, so the pointer returned here IS + * that same write-locked object and reading its state is race-free. + * Never re-lock the job inside a hook — the caller already holds it, + * and a same-thread re-lock would deadlock. */ + cluster_t* c = va_arg(ap, cluster_t*); + xce_job_rusages_t* r = va_arg(ap, xce_job_rusages_t*); + js.jobid = r->jobid; + if (c != NULL && c->jobmgr != NULL) { + job_t* jj = NULL; + lock(c->jobmgr, RTTI_LOCK_READ); + key_find(c->jobmgr, (void*)r->jobid, (void**)&jj); + unlock(c->jobmgr); + if (jj != NULL) + js.state = (uint32_t)jj->job.state; + } + break; + } default: { - /* STOP, RESUME, UPDATE, CKPT, PREASON, RUSAGE — args: cluster_t*, job_t* */ + /* STOP, RESUME, UPDATE, CKPT, PREASON — args: cluster_t*, job_t* */ (void)va_arg(ap, cluster_t*); job_t* j = va_arg(ap, job_t*); js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state; diff --git a/src/controller/testsuite/test_persist_hooks.c b/src/controller/testsuite/test_persist_hooks.c index dad9bf5..2291ed9 100644 --- a/src/controller/testsuite/test_persist_hooks.c +++ b/src/controller/testsuite/test_persist_hooks.c @@ -14,6 +14,7 @@ #include "persist.h" #include "xce/cluster_def.h" #include "xlist.h" +#include "controller.capnp.h" /* xce_job_rusages_t (bug-1502 regression test) */ /* ------------------------------------------------------------------ */ /* Test helpers */ @@ -276,9 +277,19 @@ static void test_job_hook_rusage_writes_wal_entry(void) uint64_t lsn_before = p->current_lsn; + /* bug-1502 regression: production fires RUSAGE_UPDATE with a real + * xce_job_rusages_t* (56 bytes, NO job-state field) as the second hook + * arg — NOT a job_t*. The old persist_job_hook_fun default case read + * j->job.state at offset 56, one byte past this struct → ASAN + * heap-buffer-overflow. Passing the real minimal struct here (instead of + * the old &g_stub_job, a full job_t that masked the overflow) reproduces + * the production shape exactly. */ + xce_job_rusages_t rusages = {0}; + rusages.jobid = 0x1234567890ABCDEFULL; + int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT, HOOK_DETAIL_JOB_RUSAGE_UPDATE, p, - (cluster_t*)NULL, &g_stub_job); + (cluster_t*)NULL, &rusages); TEST_ASSERT_EQUAL_INT(0, rc); TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);