forked from lei.chen/myxce
fix(controller): persist_job_hook_fun OOB read on RUSAGE_UPDATE hook (bug-1502)
The RUSAGE_UPDATE job hook fires with (cluster_t*, xce_job_rusages_t*) as its varargs — the apiserver and scheduler subscribers parse exactly that contract — but persist_job_hook_fun's default case treated the second arg as a job_t* and read j->job.state at offset 56. A xce_job_rusages_t is 56 bytes and shares only jobid@0 with xce_job_t (which is why the jobid read coincidentally worked), so the state read landed one byte past the struct: ASAN heap-buffer-overflow (READ of size 4) in thread T7 whenever a job runs and the worker pushes rusage updates. This was the recurring controller crash (term 3/4/5) blocking job-PENDING verification. Add a dedicated HOOK_DETAIL_JOB_RUSAGE_UPDATE case that reads jobid from the xce_job_rusages_t* and bakes the job's current state from a jobmgr key_find on the vararg cluster_t* (race-free: the sole caller update_job_rusages() holds that job's write lock while firing the AFTER_EVENT hook; re-locking the job inside the hook would deadlock). Regression test: test_job_hook_rusage_writes_wal_entry now passes a real 56-byte xce_job_rusages_t instead of a full job_t (which masked the overflow) — ASAN stack-buffer-overflow under the old code, PASS under the fix. Full controller unit suite: 458 tests, 0 failures. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
lei.chen
parent
ff2d784ca9
commit
2fa9bedcd3
4 changed files with 61 additions and 2 deletions
|
|
@ -14756,5 +14756,25 @@
|
|||
"related_bugs": [],
|
||||
"occurrences": 1,
|
||||
"last_seen": "2026-08-04"
|
||||
},
|
||||
{
|
||||
"id": "bug-1502",
|
||||
"timestamp": "2026-08-04",
|
||||
"error_message": "controller ASAN heap-buffer-overflow (READ of size 4, thread T7): 56-byte xce_job_rusages_t buffer from decode_JobRusages_ptr, read at offset 56 (1 byte past end). Recurring crash that kills the controller whenever a job runs and the worker pushes rusage updates (term=3/4/5 all crashed).",
|
||||
"file": "src/controller/persist.c (persist_job_hook_fun)",
|
||||
"root_cause": "RUSAGE_UPDATE job hook arg mismatch: update_job_rusages() fires HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) as its 2nd va_arg (apiserver/sched subscribers parse this exact contract), but persist_job_hook_fun default case treated it as job_t*. xce_job_rusages_t is 56 bytes and has jobid@0 (coincidentally correct) but NO state field; j->job.state lives at offset 56 of xce_job_t, so the read went 4 bytes past the rusage struct -> heap-buffer-overflow. The unit test masked it by passing a full job_t.",
|
||||
"fix": "Added case HOOK_DETAIL_JOB_RUSAGE_UPDATE in persist_job_hook_fun: read jobid from xce_job_rusages_t*, bake the job current state via a jobmgr key_find on the vararg cluster_t* (safe: sole caller update_job_rusages holds that job write lock while firing). Regression test passes a real 56-byte xce_job_rusages_t (stack-buffer-overflow under old code, PASS under fix).",
|
||||
"tags": [
|
||||
"asan",
|
||||
"heap-buffer-overflow",
|
||||
"controller",
|
||||
"persist",
|
||||
"hook",
|
||||
"job-rusage",
|
||||
"crash"
|
||||
],
|
||||
"related_bugs": [],
|
||||
"occurrences": 3,
|
||||
"last_seen": "2026-08-04"
|
||||
}
|
||||
]
|
||||
|
|
@ -7368,3 +7368,4 @@
|
|||
| 05:49 | Created ../../../../tmp/frpc-watchdog.sh | — | ~338 |
|
||||
| 13:50 | 看门狗改为纯日志模式(不自动重启): 移到 /home/dankeer/bin/frpc-watchdog.sh + crontab @reboot 开机自启; 只在隧道状态变化时写日志(DOWN抓快照/恢复记录/持续DOWN每5min) | /home/dankeer/bin/frpc-watchdog.sh | 部署完成, 用户人工监视 | ~200 |
|
||||
| 05:50 | Session end: 9 writes across 5 files (frpcdbg_mon.sh, frpc_prod.toml, frpc_mon.sh, frpc_watchdog.sh, frpc-watchdog.sh) | 2 reads | ~2219 tok |
|
||||
| 09:39 | 修复 controller 反复崩溃 bug-1502: persist_job_hook_fun 把 RUSAGE_UPDATE hook 的第2参 (xce_job_rusages_t*) 当 job_t* 用, 读 offset 56 越界 56字节结构 -> ASAN heap-buffer-overflow. 加专用 case 从 jobmgr 取当前 state | src/controller/persist.c, testsuite/test_persist_hooks.c | RED(旧代码 stack-buffer-overflow)/GREEN(修复后 PASS), 全套458测试通过 | ~350 |
|
||||
|
|
|
|||
|
|
@ -6573,8 +6573,35 @@ int persist_job_hook_fun(int type, int situation, int detail,
|
|||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||||
break;
|
||||
}
|
||||
case HOOK_DETAIL_JOB_RUSAGE_UPDATE: {
|
||||
/* args: cluster_t*, xce_job_rusages_t* — NOT job_t* (bug-1502).
|
||||
* The RUSAGE_UPDATE hook carries the rusage struct, which has no job
|
||||
* state (apiserver_job_rusage_update_fun / sched_job_rusage_update_fun
|
||||
* parse this exact contract). The old default-case path treated the
|
||||
* 2nd arg as a job_t* and read j->job.state at offset 56 — one byte
|
||||
* past a 56-byte xce_job_rusages_t → ASAN heap-buffer-overflow (the
|
||||
* only field the two structs share is jobid@0, so that read
|
||||
* coincidentally worked). Bake the job's CURRENT state from the
|
||||
* jobmgr: the sole caller update_job_rusages() fires this AFTER_EVENT
|
||||
* while holding the job's write lock, so the pointer returned here IS
|
||||
* that same write-locked object and reading its state is race-free.
|
||||
* Never re-lock the job inside a hook — the caller already holds it,
|
||||
* and a same-thread re-lock would deadlock. */
|
||||
cluster_t* c = va_arg(ap, cluster_t*);
|
||||
xce_job_rusages_t* r = va_arg(ap, xce_job_rusages_t*);
|
||||
js.jobid = r->jobid;
|
||||
if (c != NULL && c->jobmgr != NULL) {
|
||||
job_t* jj = NULL;
|
||||
lock(c->jobmgr, RTTI_LOCK_READ);
|
||||
key_find(c->jobmgr, (void*)r->jobid, (void**)&jj);
|
||||
unlock(c->jobmgr);
|
||||
if (jj != NULL)
|
||||
js.state = (uint32_t)jj->job.state;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
/* STOP, RESUME, UPDATE, CKPT, PREASON, RUSAGE — args: cluster_t*, job_t* */
|
||||
/* STOP, RESUME, UPDATE, CKPT, PREASON — args: cluster_t*, job_t* */
|
||||
(void)va_arg(ap, cluster_t*);
|
||||
job_t* j = va_arg(ap, job_t*);
|
||||
js.jobid = j->job.jobid; js.state = (uint32_t)j->job.state;
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
#include "persist.h"
|
||||
#include "xce/cluster_def.h"
|
||||
#include "xlist.h"
|
||||
#include "controller.capnp.h" /* xce_job_rusages_t (bug-1502 regression test) */
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Test helpers */
|
||||
|
|
@ -276,9 +277,19 @@ static void test_job_hook_rusage_writes_wal_entry(void)
|
|||
|
||||
uint64_t lsn_before = p->current_lsn;
|
||||
|
||||
/* bug-1502 regression: production fires RUSAGE_UPDATE with a real
|
||||
* xce_job_rusages_t* (56 bytes, NO job-state field) as the second hook
|
||||
* arg — NOT a job_t*. The old persist_job_hook_fun default case read
|
||||
* j->job.state at offset 56, one byte past this struct → ASAN
|
||||
* heap-buffer-overflow. Passing the real minimal struct here (instead of
|
||||
* the old &g_stub_job, a full job_t that masked the overflow) reproduces
|
||||
* the production shape exactly. */
|
||||
xce_job_rusages_t rusages = {0};
|
||||
rusages.jobid = 0x1234567890ABCDEFULL;
|
||||
|
||||
int rc = call_job_hook(HOOK_AFTER_EVENT, HOOK_AFTER_EVENT,
|
||||
HOOK_DETAIL_JOB_RUSAGE_UPDATE, p,
|
||||
(cluster_t*)NULL, &g_stub_job);
|
||||
(cluster_t*)NULL, &rusages);
|
||||
TEST_ASSERT_EQUAL_INT(0, rc);
|
||||
TEST_ASSERT_GREATER_THAN(lsn_before, p->current_lsn);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue