From ff9de2e0046f80df234636f2c3f78157bf9d72ea Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sun, 30 Aug 2026 15:07:49 +0000 Subject: [PATCH] fix(bug-924): prune stale /tmp test-fixture dirs in xce-build.sh 70+ testsuite/ files across controller/scheduler/lib call mkdtemp() directly with no shared fixture helper, and none clean up after themselves. Left unchecked this fills the sandbox disk (recurred three times now: 2026-06-22, 2026-08-26, 2026-08-30, each larger than the last -- 696GB at its worst), which then surfaces as unrelated "Expected Non-NULL"-style cascade failures wherever the next test happens to call mkdtemp()/persist_init(), easily mistaken for a real regression. Retrofitting cleanup into 70+ individual test files would be invasive and risks breaking tests that inspect a fixture's contents across multiple TEST_ functions in the same file. Every test run in this project goes through xce-build.sh, so pruning once there -- age-gated at 3 hours so a concurrent session's in-progress fixtures are never touched -- is the lowest-risk single choke point. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019QxaSzHRB7pmYFp4cVnwkA --- src/tools/xce-build.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/tools/xce-build.sh b/src/tools/xce-build.sh index 0137a12d..16c608a2 100755 --- a/src/tools/xce-build.sh +++ b/src/tools/xce-build.sh @@ -178,6 +178,35 @@ usage() { exit 1 } +# bug-924 (recurred 2026-06-22, 2026-08-26, 2026-08-30 -- each time at a +# larger scale, most recently 16238 dirs / 696G / disk 100% full): every +# ASAN test file in this project's testsuite/ directories that needs a +# scratch directory calls mkdtemp() on its own (70+ files, no shared +# fixture helper to retrofit centrally -- confirmed by grep), and NONE of +# them clean up their own /tmp/xce_* dir afterward. Left unchecked this +# fills the sandbox disk to ENOSPC, which then manifests as unrelated +# "Expected Non-NULL"-style cascade failures in whatever test happens to +# call mkdtemp()/persist_init() next -- easily mistaken for a real +# regression (see bug-924's own history). Retrofitting cleanup into each +# of the 70+ test files individually would be invasive and risks breaking +# tests that inspect a fixture's on-disk contents across multiple TEST_ +# functions in the same file. Pruning here instead, once per build +# invocation, is the single lowest-risk choke point: every test run in +# this project goes through this script. Age-gated (not "delete +# everything") so it never touches a fixture another CONCURRENT session's +# test run just created -- this sandbox routinely has multiple sessions +# building/testing at once. +prune_stale_tmp_fixtures() { + local stale + stale="$(find /tmp -maxdepth 1 -name 'xce_*' -mmin +180 2>/dev/null)" + if [ -n "$stale" ]; then + local count + count="$(printf '%s\n' "$stale" | wc -l)" + echo "=== Pruning $count stale (>3h old) /tmp/xce_* test-fixture dir(s) from prior runs (bug-924) ===" + printf '%s\n' "$stale" | xargs -r rm -rf -- 2>/dev/null || true + fi +} + setup_build() { if [ "$INCREMENTAL" = 1 ] && [ -f "$BUILD_DIR/build.ninja" ]; then echo "=== Incremental build: $BUILD_DIR (profile=$PROFILE) — reconfiguring in place, other components' binaries preserved. If anything looks wrong, rebuild without --incremental. ===" @@ -189,6 +218,8 @@ setup_build() { fi } +prune_stale_tmp_fixtures + case "$COMPONENT" in lib) setup_build