From afc3f4a4f3a3e85a98a676e069d9a4cdb301dd7c Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Thu, 20 Aug 2026 10:38:28 +0000 Subject: [PATCH] chore(wolf): checkpoint buglog.json (986 bugs) --- .wolf/buglog.json | 54 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index ae4667dd..2f30fd17 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -12872,7 +12872,7 @@ "error_message": "controller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run immediately after uncommitted edits to persist.c, ha_convergence.c, config_repl.c (delaymgr-inline blocking I/O moved to VTASK_LIVE_BLOCK vtasks with inline-fallback-on-failure). Compile succeeded with zero errors (only pre-existing warnings). meson test summary: Ok:0 Fail:0 Timeout:1. Last test output before the hang was test_demotion_replay_yields_committed_only (test_commit_barrier_reply.c) printing overlayfs-SKIP, followed by a burst of HA FSM / persist_apply_pump_run log lines ending in \"HA quorum lost: connected=0 required=1\", then total silence for the remaining ~294s until SIGTERM. No ASAN/crash report -- looks like a genuine hang/deadlock, not a crash.", "file": "src/controller/persist.c, src/controller/ha_convergence.c, src/controller/config_repl.c", "root_cause": "NOT YET ROOT-CAUSED. Suspect the new delaymgr-trampoline -> vtask_enq_classified(pool, task, VTASK_LIVE_BLOCK) path blocks waiting for vtask completion that never arrives inside the controller_unit_test harness (e.g. vtask thread pool not pumped/started for this test fixture, or a lock-ordering deadlock between the delaymgr thread and the vtask worker). Ruled out: disk space is not exhausted (162G free, 81% used) so this is NOT the bug-924 ENOSPC/overlayfs-fdatasync pattern despite ~8182 leftover /tmp/xce_* dirs being present.", - "fix": "NOT FIXED -- reported to user for diagnosis; no code changes made in this session.", + "fix": "NOT FIXED -- reported to user for diagnosis; no code changes made in this session. Reproduced again 2026-08-20 on scheduler-framework worktree /xce-build controller run (unrelated diff: only src/controller/main.c and testsuite/test.c argp_program_version/version-log changes) -- controller_unit_test TIMEOUT 300.14s SIGTERM, Ok:4 Fail:0 Timeout:1, hang appears to be inside/after test_commit_barrier_rollout.c's test_configack_* group. Still not root-caused; confirms this is pre-existing/environmental, not caused by the version-string diff.", "tags": [ "vtask", "VTASK_LIVE_BLOCK", @@ -12888,8 +12888,8 @@ "related_bugs": [ "bug-924" ], - "occurrences": 1, - "last_seen": "2026-07-10" + "occurrences": 2, + "last_seen": "2026-08-20T10:33:00.000Z" }, { "id": "bug-1430", @@ -17795,6 +17795,54 @@ "related_bugs": [], "occurrences": 1, "last_seen": "2026-08-20T09:20:00Z" + }, + { + "id": "bug-1604", + "timestamp": "2026-08-20T10:24:39Z", + "error_message": "rm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh setup_build()'s rm -rf \\$BUILD_DIR step aborted the script under set -euo pipefail before meson setup ran)", + "file": "src/tools/xce-build.sh", + "root_cause": "CORRECTED after further evidence (see fix note): initial overlayfs-flakiness theory was WRONG. The real cause is xce-build.sh defaulting BUILD_DIR to the same \"xce.build\" for every component and every invocation, with setup_build() doing an unconditional rm -rf \"$BUILD_DIR\" + fresh meson setup on every non-incremental run. When two xce-build.sh invocations run concurrently in the SAME repo working directory against DIFFERENT components (observed live: this session's \"xce-build.sh scheduler asan\" raced against a second, independently-running \"xce-build.sh controller asan\" process, PID 655158, started ~1 min later in the same cwd), they contend for the identical xce.build/ path -- one process's rm -rf can hit a directory the other process is actively writing into (\"Directory not empty\"), and later, a second clean-build pass can silently delete and replace a just-verified binary out from under a caller who believes their build is done. Confirmed live: xce_scheduler at src/xce.build/scheduler/xce_scheduler, verified present and fresh via ls -la and 'xce_scheduler --version' immediately after this session's scheduler build/test run, was gone minutes later -- the concurrent controller build's rm -rf had removed and was in the process of regenerating xce.build/ for its own component.", + "fix": "No code fix applied (out of scope for the build-verification task this session was doing) -- flagging as a real, reproducible hazard rather than env flakiness. Practical mitigation for future sessions/agents: never assume xce.build/ (or any non-incremental xce-build.sh output) is stable once another concurrent xce-build.sh process might be running in the same working directory/worktree -- check 'ps aux | grep xce-build.sh' (or pass a distinct BUILD_DIR / use --incremental) before relying on a binary's presence or mtime surviving past the moment it was checked. Corrects this entry's own original overlayfs-race theory, which was a plausible but incorrect guess made before the concurrent process was discovered.", + "tags": [ + "asan", + "concurrency", + "multi-agent-race", + "overlayfs", + "rm-rf", + "sandbox-flake", + "scheduler", + "shared-build-dir", + "xce-build.sh" + ], + "related_bugs": [ + "bug-1561" + ], + "occurrences": 1, + "last_seen": "2026-08-20T10:30:35Z" + }, + { + "id": "bug-1605", + "timestamp": "2026-08-20T10:27:00.000Z", + "error_message": "controller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28: error: 'RequestReply_data_probeconfigtagreplymsg' undeclared (first use in this function); did you mean 'RequestReply_data_upgradeconfigreplymsg' -- and (2) lib/controller.capnp.c:25013:1: fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory, compilation terminated; ninja: build stopped: subcommand failed.", + "file": "src/lib/controller.capnp.c (generated), src/controller/config_repl.c (consumer)", + "root_cause": "Matches the capnp custom_target's documented missing-dependency-edge race (see comment block at top of src/tools/xce-build.sh, 'retry once before debugging'): config_repl.c was compiled against a stale/partially-regenerated controller.capnp.h that lacked the 'probeconfigtagreplymsg' enumerant even though src/schema/controller.capnp:596 defines it (probeconfigtagreplymsg @20: ProbeConfigTagReply), and lib/controller.capnp.c's own compile then hit an I/O race writing its .d dependency file, most likely because the generator custom_target re-ran/rewrote lib/ mid-build without ninja having a correct dependency edge ordering the two. Not caused by the concurrent main.c/testsuite/test.c argp_program_version diff (those files are unrelated to config_repl.c/capnp).", + "fix": "Confirmed transient: re-ran the exact same command (nix develop ./src --command bash src/tools/xce-build.sh controller asan) a second time with zero source changes in between and the full clean build compiled successfully (202/202, zero errors). Not root-caused/fixed in the underlying capnp custom_target dependency graph -- xce-build.sh's own comments already document this as a known, unfixed gap and recommend exactly this retry-once workaround.", + "tags": [ + "controller", + "capnp", + "build", + "ninja", + "race", + "codegen", + "transient", + "pre-existing" + ], + "related_bugs": [ + "bug-1439", + "bug-1493" + ], + "occurrences": 1, + "last_seen": "2026-08-20T10:27:00.000Z" } ] }