diff --git a/.wolf/anatomy-index.json b/.wolf/anatomy-index.json index 7401e9c4..1cf5b33c 100644 --- a/.wolf/anatomy-index.json +++ b/.wolf/anatomy-index.json @@ -1,13 +1,13 @@ { "version": 1, "meta": { - "lastScanned": "2026-08-30T03:20:29.906Z", - "fileCount": 543, + "lastScanned": "2026-08-30T15:06:40.281Z", + "fileCount": 555, "hits": 0, "misses": 0, - "renderedHash": "c6a72a30c9736c4c6e8239e03c5d16c86b28ee46a0e226f07c495b512ef1ec45", - "storeUpdatedAt": "2026-08-30T03:20:29.913Z", - "rootHash": "28dc535af2e4da51" + "renderedHash": "ee44fa75780c1935939977f8ffaaa77b305dd8dfb1d92273e53e281cb38bb834", + "storeUpdatedAt": "2026-08-30T15:06:40.289Z", + "rootHash": "91d2c295883b912b" }, "files": { ".agent-shell/transcripts/2026-08-11-11-39-59.md": { @@ -3061,13 +3061,12 @@ }, ".kiro/specs/scheduler-snapshot-job-allocation/spec.json": { "description": "", - "tokens": 133, - "hash": "1ccf7f77f15cb274", - "size": 464, - "mtimeMs": 1787829787920.9985, - "updatedAt": "2026-08-27T19:00:00.130Z", - "source": "scan", - "importance": 0.21 + "tokens": 135, + "hash": "c9f25fe210bb27fa", + "size": 472, + "mtimeMs": 1788102400243.472, + "updatedAt": "2026-08-30T15:06:40.280Z", + "source": "hook" }, ".kiro/specs/scheduler-snapshot-job-allocation/tasks.md": { "description": "Implementation Plan", @@ -6267,17 +6266,17 @@ "tokens": 12200, "hash": "c0e97827aec51533", "size": 45750, - "mtimeMs": 1788058519050.749, - "updatedAt": "2026-08-30T02:55:19.095Z", + "mtimeMs": 1788089633567.1846, + "updatedAt": "2026-08-30T11:33:53.614Z", "source": "hook" }, "src/controller/job.c": { "description": "include ", - "tokens": 39294, - "hash": "3c05b7d63de4207d", - "size": 147501, - "mtimeMs": 1787979541114.306, - "updatedAt": "2026-08-29T04:59:01.150Z", + "tokens": 39347, + "hash": "b584def6c8be4e0b", + "size": 147700, + "mtimeMs": 1788094063748.4392, + "updatedAt": "2026-08-30T12:47:43.792Z", "source": "hook" }, "src/controller/sched.c": { @@ -6488,11 +6487,11 @@ }, "src/tools/xce-build.sh": { "description": "", - "tokens": 3774, - "hash": "903292d63fa483d1", - "size": 14170, - "mtimeMs": 1788003007490.2698, - "updatedAt": "2026-08-29T11:30:07.541Z", + "tokens": 4220, + "hash": "75d7532eda6cd754", + "size": 15844, + "mtimeMs": 1788102250218.387, + "updatedAt": "2026-08-30T15:04:10.254Z", "source": "hook" }, "src/controller/testsuite/test_apply_committed_entry.c": { @@ -6506,11 +6505,11 @@ }, "src/scheduler/src/handler.c": { "description": "include ", - "tokens": 6772, - "hash": "0164997c99c0f1a2", - "size": 25392, - "mtimeMs": 1788058540046.694, - "updatedAt": "2026-08-30T02:55:40.093Z", + "tokens": 7162, + "hash": "96c47fc30962bfd5", + "size": 26856, + "mtimeMs": 1788085225504.2131, + "updatedAt": "2026-08-30T10:20:25.539Z", "source": "hook" }, "src/controller/config_repl.c": { @@ -6530,6 +6529,114 @@ "mtimeMs": 1788060029858.196, "updatedAt": "2026-08-30T03:20:29.906Z", "source": "hook" + }, + "reports/aug23-aug29-2026.md": { + "description": "XCE Project Status — Aug 23 – Aug 29, 2026", + "tokens": 3530, + "hash": "609276740b421e22", + "size": 14294, + "mtimeMs": 1788081702821.217, + "updatedAt": "2026-08-30T09:21:42.862Z", + "source": "hook" + }, + "src/lib/controller.capnp.c": { + "description": "include \"controller.capnp.h\"", + "tokens": 167125, + "hash": "3699ca9316e50f26", + "size": 626716, + "mtimeMs": 1788089768666.0857, + "updatedAt": "2026-08-30T11:36:08.716Z", + "source": "hook" + }, + "src/lib/utils.c": { + "description": "include ", + "tokens": 7883, + "hash": "3b62d5e85b432e45", + "size": 29561, + "mtimeMs": 1788086341163.3594, + "updatedAt": "2026-08-30T10:39:01.207Z", + "source": "hook" + }, + "src/scheduler/src/driver.c": { + "description": "include ", + "tokens": 9779, + "hash": "53ef47b59a888a10", + "size": 36668, + "mtimeMs": 1788085274050.2458, + "updatedAt": "2026-08-30T10:21:14.086Z", + "source": "hook" + }, + "src/controller/include/cluster.h": { + "description": "if !defined(_CLUSTER_H_)", + "tokens": 3417, + "hash": "e29c78cb703be7f1", + "size": 12824, + "mtimeMs": 1788093629127.659, + "updatedAt": "2026-08-30T12:40:29.163Z", + "source": "hook" + }, + "src/controller/testsuite/test_host_runusage.c": { + "description": "Declares char", + "tokens": 2404, + "hash": "5c993a7a53ccfcf0", + "size": 9015, + "mtimeMs": 1788096408980.8794, + "updatedAt": "2026-08-30T13:26:49.043Z", + "source": "hook" + }, + "src/controller/meson.build": { + "description": "", + "tokens": 3852, + "hash": "9ac969fcba2cc9a6", + "size": 14444, + "mtimeMs": 1788093711080.9766, + "updatedAt": "2026-08-30T12:41:51.126Z", + "source": "hook" + }, + "src/controller/misc.c": { + "description": "include ", + "tokens": 1756, + "hash": "a6472812879cd1a8", + "size": 6582, + "mtimeMs": 1788094049916.7393, + "updatedAt": "2026-08-30T12:47:29.955Z", + "source": "hook" + }, + "src/controller/testsuite/test.c": { + "description": "include ", + "tokens": 40634, + "hash": "c44d975ff84aff74", + "size": 152465, + "mtimeMs": 1788094140047.6172, + "updatedAt": "2026-08-30T12:49:00.092Z", + "source": "hook" + }, + "src/controller/testsuite/test_commit_barrier_reply.c": { + "description": "Declares uint8_t", + "tokens": 45642, + "hash": "ab3bb63e3eaf02c2", + "size": 174257, + "mtimeMs": 1788095809277.0889, + "updatedAt": "2026-08-30T13:16:49.316Z", + "source": "hook" + }, + ".claude/skills/xce-changes/SKILL.md": { + "description": "xce-changes Skill", + "tokens": 2690, + "hash": "a20d40f546d1546e", + "size": 10818, + "mtimeMs": 1788097335404.1423, + "updatedAt": "2026-08-30T13:42:15.486Z", + "source": "hook" + }, + "CHANGES.md": { + "description": "XCE Change Log", + "tokens": 8850, + "hash": "fd71bc93fa14f8f5", + "size": 35402, + "mtimeMs": 1788098337067.5034, + "updatedAt": "2026-08-30T13:58:57.394Z", + "source": "hook" } } } \ No newline at end of file diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index dd85dfd0..855c5eb7 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,7 +1,7 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-08-30T03:20:29.906Z -> Files: 543 tracked | Anatomy hits: 0 | Misses: 0 +> Auto-maintained by OpenWolf. Last scanned: 2026-08-30T15:06:40.281Z +> Files: 555 tracked | Anatomy hits: 0 | Misses: 0 ## ./ @@ -9,6 +9,7 @@ - `.gitignore` — Git ignore rules (~402 tok) - `.mcp.json` (~7 tok) - `.opencode.json` (~7 tok) +- `CHANGES.md` — XCE Change Log (~8850 tok) - `CLAUDE.md` — OpenWolf (~1765 tok) - `README.md` — Project documentation (~612 tok) @@ -22,6 +23,10 @@ - `2026-08-23-15-56-15.md` — Agent Shell Transcript (~69291 tok) - `2026-08-26-17-36-06.md` — Agent Shell Transcript (~183 tok) +## .claude/skills/xce-changes/ + +- `SKILL.md` — xce-changes Skill (~2690 tok) + ## .codegraph/ - `.gitignore` — Git ignore rules (~61 tok) @@ -499,7 +504,7 @@ - `design.md` — Design Document: scheduler-snapshot-job-allocation (~7948 tok) - `requirements.md` — Requirements Document (~2514 tok) - `research.md` — Summary (~3634 tok) -- `spec.json` (~133 tok) +- `spec.json` (~135 tok) - `tasks.md` — Implementation Plan (~2835 tok) ## .kiro/specs/worker-join-refactor/ @@ -572,6 +577,7 @@ - `aug03-aug09-2026.md` — XCE Project Status — Aug 03 – Aug 07, 2026 (~1626 tok) - `aug08-aug15-2026.md` — XCE Project Status — Aug 08 – Aug 15, 2026 (~3603 tok) - `aug16-aug22-2026.md` — XCE Project Status — Aug 16 – Aug 22, 2026 (~2706 tok) +- `aug23-aug29-2026.md` — XCE Project Status — Aug 23 – Aug 29, 2026 (~3530 tok) - `jul06-jul11-2026.md` — Jul 6 → Jul 11 Progress Report (~2311 tok) - `jul13-jul18-2026.md` — Jul 13 → Jul 18 Progress Report (~3919 tok) - `jul19-jul26-2026.md` — Work Summary: Jul 19 – Jul 26, 2026 (~3482 tok) @@ -803,18 +809,27 @@ ## src/controller/ - `config_repl.c` — define _GNU_SOURCE (~35341 tok) -- `job.c` — include (~39294 tok) +- `job.c` — include (~39347 tok) - `main.c` — include (~5489 tok) +- `meson.build` (~3852 tok) +- `misc.c` — include (~1756 tok) - `persist.c` — define _GNU_SOURCE (~109226 tok) - `sched.c` — include (~6721 tok) +## src/controller/include/ + +- `cluster.h` — if !defined(_CLUSTER_H_) (~3417 tok) + ## src/controller/testsuite/ - `test_apply_committed_entry.c` — Declares directly (~13500 tok) - `test_apply_reapply_idempotency.c` — Declares xce_job_t (~4115 tok) +- `test_commit_barrier_reply.c` — Declares uint8_t (~45642 tok) - `test_config_repl.c` — define _GNU_SOURCE (~87218 tok) +- `test_host_runusage.c` — Declares char (~2404 tok) - `test_wal_job_finish_fact.c` — Declares as (~8980 tok) - `test_wal_job_start_fact.c` — Declares as (~12368 tok) +- `test.c` — include (~40634 tok) ## src/frontend/apiserver/backend/ @@ -822,7 +837,9 @@ ## src/lib/ +- `controller.capnp.c` — include "controller.capnp.h" (~167125 tok) - `meson.build` (~1757 tok) +- `utils.c` — include (~7883 tok) - `vtaskmgr.c` — include (~3978 tok) ## src/lib/testsuite/ @@ -838,7 +855,8 @@ ## src/scheduler/src/ - `core_events.c` — include (~12200 tok) -- `handler.c` — include (~6772 tok) +- `driver.c` — include (~9779 tok) +- `handler.c` — include (~7162 tok) - `main.c` — include (~5330 tok) - `snapshot_load.c` — include (~5920 tok) - `snapshot.c` — include (~8498 tok) @@ -854,7 +872,7 @@ ## src/tools/ -- `xce-build.sh` (~3774 tok) +- `xce-build.sh` (~4220 tok) ## src/worker/src/ diff --git a/.wolf/buglog.json b/.wolf/buglog.json index f409a1c0..7953ffd3 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -7847,7 +7847,7 @@ "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", "file": "environment (/ overlay filesystem)", "root_cause": "Disk filled to 100% (7466 leftover /tmp/xce_* WAL dirs from repeated ASAN test runs). make_fresh_persist's persist_init mkdir_p hit ENOSPC -> returned NULL -> cascade of 'Expected Non-NULL'. Also overlayfs returned fdatasync EIO under disk pressure. NOT a code regression: baseline full suite passes; test_config_repl catchup tests are order-dependent and fail in isolation on baseline too.", - "fix": "rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles.\n\n[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed at much larger scale (14731 leftover /tmp/xce_* dirs, 653.6G, filesystem at 100% used / 0 available) and a MORE SEVERE manifestation than the original 'Expected Non-NULL' assertion failures: mkdtemp() failing inside test_repl_apply.c's make_test_persist_for_apply()/make_test_persist_in() hit ENOSPC, and that pre-existing test file's own hard exit(1) on mkdtemp failure silently killed the ENTIRE controller_unit_test process mid-run (right after test_repl_unit's last test, well before test_repl_apply's own output ever printed) -- exit code 1, no crash/ASAN report, no visible error in the redirected log until grepped for the literal 'mkdtemp failed' stderr line. A leftover orphaned controller_unit_test process (PID from an earlier invocation, blocked in futex_do_wait, presumably stalled retrying a WAL fsync against the full disk) was also found and killed. Fixed for this session via `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` (630G reclaimed, 100% -> 29% used) -- confirmed no live xce process had any of those dirs open before deleting. General lesson: when controller_unit_test dies silently/instantly with exit code 1 and the log just stops mid-file with no crash signature, check `df -h /tmp` and grep for 'mkdtemp failed'/'calloc failed' before assuming a real regression -- this sandbox's /tmp accumulates leftover unit-test mkdtemp() fixture dirs across EVERY session indefinitely (no test file in this suite cleans up its own /tmp/xce_* dirs), so this will keep recurring until either the testsuite adds its own cleanup or something periodically clears /tmp.", + "fix": "rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles.\n\n[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed at much larger scale (14731 leftover /tmp/xce_* dirs, 653.6G, filesystem at 100% used / 0 available) and a MORE SEVERE manifestation than the original 'Expected Non-NULL' assertion failures: mkdtemp() failing inside test_repl_apply.c's make_test_persist_for_apply()/make_test_persist_in() hit ENOSPC, and that pre-existing test file's own hard exit(1) on mkdtemp failure silently killed the ENTIRE controller_unit_test process mid-run (right after test_repl_unit's last test, well before test_repl_apply's own output ever printed) -- exit code 1, no crash/ASAN report, no visible error in the redirected log until grepped for the literal 'mkdtemp failed' stderr line. A leftover orphaned controller_unit_test process (PID from an earlier invocation, blocked in futex_do_wait, presumably stalled retrying a WAL fsync against the full disk) was also found and killed. Fixed for this session via `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` (630G reclaimed, 100% -> 29% used) -- confirmed no live xce process had any of those dirs open before deleting. General lesson: when controller_unit_test dies silently/instantly with exit code 1 and the log just stops mid-file with no crash signature, check `df -h /tmp` and grep for 'mkdtemp failed'/'calloc failed' before assuming a real regression -- this sandbox's /tmp accumulates leftover unit-test mkdtemp() fixture dirs across EVERY session indefinitely (no test file in this suite cleans up its own /tmp/xce_* dirs), so this will keep recurring until either the testsuite adds its own cleanup or something periodically clears /tmp.\n\n[UPDATE 2026-08-30, bug-1735 verification build] Re-observed a THIRD time, at the largest scale yet: 16238 leftover /tmp/xce_* dirs, 696.1G, filesystem at 100% used / 0 available. This time it manifested as a `controller_unit_test` run reporting `1220 Tests 282 Failures` (mostly test_dispatch_commit.c/test_dispatch_fail_terminal.c/test_dispatch_fail_retry.c/test_jobdispatch_no_double_notify.c 'Expected Non-NULL' -- the exact same fixture-setup-ENOSPC cascade signature as both prior occurrences, on files wholly unrelated to whatever the run was actually verifying) plus a meson-internal `OSError: [Errno 28] No space left on device` while serializing testlog.json/testlog.junit.xml (corrupting those log artifacts, though the console summary line survived). No live xce process was running at the time (`ps aux` confirmed clean) so `find /tmp -maxdepth 1 -name 'xce_*' -exec rm -rf {} +` was safe; reclaimed 696.1G (100% -> 22% used). Re-running the full suite after cleanup is required before trusting any failure count from a run that coincided with disk exhaustion. Still true three occurrences in: nothing in this test suite cleans up its own /tmp/xce_* dirs, so this WILL keep recurring -- the durable fix is adding cleanup to the shared test fixture helpers, not another one-off `rm -rf`.", "tags": [ "disk-space", "enospc", @@ -7855,11 +7855,12 @@ "fdatasync-eio", "controller-tests", "flaky", - "ha-term-safety" + "ha-term-safety", + "recurring" ], "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-08-26T00:00:00Z" + "occurrences": 3, + "last_seen": "2026-08-30T06:24:00" }, { "id": "bug-925", @@ -18377,21 +18378,22 @@ "error_message": "buglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948). The Stop hook's duplicate-ID validation flags this every turn. bug-908, for example, is BOTH a 2026-06-21 raft_sim.c missing-guard-clause entry AND a 2026-08-16 config-upgrade-snapshot/spec.json refactor note -- two unrelated bugs sharing one id.", "file": ".wolf/buglog.json", "root_cause": "A session on 2026-08-15..17 appended new entries using an ID counter that had been restarted (or derived from a stale/remembered max) instead of computed from the file's actual maximum numeric ID. New bugs were therefore written into the already-occupied bug-875..bug-949 range: 8 collisions dated 2026-08-15, 14 dated 2026-08-16, 22 dated 2026-08-17. This is precisely the failure mode the OpenWolf bug-logging rule warns about ('never restart from bug-001; a restarted counter collides with existing history and is unrecoverable without a git-history merge'). Discovered while generating the Aug 16-22 weekly report, where an in-window timestamp query returned an implausible id range of bug-908..bug-1624.", - "fix": "", + "fix": "Housekeeping only, no code change needed: renumbering commit 755976fc (2026-08-24) already repaired all 39 duplicate IDs. Re-verified 2026-08-30 -- 1077 bug entries, 1077 unique IDs, zero collisions. This entry's own `fix` field was simply never backfilled after the repair landed.", "tags": [ "buglog", "openwolf", "data-integrity", "duplicate-id", "tooling", - "unfixed", - "needs-git-history-merge" + "needs-git-history-merge", + "fixed", + "verified-2026-08-30" ], "related_bugs": [ "bug-1628" ], "occurrences": 1, - "last_seen": "2026-08-23T08:35:55" + "last_seen": "2026-08-30T05:40:00" }, { "id": "bug-1631", @@ -19494,8 +19496,8 @@ "timestamp": "2026-08-26T18:35:00Z", "error_message": "User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically.", "file": "src/scheduler/ (exact crash site unknown -- no core dump, no backtrace available for either occurrence)", - "root_cause": "NOT ROOT-CAUSED -- confirmed real (kernel dmesg segfault line, error code 4 = user-mode read fault, not a random/unrelated event) but no core dump or backtrace exists for any xce_scheduler crash observed so far, including the Aug 25 19:21:39 kernel-confirmed occurrence and today's (Aug 26, ~18:00-18:02) restart (PID 3964918 -> 4137745), which may or may not be the SAME underlying bug -- the two occurrences have not been correlated beyond both being unexplained scheduler deaths. Notable secondary finding: systemd-coredump is confirmed working for xce_controller (4 real core.xce_controller.*.zst files captured and used earlier this session, see bug-1699) but has captured ZERO xce_scheduler crashes despite at least one kernel-confirmed scheduler segfault -- needs investigation on the user's real machine (check core_pattern/ulimit -c/systemd-coredump size or process-name filtering specifically for the scheduler binary; also worth checking whether the scheduler's own launch wrapper -- start-scheduler.sh, a bare unwrapped invocation with no core-dump-enabling ulimit call -- inherits a different ulimit -c than however the controller happens to get its unlimited setting). FOLLOW-UP (2026-08-26, disambiguating today's restart from the Aug-25 segfault): checked dmesg for today's exact restart window (17:45-18:15) -- zero xce_scheduler entries (the ONLY dmesg segfault hit remains the single Aug 25 19:21:39 occurrence). journalctl for the same window -- zero xce_scheduler entries at all. dmesg confirms OOM killer was never actually invoked against any process in this window (only periodic OOM-killer enable/disable toggling, unrelated). ulimit -c is unlimited and core_pattern correctly routes to systemd-coredump (confirmed working for xce_controller, 4 real cores captured this session). The combination of (a) unlimited core-dump capacity + correct core_pattern, (b) zero kernel fault-signal record, (c) zero OOM record, (d) zero journalctl record for today's specific window strongly indicates today's PID change (3964918 -> 4137745) was NOT a crash of any kind -- a genuine fault signal (SIGSEGV/SIGABRT/SIGBUS/SIGILL/SIGFPE) is unconditionally kernel-logged regardless of which process triggers it, and none was. The scheduler is launched as a bare backgrounded process (start-scheduler.sh, no systemd unit), so an ordinary kill/pkill sent by an unrelated concurrent process on this shared machine would leave NO trace in dmesg or journalctl -- most likely explanation for today's specific restart, DECOUPLED from this bug going forward. This buglog entry now tracks ONLY the confirmed, kernel-verified Aug 25 19:21:39 xce_scheduler segfault (still unreproduced, still no core available, still worth investigating on recurrence) -- not today's restart, which is environmental noise, not a code bug.\n\nUPDATE (2026-08-29, bug-1731 live gdb-wrapped repro session): CRASH CAPTURED LIVE, first-ever backtrace for this crash family. Root cause of the earlier no-core-dump gap circumvented by launching xce_scheduler wrapped in `gdb -batch -ex run` FROM PROCESS START (not attach-after-the-fact) during a heavy burst-array workload (6 concurrent `jobsub -a 50` array submissions, 300 near-instant /bin/true elements, on a live 3-node HA cluster, debug build profile). Reproduced at T+150s into the burst. Full backtrace:\nSIGSEGV in sched_get_project(core, name=0x41d0000000000000) at scheduler/src/core.c:412, called from core_build_job() at scheduler/src/core_events.c:221, called from xce_core_push_events() at scheduler/src/core_events.c:725 (crashed on the 3rd of a 6-event batch), called from xce_sched_driver_translate_pending_events() (driver.c:555) <- xce_sched_driver_run_normal_mode_round() (driver.c:749) <- main() (main.c:465). Registers: rax=rdx=rsi=0x41d0000000000000 (corrupted `name` arg present in 3 registers), rdi=0x555555695da0 (valid `core` arg). Single-thread fault in the main scheduling-round thread (all 9 other threads idle in pthread_cond_wait/epoll_wait) -- not a cross-thread race at the fault site itself, though the corruption's origin is not yet located.\ncore_build_job() (core_events.c:221) calls sched_get_project(core, j->submit->project_name). The two preceding calls in the same function (sched_get_user, sched_get_queue, reading earlier fields of the same job_submit_data_t*) succeeded, so j->submit itself is a valid pointer -- but its project_name field held 0x41d0000000000000, not NULL and not a plausible string pointer. clone_job_submit_data() (handler.c:102-150) calloc()s the destination and only strdup()s project_name when the source is non-NULL, so a legitimately-absent project name would be NULL (which sched_get_project's own guard at core.c:412 handles safely) -- this crash requires the field to have been overwritten with garbage AFTER allocation. Points to memory corruption or a stale/reused pointer in the array-job event-translation/batching path under heavy concurrent load, not a missing-null-check. NOT YET further isolated -- the corruption's write site is still unknown. Fault class (SIGSEGV, invalid-pointer read via strlen()) matches the original kernel dmesg description (`error 4` = user-mode read fault) closely enough that this is plausibly the same crash family, now finally attributable to an exact function/line/call-chain instead of an unattributed dmesg line.\n\nUPDATE (2026-08-29, post-fix live-repro session -- bug-1730/1731/1733 fixes now all landed and included): two-phase reproduction attempt, both phases succeeded.\n\nPHASE 1 (ASAN, fresh clean builds of controller/scheduler/worker/xceadmin including commits 42eeaa1e[bug-1733 vtaskmgr UAF fix], b1323f8d[bug-1730], ebe691c3+67009815[bug-1731]): deployed fresh 3-node HA cluster (/tmp/xce-deploy-bug1714-asan2), launched xce_scheduler directly under ASAN (ASAN_OPTIONS=detect_leaks=0:halt_on_error=1:abort_on_error=1), ran a 10x50-element concurrent array-job burst (500 elements, wave 1 of a planned two-wave/1000-element burst). CRASH REPRODUCED WITHIN WAVE 1 ITSELF -- before all 10 clients' submissions even completed, after only 2 of 10 array pushes had been expanded by the scheduler (100 of the intended 500 elements). This is FASTER than the original bug-1731-session capture (T+150s) and definitively proves the bug is STILL PRESENT after all three recent fixes, not masked or perturbed by ASAN instrumentation. Verbatim ASAN report:\n AddressSanitizer:DEADLYSIGNAL\n ==562488==ERROR: AddressSanitizer: SEGV on unknown address (pc 0x64b0be11076e ...)\n ==562488==The signal is caused by a READ memory access.\n #0 sched_get_project ../scheduler/src/core.c:412\n #1 core_build_job ../scheduler/src/core_events.c:221\n #2 xce_core_push_events ../scheduler/src/core_events.c:725\n #3 xce_sched_driver_translate_pending_events ../scheduler/src/driver.c:555\n #4 xce_sched_driver_run_normal_mode_round ../scheduler/src/driver.c:749\n #5 main ../scheduler/src/main.c:465\n Register values: rax=rdx approx corrupted; rax = 0x41d0000000000000 (still exactly 2^30=1073741824.0 as an IEEE-754 double, matching the original capture bit-for-bit) -- IDENTICAL crash signature, file, line, and call chain to the original bug-1731-session capture. SUMMARY: AddressSanitizer: SEGV core.c:412 in sched_get_project.\n Diagnostically important: ASAN's own heap-buffer-overflow and use-after-free shadow-memory detectors did NOT fire -- only the raw SIGSEGV/DEADLYSIGNAL handler caught this, exactly as a non-ASAN build would. This RULES OUT a classic redzone-violating heap overflow into an adjacent allocation (ASAN would have caught that at the WRITE) and argues for either (a) a wrong-offset/type-confusion write within a still-valid allocation, or (b) a use-after-free where the stale read happens only after the freed chunk has already left ASAN's quarantine and been legitimately reallocated for a new, unrelated live object -- which Phase 2 then confirmed directly.\n\nPHASE 2 (gdb scripted watchpoint, debug -O0 -g3 profile, same fix set, fresh cluster /tmp/xce-deploy-bug1714-dbg2): launched xce_scheduler under `gdb -batch -x scheduler-watchpoint.gdb`, breaking on clone_job_submit_data() (handler.c:102, called once per array element from make_array_element), sampling every 31st call and arming a hardware watchpoint on the returned job_submit_data_t's ->project_name field (via `finish` + cast of $rax to the known x86-64 System V pointer-return-value register), capped at 3 simultaneous watchpoints to stay within HW watchpoint budget. Ran the same burst pattern. ALL THREE armed watchpoints fired within the first burst wave. Full verbatim capture in this session's transcript; key facts:\n - All three watched job_submit_data_t->project_name fields transitioned from a plausible-looking heap string pointer (e.g. 'Old value = 0x7fffe000bc60 \"0\\275\"') to exactly NULL (0x0) -- i.e., we caught a real WRITE, but a NULL-out, not the resource-double corruption pattern itself.\n - Root cause of that write, nailed by backtrace: the write instruction was __memset_avx2_unaligned_erms, called from libCapnC_Runtime.so's capn_init_mem()/init_fp(), called from xce_decode() (lib/request.c:103) -- i.e. Cap'n Proto's own message-decode-buffer zero-initialization for a BRAND NEW INCOMING NETWORK REQUEST, executing on a COMPLETELY DIFFERENT thread (LWP 580496, decoding its own client's array submission) than the thread that had originally cloned the watched job_submit_data_t (LWP 580499, decoding a DIFFERENT client's array submission, job_array pointer 0x7fffe0000cf0 vs 0x7fffe4002fe0).\n - This is DEFINITIVE PROOF of the corruption mechanism: a job_submit_data_t's heap chunk, after being freed, gets immediately recycled by glibc malloc to serve an UNRELATED incoming request's Cap'n Proto decode buffer on a different thread. Any stale pointer still referencing the old (freed) job_submit_data_t will, from that point on, read whatever the CURRENT occupant of that memory happens to contain -- first zeros (as caught here), but on a LATER reuse cycle for something that stores a raw IEEE-754 double (e.g. an xce_alloc_t's `value` field, src/include/xce/types.h:270), it would read exactly the 2^30/1GiB bit pattern seen in both the original and this session's crash.\n - IMPORTANT CAVEAT / what remains UNRESOLVED: the free() itself, in this specific capture, was almost certainly LEGITIMATE, not a bug by itself. Verified by static reading of driver.c's xce_sched_driver_translate_pending_events(): within one drain/translate call, xce_core_push_events(core, events, n) (driver.c:555, which runs every core_build_job() call for the whole batch) unconditionally executes BEFORE the cleanup loop that calls release_translated_payload()->free_Job_ptr()->free_JobSubmitData_ptr() (driver.c:585-588) -- ownership/free ordering within a single batch is correct by inspection. sched_on_job_array() (handler.c) also does not push a client's array to the scheduler until its ENTIRE 50-element clone loop completes, so the specific object we watched (armed at clone call #31, i.e. mid-loop) could only have been freed AFTER its owning thread's full loop finished, was submitted as one combined XCE_SCHED_OP_UPDATE_JOB request, and was drained+processed+freed by an EARLIER, already-completed scheduling round -- i.e. a normal, successful lifecycle, not an early/premature free caught in the act. The actual bug is therefore NOT in this ownership-transfer/free ordering itself, but in some OTHER, not-yet-identified code path that reads a job_submit_data_t (or the containing xce_job_t) a SECOND time, AFTER it has already been correctly processed and freed once. Did not have time this session to isolate that second/stale reader. Two concrete leads for the next session: (1) whether a redelivered/retried array push can race sched_on_job_array()'s per-element `sched_get_job(&(sched->core), jobid) != NULL` dedup check (handler.c ~212) against the window before a just-submitted job is actually added to sched->jobs (which only happens later, inside core_build_job()/sched_add_job(), not at submit time) -- if so, two independent xce_job_t/job_submit_data_t clones could exist for the same jobid, and it's worth checking whether anything then conflates or double-references them; (2) core_build_schedobj() (core_events.c:174-197, called from core_build_job() at core_events.c:253 with the ORIGINAL `j` still valid) does `obj->project = makeString(j->submit->project_name)` -- a SECOND independent read of j->submit->project_name beyond the sched_get_project() call at core_events.c:221 that actually crashes; worth double-checking core_build_schedobj/its 3 tag-dispatch helpers for any raw pointer-aliasing (vs. deep copy) into the resulting long-lived xce_sched_job_t/xce_sched_obj_t that could keep referencing freed submit-owned memory after `j` itself is gone.\n\nBoth clusters deployed this session (/tmp/xce-deploy-bug1714-asan2, /tmp/xce-deploy-bug1714-dbg2) were torn down cleanly after their captures; the crash/watchpoint logs remain on disk at /tmp/xce-deploy-bug1714-asan2/logs/scheduler_asan.log and /tmp/xce-deploy-bug1714-dbg2/logs/scheduler_gdb_watchpoint.log for reference (not committed, local /tmp only -- copy out if this environment gets reset before a follow-up session).", - "fix": "STILL NOT FIXED -- but no longer blocked on evidence collection; see root_cause UPDATE above for the first captured backtrace. NEXT STEP: find what writes garbage into job_submit_data_t->project_name after clone_job_submit_data() allocates it -- likely somewhere in the array-job batching/event-translation path (xce_core_push_events / xce_sched_driver_translate_pending_events) given the crash hit specifically while processing event index 2 of a 6-event batch under heavy concurrent array-job load. Reproduction recipe confirmed reliable enough to use again: debug-profile build, xce_scheduler launched via `gdb -batch -ex run -ex \"thread apply all bt full\"` from process start (script preserved at /tmp/xce-deploy-bug1731/scripts/start-scheduler-gdb.sh), 6 concurrent 50-element array submissions against a live 3-node HA cluster -- crashed within 150s this run.\n\nUPDATE (2026-08-29): STILL NOT FIXED. This session's two-phase live-repro (ASAN + gdb watchpoint, see root_cause UPDATE) upgrades the diagnosis from 'read-site backtrace only' to 'corruption mechanism proven' (freed job_submit_data_t memory gets recycled by malloc for unrelated Cap'n Proto decode buffers on other threads; any surviving stale pointer to the old object then reads garbage matching whatever now occupies that memory) but the actual SOURCE of the stale pointer/second-reader is still not isolated. NEXT STEP, sharpened from this session's two leads: (1) audit sched_on_job_array()'s per-element dedup check (handler.c ~212) for a race window against a redelivered/retried array push (job not yet added to sched->jobs at submit time, only later inside core_build_job()/sched_add_job()); (2) audit core_build_schedobj() and its 3 tag-dispatch helpers (core_events.c:110-197) for any raw pointer stored into the long-lived xce_sched_obj_t/xce_sched_job_t that aliases (rather than deep-copies via makeString()) memory owned by the short-lived original xce_job_t/job_submit_data_t, which release_translated_payload() frees shortly after core_build_job() returns.\n\nUPDATE (2026-08-29, two side-bugs found during this investigation FIXED -- bug-1714's OWN root cause remains UNFIXED, see this session's root_cause UPDATE above and the still-open leads there): while chasing this bug's stale-pointer UAF, static code reading (not live repro) surfaced two SEPARATE, independently-confirmed defects, real bugs on their own merits, not proven to BE this bug's own root cause. Both now fixed with TDD (RED->GREEN) and full regression evidence.\n\nBug A -- xce_core_push_events()'s JOBS_PUSH case (src/scheduler/src/core_events.c, the XCE_CORE_EVENT_JOBS_PUSH switch case) never checked sched_add_job()'s return value. sched_add_job() (core.c:219) is a thin wrapper around key_add(core->jobs, jobid, job), which returns a negative errno (duplicate-jobid class failure) if the jobid already exists -- previously silently discarded, so a freshly-built xce_sched_job_t that FAILED to register in core->jobs was still handed to picker->picker_iface->on_job_push() anyway: an orphaned job object the picker now tracks but that sched_get_job()/any other core->jobs-keyed lookup can never find again. FIX: check sched_add_job()'s return value; on failure, skip the on_job_push() call, log a WARN (matching this case's existing WARN convention for the job==NULL branch), and free the orphaned job at the same granularity core_build_job()'s own early-failure paths already use (FREEUP(job->resreq); FREEUP(job->schedobj); FREEUP(job);) -- job->user/queue/project deliberately left alone (kvmgr-owned get-or-create lookups, not owned by this job instance); job->allocs/accounts deliberately left alone too (out of scope -- xce_sched_job_t has no complete destructor anywhere in this codebase; kill_sched_job() in schedobj.c only frees job->rusages, a genuine pre-existing gap this fix does not attempt to close). TDD: RED -- added test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak to src/scheduler/testsuite/test_core_events.c (pushes one job successfully, baseline on_job_push count 1, then pushes a second JOBS_PUSH batch with the SAME jobid); on unfixed code this failed with 'Expected 1 Was 2' (on_job_push fired a second time for the orphaned duplicate), confirming the bug. GREEN -- after the fix, the same test passes: on_job_push count stays at 1, and sched_get_job() for that jobid still returns the ORIGINAL job object, not the duplicate.\n\nBug B -- sched_on_job_array()'s per-element dedup check (src/scheduler/src/handler.c ~line 179-226) has a TOCTOU race window: it runs on the network request-handling thread and checks sched_get_job() against core->jobs, but a job is only actually inserted into core->jobs later, when the separate DRIVER thread drains the pending-update queue and calls xce_core_push_events() -> JOBS_PUSH -> sched_add_job(). A redelivered (e.g. network-retried) copy of the same array push arriving on the request thread during that window is not caught by this check, so a second, independent set of xce_job_t/job_submit_data_t clones for the same jobids gets constructed and enqueued too. Per this session's analysis, eliminating the race itself (e.g. a new synchronized pending-jobids set) was deliberately NOT attempted -- out of scope, bigger architectural change, own risk -- because it is not actually necessary: once Bug A is fixed, this race is SAFE rather than dangerous, since the redelivered duplicate is now correctly and safely rejected at the authoritative uniqueness boundary (sched_add_job()'s key_add()), freed, and never reaches the picker. FIX: documentation only, no logic change -- added a comment on the dedup check explaining the race window, that it is a best-effort optimization (not a correctness boundary), and that sched_add_job()'s key_add() (now correctly handled per Bug A) is the actual authoritative dedup boundary a redelivered duplicate ultimately hits. No test needed (no behavior change); covered indirectly by Bug A's test, which models exactly the duplicate-jobid scenario this race can produce.\n\nFull regression (nix develop ./src --command bash src/tools/xce-build.sh scheduler asan --incremental): 54/55 scheduler test suites pass. The one remaining failure, test_recordstart_symlink_escape_replies_error in test_admin_request.c (expected 'invalid record file name', got 'permission denied accessing record file'), is the already-known, pre-existing bug-1561-adjacent sandbox filesystem-permissions flake, unrelated to and unaffected by this change -- present identically before these fixes were applied (also seen in the RED run). IMPORTANT: these two fixes are scoped, standalone correctness fixes for real, independently-confirmed defects -- they do NOT resolve bug-1714's own root cause (the stale-pointer UAF into a freed job_submit_data_t, see root_cause above). That remains open; the two still-open leads listed in this entry's prior fix UPDATE (2026-08-29) are what a future session should pursue next, now narrowed since lead (1) [sched_on_job_array()'s dedup race] has been read closely enough to conclude it is not itself hazardous post-Bug-A -- lead (2) [core_build_schedobj()'s makeString()-vs-alias handling] remains the live suspect.", + "root_cause": "NOT ROOT-CAUSED -- confirmed real (kernel dmesg segfault line, error code 4 = user-mode read fault, not a random/unrelated event) but no core dump or backtrace exists for any xce_scheduler crash observed so far, including the Aug 25 19:21:39 kernel-confirmed occurrence and today's (Aug 26, ~18:00-18:02) restart (PID 3964918 -> 4137745), which may or may not be the SAME underlying bug -- the two occurrences have not been correlated beyond both being unexplained scheduler deaths. Notable secondary finding: systemd-coredump is confirmed working for xce_controller (4 real core.xce_controller.*.zst files captured and used earlier this session, see bug-1699) but has captured ZERO xce_scheduler crashes despite at least one kernel-confirmed scheduler segfault -- needs investigation on the user's real machine (check core_pattern/ulimit -c/systemd-coredump size or process-name filtering specifically for the scheduler binary; also worth checking whether the scheduler's own launch wrapper -- start-scheduler.sh, a bare unwrapped invocation with no core-dump-enabling ulimit call -- inherits a different ulimit -c than however the controller happens to get its unlimited setting). FOLLOW-UP (2026-08-26, disambiguating today's restart from the Aug-25 segfault): checked dmesg for today's exact restart window (17:45-18:15) -- zero xce_scheduler entries (the ONLY dmesg segfault hit remains the single Aug 25 19:21:39 occurrence). journalctl for the same window -- zero xce_scheduler entries at all. dmesg confirms OOM killer was never actually invoked against any process in this window (only periodic OOM-killer enable/disable toggling, unrelated). ulimit -c is unlimited and core_pattern correctly routes to systemd-coredump (confirmed working for xce_controller, 4 real cores captured this session). The combination of (a) unlimited core-dump capacity + correct core_pattern, (b) zero kernel fault-signal record, (c) zero OOM record, (d) zero journalctl record for today's specific window strongly indicates today's PID change (3964918 -> 4137745) was NOT a crash of any kind -- a genuine fault signal (SIGSEGV/SIGABRT/SIGBUS/SIGILL/SIGFPE) is unconditionally kernel-logged regardless of which process triggers it, and none was. The scheduler is launched as a bare backgrounded process (start-scheduler.sh, no systemd unit), so an ordinary kill/pkill sent by an unrelated concurrent process on this shared machine would leave NO trace in dmesg or journalctl -- most likely explanation for today's specific restart, DECOUPLED from this bug going forward. This buglog entry now tracks ONLY the confirmed, kernel-verified Aug 25 19:21:39 xce_scheduler segfault (still unreproduced, still no core available, still worth investigating on recurrence) -- not today's restart, which is environmental noise, not a code bug.\n\nUPDATE (2026-08-29, bug-1731 live gdb-wrapped repro session): CRASH CAPTURED LIVE, first-ever backtrace for this crash family. Root cause of the earlier no-core-dump gap circumvented by launching xce_scheduler wrapped in `gdb -batch -ex run` FROM PROCESS START (not attach-after-the-fact) during a heavy burst-array workload (6 concurrent `jobsub -a 50` array submissions, 300 near-instant /bin/true elements, on a live 3-node HA cluster, debug build profile). Reproduced at T+150s into the burst. Full backtrace:\nSIGSEGV in sched_get_project(core, name=0x41d0000000000000) at scheduler/src/core.c:412, called from core_build_job() at scheduler/src/core_events.c:221, called from xce_core_push_events() at scheduler/src/core_events.c:725 (crashed on the 3rd of a 6-event batch), called from xce_sched_driver_translate_pending_events() (driver.c:555) <- xce_sched_driver_run_normal_mode_round() (driver.c:749) <- main() (main.c:465). Registers: rax=rdx=rsi=0x41d0000000000000 (corrupted `name` arg present in 3 registers), rdi=0x555555695da0 (valid `core` arg). Single-thread fault in the main scheduling-round thread (all 9 other threads idle in pthread_cond_wait/epoll_wait) -- not a cross-thread race at the fault site itself, though the corruption's origin is not yet located.\ncore_build_job() (core_events.c:221) calls sched_get_project(core, j->submit->project_name). The two preceding calls in the same function (sched_get_user, sched_get_queue, reading earlier fields of the same job_submit_data_t*) succeeded, so j->submit itself is a valid pointer -- but its project_name field held 0x41d0000000000000, not NULL and not a plausible string pointer. clone_job_submit_data() (handler.c:102-150) calloc()s the destination and only strdup()s project_name when the source is non-NULL, so a legitimately-absent project name would be NULL (which sched_get_project's own guard at core.c:412 handles safely) -- this crash requires the field to have been overwritten with garbage AFTER allocation. Points to memory corruption or a stale/reused pointer in the array-job event-translation/batching path under heavy concurrent load, not a missing-null-check. NOT YET further isolated -- the corruption's write site is still unknown. Fault class (SIGSEGV, invalid-pointer read via strlen()) matches the original kernel dmesg description (`error 4` = user-mode read fault) closely enough that this is plausibly the same crash family, now finally attributable to an exact function/line/call-chain instead of an unattributed dmesg line.\n\nUPDATE (2026-08-29, post-fix live-repro session -- bug-1730/1731/1733 fixes now all landed and included): two-phase reproduction attempt, both phases succeeded.\n\nPHASE 1 (ASAN, fresh clean builds of controller/scheduler/worker/xceadmin including commits 42eeaa1e[bug-1733 vtaskmgr UAF fix], b1323f8d[bug-1730], ebe691c3+67009815[bug-1731]): deployed fresh 3-node HA cluster (/tmp/xce-deploy-bug1714-asan2), launched xce_scheduler directly under ASAN (ASAN_OPTIONS=detect_leaks=0:halt_on_error=1:abort_on_error=1), ran a 10x50-element concurrent array-job burst (500 elements, wave 1 of a planned two-wave/1000-element burst). CRASH REPRODUCED WITHIN WAVE 1 ITSELF -- before all 10 clients' submissions even completed, after only 2 of 10 array pushes had been expanded by the scheduler (100 of the intended 500 elements). This is FASTER than the original bug-1731-session capture (T+150s) and definitively proves the bug is STILL PRESENT after all three recent fixes, not masked or perturbed by ASAN instrumentation. Verbatim ASAN report:\n AddressSanitizer:DEADLYSIGNAL\n ==562488==ERROR: AddressSanitizer: SEGV on unknown address (pc 0x64b0be11076e ...)\n ==562488==The signal is caused by a READ memory access.\n #0 sched_get_project ../scheduler/src/core.c:412\n #1 core_build_job ../scheduler/src/core_events.c:221\n #2 xce_core_push_events ../scheduler/src/core_events.c:725\n #3 xce_sched_driver_translate_pending_events ../scheduler/src/driver.c:555\n #4 xce_sched_driver_run_normal_mode_round ../scheduler/src/driver.c:749\n #5 main ../scheduler/src/main.c:465\n Register values: rax=rdx approx corrupted; rax = 0x41d0000000000000 (still exactly 2^30=1073741824.0 as an IEEE-754 double, matching the original capture bit-for-bit) -- IDENTICAL crash signature, file, line, and call chain to the original bug-1731-session capture. SUMMARY: AddressSanitizer: SEGV core.c:412 in sched_get_project.\n Diagnostically important: ASAN's own heap-buffer-overflow and use-after-free shadow-memory detectors did NOT fire -- only the raw SIGSEGV/DEADLYSIGNAL handler caught this, exactly as a non-ASAN build would. This RULES OUT a classic redzone-violating heap overflow into an adjacent allocation (ASAN would have caught that at the WRITE) and argues for either (a) a wrong-offset/type-confusion write within a still-valid allocation, or (b) a use-after-free where the stale read happens only after the freed chunk has already left ASAN's quarantine and been legitimately reallocated for a new, unrelated live object -- which Phase 2 then confirmed directly.\n\nPHASE 2 (gdb scripted watchpoint, debug -O0 -g3 profile, same fix set, fresh cluster /tmp/xce-deploy-bug1714-dbg2): launched xce_scheduler under `gdb -batch -x scheduler-watchpoint.gdb`, breaking on clone_job_submit_data() (handler.c:102, called once per array element from make_array_element), sampling every 31st call and arming a hardware watchpoint on the returned job_submit_data_t's ->project_name field (via `finish` + cast of $rax to the known x86-64 System V pointer-return-value register), capped at 3 simultaneous watchpoints to stay within HW watchpoint budget. Ran the same burst pattern. ALL THREE armed watchpoints fired within the first burst wave. Full verbatim capture in this session's transcript; key facts:\n - All three watched job_submit_data_t->project_name fields transitioned from a plausible-looking heap string pointer (e.g. 'Old value = 0x7fffe000bc60 \"0\\275\"') to exactly NULL (0x0) -- i.e., we caught a real WRITE, but a NULL-out, not the resource-double corruption pattern itself.\n - Root cause of that write, nailed by backtrace: the write instruction was __memset_avx2_unaligned_erms, called from libCapnC_Runtime.so's capn_init_mem()/init_fp(), called from xce_decode() (lib/request.c:103) -- i.e. Cap'n Proto's own message-decode-buffer zero-initialization for a BRAND NEW INCOMING NETWORK REQUEST, executing on a COMPLETELY DIFFERENT thread (LWP 580496, decoding its own client's array submission) than the thread that had originally cloned the watched job_submit_data_t (LWP 580499, decoding a DIFFERENT client's array submission, job_array pointer 0x7fffe0000cf0 vs 0x7fffe4002fe0).\n - This is DEFINITIVE PROOF of the corruption mechanism: a job_submit_data_t's heap chunk, after being freed, gets immediately recycled by glibc malloc to serve an UNRELATED incoming request's Cap'n Proto decode buffer on a different thread. Any stale pointer still referencing the old (freed) job_submit_data_t will, from that point on, read whatever the CURRENT occupant of that memory happens to contain -- first zeros (as caught here), but on a LATER reuse cycle for something that stores a raw IEEE-754 double (e.g. an xce_alloc_t's `value` field, src/include/xce/types.h:270), it would read exactly the 2^30/1GiB bit pattern seen in both the original and this session's crash.\n - IMPORTANT CAVEAT / what remains UNRESOLVED: the free() itself, in this specific capture, was almost certainly LEGITIMATE, not a bug by itself. Verified by static reading of driver.c's xce_sched_driver_translate_pending_events(): within one drain/translate call, xce_core_push_events(core, events, n) (driver.c:555, which runs every core_build_job() call for the whole batch) unconditionally executes BEFORE the cleanup loop that calls release_translated_payload()->free_Job_ptr()->free_JobSubmitData_ptr() (driver.c:585-588) -- ownership/free ordering within a single batch is correct by inspection. sched_on_job_array() (handler.c) also does not push a client's array to the scheduler until its ENTIRE 50-element clone loop completes, so the specific object we watched (armed at clone call #31, i.e. mid-loop) could only have been freed AFTER its owning thread's full loop finished, was submitted as one combined XCE_SCHED_OP_UPDATE_JOB request, and was drained+processed+freed by an EARLIER, already-completed scheduling round -- i.e. a normal, successful lifecycle, not an early/premature free caught in the act. The actual bug is therefore NOT in this ownership-transfer/free ordering itself, but in some OTHER, not-yet-identified code path that reads a job_submit_data_t (or the containing xce_job_t) a SECOND time, AFTER it has already been correctly processed and freed once. Did not have time this session to isolate that second/stale reader. Two concrete leads for the next session: (1) whether a redelivered/retried array push can race sched_on_job_array()'s per-element `sched_get_job(&(sched->core), jobid) != NULL` dedup check (handler.c ~212) against the window before a just-submitted job is actually added to sched->jobs (which only happens later, inside core_build_job()/sched_add_job(), not at submit time) -- if so, two independent xce_job_t/job_submit_data_t clones could exist for the same jobid, and it's worth checking whether anything then conflates or double-references them; (2) core_build_schedobj() (core_events.c:174-197, called from core_build_job() at core_events.c:253 with the ORIGINAL `j` still valid) does `obj->project = makeString(j->submit->project_name)` -- a SECOND independent read of j->submit->project_name beyond the sched_get_project() call at core_events.c:221 that actually crashes; worth double-checking core_build_schedobj/its 3 tag-dispatch helpers for any raw pointer-aliasing (vs. deep copy) into the resulting long-lived xce_sched_job_t/xce_sched_obj_t that could keep referencing freed submit-owned memory after `j` itself is gone.\n\nBoth clusters deployed this session (/tmp/xce-deploy-bug1714-asan2, /tmp/xce-deploy-bug1714-dbg2) were torn down cleanly after their captures; the crash/watchpoint logs remain on disk at /tmp/xce-deploy-bug1714-asan2/logs/scheduler_asan.log and /tmp/xce-deploy-bug1714-dbg2/logs/scheduler_gdb_watchpoint.log for reference (not committed, local /tmp only -- copy out if this environment gets reset before a follow-up session).\n\nUPDATE (2026-08-30, Part 1/Part 2 investigation session): PART 1 (static-only, XceData_data_schedjobs dataset->rows aliasing) -- RULED OUT, not a bug. Traced the full decode chain for the schedjobs/jobs table (src/lib/xcectx.c:139 xce_decode() -> src/lib/request.c:1786 extract_data() -> src/lib/controller.capnp.c decode_Job_list()/decode_Job_ptr() (line 6348/6369) -> decode_JobSubmitData_ptr() (line 12427) -> decode_JobSubmitData() (line 12223)). Confirmed every xce_job_t (calloc, decode_Job_ptr/decode_Job_list) and every job_submit_data_t (calloc, decode_JobSubmitData_ptr) is an independent heap allocation, and every job_submit_data_t string field uses STRING_DUP (= strdup, controller.capnp.h:9) -- structurally identical to clone_job_submit_data()/decode_pidinfo(). extract_data() (request.c:1786-1815) only memcpy's the already-decoded xce_job_t* POINTER VALUES into dataset->rows (then memsets the source array to prevent double-free) -- it never aliases into the capnp segment/request buffer, and this happens entirely INSIDE xce_decode(), before capn_free(&c) at request.c:281 destroys the wire buffer. So sched_handle_dataset()'s XceData_data_schedjobs case handing dataset->rows to sched_on_jobs() for deferred driver-thread processing is a correct, safe zero-copy ownership transfer of already-independent objects, not a second UAF class. The XceData_data_jobArray case's contrasting comment ('we never alias... unlike the list-shaped cases above') is about a different concern (N-way sharing of one spec across array elements), not capnp-buffer lifetime -- both cases are equally safe on that specific axis.\n\nPART 2 (live pointer-lifecycle tracking, main task) -- did NOT capture a fresh crash this session despite extensive effort (see below), but surfaced a significant, previously-undocumented, independently-reproducible finding that materially narrows the investigation and directly answers this entry's own Question 3 lead about a redelivery/retry mechanism. Full method: added temporary mutex-guarded fprintf/fflush diagnostic logging (address, jobid, tid, timestamp) at three sites -- make_array_element() (handler.c, ALLOC), every free_Job_ptr()/free_JobSubmitData_ptr() call site in driver.c's release_translated_payload() (FREE), and core_build_job() (core_events.c) immediately before the historically-crashing sched_get_project() line (READ). The shared logging implementation was placed in src/lib/utils.c (a genuine, non-generated libxce.a source file) rather than controller.capnp.c, after discovering mid-session that src/lib/controller.capnp.c is NOT actually compiled -- meson's custom_target regenerates lib/controller.capnp.c/.h from src/schema/controller.capnp fresh into the BUILD directory on every build, silently discarding any manual source-tree edit to that file (confirmed empirically: an initial edit there vanished from the compiled object with no error). Built (debug profile), deployed multiple fresh 3-node HA clusters, launched xce_scheduler under `gdb -batch -ex run -ex \"thread apply all bt full\"`, and drove very heavy concurrent `jobsub -a 50` array-job bursts -- far exceeding the original recipe's intensity: 6-15 concurrent submissions per wave, 20+ waves across several redeploys (both -v and -vvv controller verbosity), well over 100 total concurrent array submissions / 5000+ array elements pushed. The scheduler did NOT crash in any of these attempts (longest single continuous run: ~16 minutes / 5 waves without a restart), a notably higher bar than the original session's 150s/6-concurrent capture -- see 'What changed / open question' below.\n\nKEY FINDING (from the diagnostic log, cross-referenced against the scheduler's own pre-existing Bug-A WARN logging): every array-job jobid gets independently, repeatedly RE-CONSTRUCTED and pushed to the scheduler via the ORDINARY (non-array) XceData_data_schedjobs/JOBS_PUSH bulk path -- confirmed by cross-referencing FREE events against ALLOC events by jobid: every array-path jobid showed MORE free events than array-path (make_array_element) allocs, each with a DISTINCT heap address (never the same address freed twice -- ruling out a literal double-free), e.g. jobid 4658596961282883584 was array-allocated once but freed 18 times from 18 distinct addresses over a 16-minute run. Each of these extra frees is preceded by a matching WARN in scheduler.log: 'xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid ... (rc=-17), discarding duplicate/orphaned job' -- i.e. Bug A's fix (this same buglog entry, prior session) is correctly catching and safely discarding every one of these redelivered duplicates before they reach the picker. CRITICALLY, this was reproduced in a run with ZERO client-side submission failures/retries (confirmed: grep for 'jobsubmit failed'/'retry' across all burst client output files in the clean final run returned zero matches, yet the SAME redelivery pattern still occurred, up to 18x per jobid) -- this RULES OUT the client-retry-on-timeout hypothesis (XCECLI_FAILOVER_CALL's retry loop, client_internal.h) as the trigger; the redelivery is NOT a symptom of client-observed transport failure, it happens under clean, fully-acknowledged submissions too. Static reading of the two most likely controller-side mechanisms did not find an obvious culprit: (1) persist_apply_pump_run() (persist.c:5521) advances last_applied_lsn strictly one entry at a time under apply_pump_lock and only ever processes next=last_applied_lsn+1, so a genuine re-walk of an already-applied LSN range looks structurally impossible by construction; (2) apply_committed_entry()'s WAL_JOB_ARRAY_SUBMIT case (persist.c:4391-4461) explicitly fires HOOK_DETAIL_JOB_NEW exactly once per successful apply, with an EEXIST short-circuit for an idempotent whole-record re-apply. Neither explains a genuinely repeated, single-job-shaped (l->count==1, routed through xcesched_push_joblist() per sched.c:326-327's dispatch, not xcesched_push_jobarray()) HOOK_DETAIL_JOB_NEW fire for jobids that were originally created via the array path. bug-1419's in-flight submission registry (job.c:71-140, tracks [base_jobid, base_jobid+array_size) ranges) is the most likely place a gap could exist for this specific pattern but was not dynamically traced this session -- NEXT STEP for a follow-up session: either a live gdb breakpoint on sched_job_new_fun()/xcesched_push_joblist() (sched.c) capturing the call's l->count and full backtrace, or controller-side instrumentation logging out_new_jobs.count at every HOOK_DETAIL_JOB_NEW fire site, to determine whether this is a genuine second independent WAL commit for an overlapping jobid range (a real gap in bug-1419's registry or jobid_range_exists()) or some other apply-side effect not yet identified by static reading.\n\n(a) vs (b) QUESTION -- NOT SETTLED this session (no fresh crash captured to inspect). The redelivery-storm finding above establishes a plausible AMPLIFYING mechanism (far more job_submit_data_t alloc/free churn under array-job load than previously suspected, all safely discarded by Bug A but still consuming and freeing large volumes of same-shaped heap chunks under concurrent capnp-decode pressure on other threads) but does not by itself prove a stale-pointer read, since every redelivered object observed in this session's log was a freshly-decoded, valid object at its own READ time (not a stale/freed one) -- the two prior sessions' watchpoint-proven UAF mechanism (freed job_submit_data_t memory reused by malloc for an unrelated concurrent capnp decode buffer) still requires a genuinely stale SECOND reference somewhere, which this session did not catch live.\n\nWhat changed / open question: this session's burst intensity substantially exceeded the recipe that reliably crashed within 150s-2.5min in the two prior sessions (same debug build profile, same gdb wrapper, same jobsub -a 50 command), yet produced zero crashes across ~20+ waves / 100+ concurrent submissions / 5000+ array elements over multiple redeploys. Possible explanations, none confirmed: (1) the two prior sessions' side-bug fixes (bug-1730's core_events.c fix, bug-1731's HOOK_DETAIL_JOB_FAIL fix, and this entry's own Bug A/Bug B, all landed 2026-08-29 per git log -- 'fix(scheduler): don't hand an orphaned duplicate job to the picker' at commit 38437e57 among them) may have shifted timing enough to reduce (not eliminate) the race window; (2) this session's burst driver script (piped `xce_test --norc` REPL commands, one process per concurrent submission) may differ subtly in timing/pacing from whatever exact mechanics the original crash-capturing sessions used; (3) simple bad luck across a large but still finite number of attempts for a genuinely rare race. All temporary diagnostic instrumentation (handler.c, driver.c, core_events.c, src/lib/utils.c) was fully reverted before this session ended -- confirmed via `git diff` returning clean/identical-to-HEAD for all four tracked files, plus a manual re-check of the untracked/regenerated src/lib/controller.capnp.c (git-invisible, confirmed no leftover bug1714_diag_log references anywhere under src/). Full scheduler ASAN test suite re-run after revert: 54/55 pass, the one failure (test_recordstart_symlink_escape_replies_error) is the already-documented pre-existing sandbox filesystem-permissions flake, unrelated to this session. All deployed clusters (3x) torn down; zero xce_controller/xce_scheduler/xcecd_virtual/gdb processes left running. NEXT STEP for a follow-up session: (a) pursue the redelivery-storm's exact trigger per the NEXT STEP above (this is now the most concrete, evidence-backed lead); (b) if reproduction is attempted again, consider a substantially longer single continuous run (30+ min) rather than many shorter redeploys, since this session's longest unbroken run was only ~16 minutes and the original captures may have benefited from cumulative state a fresh redeploy resets each time.\n\nUPDATE (2026-08-30, later same day, large-ASAN-quarantine live-repro session): NEW APPROACH TRIED, NOT YET DOCUMENTED IN THIS ENTRY -- launched the ALREADY-BUILT ASAN xce_scheduler (src/xce.build/scheduler/xce_scheduler, built 2026-08-30 11:36, already including every fix through commit 22aa16bf/38437e57 -- no rebuild needed since ASAN_OPTIONS is a runtime env var, not compile-time) with a deliberately oversized quarantine/redzone (ASAN_OPTIONS=quarantine_size_mb=256:redzone=256:max_redzone=2048:detect_leaks=0:abort_on_error=1:halt_on_error=1) specifically to test whether the documented \"freed block reused too fast for ASAN's default quarantine\" explanation for plain-ASAN's prior silence (see the 2026-08-29 post-fix update above) would resolve with a much larger quarantine. Deployed a controller-only + scheduler 3-node HA cluster (NO worker -- confirmed by static reading of the crash call chain, in this same entry, that sched_get_project()/core_build_job()/xce_core_push_events() all fire at JOBS_PUSH submission-ingestion time, before any picker/allocator/worker involvement, so a worker is not required to trigger this specific crash; xcecd_virtual was also unavailable to build this session, see environmental note below, making this a load-bearing simplification, not just a shortcut). Drove a sustained, unthrottled burst-submission loop (8 concurrent `client`+`jobsub -u root -q normal -c hostname -a 50` xce_test invocations per wave, back-to-back waves, ~1-8s apart) for ~8 minutes continuous wall clock, reaching 64 waves = 25,600 array elements pushed (verified via a final controller `joblist` count of 25,756 PEND jobs, consistent with no jobs lost at the controller level) -- substantially higher volume and, without a worker providing any execution-time backpressure, likely higher submission RATE than any prior session's recipe (the original crash-capturing sessions used 6 concurrent x 50 = 300 elements in 150s; this session pushed 25,600 in ~480s, roughly an order of magnitude more elements per unit time). RESULT: the scheduler did NOT crash and produced ZERO AddressSanitizer reports of any kind (grep -ci AddressSanitizer scheduler.log == 0) across the full run. ADDITIONAL NEGATIVE FINDING: the 2026-08-30-morning session's \"redelivery-storm\" (array-job jobids redelivered via the ordinary JOBS_PUSH path, discarded by Bug A with a \"discarding duplicate/orphaned job\" WARN, observed up to 18x per jobid that session) did NOT reproduce AT ALL this run either -- zero occurrences of that WARN string anywhere in 94,794 scheduler.log lines, despite comparable-or-higher load. This is a second, independent \"previously-reliable symptom now absent\" data point in addition to bug-1714's own crash, on the same day, suggesting SOMETHING about either the exact submission-driver mechanics (this session used direct `client`+`jobsub` xce_test invocations; the morning session used a different piped-REPL driver script whose exact mechanics were not preserved) or genuine environment/timing variance materially changes whether either symptom fires -- not just bad luck on the crash alone. CAVEAT (important, do not skip): a controller build from an EARLIER point in this same work session (commit 38437e57 onward, built 2026-08-30 03:21) and a scheduler build from 11:36 the same day were reused as-is (not rebuilt) for this attempt -- both already post-date every fix this entry and its siblings (bug-1730/1731/1733) describe, so this is a like-for-like continuation of the \"post-fix\" investigation, not a regression to older code.\n\nENVIRONMENTAL INCIDENT (important for future sessions on this sandbox): this session began by finding the root filesystem at 0 bytes available (936.8G/936.8G used), which blocked ALL work (builds, deploys, even log writes) until resolved. `nix-collect-garbage` (no -d) was run to reclaim space, which worked (freed ~790GB, root cause was almost certainly months of accumulated `nix develop` closures across many past sessions' worth of ASAN toolchains, never garbage-collected) but had a serious side effect: it evicted this flake's own cached bootstrap-toolchain closure (gnum4/bison/gmp/mpfr/isl/libmpc/autoconf/a full multi-stage GCC bootstrap), forcing EVERY subsequent `nix develop ./src` invocation on this box -- including a DIFFERENT, concurrently-running agent's own controller build for an unrelated bug (bug-1735) -- into a full from-source toolchain rebuild. This cost roughly 15-20 minutes of wall clock before ANY build (mine or the other agent's) could proceed, and materially ate into this session's time budget. LESSON for next time: if disk space must be reclaimed on this box again, prefer `nix-store --gc --print-dead` / a bounded `nix-collect-garbage --max-freed ` first to gauge what will be evicted, since this flake's own dev-shell closure is apparently NOT protected by a persistent gcroot between sessions and is fully re-buildable-from-scratch (not just re-fetchable) when evicted -- a full unbounded `nix-collect-garbage` is disproportionately expensive here. Once resolved, this session worked around not having a fresh full build by (a) reusing the ALREADY-BUILT src/xce.build/{controller,scheduler} binaries directly (their /nix/store runtime deps were, fortunately, NOT evicted -- `ldd` still resolved cleanly) with the new ASAN_OPTIONS applied purely at launch time, and (b) building ONLY the missing `xceadmin` component fresh (into a separate dir, src/xce.build.bug1714xa) since a cluster.bin config cannot be produced without it -- this minimal-rebuild strategy is worth reusing directly if this situation recurs, rather than a full `xce-build.sh all`.\n\nbug-1734 was NOT reproduced this session: its repro recipe needs a worker to actually run jobs to DONE/EXIT so the scheduler-vs-controller view can be compared; no `xcecd_virtual` binary exists in any build dir on this box currently (checked src/xce.build, src/xce.build.validate, src/build-asan, src/build-asan-1696repro -- only worker *test* binaries exist, no `xcecd_virtual` itself), and this session's bounded time budget (already consumed by the disk/nix incident above) did not allow building `worker` fresh on top of the already-spent time. See bug-1734's own entry for a same-day cross-reference note.", + "fix": "STILL NOT FIXED -- but no longer blocked on evidence collection; see root_cause UPDATE above for the first captured backtrace. NEXT STEP: find what writes garbage into job_submit_data_t->project_name after clone_job_submit_data() allocates it -- likely somewhere in the array-job batching/event-translation path (xce_core_push_events / xce_sched_driver_translate_pending_events) given the crash hit specifically while processing event index 2 of a 6-event batch under heavy concurrent array-job load. Reproduction recipe confirmed reliable enough to use again: debug-profile build, xce_scheduler launched via `gdb -batch -ex run -ex \"thread apply all bt full\"` from process start (script preserved at /tmp/xce-deploy-bug1731/scripts/start-scheduler-gdb.sh), 6 concurrent 50-element array submissions against a live 3-node HA cluster -- crashed within 150s this run.\n\nUPDATE (2026-08-29): STILL NOT FIXED. This session's two-phase live-repro (ASAN + gdb watchpoint, see root_cause UPDATE) upgrades the diagnosis from 'read-site backtrace only' to 'corruption mechanism proven' (freed job_submit_data_t memory gets recycled by malloc for unrelated Cap'n Proto decode buffers on other threads; any surviving stale pointer to the old object then reads garbage matching whatever now occupies that memory) but the actual SOURCE of the stale pointer/second-reader is still not isolated. NEXT STEP, sharpened from this session's two leads: (1) audit sched_on_job_array()'s per-element dedup check (handler.c ~212) for a race window against a redelivered/retried array push (job not yet added to sched->jobs at submit time, only later inside core_build_job()/sched_add_job()); (2) audit core_build_schedobj() and its 3 tag-dispatch helpers (core_events.c:110-197) for any raw pointer stored into the long-lived xce_sched_obj_t/xce_sched_job_t that aliases (rather than deep-copies via makeString()) memory owned by the short-lived original xce_job_t/job_submit_data_t, which release_translated_payload() frees shortly after core_build_job() returns.\n\nUPDATE (2026-08-29, two side-bugs found during this investigation FIXED -- bug-1714's OWN root cause remains UNFIXED, see this session's root_cause UPDATE above and the still-open leads there): while chasing this bug's stale-pointer UAF, static code reading (not live repro) surfaced two SEPARATE, independently-confirmed defects, real bugs on their own merits, not proven to BE this bug's own root cause. Both now fixed with TDD (RED->GREEN) and full regression evidence.\n\nBug A -- xce_core_push_events()'s JOBS_PUSH case (src/scheduler/src/core_events.c, the XCE_CORE_EVENT_JOBS_PUSH switch case) never checked sched_add_job()'s return value. sched_add_job() (core.c:219) is a thin wrapper around key_add(core->jobs, jobid, job), which returns a negative errno (duplicate-jobid class failure) if the jobid already exists -- previously silently discarded, so a freshly-built xce_sched_job_t that FAILED to register in core->jobs was still handed to picker->picker_iface->on_job_push() anyway: an orphaned job object the picker now tracks but that sched_get_job()/any other core->jobs-keyed lookup can never find again. FIX: check sched_add_job()'s return value; on failure, skip the on_job_push() call, log a WARN (matching this case's existing WARN convention for the job==NULL branch), and free the orphaned job at the same granularity core_build_job()'s own early-failure paths already use (FREEUP(job->resreq); FREEUP(job->schedobj); FREEUP(job);) -- job->user/queue/project deliberately left alone (kvmgr-owned get-or-create lookups, not owned by this job instance); job->allocs/accounts deliberately left alone too (out of scope -- xce_sched_job_t has no complete destructor anywhere in this codebase; kill_sched_job() in schedobj.c only frees job->rusages, a genuine pre-existing gap this fix does not attempt to close). TDD: RED -- added test_jobs_push_duplicate_jobid_does_not_call_on_job_push_or_leak to src/scheduler/testsuite/test_core_events.c (pushes one job successfully, baseline on_job_push count 1, then pushes a second JOBS_PUSH batch with the SAME jobid); on unfixed code this failed with 'Expected 1 Was 2' (on_job_push fired a second time for the orphaned duplicate), confirming the bug. GREEN -- after the fix, the same test passes: on_job_push count stays at 1, and sched_get_job() for that jobid still returns the ORIGINAL job object, not the duplicate.\n\nBug B -- sched_on_job_array()'s per-element dedup check (src/scheduler/src/handler.c ~line 179-226) has a TOCTOU race window: it runs on the network request-handling thread and checks sched_get_job() against core->jobs, but a job is only actually inserted into core->jobs later, when the separate DRIVER thread drains the pending-update queue and calls xce_core_push_events() -> JOBS_PUSH -> sched_add_job(). A redelivered (e.g. network-retried) copy of the same array push arriving on the request thread during that window is not caught by this check, so a second, independent set of xce_job_t/job_submit_data_t clones for the same jobids gets constructed and enqueued too. Per this session's analysis, eliminating the race itself (e.g. a new synchronized pending-jobids set) was deliberately NOT attempted -- out of scope, bigger architectural change, own risk -- because it is not actually necessary: once Bug A is fixed, this race is SAFE rather than dangerous, since the redelivered duplicate is now correctly and safely rejected at the authoritative uniqueness boundary (sched_add_job()'s key_add()), freed, and never reaches the picker. FIX: documentation only, no logic change -- added a comment on the dedup check explaining the race window, that it is a best-effort optimization (not a correctness boundary), and that sched_add_job()'s key_add() (now correctly handled per Bug A) is the actual authoritative dedup boundary a redelivered duplicate ultimately hits. No test needed (no behavior change); covered indirectly by Bug A's test, which models exactly the duplicate-jobid scenario this race can produce.\n\nFull regression (nix develop ./src --command bash src/tools/xce-build.sh scheduler asan --incremental): 54/55 scheduler test suites pass. The one remaining failure, test_recordstart_symlink_escape_replies_error in test_admin_request.c (expected 'invalid record file name', got 'permission denied accessing record file'), is the already-known, pre-existing bug-1561-adjacent sandbox filesystem-permissions flake, unrelated to and unaffected by this change -- present identically before these fixes were applied (also seen in the RED run). IMPORTANT: these two fixes are scoped, standalone correctness fixes for real, independently-confirmed defects -- they do NOT resolve bug-1714's own root cause (the stale-pointer UAF into a freed job_submit_data_t, see root_cause above). That remains open; the two still-open leads listed in this entry's prior fix UPDATE (2026-08-29) are what a future session should pursue next, now narrowed since lead (1) [sched_on_job_array()'s dedup race] has been read closely enough to conclude it is not itself hazardous post-Bug-A -- lead (2) [core_build_schedobj()'s makeString()-vs-alias handling] remains the live suspect.\n\nUPDATE (2026-08-30): STILL NOT FIXED -- no fix attempted this session (per this entry's own 'do not attempt a speculative fix without a confirmed root cause' guidance). This session's Part 1 static audit ruled out the XceData_data_schedjobs dataset->rows aliasing as a second UAF class (see root_cause UPDATE). Part 2's live pointer-lifecycle instrumentation did not capture a fresh crash despite ~20+ burst waves substantially exceeding the historically-successful recipe's intensity, but did surface a new, real, independently-reproducible finding: array-job jobids get redelivered to the scheduler via the ordinary single-job XceData_data_schedjobs/JOBS_PUSH path many times (up to 18x observed) after their original array-path submission, confirmed unrelated to client-side retries (reproduced with zero client submission failures). Bug A (this same entry, prior session) correctly discards every one of these before they reach the picker, so this redelivery is not itself a proven crash cause, but it is a strong candidate AMPLIFYING factor for the previously-proven malloc-reuse race (more job_submit_data_t churn = more opportunities to hit the race) and a concrete, scoped lead for a follow-up session (see root_cause UPDATE's 'NEXT STEP'). All temporary diagnostic instrumentation was fully reverted (verified via git diff + full re-run of the scheduler ASAN test suite: 54/55, same pre-existing flake as before). No code changes remain in the working tree from this session.\n\nUPDATE (2026-08-30, later same day): STILL NOT FIXED, STILL NOT REPRODUCED. Tried the large-ASAN-quarantine idea this entry's own root_cause section flagged as untried (quarantine_size_mb=256:redzone=256:max_redzone=2048, launched via already-built binaries, no rebuild) against a controller+scheduler-only cluster (no worker -- confirmed unnecessary for this specific crash's call chain, which fires entirely at JOBS_PUSH submission-ingestion time). 64 waves / 25,600 array elements over ~8 minutes, an order of magnitude more throughput than the original 150s/300-element capture: zero crash, zero ASAN reports of any kind, and (new, separate negative finding) zero recurrence of the morning session's reliably-observed redelivery-storm WARN either. Ruling nothing new out beyond what's already documented -- this is a THIRD consecutive session-day attempt (after the 08-29 post-fix session and the 08-30-morning session) where heavy live load did not reproduce a symptom that DID reproduce reliably in the two original capturing sessions (bug-1731's live-repro session and this entry's own 08-29 post-fix Phase-1/Phase-2 session), now across three different techniques (plain heavy load, gdb-wrapped heavy load with diagnostic instrumentation, and now large-quarantine ASAN). NEXT STEP suggestion, sharpened by three failed-to-reproduce sessions in a row: the two ORIGINAL successful captures both happened on 2026-08-29, before commit 38437e57 (Bug A) and its sibling Bug B documentation landed the same day, and EVERY session since (including this one) has been AFTER those fixes landed. While Bug A/B were carefully reasoned to be independent of bug-1714's own root cause (see this entry's history), the fact that reproduction rate appears to have dropped sharply coincident with their landing -- both the crash itself AND the previously-reliable redelivery-storm symptom now failing to reproduce under comparable or heavier load -- is a pattern worth taking seriously rather than dismissing as noise. Concrete suggestion for a follow-up session: (a) try reverting JUST commit 38437e57 (Bug A's fix) in a throwaway branch/build and re-run this exact large-quarantine-ASAN recipe against pre-Bug-A code, to directly test whether Bug A's fix (or the TOCTOU race it interacts with) was itself part of what made the race window practically hittable, rather than assuming independence holds; (b) if pre-Bug-A code DOES still crash under this recipe while post-Bug-A does not, that would suggest Bug A's fix (checking sched_add_job()'s return and discarding the duplicate instead of silently handing it to the picker) also incidentally narrows or closes bug-1714's own race window as a side effect worth understanding precisely rather than just accepting as an unexplained improvement. Both this session's build artifacts (src/xce.build.bug1714xa, a from-scratch xceadmin build with wheel+venv) were left in place (not cleaned up, unlike the temp deploy dir and processes) since they represent real, reusable build work with no live-process footprint.", "tags": [ "scheduler", "segfault", @@ -19513,14 +19515,19 @@ "uaf-mechanism-confirmed", "malloc-reuse-confirmed", "watchpoint-captured", - "side-bugs-fixed" + "side-bugs-fixed", + "dataset-schedjobs-ruled-out", + "redelivery-storm-found", + "live-repro-not-captured-2026-08-30", + "large-quarantine-asan-tried-no-repro-2026-08-30", + "redelivery-storm-also-not-reproduced-2026-08-30" ], "related_bugs": [ "bug-1699", "bug-1731" ], - "occurrences": 4, - "last_seen": "2026-08-29T15:05:00Z" + "occurrences": 6, + "last_seen": "2026-08-30T14:00:00Z" }, { "id": "bug-1715", @@ -19973,6 +19980,104 @@ ], "occurrences": 2, "last_seen": "2026-08-29T07:23:24Z" + }, + { + "id": "bug-1734", + "timestamp": "2026-08-30T10:15:00Z", + "error_message": "User-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` / xce_scheduler -i -s) diverges sharply from the controller's own joblist for the SAME array, well after every element genuinely finished. Array 0-4-1788084587-0 (50 elements, command `hostname`): controller's `joblist` shows ALL 50 elements as DONE with real, monotonically-increasing StartTime/FinishTime timestamps (1788084587..1788084602) -- the array is genuinely, fully complete. The scheduler's own concurrent view of the identical array shows a scrambled mix: [0-4]=RUN, [5-16]=DONE, [17]=EXIT, [18-19]=DONE, [20]=EXIT, [21]=DONE, [22-24]=EXIT, [25]=DONE, [26-29]=EXIT, [30-46]=RUN, [47-49]=PEND. Two things about this pattern are themselves suspicious, not just 'the scheduler is behind': (a) elements [30-46] show RUN and [47-49] show PEND, i.e. the scheduler believes 20 elements haven't even STARTED yet, even though the controller's timestamps show the whole array finished by 1788084602 -- this is not a small lag, the scheduler's view is drastically behind reality for over a third of the array; (b) EXIT appears interspersed with DONE for a uniform array of a single reliably-successful command (`hostname`, not `/bin/false` or anything failure-prone) -- elements 17/20/22-24/26-29 show EXIT while their immediate neighbors show DONE, which is not what a simple 'scheduler hasn't caught up yet' lag would produce (a lagging-but-otherwise-correct view should show a clean PEND/RUN prefix behind a DONE suffix, not DONE and EXIT interleaved for what should be uniform outcomes). CORRELATED evidence from the SAME run: xce_scheduler's own log shows 10 consecutive WARN lines at the very same timeframe -- `xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), discarding duplicate/orphaned job` for jobid 4631575359520768000 through 4631575359520768009 -- which decode (JobID bit layout: tag/cluster/apiserver/timestamp/array_idx) to EXACTLY array elements 0-4-1788084587-0[0] through [9] of this SAME array. This WARN is emitted by the bug-1714-adjacent fix landed earlier this session (commit 38437e57, 'fix(scheduler): don't hand an orphaned duplicate job to the picker') -- confirms live, in this exact reproduction, that this array's first 10 elements were pushed to the scheduler TWICE (the TOCTOU dedup race in sched_on_job_array()/handler.c that commit documented as theoretically possible is firing for real here), and the second copy is now being correctly discarded rather than corrupting state or crashing (that part of the fix is working as designed). User also separately requested (logged here as part of the same report, lower priority, not a correctness bug): xce_core_push_events's WARN line above prints the raw numeric jobid (`4631575359520768000`) instead of this project's human-readable fmt_jobid() form (`0-4-1788084587-0[0]`) that every other job-related log line in this codebase uses -- a logging-usability gap in that specific WARN, worth a one-line fix.", + "file": "src/scheduler/src/handler.c (sched_on_job_array's dedup-race, already documented as a known TOCTOU race by commit 38437e57), src/scheduler/src/core_events.c (xce_core_push_events, WARN log site + the JOBS_PUSH duplicate-discard path from commit 38437e57), src/scheduler/src/driver.c / src/controller/sched.c / src/controller/persist.c (the job-update notify pipeline this session's bug-1731/bug-1714 investigations already covers)", + "root_cause": "NOT YET ROOT-CAUSED -- logged for follow-up investigation per user request, not investigated live this session (a separate, still-running background investigation was mid-flight on bug-1714's own stale-reader when this was reported). Working hypothesis, NOT confirmed: this looks like more than one mechanism compounding. (1) The already-known, already-partially-mitigated sched_on_job_array() dedup race (see bug-1714's own root_cause history and commit 38437e57's message) is CONFIRMED firing live here (the WARN evidence decodes to exactly this array's first 10 elements) -- but discarding a genuine duplicate PUSH should be harmless on its own (both copies start from identical fresh JOB_STAT_PEND state; sched_add_job()'s key_add() correctly keeps the first copy), so this alone should not explain elements staying stuck at RUN/PEND when the controller confirms DONE. (2) The DONE/EXIT interleaving for a uniform, reliably-successful command (hostname) suggests either an ordering bug in how terminal-state notifications get applied (an older notification overwriting a newer one, or vice versa, for different elements) -- which would tie into bug-1731's own notify-drop/redelivery investigation and its live-captured evidence of scheduler-endpoint-NULL notify drops -- or a distinct bug in how JOB_STATE_CHANGE vs JOB_UPDATE detail routing interacts for array elements specifically (both hook-detail paths were touched this session by bug-1731's fixes; worth checking whether an array element's terminal notification can race between the two). (3) The ~20 elements the scheduler shows as never having started (RUN/PEND for [30-49]) despite controller-confirmed completion by 1788084602 is the most severe and least explained part of this report -- if genuinely not just a stale UI snapshot, this suggests SOME elements' notifications are being lost entirely, not just delayed, distinct from anything bug-1731's fix addressed (bug-1731 fixed the CONTROLLER never firing the right hook; this could be a SEPARATE loss further downstream, e.g. in delivery to the scheduler, or in how the scheduler's REPL/inspect view is rendered from its own internal state). NOT RULED OUT: this could also be nothing more than a snapshot-timing artifact if `[sim] jobs`/`joblist` were captured at genuinely different moments and the scheduler was still catching up -- but the user's own framing ('still have some issues') and the DONE/EXIT interleaving for a uniform command argue against pure timing lag being the whole explanation. Needs live reproduction with synchronized capture (query both views at the same instant, ideally with sequence numbers/timestamps on the query itself) to distinguish 'genuinely stuck' from 'was mid-catch-up when sampled'.\n\nADDITIONAL CONFIRMATION (same report, user follow-up): xce_test itself (the submitting client) also reports every job in this array finished successfully -- i.e. all three independent vantage points agree the work genuinely completed (worker execution, controller's own joblist/WAL state, and the submitting client's own completion tracking). This rules out worker-side execution failure and controller-side completion-tracking failure as explanations, narrowing this specifically to the scheduler-side notification delivery/application path (or, less likely, the scheduler's own REPL/inspect rendering of otherwise-correct internal state) -- consistent with, but not identical to, bug-1731's already-fixed dead-hook gap and its still-open notify-drop-on-disconnect finding.", + "fix": "NOT FIXED -- not yet investigated beyond this initial evidence capture, logged per explicit user request to defer to a later session. RECOMMENDED NEXT STEPS, in order of expected signal: (1) Reproduce with synchronized before/after captures of both `[sim] jobs` (scheduler) and `joblist` (controller) for the same array, timestamped precisely, to rule in/out pure snapshot-timing lag as the explanation for the [30-49] never-started-per-scheduler symptom. (2) If genuinely stuck (not just lagging), check whether the DEBUG3-gated diagnostic TRACE lines already added this session (job.c's update_job_rusages, sched.c's sched_job_update_fun, both from bug-1731's investigation) fire for the specific stuck elements' jobids -- if the TRACE never fires for them at all, the loss is upstream of the scheduler-notify layer (controller-side); if it fires but the scheduler's view still doesn't update, the loss is scheduler-side (delivery or apply). (3) Investigate whether the dedup-race discard path (commit 38437e57) has any interaction with the SPECIFIC elements that show the DONE/EXIT interleaving -- decode the jobids in scheduler.log's other DIAG/WARN lines around this same timeframe to see if the interleaved-EXIT elements correlate with duplicate-discard events too, the way [0-9] did here. (4) Small, separately-scoped fix for the human-readable-jobid logging request: xce_core_push_events's WARN in core_events.c (added by commit 38437e57) should use fmt_jobid() (already used throughout sched.c/handler.c/scheduler.c for jobid formatting in logs) instead of a raw %llu -- trivial, no TDD complexity, safe to land independently of the rest of this investigation.\n\nUPDATE (2026-08-30, later same day): NOT ATTEMPTED this session (documenting the gap rather than leaving it silent). Live repro of THIS entry specifically needs a worker to actually run array elements to DONE/EXIT so the scheduler-vs-controller views can diverge and be compared -- no `xcecd_virtual` binary exists in any build directory currently on this box (checked src/xce.build, src/xce.build.validate, src/build-asan, src/build-asan-1696repro -- only worker *unit test* binaries are present, never the daemon itself), and this session's time budget was consumed by an unrelated disk-space/nix-environment incident (see bug-1714's same-day update) plus a bug-1714-focused live-ASAN repro attempt, leaving no bounded time remaining to build `worker` fresh and pursue this entry's own recommended synchronized-capture plan. NEXT SESSION: build `worker` (`/xce-build worker`) alongside controller/scheduler before attempting this entry's Step 1 (synchronized `[sim] jobs` vs `joblist` capture) -- everything else in this entry's own recommended-next-steps list still stands unchanged.", + "tags": [ + "scheduler", + "controller", + "job-state-sync", + "state-desync", + "found-live", + "array-job", + "notify-loss-suspected", + "ordering-suspected", + "unfixed", + "unreproduced-with-sync-capture", + "needs-owner", + "session-sibling-of-bug-1714", + "session-sibling-of-bug-1731", + "correlated-with-dedup-race-warn", + "human-readable-jobid-logging-gap", + "low-priority-logging-followup", + "not-attempted-no-worker-binary-2026-08-30" + ], + "related_bugs": [ + "bug-1714", + "bug-1731", + "bug-1733" + ], + "occurrences": 2, + "last_seen": "2026-08-30T14:00:00Z" + }, + { + "id": "bug-1735", + "timestamp": "2026-08-30T10:30:00Z", + "error_message": "PRIORITY: HIGH -- affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note below). Split 2026-08-30 from a broader initial report into this (core usage: memory + CPU time) and bug-1736 (display-only statistics: swap, task count, peak/efficiency percentages) at user request, since the two have very different urgency and the broader one should not block this one.\n\nUser-reported: job memory usage always shows 0 in the scheduler's simulation record file (.replay), despite the worker daemon presumably measuring real resource usage. User asked to check whether the worker even reports this data at all -- CONFIRMED: it does, but the value never survives to the controller due to a plain resource-name string mismatch, plus a second, independent gap in how the controller aggregates whatever it does receive.", + "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages, line ~1418-1436 -- the worker-side cgroup-based measurement and resname assignment), src/controller/misc.c (update_host_runusage, line ~128-138 -- the controller-side parser), src/controller/job.c (update_job_rusages, line ~3200 -- pre-existing 'FIXME update other fields of job rusage later' comment), src/scheduler/include/sched/types.h:115 (xce_sched_job_t.rusages -- confirmed the scheduler stores/forwards the RAW xce_job_rusages_t wire report unmodified, does not itself re-parse or aggregate it), src/scheduler/src/recording_events.c:142 (wire.body.job_rusage = ev->u.job_rusage.rusages -- confirms the .replay file records whatever raw resname/value pairs it received, unchanged)", + "root_cause": "TWO CONFIRMED, INDEPENDENT GAPS -- either alone would produce 'memory usage always 0'; both are real. Neither was live-verified this session (no real cgroup-backed xcecd deployment was exercised -- see the environment-limitation note below), but both are confirmed by direct code reading with exact citations, and there is NO translation/mapping layer anywhere between the two mismatched naming conventions (grepped the whole worker/controller/lib/scheduler source tree for both string sets -- only the two sites below reference them at all).\n\nFINDING 1 (the primary, most likely explanation): resource-name string mismatch between producer and consumer. collect_job_rusages() (xcecd_cgroups.c, the REAL, cgroup/systemd-backed worker's rusage collector -- queries a job's own 'xcecd-worker-.service' systemd unit via sd_bus_open_system()) reports exactly 4 resnames: 'cpu_time_sec', 'memory_rss', 'memory_swap', 'tasks_current'. update_host_runusage() (misc.c, the controller-side parser that receives this over the wire) only recognizes the LITERAL strings 'cputime' and 'mem' -- neither of which matches ANY of the 4 names the worker actually sends. Every rusage report from the real worker therefore falls through both `if (strcmp(...))` checks doing nothing, silently, for every single field -- host_runusage[j]->cpu_time and ->mem_usage stay at their calloc-zeroed default FOREVER, regardless of what the worker's cgroup query actually measured. No error, no log, no indication anything was dropped -- this is why it reads as 'always 0' rather than an obvious failure.\n\nFINDING 2 (independent, compounding): even if Finding 1 were fixed and host_runusage[]'s PER-HOST fields started populating correctly, the TOP-LEVEL AGGREGATE fields of xce_job_runusage_t (max_mem, avg_mem, used_mem, cpu_time, peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) are NEVER computed from the per-host breakdown anywhere -- update_job_rusages() (job.c) calls update_host_runusage() (which only touches host_runusage[]) then has an explicit, still-open '/* FIXME update other fields of job rusage later */' comment immediately after, with no code implementing that FIXME anywhere in this codebase (grepped). Any downstream consumer that reads the AGGREGATE fields (rather than iterating host_runusage[] itself) would see 0 regardless of Finding 1's fix.\n\nSCHEDULER-SIDE STRUCTURE NOTE (relevant context, not itself a bug): the scheduler does NOT receive or store the aggregated xce_job_runusage_t at all -- xce_sched_job_t.rusages (sched/types.h:115) is typed xce_job_rusages_t*, the SAME flat wire-report struct the worker originally sent (jobid/npids/pids/count/usages[] of raw {resname,value} pairs) -- forwarded/stored as-is, and recording_events.c's capnp encode for the .replay file writes that same raw struct unchanged. So the .replay file the user is inspecting genuinely contains whatever resname/value pairs the worker sent, byte for byte -- if Finding 1 is fixed (or the worker already sent matching names), the RAW data WOULD flow all the way into the recording faithfully. Grepped src/scheduler/src/repl*.c for any resname-aware rendering of memory usage in the interactive REPL/inspect commands and found NONE -- could not determine within this session's time budget whether the specific tool/view the user used to inspect the .replay file (unclear from the report whether it's the live REPL's own display, a separate dump/analysis tool, or something else) has ITS OWN resname expectation that might differ yet again from both 'mem' and 'memory_rss' -- flagged as an open question for whoever picks this up, not confirmed either way.\n\nENVIRONMENT LIMITATION NOTE: this session's own live burst-testing (bug-1714 investigation) exclusively used xcecd_virtual (the no-root virtual worker), never the real, cgroup/systemd-backed xcecd binary collect_job_rusages() lives in -- this sandbox has no D-Bus/systemd (already confirmed in this session's bug-1731 follow-up work: get_systemd_job_exit_code() always returns -1 here), so sd_bus_open_system() at collect_job_rusages()'s own first real step would itself fail immediately in this environment (returns NULL before ever reaching the resname assignment code) -- this bug could not have been, and was not, live-reproduced in this sandbox; it is a pure static-code finding, applicable specifically to the user's own real deployment (which does have real cgroups/systemd, matching the user's own 'live deployment' framing throughout this investigation).\n\nPRIORITY NOTE (2026-08-30, user request): this half of the original finding -- memory (memory_rss/'mem') and CPU time (cpu_time_sec/'cputime') specifically -- is the HIGH-PRIORITY half. These are exactly the two metrics xce_host_runusage_t already has dedicated fields for (cpu_time, mem_usage) and the two the controller's parser already has matching (if misspelled) if-branches for -- clearly the originally-intended 'core' usage metrics, as opposed to the swap/task-count/peak/efficiency fields split out to bug-1736, which have no controller-side field or computation logic at all today. IMPORTANT CAVEAT, checked directly this session: grepped the full scheduler source tree for any consumer of rusage/usage data in scheduling/picker/allocation logic and found NONE -- xce_sched_job_t.rusages (sched/types.h:115) is stored and forwarded into recordings but not read by any picker plugin or allocation code today. So this fix corrects the DATA PIPELINE (worker measurement -> controller aggregation -> scheduler storage/recording), the necessary prerequisite for any future usage-aware scheduling policy (e.g. backfill sizing, over-commit detection, preemption on real memory pressure) -- it does not itself wire usage data into a scheduling decision, since no such consumer exists yet. Marked high priority on that basis (unblocks correct data for future scheduling work, and is also simply a correctness bug in its own right for any operator-facing usage reporting), not because it changes scheduling behavior today.", + "fix": "NOT FIXED -- logged for follow-up, scope narrowed 2026-08-30 to memory + CPU time only (see bug-1736 for the split-off lower-priority statistics). RECOMMENDED FIX, Finding 1 (resname mismatch): align 'cpu_time_sec'/'memory_rss' (worker, xcecd_cgroups.c) with 'cputime'/'mem' (controller, misc.c) -- pick a direction (see the two options in the original finding, not repeated here) and apply consistently. RECOMMENDED FIX, Finding 2 (aggregation): implement update_job_rusages()'s (job.c) still-open FIXME specifically for max_mem/avg_mem/used_mem/cpu_time -- derive from host_runusage[]'s per-host entries each time a report merges in (max/avg/sum per field semantics). Both need TDD (unit tests, not just a live check) plus live verification against a REAL cgroup-backed xcecd deployment -- this session's sandbox has no D-Bus/systemd and cannot provide that verification itself (see the original environment-limitation note).", + "tags": [ + "worker", + "controller", + "resource-usage", + "rusage", + "memory-tracking", + "resname-mismatch", + "cgroups", + "xcecd_cgroups", + "unfixed", + "found-via-user-report", + "static-analysis-only", + "unreproduced-in-sandbox", + "no-dbus-systemd-in-sandbox", + "needs-real-deployment-verification", + "needs-owner", + "session-sibling-of-bug-1734", + "priority-high", + "split-2026-08-30", + "core-usage-metrics-only" + ], + "related_bugs": [ + "bug-1734", + "bug-1736" + ], + "occurrences": 1, + "last_seen": "2026-08-30T10:45:00Z" + }, + { + "id": "bug-1736", + "timestamp": "2026-08-30T10:45:00Z", + "error_message": "PRIORITY: LOW -- display/reporting only, not consumed by any scheduling logic. Split 2026-08-30 from bug-1735 at user request: the 'statistics-only' half of that finding -- resource usage fields with no scheduling relevance, purely for operator-facing job accounting/reporting (bjobs -l/bhist-style views), as opposed to bug-1735's memory+CPU-time core, which is a prerequisite for future scheduling decisions. Worker (collect_job_rusages(), xcecd_cgroups.c) already measures and sends 'memory_swap' and 'tasks_current' per job, but the controller has NO field to receive them at all (xce_host_runusage_t only has cpu_time/mem_usage -- no swap or task-count field exists) -- these two are silently dropped entirely, not just misnamed. Separately, xce_job_runusage_t's peak/efficiency fields (peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) have no computation logic anywhere in the codebase (grepped) -- nothing populates them, ever, regardless of bug-1735's fix.", + "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages -- already sends memory_swap/tasks_current, no controller-side field to receive them), src/include/xce/types.h (xce_host_runusage_t -- would need new fields for swap/task-count; xce_job_runusage_t -- the peak/efficiency fields with no producer), src/controller/misc.c (update_host_runusage -- would need new parse branches), src/controller/job.c (update_job_rusages -- same open FIXME as bug-1735, this is the other half of what it would need to cover)", + "root_cause": "See bug-1735's root_cause for the full original investigation (resname-mismatch mechanism, environment-limitation note re: this session's sandbox having no real cgroup-backed worker to live-verify against) -- this entry covers only the subset of that finding split out as lower-priority. Unlike bug-1735's memory/cpu_time (which have existing-but-mismatched fields), memory_swap/tasks_current have NO receiving field in xce_host_runusage_t at all -- the worker's measurement is a complete dead end on the controller side, not a naming mismatch. The peak/efficiency fields (xce_job_runusage_t's own top-level fields) are simply never computed by anything, anywhere -- confirmed via grep, no partial implementation exists to build on.", + "fix": "NOT FIXED -- low priority, no scheduling impact identified, logged for whenever operator-facing job-accounting/reporting detail becomes a priority. RECOMMENDED SCOPE if picked up: (1) extend xce_host_runusage_t with swap/task-count fields and wire update_host_runusage() to parse 'memory_swap'/'tasks_current' alongside bug-1735's mem/cputime fix; (2) design and implement the peak/efficiency computations (peak_cpu_time/peak_cpu_duration would need tracking the running max across reports, not just the latest value; mem_efficiency/avg_cpu_efficiency/peak_cpu_efficiency need a defined formula against requested resources that doesn't exist yet -- this is closer to a small design task than a bug fix, unlike bug-1735's two clean, well-scoped corrections).", + "tags": [ + "worker", + "controller", + "resource-usage", + "rusage", + "display-only", + "reporting-only", + "priority-low", + "split-2026-08-30", + "no-scheduling-impact", + "unfixed", + "static-analysis-only", + "unreproduced-in-sandbox", + "needs-owner", + "needs-design-not-just-fix" + ], + "related_bugs": [ + "bug-1735" + ], + "occurrences": 1, + "last_seen": "2026-08-30T10:45:00Z" } ] -} +} \ No newline at end of file diff --git a/.wolf/memory.md b/.wolf/memory.md index 4d545b6c..da033a48 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -3123,3 +3123,57 @@ | 20:20 | Edited src/controller/config_repl.c | modified struct() | ~134 | | 20:20 | Edited src/controller/config_repl.c | 4→4 lines | ~46 | | 20:20 | Edited src/controller/testsuite/test_config_repl.c | modified cr_setup() | ~54 | +| 02:20 | Created reports/aug23-aug29-2026.md | — | ~3656 | +| 02:21 | Edited reports/aug23-aug29-2026.md | 1→2 lines | ~198 | +| 02:21 | Edited reports/aug23-aug29-2026.md | inline fix | ~101 | +| 02:21 | Edited reports/aug23-aug29-2026.md | inline fix | ~72 | +| 03:09 | Edited src/scheduler/src/handler.c | added 1 condition(s) | ~552 | +| 03:10 | Edited src/scheduler/src/handler.c | modified if() | ~116 | +| 03:10 | Edited src/lib/controller.capnp.c | expanded (+6 lines) | ~129 | +| 03:10 | Edited src/lib/controller.capnp.c | modified free_JobSubmitData_ptr() | ~118 | +| 03:10 | Edited src/scheduler/src/core_events.c | expanded (+11 lines) | ~198 | +| 03:11 | Edited src/scheduler/src/core_events.c | modified diagnostic() | ~244 | +| 03:14 | Edited src/scheduler/src/handler.c | reduced (-20 lines) | ~314 | +| 03:14 | Edited src/scheduler/src/handler.c | 3→2 lines | ~10 | +| 03:14 | Edited src/lib/controller.capnp.c | added 1 condition(s) | ~561 | +| 03:15 | Edited src/scheduler/src/handler.c | 2→2 lines | ~40 | +| 03:19 | Edited src/lib/controller.capnp.c | removed 51 lines | ~35 | +| 03:20 | Edited src/lib/utils.c | added 1 condition(s) | ~670 | +| 03:20 | Edited src/scheduler/src/handler.c | 17→17 lines | ~314 | +| 03:20 | Edited src/scheduler/src/driver.c | expanded (+7 lines) | ~132 | +| 03:20 | Edited src/scheduler/src/driver.c | added 1 condition(s) | ~205 | +| 03:21 | Edited src/scheduler/src/driver.c | added 1 condition(s) | ~135 | +| 03:39 | Edited src/lib/utils.c | added 1 condition(s) | ~377 | +| 04:33 | Edited src/scheduler/src/core_events.c | removed 16 lines | ~29 | +| 04:33 | Edited src/scheduler/src/core_events.c | reduced (-10 lines) | ~52 | +| 04:36 | Edited src/lib/controller.capnp.c | modified free_JobSubmitData_ptr() | ~35 | +| 05:30 | Session end: 71 writes across 22 files (job.c, sched.c, test_vtask_enq_race.c, meson.build, test.c) | 73 reads | ~69911 tok | + +## Session: 2026-08-30 05:30 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 05:40 | Edited src/controller/include/cluster.h | 3→4 lines | ~66 | +| 05:41 | Created src/controller/testsuite/test_host_runusage.c | — | ~2270 | +| 05:41 | Edited src/controller/meson.build | expanded (+24 lines) | ~327 | +| 05:47 | Edited src/controller/misc.c | added 4 condition(s) | ~586 | +| 05:47 | Edited src/controller/job.c | 10→11 lines | ~138 | +| 05:49 | Edited src/controller/testsuite/test.c | 4→7 lines | ~86 | +| 06:16 | Edited src/controller/testsuite/test_commit_barrier_reply.c | modified update_pgids() | ~42 | +| 06:25 | Edited src/controller/testsuite/test_host_runusage.c | expanded (+11 lines) | ~260 | +| 06:26 | Edited src/controller/testsuite/test_host_runusage.c | TEST_ASSERT_EQUAL_DOUBLE_MESSAGE() → ASSERT_DOUBLE_EQ_MSG() | ~62 | +| 06:26 | Edited src/controller/testsuite/test_host_runusage.c | TEST_ASSERT_EQUAL_DOUBLE() → ASSERT_DOUBLE_EQ() | ~36 | +| 06:26 | Edited src/controller/testsuite/test_host_runusage.c | TEST_ASSERT_EQUAL_DOUBLE() → ASSERT_DOUBLE_EQ() | ~45 | +| 06:26 | Edited src/controller/testsuite/test_host_runusage.c | TEST_ASSERT_EQUAL_DOUBLE_MESSAGE() → ASSERT_DOUBLE_EQ_MSG() | ~95 | +| 06:26 | Edited src/controller/testsuite/test_host_runusage.c | inline fix | ~12 | + +## Session: 2026-08-30 06:28 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 06:42 | Created .claude/skills/xce-changes/SKILL.md | — | ~2869 | +| 06:48 | Created CHANGES.md | — | ~5924 | +| 06:58 | Edited CHANGES.md | modified off() | ~3537 | +| 08:03 | Edited src/tools/xce-build.sh | modified 924() | ~444 | +| 08:04 | Edited src/tools/xce-build.sh | 3→5 lines | ~23 | +| 08:06 | Edited .kiro/specs/scheduler-snapshot-job-allocation/spec.json | 3→3 lines | ~28 |