From 25778ee8a2f1dd6524d543515d6ea600520d9316 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Sat, 29 Aug 2026 11:29:23 +0000 Subject: [PATCH] docs: sync OpenWolf state for bug-1730/1733/1731 investigation Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_019QxaSzHRB7pmYFp4cVnwkA --- .wolf/anatomy-index.json | 504 +++++++++++++++++++++++++++++++++++++-- .wolf/anatomy.md | 74 +++++- .wolf/buglog.json | 271 ++++++++++++++++++++- .wolf/memory.md | 136 +++++++++++ 4 files changed, 948 insertions(+), 37 deletions(-) diff --git a/.wolf/anatomy-index.json b/.wolf/anatomy-index.json index 6d9d9e32..ac77c057 100644 --- a/.wolf/anatomy-index.json +++ b/.wolf/anatomy-index.json @@ -1,13 +1,13 @@ { "version": 1, "meta": { - "lastScanned": "2026-08-28T05:12:31.323Z", - "fileCount": 507, + "lastScanned": "2026-08-29T09:04:17.242Z", + "fileCount": 538, "hits": 0, "misses": 0, - "renderedHash": "be1690fc464dd0edbd5f78cdbbd688f2a7a07e6bcbdde3efc68fa89a47e7210b", - "storeUpdatedAt": "2026-08-28T05:12:31.332Z", - "rootHash": "0eca8ae935c845b1" + "renderedHash": "995babd82e796eff0c8bbf7080b294d1bec36d670435362804d2501052109ed0", + "storeUpdatedAt": "2026-08-29T09:04:17.250Z", + "rootHash": "0107489d3ff2c703" }, "files": { ".agent-shell/transcripts/2026-08-11-11-39-59.md": { @@ -422,13 +422,12 @@ }, ".kiro/specs/admin-config-tool/design.md": { "description": "Technical Design: admin-config-tool", - "tokens": 13515, - "hash": "c0bbf00563786b85", - "size": 54684, - "mtimeMs": 1785507539000, - "updatedAt": "2026-08-27T19:00:00.054Z", - "source": "scan", - "importance": 0.21 + "tokens": 13800, + "hash": "6ecaf6b114410700", + "size": 55833, + "mtimeMs": 1787902994953.2322, + "updatedAt": "2026-08-28T07:43:14.998Z", + "source": "hook" }, ".kiro/specs/admin-config-tool/requirements.md": { "description": "Requirements Document", @@ -3341,13 +3340,12 @@ }, ".kiro/steering/roadmap.md": { "description": "Roadmap", - "tokens": 15324, - "hash": "8ab4780b1d77d4ed", - "size": 61757, - "mtimeMs": 1787815056430.2776, - "updatedAt": "2026-08-27T19:00:00.136Z", - "source": "scan", - "importance": 0.21 + "tokens": 15320, + "hash": "5fdc4e8d0973de92", + "size": 61740, + "mtimeMs": 1787903021394.5676, + "updatedAt": "2026-08-28T07:43:41.441Z", + "source": "hook" }, ".kiro/steering/scheduler-driver-equivalence.md": { "description": "Scheduler Driver Equivalence", @@ -6014,11 +6012,11 @@ }, "src/scheduler/src/snapshot_load.c": { "description": "include ", - "tokens": 5573, - "hash": "b55b62f95438f057", - "size": 20898, - "mtimeMs": 1787890430318.6338, - "updatedAt": "2026-08-28T04:13:50.366Z", + "tokens": 5920, + "hash": "e29f37e83c6b6713", + "size": 22198, + "mtimeMs": 1787911493192.2705, + "updatedAt": "2026-08-28T10:04:53.240Z", "source": "hook" }, "src/scheduler/src/snapshot.c": { @@ -6029,6 +6027,464 @@ "mtimeMs": 1787893051624.2847, "updatedAt": "2026-08-28T04:57:31.668Z", "source": "hook" + }, + "src/xceadmin/meson-build.sh": { + "description": "Meson-driven xceadmin build: install + test out-of-tree, in the meson", + "tokens": 2445, + "hash": "b52180cef5ca681a", + "size": 9168, + "mtimeMs": 1787900878930.0732, + "updatedAt": "2026-08-28T07:07:58.972Z", + "source": "hook" + }, + "src/xceadmin/tests/test_e2e_live_ha_push.py": { + "description": "", + "tokens": 8887, + "hash": "e5d95f41487c639d", + "size": 31104, + "mtimeMs": 1787903632275.3694, + "updatedAt": "2026-08-28T07:53:52.313Z", + "source": "hook", + "symbols": [ + { + "name": "HaNode", + "kind": "class", + "startLine": 245, + "endLine": 251, + "tokens": 25 + }, + { + "name": "HaCluster", + "kind": "class", + "startLine": 252, + "endLine": 256, + "tokens": 19 + }, + { + "name": "_wait_for_primary", + "kind": "fn", + "startLine": 257, + "endLine": 274, + "tokens": 209 + }, + { + "name": "_wait_for_standbys", + "kind": "fn", + "startLine": 275, + "endLine": 302, + "tokens": 381 + }, + { + "name": "live_ha_cluster", + "kind": "fn", + "startLine": 303, + "endLine": 413, + "tokens": 1480 + }, + { + "name": "_build_cluster_bin", + "kind": "fn", + "startLine": 414, + "endLine": 425, + "tokens": 118 + }, + { + "name": "_assert_no_decode_crash", + "kind": "fn", + "startLine": 426, + "endLine": 444, + "tokens": 286 + }, + { + "name": "_assert_primary_committed", + "kind": "fn", + "startLine": 445, + "endLine": 464, + "tokens": 261 + }, + { + "name": "test_live_config_push_against_primary_reaches_success_path", + "kind": "fn", + "startLine": 465, + "endLine": 537, + "tokens": 1039 + }, + { + "name": "test_live_config_push_against_standby_redirects_and_succeeds", + "kind": "fn", + "startLine": 538, + "endLine": 609, + "tokens": 984 + } + ] + }, + "src/scheduler/testsuite/test_snapshot_save_load.c": { + "description": "", + "tokens": 28712, + "hash": "48ddaa21e62b753a", + "size": 107668, + "mtimeMs": 1787908872470.1384, + "updatedAt": "2026-08-28T09:21:12.513Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_startup.c": { + "description": "Declares char", + "tokens": 5708, + "hash": "3fd13240aa5a3af0", + "size": 21404, + "mtimeMs": 1787908902301.5325, + "updatedAt": "2026-08-28T09:21:42.341Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_session_ordering.c": { + "description": "", + "tokens": 8092, + "hash": "2010da22fa29c474", + "size": 30345, + "mtimeMs": 1787908989473.972, + "updatedAt": "2026-08-28T09:23:09.518Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_contention_parity.c": { + "description": "Declares calls", + "tokens": 18713, + "hash": "8160ccd11dfd948b", + "size": 70171, + "mtimeMs": 1787910165664.212, + "updatedAt": "2026-08-28T09:42:45.704Z", + "source": "hook" + }, + "src/scheduler/tools_verify_fixture.c": { + "description": "include ", + "tokens": 1550, + "hash": "b4ca3728217b5a73", + "size": 5810, + "mtimeMs": 1787911395334.8425, + "updatedAt": "2026-08-28T10:03:15.379Z", + "source": "hook" + }, + "src/scheduler/meson.build": { + "description": "", + "tokens": 1426, + "hash": "777690ff5feddf78", + "size": 5345, + "mtimeMs": 1787911519457.4392, + "updatedAt": "2026-08-28T10:05:19.494Z", + "source": "hook" + }, + "src/worker/src/uv_ipc.c": { + "description": "include \"uv_ipc.h\"", + "tokens": 12439, + "hash": "26349411920cd7f7", + "size": 46645, + "mtimeMs": 1787913057760.9097, + "updatedAt": "2026-08-28T10:30:57.814Z", + "source": "hook" + }, + "src/worker/testsuite/test_uv_ipc_job_exit_severity.c": { + "description": "Declares with", + "tokens": 863, + "hash": "5a73c47760fdc6b8", + "size": 3235, + "mtimeMs": 1787913014122.3564, + "updatedAt": "2026-08-28T10:30:14.167Z", + "source": "hook" + }, + "src/worker/testsuite/meson.build": { + "description": "Worker unit test suite (component-failover-adoption, task 2.1). White-box", + "tokens": 1380, + "hash": "5f46f8484b5aa279", + "size": 5175, + "mtimeMs": 1787912991931.566, + "updatedAt": "2026-08-28T10:29:51.973Z", + "source": "hook" + }, + ".kiro/specs/scheduler-config-version-tag/spec.json": { + "description": "", + "tokens": 131, + "hash": "104687efcabaf71b", + "size": 458, + "mtimeMs": 1787914808176.6797, + "updatedAt": "2026-08-28T11:00:08.219Z", + "source": "hook" + }, + ".kiro/specs/scheduler-config-version-tag/requirements.md": { + "description": "Requirements Document", + "tokens": 2371, + "hash": "7186b200e9939662", + "size": 9490, + "mtimeMs": 1787913868680.594, + "updatedAt": "2026-08-28T10:44:28.731Z", + "source": "hook" + }, + ".kiro/specs/scheduler-config-version-tag/research.md": { + "description": "Research & Design Decisions", + "tokens": 2711, + "hash": "8d9331b88d91308e", + "size": 10887, + "mtimeMs": 1787914220305.0403, + "updatedAt": "2026-08-28T10:50:20.352Z", + "source": "hook" + }, + ".kiro/specs/scheduler-config-version-tag/design.md": { + "description": "Design Document", + "tokens": 6413, + "hash": "bd7a98f6f26fc86f", + "size": 25777, + "mtimeMs": 1787914315764.995, + "updatedAt": "2026-08-28T10:51:55.813Z", + "source": "hook" + }, + ".kiro/specs/scheduler-config-version-tag/tasks.md": { + "description": "Implementation Plan", + "tokens": 1764, + "hash": "ffab2744dc79a119", + "size": 7064, + "mtimeMs": 1787914731794.2122, + "updatedAt": "2026-08-28T10:58:51.838Z", + "source": "hook" + }, + "src/controller/persist.c": { + "description": "define _GNU_SOURCE", + "tokens": 107488, + "hash": "9e453f8e5abe824d", + "size": 403751, + "mtimeMs": 1787990648017.7603, + "updatedAt": "2026-08-29T08:04:08.068Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_core_events.c": { + "description": "Declares and", + "tokens": 10371, + "hash": "494c0ffb7e0a7185", + "size": 38888, + "mtimeMs": 1787924309248.4097, + "updatedAt": "2026-08-28T13:38:29.298Z", + "source": "hook" + }, + "src/scheduler/src/core_events.c": { + "description": "include ", + "tokens": 11705, + "hash": "c200515f9b89116e", + "size": 43893, + "mtimeMs": 1787924333244.3672, + "updatedAt": "2026-08-28T13:38:53.280Z", + "source": "hook" + }, + "src/controller/job.c": { + "description": "include ", + "tokens": 39294, + "hash": "3c05b7d63de4207d", + "size": 147501, + "mtimeMs": 1787979541114.306, + "updatedAt": "2026-08-29T04:59:01.150Z", + "source": "hook" + }, + "src/controller/sched.c": { + "description": "include ", + "tokens": 6721, + "hash": "3613637e5df87619", + "size": 25211, + "mtimeMs": 1787979549837.7444, + "updatedAt": "2026-08-29T04:59:09.885Z", + "source": "hook" + }, + "src/controller/main.c": { + "description": "include ", + "tokens": 5489, + "hash": "fecdcb3202376803", + "size": 20594, + "mtimeMs": 1787928397591.543, + "updatedAt": "2026-08-28T14:46:37.639Z", + "source": "hook" + }, + "src/scheduler/src/main.c": { + "description": "include ", + "tokens": 5330, + "hash": "46c890f07e394311", + "size": 19984, + "mtimeMs": 1787928433129.828, + "updatedAt": "2026-08-28T14:47:13.174Z", + "source": "hook" + }, + "src/lib/testsuite/test_vtask_enq_race.c": { + "description": "Declares under", + "tokens": 1430, + "hash": "7b147a32b91c1b6c", + "size": 5375, + "mtimeMs": 1787987724632.4092, + "updatedAt": "2026-08-29T07:15:24.674Z", + "source": "hook" + }, + "src/lib/meson.build": { + "description": "", + "tokens": 1757, + "hash": "eaefdf03aaca6d3d", + "size": 6588, + "mtimeMs": 1787985733006.7087, + "updatedAt": "2026-08-29T06:42:13.039Z", + "source": "hook" + }, + "src/lib/testsuite/test.c": { + "description": "include ", + "tokens": 1279, + "hash": "64a7da6be607287e", + "size": 4793, + "mtimeMs": 1787985735215.0815, + "updatedAt": "2026-08-29T06:42:15.260Z", + "source": "hook" + }, + "src/lib/vtaskmgr.c": { + "description": "include ", + "tokens": 3978, + "hash": "8150c77d93be408b", + "size": 14930, + "mtimeMs": 1787987830192.2258, + "updatedAt": "2026-08-29T07:17:10.237Z", + "source": "hook" + }, + "src/controller/testsuite/test_wal_job_finish_fact.c": { + "description": "Declares as", + "tokens": 8980, + "hash": "5f5635ccc06f0543", + "size": 33919, + "mtimeMs": 1787991941054.7356, + "updatedAt": "2026-08-29T08:25:41.090Z", + "source": "hook" + }, + "src/controller/testsuite/test_wal_job_start_fact.c": { + "description": "Declares as", + "tokens": 12368, + "hash": "5e6485d7caae7c0a", + "size": 46621, + "mtimeMs": 1787991925242.866, + "updatedAt": "2026-08-29T08:25:25.291Z", + "source": "hook" + }, + "src/controller/testsuite/test_apply_reapply_idempotency.c": { + "description": "Declares xce_job_t", + "tokens": 4115, + "hash": "0a6b27cc52f6eaf5", + "size": 15643, + "mtimeMs": 1787991975200.8542, + "updatedAt": "2026-08-29T08:26:15.238Z", + "source": "hook" + }, + "src/frontend/apiserver/backend/handler.go": { + "description": "Struct: DefaultHandler", + "tokens": 3107, + "hash": "f6a24d8fe38fefbc", + "size": 11656, + "mtimeMs": 1787994257192.1926, + "updatedAt": "2026-08-29T09:04:17.242Z", + "source": "hook", + "symbols": [ + { + "name": "DefaultHandler", + "kind": "class", + "startLine": 13, + "endLine": 39, + "tokens": 394 + }, + { + "name": "resetHostConfigCache", + "kind": "fn", + "startLine": 40, + "endLine": 58, + "tokens": 260 + }, + { + "name": "applyDefaultHostConfigFallback", + "kind": "fn", + "startLine": 59, + "endLine": 74, + "tokens": 104 + }, + { + "name": "OnJobCreated", + "kind": "fn", + "startLine": 75, + "endLine": 78, + "tokens": 32 + }, + { + "name": "OnJobUpdated", + "kind": "fn", + "startLine": 79, + "endLine": 82, + "tokens": 32 + }, + { + "name": "OnJobDeleted", + "kind": "fn", + "startLine": 83, + "endLine": 94, + "tokens": 188 + }, + { + "name": "OnJobFinished", + "kind": "fn", + "startLine": 95, + "endLine": 112, + "tokens": 165 + }, + { + "name": "OnQueueUpdated", + "kind": "fn", + "startLine": 113, + "endLine": 122, + "tokens": 155 + }, + { + "name": "OnHostUpdated", + "kind": "fn", + "startLine": 123, + "endLine": 143, + "tokens": 282 + }, + { + "name": "OnHostConfigUpdated", + "kind": "fn", + "startLine": 144, + "endLine": 182, + "tokens": 329 + }, + { + "name": "OnClusterUpdated", + "kind": "fn", + "startLine": 183, + "endLine": 194, + "tokens": 137 + }, + { + "name": "OnResourceUpdated", + "kind": "fn", + "startLine": 195, + "endLine": 198, + "tokens": 37 + }, + { + "name": "OnResourceLocationUpdated", + "kind": "fn", + "startLine": 199, + "endLine": 254, + "tokens": 854 + }, + { + "name": "dispatchJobArray", + "kind": "fn", + "startLine": 255, + "endLine": 280, + "tokens": 177 + }, + { + "name": "dispatchConfigPush", + "kind": "fn", + "startLine": 281, + "endLine": 301, + "tokens": 130 + } + ] } } } \ No newline at end of file diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index b69fa168..3ce1aa9b 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,7 +1,7 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-08-28T05:12:31.323Z -> Files: 507 tracked | Anatomy hits: 0 | Misses: 0 +> Auto-maintained by OpenWolf. Last scanned: 2026-08-29T09:04:17.242Z +> Files: 538 tracked | Anatomy hits: 0 | Misses: 0 ## ./ @@ -74,7 +74,7 @@ ## .kiro/specs/admin-config-tool/ - `brief.md` — Brief: admin-config-tool (~10613 tok) -- `design.md` — Technical Design: admin-config-tool (~13515 tok) +- `design.md` — Technical Design: admin-config-tool (~13800 tok) - `requirements.md` — Requirements Document (~5992 tok) - `research.md` — Research & Design Decisions: admin-config-tool (~8886 tok) - `spec.json` (~1040 tok) @@ -415,6 +415,14 @@ - `requirements.md` — Requirements Document (~9034 tok) - `spec.json` (~338 tok) +## .kiro/specs/scheduler-config-version-tag/ + +- `design.md` — Design Document (~6413 tok) +- `requirements.md` — Requirements Document (~2371 tok) +- `research.md` — Research & Design Decisions (~2711 tok) +- `spec.json` (~131 tok) +- `tasks.md` — Implementation Plan (~1764 tok) + ## .kiro/specs/scheduler-controller-msg-gap/ - `design.md` — Design: scheduler-controller-msg-gap (~5006 tok) @@ -537,7 +545,7 @@ - `ha-async-io.md` — HA Protocol & Async I/O (~4721 tok) - `persistence.md` — WAL & Persistence Durability (~3851 tok) - `product.md` — Product Overview (~1509 tok) -- `roadmap.md` — Roadmap (~15324 tok) +- `roadmap.md` — Roadmap (~15320 tok) - `scheduler-driver-equivalence.md` — Scheduler Driver Equivalence (~1520 tok) - `security.md` — Security Standards (~1960 tok) - `structure.md` — Project Structure (~1348 tok) @@ -792,11 +800,67 @@ - `dataex.c.o` (~255400 tok) +## src/controller/ + +- `job.c` — include (~39294 tok) +- `main.c` — include (~5489 tok) +- `persist.c` — define _GNU_SOURCE (~107488 tok) +- `sched.c` — include (~6721 tok) + +## src/controller/testsuite/ + +- `test_apply_reapply_idempotency.c` — Declares xce_job_t (~4115 tok) +- `test_wal_job_finish_fact.c` — Declares as (~8980 tok) +- `test_wal_job_start_fact.c` — Declares as (~12368 tok) + +## src/frontend/apiserver/backend/ + +- `handler.go` — Struct: DefaultHandler (~3107 tok) + +## src/lib/ + +- `meson.build` (~1757 tok) +- `vtaskmgr.c` — include (~3978 tok) + +## src/lib/testsuite/ + +- `test_vtask_enq_race.c` — Declares under (~1430 tok) +- `test.c` — include (~1279 tok) + +## src/scheduler/ + +- `meson.build` (~1426 tok) +- `tools_verify_fixture.c` — include (~1550 tok) + ## src/scheduler/src/ -- `snapshot_load.c` — include (~5573 tok) +- `core_events.c` — include (~11705 tok) +- `main.c` — include (~5330 tok) +- `snapshot_load.c` — include (~5920 tok) - `snapshot.c` — include (~8498 tok) ## src/scheduler/testsuite/ +- `test_core_events.c` — Declares and (~10371 tok) +- `test_sim_contention_parity.c` — Declares calls (~18713 tok) - `test_sim_e2e_original_motivation.c` (~17120 tok) +- `test_sim_session_ordering.c` (~8092 tok) +- `test_sim_startup.c` — Declares char (~5708 tok) +- `test_snapshot_save_load.c` (~28712 tok) + +## src/worker/src/ + +- `uv_ipc.c` — include "uv_ipc.h" (~12439 tok) + +## src/worker/testsuite/ + +- `meson.build` — Worker unit test suite (component-failover-adoption, task 2.1). White-box (~1380 tok) +- `test_uv_ipc_job_exit_severity.c` — Declares with (~863 tok) + +## src/xceadmin/ + +- `meson-build.sh` — Meson-driven xceadmin build: install + test out-of-tree, in the meson (~2445 tok) + +## src/xceadmin/tests/ + +- `test_e2e_live_ha_push.py` (~8887 tok) diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 23dcf051..afc7f780 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -16304,7 +16304,7 @@ "error_message": "controller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config (test_config_repl.c) FAIL during an unrelated ASAN controller build/test run (bug-451 fix session)", "file": "src/controller/testsuite/test_config_repl.c", "root_cause": "Not investigated -- discovered incidentally while verifying an unrelated fix (bug-451, persist_job_hook_fun's RUSAGE arm). Neither test_config_repl.c nor config_repl.c were touched this session (confirmed via git diff, zero changes). Coincides with a burst of 'persist: flusher fdatasync FAILED (errno=5 Input/output error)' ERROR lines in the same test run, matching this project's well-documented overlayfs sandbox fdatasync artifact class (same family as bug-924, bug-1470, bug-1553) rather than a genuine regression.", - "fix": "NOT FIXED -- out of scope for the bug-451 session that surfaced it. Needs confirmation this is the same overlayfs artifact (re-run on a real fs) before treating it as a real regression in test_config_repl.c's config-version-trim/tag-save logic. Re-observed 2026-08-17 during resource-value-verification task 1.1 (resval_typeresolve extraction) full controller ASAN build/test run: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was ), same accompanying persist_flush/flusher fdatasync errno=5 burst. Zero diff in test_config_repl.c/config_repl.c this session either (task 1.1 boundary was testsuite/resval_typeresolve.{c,h}, testsuite/test_resval_typeresolve.c, meson.build only) -- further corroborates the overlayfs sandbox artifact theory over a genuine regression, still unconfirmed on a real fs. Re-observed again 2026-08-17 during resource-value-verification task 2.1 (resupdate fallback + -t flag wiring, GREEN phase): identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst, in the same full `/xce-build controller` run that also confirmed resval_typeresolve_unit_test's own Unity summary at 4 Tests 0 Failures (task 2.1's GREEN evidence). Task 2.1's diff was scoped to testsuite/resval_typeresolve.{c,h} and testsuite/test.c only (confirmed via git diff --stat) -- test_config_repl.c/config_repl.c untouched, per this bug's own do-not-chase guidance in tasks.md's Implementation Notes.\n\nRe-observed a 5th time 2026-08-18 during a full `nix develop ./src --command bash src/tools/xce-build.sh all asan build-asan --incremental` verification run on the scheduler-framework branch (post bug-1584/1585 scheduler fixes). Identical signature: same 2 tests, same Expected/Was values (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. This run recompiled ZERO C source files (ninja reported \"no work to do\" for all C targets -- only the xceadmin custom target/wheel regenerated), so the failure cannot be attributed to any change in this session -- strongest evidence yet that this is purely an environment (overlayfs fdatasync) artifact, not a code regression. Scheduler suite's own single failure in the same run (test_recordstart_symlink_escape_replies_error, bug-1560) is separate and expected. Re-observed a 6th time 2026-08-19 during a controller ASAN build/test run verifying the StartTime/FinishTime persist.c fix (raft-apply-decoupling job-fact timestamp stamping, unrelated files persist.c/test_apply_committed_entry.c) -- identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst. Session diff confirmed scoped to persist.c/test_apply_committed_entry.c only -- further corroborates environment-artifact theory.\n\nRe-observed a 7th time 2026-08-21 verifying a pure output-formatting edit to src/controller/testsuite/test.c (raw %lu jobid printf sites replaced with fmt_jobid()+%s, 10 call sites, zero logic change) via `xce-build.sh controller asan --incremental` on the scheduler-framework branch. Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to test.c only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. controller/xce_test (the binary test.c itself compiles into) built and linked cleanly with zero warnings/errors tied to the fmt_jobid change; only controller_unit_test (a separate binary, testsuite_test_config_repl.c.o) carries these 2 pre-existing failures. Further corroborates environment-artifact theory over regression.\n\nRe-observed an 8th time 2026-08-23 during scheduler-dispatch-commit task 3.3's full controller ASAN build/test verification run (job.c dispatch-failure retry restructuring + new test_dispatch_fail_retry.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c and testsuite/ files only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression.\n\nRe-observed a 9th time 2026-08-23 during scheduler-dispatch-commit task 4.1's full controller ASAN build/test verification run (job.c handle_jobdispatch() double-notify removal + new test_jobdispatch_no_double_notify.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c (handle_jobdispatch() loop body only) and new/edited testsuite files -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression.\n\nRe-observed a 10th time 2026-08-23 during the scheduler-dispatch-commit feature's final GO/NO-GO release-gate verification run (`xce-build.sh controller asan build-asan --incremental`, tree fully committed, zero uncommitted diff in src/controller/job.c, handler.c, or testsuite/). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. A dedicated scoped rerun of controller_unit_test with -t filters covering only the scheduler-dispatch-commit feature's own groups (test_dispatch_commit, test_dispatch_fail_terminal, test_dispatch_fail_retry, test_jobdispatch_no_double_notify, test_commit_barrier_reply, test_wal_job_dispatch_fact, test_job_dispatch, test_job_dispatch_fail_count) came back 62 Tests 0 Failures 0 Ignored, exit 0 -- confirms these 2 pre-existing failures are fully isolated to test_config_repl.c and do not touch this feature's own test surface.\n\n[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed again, identical signature (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was , same fdatasync errno=5 burst) in both the meson-driven controller test run and a standalone direct controller_unit_test invocation, in a session whose only diff was adding a new, unrelated rcfile.c/rcfile.h/test_rcfile.c module (xce-test-rc-file spec task 1.1) plus meson.build/controller_test.c wiring -- config_repl.c/test_config_repl.c untouched by this session's own diff (though config_repl.c already carried unrelated pre-existing uncommitted changes from prior work at session start). Further corroborates: not caused by this task.", + "fix": "NOT FIXED -- out of scope for the bug-451 session that surfaced it. Needs confirmation this is the same overlayfs artifact (re-run on a real fs) before treating it as a real regression in test_config_repl.c's config-version-trim/tag-save logic. Re-observed 2026-08-17 during resource-value-verification task 1.1 (resval_typeresolve extraction) full controller ASAN build/test run: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was ), same accompanying persist_flush/flusher fdatasync errno=5 burst. Zero diff in test_config_repl.c/config_repl.c this session either (task 1.1 boundary was testsuite/resval_typeresolve.{c,h}, testsuite/test_resval_typeresolve.c, meson.build only) -- further corroborates the overlayfs sandbox artifact theory over a genuine regression, still unconfirmed on a real fs. Re-observed again 2026-08-17 during resource-value-verification task 2.1 (resupdate fallback + -t flag wiring, GREEN phase): identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst, in the same full `/xce-build controller` run that also confirmed resval_typeresolve_unit_test's own Unity summary at 4 Tests 0 Failures (task 2.1's GREEN evidence). Task 2.1's diff was scoped to testsuite/resval_typeresolve.{c,h} and testsuite/test.c only (confirmed via git diff --stat) -- test_config_repl.c/config_repl.c untouched, per this bug's own do-not-chase guidance in tasks.md's Implementation Notes.\n\nRe-observed a 5th time 2026-08-18 during a full `nix develop ./src --command bash src/tools/xce-build.sh all asan build-asan --incremental` verification run on the scheduler-framework branch (post bug-1584/1585 scheduler fixes). Identical signature: same 2 tests, same Expected/Was values (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. This run recompiled ZERO C source files (ninja reported \"no work to do\" for all C targets -- only the xceadmin custom target/wheel regenerated), so the failure cannot be attributed to any change in this session -- strongest evidence yet that this is purely an environment (overlayfs fdatasync) artifact, not a code regression. Scheduler suite's own single failure in the same run (test_recordstart_symlink_escape_replies_error, bug-1560) is separate and expected. Re-observed a 6th time 2026-08-19 during a controller ASAN build/test run verifying the StartTime/FinishTime persist.c fix (raft-apply-decoupling job-fact timestamp stamping, unrelated files persist.c/test_apply_committed_entry.c) -- identical 2 tests, identical Expected/Was values, identical fdatasync errno=5 burst. Session diff confirmed scoped to persist.c/test_apply_committed_entry.c only -- further corroborates environment-artifact theory.\n\nRe-observed a 7th time 2026-08-21 verifying a pure output-formatting edit to src/controller/testsuite/test.c (raw %lu jobid printf sites replaced with fmt_jobid()+%s, 10 call sites, zero logic change) via `xce-build.sh controller asan --incremental` on the scheduler-framework branch. Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to test.c only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. controller/xce_test (the binary test.c itself compiles into) built and linked cleanly with zero warnings/errors tied to the fmt_jobid change; only controller_unit_test (a separate binary, testsuite_test_config_repl.c.o) carries these 2 pre-existing failures. Further corroborates environment-artifact theory over regression.\n\nRe-observed an 8th time 2026-08-23 during scheduler-dispatch-commit task 3.3's full controller ASAN build/test verification run (job.c dispatch-failure retry restructuring + new test_dispatch_fail_retry.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c and testsuite/ files only (git diff --stat) -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression.\n\nRe-observed a 9th time 2026-08-23 during scheduler-dispatch-commit task 4.1's full controller ASAN build/test verification run (job.c handle_jobdispatch() double-notify removal + new test_jobdispatch_no_double_notify.c). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. Session diff confirmed scoped to job.c (handle_jobdispatch() loop body only) and new/edited testsuite files -- test_config_repl.c/config_repl.c untouched. Further corroborates environment-artifact theory over regression.\n\nRe-observed a 10th time 2026-08-23 during the scheduler-dispatch-commit feature's final GO/NO-GO release-gate verification run (`xce-build.sh controller asan build-asan --incremental`, tree fully committed, zero uncommitted diff in src/controller/job.c, handler.c, or testsuite/). Identical signature: same 2 tests (test_conf_trim_keeps_only_n_versions Expected 2 Was 3; test_conf_get_saved_tag_no_config Expected 0 Was large-garbage-uint64), same accompanying persist_flush/flusher fdatasync errno=5 burst. A dedicated scoped rerun of controller_unit_test with -t filters covering only the scheduler-dispatch-commit feature's own groups (test_dispatch_commit, test_dispatch_fail_terminal, test_dispatch_fail_retry, test_jobdispatch_no_double_notify, test_commit_barrier_reply, test_wal_job_dispatch_fact, test_job_dispatch, test_job_dispatch_fail_count) came back 62 Tests 0 Failures 0 Ignored, exit 0 -- confirms these 2 pre-existing failures are fully isolated to test_config_repl.c and do not touch this feature's own test surface.\n\n[UPDATE 2026-08-26, xce-test-rc-file task 1.1] Re-observed again, identical signature (test_conf_trim_keeps_only_n_versions Expected 2 Was 3, test_conf_get_saved_tag_no_config Expected 0 Was , same fdatasync errno=5 burst) in both the meson-driven controller test run and a standalone direct controller_unit_test invocation, in a session whose only diff was adding a new, unrelated rcfile.c/rcfile.h/test_rcfile.c module (xce-test-rc-file spec task 1.1) plus meson.build/controller_test.c wiring -- config_repl.c/test_config_repl.c untouched by this session's own diff (though config_repl.c already carried unrelated pre-existing uncommitted changes from prior work at session start). Further corroborates: not caused by this task.\n\nRe-observed 2026-08-29 during a debug-profile (non-ASAN, -O0 -g3) controller build/test run -- same 2 tests, same failure shape, but this time WITHOUT ASAN instrumentation and without the accompanying fdatasync errno=5 burst previously blamed on overlayfs -- weakens the overlayfs-artifact theory as the sole explanation. New candidate root cause found by reading the actual test/production code: cr_setup() (test_config_repl.c:62-71) keys its temp dir on getpid()+time(NULL) (second resolution), and conf_trim_old_versions() (config_repl.c:1027-1079) sorts by st.st_mtime (also second resolution) -- both can collide within one wall-clock second despite the test's own writes being only ~15ms apart. NOT YET fixed or confirmed as the sole cause (both theories -- overlayfs fdatasync artifact and second-resolution timestamp collision -- remain plausible and are not mutually exclusive); still open.", "tags": [ "controller", "testsuite", @@ -16320,8 +16320,8 @@ "bug-1470", "bug-1553" ], - "occurrences": 12, - "last_seen": "2026-08-26T00:00:00Z" + "occurrences": 13, + "last_seen": "2026-08-29T05:21:00Z" }, { "id": "bug-912", @@ -19494,8 +19494,8 @@ "timestamp": "2026-08-26T18:35:00Z", "error_message": "User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically.", "file": "src/scheduler/ (exact crash site unknown -- no core dump, no backtrace available for either occurrence)", - "root_cause": "NOT ROOT-CAUSED -- confirmed real (kernel dmesg segfault line, error code 4 = user-mode read fault, not a random/unrelated event) but no core dump or backtrace exists for any xce_scheduler crash observed so far, including the Aug 25 19:21:39 kernel-confirmed occurrence and today's (Aug 26, ~18:00-18:02) restart (PID 3964918 -> 4137745), which may or may not be the SAME underlying bug -- the two occurrences have not been correlated beyond both being unexplained scheduler deaths. Notable secondary finding: systemd-coredump is confirmed working for xce_controller (4 real core.xce_controller.*.zst files captured and used earlier this session, see bug-1699) but has captured ZERO xce_scheduler crashes despite at least one kernel-confirmed scheduler segfault -- needs investigation on the user's real machine (check core_pattern/ulimit -c/systemd-coredump size or process-name filtering specifically for the scheduler binary; also worth checking whether the scheduler's own launch wrapper -- start-scheduler.sh, a bare unwrapped invocation with no core-dump-enabling ulimit call -- inherits a different ulimit -c than however the controller happens to get its unlimited setting). FOLLOW-UP (2026-08-26, disambiguating today's restart from the Aug-25 segfault): checked dmesg for today's exact restart window (17:45-18:15) -- zero xce_scheduler entries (the ONLY dmesg segfault hit remains the single Aug 25 19:21:39 occurrence). journalctl for the same window -- zero xce_scheduler entries at all. dmesg confirms OOM killer was never actually invoked against any process in this window (only periodic OOM-killer enable/disable toggling, unrelated). ulimit -c is unlimited and core_pattern correctly routes to systemd-coredump (confirmed working for xce_controller, 4 real cores captured this session). The combination of (a) unlimited core-dump capacity + correct core_pattern, (b) zero kernel fault-signal record, (c) zero OOM record, (d) zero journalctl record for today's specific window strongly indicates today's PID change (3964918 -> 4137745) was NOT a crash of any kind -- a genuine fault signal (SIGSEGV/SIGABRT/SIGBUS/SIGILL/SIGFPE) is unconditionally kernel-logged regardless of which process triggers it, and none was. The scheduler is launched as a bare backgrounded process (start-scheduler.sh, no systemd unit), so an ordinary kill/pkill sent by an unrelated concurrent process on this shared machine would leave NO trace in dmesg or journalctl -- most likely explanation for today's specific restart, DECOUPLED from this bug going forward. This buglog entry now tracks ONLY the confirmed, kernel-verified Aug 25 19:21:39 xce_scheduler segfault (still unreproduced, still no core available, still worth investigating on recurrence) -- not today's restart, which is environmental noise, not a code bug.", - "fix": "NOT ATTEMPTED -- cannot fix an uninvestigated crash responsibly. Next steps if revisited: (a) fix the core-dump capture gap first (confirm ulimit -c/core_pattern actually applies to xce_scheduler processes the same way it does to xce_controller, on the user's real machine -- this blocks ALL future scheduler crash forensics, not just this one); (b) once captured, decode the NEXT scheduler core the same way bug-1699's controller cores were analyzed (systemd-coredump or manual zstd -d + gdb against the matching build.dbg/scheduler/xce_scheduler binary); (c) correlate against scheduler.log's own last lines before each death for behavioral context (what was the scheduler doing in the moment it died -- dispatch round, resource accounting, a specific request type).", + "root_cause": "NOT ROOT-CAUSED -- confirmed real (kernel dmesg segfault line, error code 4 = user-mode read fault, not a random/unrelated event) but no core dump or backtrace exists for any xce_scheduler crash observed so far, including the Aug 25 19:21:39 kernel-confirmed occurrence and today's (Aug 26, ~18:00-18:02) restart (PID 3964918 -> 4137745), which may or may not be the SAME underlying bug -- the two occurrences have not been correlated beyond both being unexplained scheduler deaths. Notable secondary finding: systemd-coredump is confirmed working for xce_controller (4 real core.xce_controller.*.zst files captured and used earlier this session, see bug-1699) but has captured ZERO xce_scheduler crashes despite at least one kernel-confirmed scheduler segfault -- needs investigation on the user's real machine (check core_pattern/ulimit -c/systemd-coredump size or process-name filtering specifically for the scheduler binary; also worth checking whether the scheduler's own launch wrapper -- start-scheduler.sh, a bare unwrapped invocation with no core-dump-enabling ulimit call -- inherits a different ulimit -c than however the controller happens to get its unlimited setting). FOLLOW-UP (2026-08-26, disambiguating today's restart from the Aug-25 segfault): checked dmesg for today's exact restart window (17:45-18:15) -- zero xce_scheduler entries (the ONLY dmesg segfault hit remains the single Aug 25 19:21:39 occurrence). journalctl for the same window -- zero xce_scheduler entries at all. dmesg confirms OOM killer was never actually invoked against any process in this window (only periodic OOM-killer enable/disable toggling, unrelated). ulimit -c is unlimited and core_pattern correctly routes to systemd-coredump (confirmed working for xce_controller, 4 real cores captured this session). The combination of (a) unlimited core-dump capacity + correct core_pattern, (b) zero kernel fault-signal record, (c) zero OOM record, (d) zero journalctl record for today's specific window strongly indicates today's PID change (3964918 -> 4137745) was NOT a crash of any kind -- a genuine fault signal (SIGSEGV/SIGABRT/SIGBUS/SIGILL/SIGFPE) is unconditionally kernel-logged regardless of which process triggers it, and none was. The scheduler is launched as a bare backgrounded process (start-scheduler.sh, no systemd unit), so an ordinary kill/pkill sent by an unrelated concurrent process on this shared machine would leave NO trace in dmesg or journalctl -- most likely explanation for today's specific restart, DECOUPLED from this bug going forward. This buglog entry now tracks ONLY the confirmed, kernel-verified Aug 25 19:21:39 xce_scheduler segfault (still unreproduced, still no core available, still worth investigating on recurrence) -- not today's restart, which is environmental noise, not a code bug.\n\nUPDATE (2026-08-29, bug-1731 live gdb-wrapped repro session): CRASH CAPTURED LIVE, first-ever backtrace for this crash family. Root cause of the earlier no-core-dump gap circumvented by launching xce_scheduler wrapped in `gdb -batch -ex run` FROM PROCESS START (not attach-after-the-fact) during a heavy burst-array workload (6 concurrent `jobsub -a 50` array submissions, 300 near-instant /bin/true elements, on a live 3-node HA cluster, debug build profile). Reproduced at T+150s into the burst. Full backtrace:\nSIGSEGV in sched_get_project(core, name=0x41d0000000000000) at scheduler/src/core.c:412, called from core_build_job() at scheduler/src/core_events.c:221, called from xce_core_push_events() at scheduler/src/core_events.c:725 (crashed on the 3rd of a 6-event batch), called from xce_sched_driver_translate_pending_events() (driver.c:555) <- xce_sched_driver_run_normal_mode_round() (driver.c:749) <- main() (main.c:465). Registers: rax=rdx=rsi=0x41d0000000000000 (corrupted `name` arg present in 3 registers), rdi=0x555555695da0 (valid `core` arg). Single-thread fault in the main scheduling-round thread (all 9 other threads idle in pthread_cond_wait/epoll_wait) -- not a cross-thread race at the fault site itself, though the corruption's origin is not yet located.\ncore_build_job() (core_events.c:221) calls sched_get_project(core, j->submit->project_name). The two preceding calls in the same function (sched_get_user, sched_get_queue, reading earlier fields of the same job_submit_data_t*) succeeded, so j->submit itself is a valid pointer -- but its project_name field held 0x41d0000000000000, not NULL and not a plausible string pointer. clone_job_submit_data() (handler.c:102-150) calloc()s the destination and only strdup()s project_name when the source is non-NULL, so a legitimately-absent project name would be NULL (which sched_get_project's own guard at core.c:412 handles safely) -- this crash requires the field to have been overwritten with garbage AFTER allocation. Points to memory corruption or a stale/reused pointer in the array-job event-translation/batching path under heavy concurrent load, not a missing-null-check. NOT YET further isolated -- the corruption's write site is still unknown. Fault class (SIGSEGV, invalid-pointer read via strlen()) matches the original kernel dmesg description (`error 4` = user-mode read fault) closely enough that this is plausibly the same crash family, now finally attributable to an exact function/line/call-chain instead of an unattributed dmesg line.", + "fix": "STILL NOT FIXED -- but no longer blocked on evidence collection; see root_cause UPDATE above for the first captured backtrace. NEXT STEP: find what writes garbage into job_submit_data_t->project_name after clone_job_submit_data() allocates it -- likely somewhere in the array-job batching/event-translation path (xce_core_push_events / xce_sched_driver_translate_pending_events) given the crash hit specifically while processing event index 2 of a 6-event batch under heavy concurrent array-job load. Reproduction recipe confirmed reliable enough to use again: debug-profile build, xce_scheduler launched via `gdb -batch -ex run -ex \"thread apply all bt full\"` from process start (script preserved at /tmp/xce-deploy-bug1731/scripts/start-scheduler-gdb.sh), 6 concurrent 50-element array submissions against a live 3-node HA cluster -- crashed within 150s this run.", "tags": [ "scheduler", "segfault", @@ -19510,10 +19510,11 @@ "kernel-confirmed" ], "related_bugs": [ - "bug-1699" + "bug-1699", + "bug-1731" ], - "occurrences": 1, - "last_seen": "2026-08-26T18:35:00Z" + "occurrences": 2, + "last_seen": "2026-08-29T05:21:00Z" }, { "id": "bug-1715", @@ -19712,6 +19713,260 @@ ], "occurrences": 1, "last_seen": "2026-08-27T00:00:00Z" + }, + { + "id": "bug-1724", + "timestamp": "2026-08-28T00:00:00Z", + "error_message": "3 xceadmin live E2E tests (test_e2e_live_push.py::test_live_config_push_against_real_single_node_controller, test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path, test_e2e_live_ha_push.py::test_live_config_push_against_standby_returns_standby_rejection_not_redirect) always SKIPPED under the meson-driven build path (`/xce-build xceadmin`, `nix build .#xcecore`) with 'no xce_controller binary found under xce.build/build.release/build.dbg', regardless of whether a real xce_controller/xceadmin binary actually existed anywhere in the checkout.", + "file": "src/xceadmin/meson-build.sh", + "root_cause": "tests/_live_e2e_common.py's CONTROLLER_CANDIDATES/XCEADMIN_CANDIDATES/DEPLOY_HA_SH resolve paths relative to the test file's own on-disk location (Path(__file__).resolve().parents[2]). That works when pytest runs directly from the real checkout, but meson-build.sh copies the whole xceadmin test tree into an isolated out-of-tree repo-mirror before invoking pytest (to keep the meson build pure), and only ever symlinked ha/, src/configs, src/schema into that mirror -- never the sibling xce.build/build.release/build.dbg controller-binary directories, nor src/tools/ (deploy-ha.sh). So the mirror's own SRC_DIR (repo-mirror/src/) never had a controller subdir, a deploy-ha.sh, or a matching xceadmin binary at the expected relative path, no matter what was actually built in the real repo -- the skip was structural, not a signal about the real build state.", + "fix": "Two changes to meson-build.sh: (1) symlink each profile's `controller` subdirectory (xce.build/controller, build.release/controller, build.dbg/controller -- only the controller subdir, never the whole profile-root, since the profile-root can be an ancestor of the mirror itself when xceadmin is built in that same profile, which would create a self-referential symlink loop) into the mirror so CONTROLLER_CANDIDATES can find a real xce_controller wherever it was built; (2) symlink src/tools (checked-in, not a build-output dir, so a plain safe symlink like configs/schema) so DEPLOY_HA_SH resolves. For xceadmin's OWN binary, which genuinely is self-referential (the mirror lives inside that very build's own xceadmin output), routed around it via the existing XCEADMIN_BIN env-var override (resolve_binary_candidates in _live_e2e_common.py), set to $VENV_DIR/bin/xceadmin -- the binary this very meson-build.sh invocation just pip-installed -- before invoking pytest. Verified end-to-end via a real `xce-build.sh xceadmin debug build.dbg --incremental` run: all 3 tests now execute for real against a live controller/cluster instead of skipping.", + "tags": [ + "xceadmin", + "meson-build.sh", + "repo-mirror", + "live-e2e", + "test-discovery", + "symlink", + "build-infra", + "found-live", + "root-caused", + "fixed", + "live-verified" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-28T00:00:00Z" + }, + { + "id": "bug-1725", + "timestamp": "2026-08-28T00:00:00Z", + "error_message": "After bug-1724's fix made test_live_config_push_against_standby_returns_standby_rejection_not_redirect actually execute (previously always skipped), it FAILED for real: pushing config to a standby now redirects to the primary and succeeds, contradicting the test's own hard-coded 'standby flatly rejects, no redirect' assertions. Separately, the rewritten success-path test's reused _assert_primary_committed() helper intermittently failed its 4th assertion ('config activated from committed marker tag=...' not found in the primary's log immediately after the push CLI returned success).", + "file": "src/xceadmin/tests/test_e2e_live_ha_push.py", + "root_cause": "(1) The test's assumption was stale, not wrong test infra: bug-1700/1701 (fixed 2026-08-26) closed the exact standby-redirect gap this test was written to regression-lock, at the generic ha_dispatch_gate() rejection level in cluster.c -- the test's own docstring had explicitly predicted this exact failure as the correct signal to replace it with a success-path assertion. design.md ('Boundary Commitments'/'Revalidation Triggers') and roadmap.md still described the gap as open. (2) config_activate_committed() on the primary runs off the WAL apply-pump dispatch (persist.c ~5111, the bug-1469 fix), asynchronously relative to the majority-ack-gated client reply (commitIndex vs lastApplied are distinct steps) -- the CLI can legitimately return success before the primary's own local apply of that marker has logged its activation line, a latent race in the existing single-shot log read shared by the direct-to-primary test too.", + "fix": "Renamed and rewrote the test as test_live_config_push_against_standby_redirects_and_succeeds, asserting the redirect line IS present and the push succeeds (same success bar as the direct-to-primary test: positive ackedCount, exit 0, primary commit evidence). Updated the module docstring, design.md (Boundary Commitments, Revalidation Triggers, sequence-diagram note, redirect-implementation rationale, Testing Strategy), and roadmap.md's corresponding item to describe the gap as closed instead of open -- also corrected an adjacent, independently-verified stale claim (bug-1468, fixed 2026-07-25, already resolved but never reflected in design.md) that no controller role could reach the upgradeConfig handler at all. For the activation race, changed _assert_primary_committed() from a single-shot log_text argument to a Path-based helper that polls (up to 5s, 0.2s interval) for the activation line instead of assuming synchronous ordering, and updated both call sites. Verified end-to-end: full xce-build.sh xceadmin run now passes 288/288 with zero skips/failures.", + "tags": [ + "xceadmin", + "config-push", + "redirect", + "OPCode_upgradeConfig", + "stale-test", + "design-doc-sync", + "roadmap-sync", + "race-condition", + "config_activate_committed", + "found-live", + "root-caused", + "fixed", + "live-verified" + ], + "related_bugs": [ + "bug-1724", + "bug-1700", + "bug-1701", + "bug-1468" + ], + "occurrences": 1, + "last_seen": "2026-08-28T00:00:00Z" + }, + { + "id": "bug-1726", + "timestamp": "2026-08-28T08:15:00Z", + "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at live_ha_cluster fixture setup (single occurrence, user-run via `nix develop ./src` + direct pytest against the built build.dbg mirror, not via any Claude-driven build): _wait_for_primary(nodes, 30.0) raised TimeoutError -- none of the 3 freshly-spawned xce_controller processes ever printed the CTRL_PRIMARY election-complete marker within the 30s timeout (normal election completes ~10.2s per this file's own docstring, keyed to HA_DEFAULT_HEARTBEAT_TIMEOUT_MS=10000).", + "file": "src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture, _wait_for_primary)", + "root_cause": "NOT ISOLATED -- deliberately distinct from bug-1485 (do not merge into it): bug-1485's signature is election COMPLETING (all 3 markers print) then the freshly-promoted primary going /unreachable shortly after; this occurrence is election never visibly settling AT ALL within 3x the normal budget, a different symptom with a different plausible cause. XCE_KEEP_DEPLOY_DIR was not set for this run, so the 3 nodes' stdout/stderr logs from the actual failure were deleted by the fixture's own cleanup before this could be forensically investigated -- no diagnostic evidence survives from this specific occurrence. Immediate reruns (5/5, by Claude directly after the report, same build.dbg binaries, same test, quiet system: load avg ~0.6-1.5/22 cores, no concurrent xce_controller/meson/nix processes found) all passed cleanly in ~14-15s each, well inside the 30s budget -- consistent with (but not proof of) transient CPU/IO contention from another concurrent session on this shared multi-agent sandbox (the established pattern documented at length in bug-1486 and this same file's bug-1485 investigation) delaying all 3 nodes' heartbeat timers past the window, rather than a deterministic protocol liveness bug. Not ruled out: a genuine, rare election-liveness edge case (e.g. a PreVote/RequestVote split-vote condition) that happens not to leave a marker before some other transient condition times it out.", + "fix": "NOT FIXED -- single occurrence, unreproduced on 5 immediate retries, no surviving log evidence to root-cause further. Logged per this project's own established convention (bug-1485) of tracking an inconclusive live-cluster-election flake rather than silently dismissing it. Next step if this recurs: re-run with XCE_KEEP_DEPLOY_DIR=1 (already supported by the fixture, added for exactly this class of investigation per bug-1485's own remediation) to preserve deploy_dir/logs/*.log and *.stderr.log for real forensic inspection instead of losing them to the fixture's own cleanup.", + "tags": [ + "flaky-test", + "ha", + "raft-election", + "live-ha-push", + "unresolved", + "unreproduced", + "low-priority", + "distinct-from-bug-1485", + "found-live" + ], + "related_bugs": [ + "bug-1485", + "bug-1486" + ], + "occurrences": 1, + "last_seen": "2026-08-28T08:15:00Z" + }, + { + "id": "bug-1727", + "timestamp": "2026-08-28T10:25:14Z", + "error_message": "User-reported (live REPL, real cluster-captured replay /tmp/scheduler-replay-20260828-085356.replay, 40 sessions, 60 jobs): after `xce_scheduler -i -s ` + `step`, `info worker all` shows `resource slots: value=44.00 used=22.00` for worker pc13v, but the worker's real installed capacity is 22 slots (confirmed by the same worker's own `resource maxcpus: value=22.00` line) and all 22 running jobs are 1-slot-each, so the correct display is `value=22.00 used=22.00` (fully packed, 0 free) -- `used=22.00` is correct, `value=44.00` is double the true installed capacity.", + "file": "src/scheduler/src/repl_info.c (repl_info_worker_one, ~line 281-390)", + "root_cause": "NOT FIXED -- recorded for future investigation, but well-narrowed: repl_info_worker_one()'s own header comment (repl_info.c:281-291) asserts w->resvals holds 'currently-AVAILABLE' capacity that shrinks as jobs are charged, and that 'value=' should be printed as avail+inUse ('a conservation invariant this codebase's own resource accounting already relies on'), implemented at repl_info.c:376-390's Value_u_fvalue branch: `avail_v->u.fvalue + used` printed as value=. Traced the actual charge path (sched_account_jobs -> sched_charge_account, resource.c:586 -> sched_update_account, resource.c:559-583 -> sched_update_res_accounts, resource.c:401-470) and confirmed it ONLY ever writes to w->resInUse (`sched_update_res_accounts(core, &(w->resInUse), 0, haccount->naccount, haccount->accounts, charge)`, resource.c:580-581) -- it never touches w->resvals. Exhaustively grepped every `zarray_put(&(w->resvals), ...)`/`zarray_put(&(worker->resvals), ...)` call site in the scheduler (core_events.c:500, core_events.c:848, snapshot_load.c:333, schedobj.c:311): all four are WORKERS_PUSH / worker-update / snapshot-restore INGESTION paths only -- i.e. w->resvals is populated purely from what the worker daemon self-reports (its own configured/installed capacity, e.g. 'slots: 22'), and is architecturally NEVER decremented when a job is dispatched or charged. So for a live worker's CONSUMABLE resource like 'slots', w->resvals already IS the total installed capacity (not a live 'remaining free' counter), making repl_info.c's own documented 'avail+used = value, a conservation invariant' assumption appear to be FALSE in practice -- the comment's claimed invariant does not match how w->resvals is actually populated for this resource type. This makes `value = avail + used` double-count: true installed capacity (22) gets added to itself once resInUse also reads 22, yielding 44. CONFIRMED PRE-EXISTING, NOT caused by today's scheduler-snapshot-job-allocation task 4.6 fix (the sched_account_jobs()-at-load-time change earlier this session): this exact 44/22 pattern already appeared in the user's ORIGINAL first-ever paste this session, produced via the normal `step` -> sched_account_jobs() path (not the load-time path) -- task 4.6 only changed WHEN the display first goes wrong (now visible immediately at load, previously only after the first step/session, since sched_account_jobs() previously never ran until then). Two competing hypotheses for the real fix, not yet distinguished: (a) repl_info.c's display formula is simply wrong for how resvals is actually populated -- should print avail_v directly as value= (already the total) with used= from resInUse, no addition; or (b) w->resvals was INTENDED to be a live 'remaining free' counter (matching the comment's claimed invariant) and something is missing a `w->resvals -= charged_amount` step at dispatch/charge time that was never implemented -- but no evidence of (b) was found anywhere in resource.c's charge/uncharge functions, so (a) is the better-supported hypothesis pending closer investigation of what a live worker daemon (xcecd) actually reports for a 'slots' resval on each heartbeat (constant install size vs. something already load-adjusted).", + "fix": "NOT FIXED -- user asked to record this for future investigation only, no code change made. Likely fix direction (per root_cause hypothesis (a), needs confirming first): repl_info_worker_one()'s Value_u_fvalue branch (repl_info.c ~376-390) should print `avail_v->u.fvalue` directly as value= (not avail+used) for a CONSUMABLE resource, same as the non-CONSUMABLE branch already does -- verify against repl_info_resource_one()'s cluster-wide CONSUMABLE case (mentioned in the same comment as sharing 'an identical gate') to see if it has the same double-count bug or got this right, before changing anything. Also worth checking test_repl_breakpoints_inspection.c / any existing `info worker`-covering test for a hardcoded expectation that already bakes in the avail+used formula (would need updating alongside any fix, same pattern as this session's earlier test_sim_contention_parity.c/test_sim_session_ordering.c hardcoded-index fixes for task 4.6).", + "tags": [ + "scheduler", + "repl", + "display-bug", + "resource-accounting", + "resInUse", + "resvals", + "unfixed", + "found-live", + "pre-existing", + "not-caused-by-task-4.6" + ], + "related_bugs": [ + "bug-1568", + "bug-1590" + ], + "occurrences": 1, + "last_seen": "2026-08-28T10:25:14Z" + }, + { + "id": "bug-1728", + "timestamp": "2026-08-28T10:25:14Z", + "error_message": "Controller logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live HA cluster (ctrl-1.log, ctrl-2.log, ctrl-3.log), always immediately after the same two-line sequence: `WARN [commit_submit] commit: WAIT deadline expired without commit (lsn=X committed_lsn=Y writable_lsn=1 timeout_ms=500)` then `WARN [handle_worker_join] handle_worker_join commit barrier failed for pc13v, rc -110`. All three occurrences in the sampled log correlate exactly with an in-progress PreVote/failover window (liveness poll timeout -> primary_ctrl_fd RDHUP -> PreVote begin, immediately preceding each ERROR).", + "file": "src/controller/cluster.c (handle_endpoint_error, ~line 52-77); src/controller/worker.c or join.c (handle_worker_join, commit-barrier caller -- not yet located)", + "root_cause": "NOT FIXED -- recorded for future investigation. Read handle_endpoint_error() (cluster.c:52-119): on any endpoint teardown/error, it inspects priv->role; for HostRole_worker it branches on priv->u.idx -- if >=0, calls handle_worker_leave(c, idx) (the normal, silent teardown path); if <0 (the sentinel for 'never finished joining'), it logs ERROR('incorrect worker idx %d', priv->u.idx) instead (cluster.c:75). Working hypothesis (not yet confirmed by reading handle_worker_join() itself): a worker's endpoint gets `priv->u.idx` assigned only once its join fully completes; the commit barrier inside handle_worker_join() is what actually blocks that completion, and it just failed (rc -110 = -ETIMEDOUT) because quorum was mid-PreVote-failover and couldn't ack the WAL entry within the fixed 500ms commit_submit deadline (already WARNed one line above, correctly). So the worker's join never finishes, its endpoint's idx sentinel never advances off -1, and when that half-joined endpoint then errors out (e.g. the join's own failure path tearing the connection down, or the worker retrying), handle_endpoint_error() sees role==worker && idx==-1 and logs it as a hard, unexplained ERROR -- even though it is the direct, already-WARNed consequence of the commit-barrier timeout two lines above, not a new/distinct fault. Same shape as bug-1727's session sibling finding for scheduler sched_init_resources() and the worker kill-job ERROR-logging issue: an expected, self-healing HA-transition condition (worker retries the join once quorum stabilizes -- confirmed later in the same logs by 'HA quorum restored') surfacing as an unexplained hard ERROR instead of being correlated with the WARN that actually explains it. NOT YET CONFIRMED: have not read handle_worker_join()'s own body (location not pinned down -- likely worker.c or join.c) to verify priv->u.idx really stays -1 specifically because of THIS commit-barrier failure path (vs. some other reason), nor traced exactly what tears the endpoint down afterward to trigger handle_endpoint_error().", + "fix": "NOT FIXED -- recorded for future investigation only, no code change made. Next steps: (1) locate and read handle_worker_join()'s full body to confirm priv->u.idx is left at -1 specifically on the commit-barrier-timeout path (rc -110) and identify what closes/errors the endpoint afterward; (2) if confirmed, either (a) downgrade cluster.c:75's ERROR to WARN with a message that explicitly references the join-in-progress/commit-barrier-timeout context (avoids the unexplained-hard-ERROR read), or (b) have the commit-barrier-timeout failure path in handle_worker_join() itself suppress/mark the endpoint so handle_endpoint_error() can recognize 'this was a known, already-logged join failure' and skip the redundant ERROR entirely; (3) verify against a live 3-node HA soak (xce-test-ha) that downgrading doesn't hide a genuinely-different idx<0 cause (e.g. a real accounting bug elsewhere assigning idx incorrectly) -- keep ERROR (or WARN-with-detail) for any idx<-1 or any case NOT immediately preceded by a commit-barrier-timeout WARN for the same worker, so a genuinely new failure mode doesn't get silently swallowed by too broad a downgrade.", + "tags": [ + "controller", + "ha", + "logging-severity", + "false-error", + "worker-join", + "commit-barrier", + "unfixed", + "found-live", + "session-sibling-of-bug-1727" + ], + "related_bugs": [ + "bug-1727" + ], + "occurrences": 3, + "last_seen": "2026-08-28T10:25:14Z" + }, + { + "id": "bug-1729", + "timestamp": "2026-08-28T10:25:14Z", + "error_message": "xcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary, deliberate job kill/cancel (exit code 137 = 128+SIGKILL) -- user-reported as misleading, since 'kill job' is a normal admin operation, not a failure.", + "file": "src/worker/src/uv_ipc.c (ipc_worker_send_status, JOB_EXITED branch)", + "root_cause": "CONFIRMED: process_exit_callback() (xcecd_worker.c:467-502) already treats a signal-terminated job exit as informational -- `XCECD_WORKER_LOG(XCE_LOG_INFO, \"Child %d killed by signal %d\\n\", pid, term_signal)` at line 497 -- immediately before calling ipc_worker_send_status(&job_runtime_info, \"Killed\"). But ipc_worker_send_status() (uv_ipc.c, JOB_EXITED branch) re-derived its own severity purely from job_status, hardcoded to XCE_LOG_ERROR for every JOB_EXITED regardless of cause -- so the SAME kill event logged INFO two lines earlier and ERROR two lines later, in the same call chain. There is no JOB_KILLED status distinct from JOB_EXITED, so a deliberate xcecli kill (worker_signal_callback(), xcecd_worker.c:1118, itself logs INFO: 'Terminating job %s with signal %d') produced an ERROR-level status log downstream regardless of intent. worker_signal_callback() only ever sends SIGTERM or SIGKILL for a normal cancel/force-kill (xcecd_worker.c:1116-1125) -- exit_code is always exactly 128+term_signal (xcecd_worker.c:493), so exit_code==143 (SIGTERM) or 137 (SIGKILL) reliably identifies a deliberate worker-initiated termination, distinct from a genuine crash signal (SIGSEGV=139, SIGBUS=135, SIGABRT=134, etc.).", + "fix": "FIXED and unit-tested (TDD red->green, negative-control verified: temporarily forced job_exit_log_level() to always return XCE_LOG_ERROR, confirmed 2/4 new tests failed as expected, then restored the real fix and confirmed all 4 pass). Extracted the severity decision into a small pure function, `static LogLevel job_exit_log_level(int exit_code)` (uv_ipc.c, just above ipc_worker_send_status()): returns XCE_LOG_INFO for exit_code == 128+SIGTERM or 128+SIGKILL (matches process_exit_callback()'s own INFO judgment for the identical event), XCE_LOG_ERROR otherwise (preserves ERROR for genuine crash signals and any other/unrecognized exit code). Wired into the JOB_EXITED branch: `XCECD_WORKER_LOG(job_exit_log_level(job_info->exit_code), \"Job %s exited with code %d: %s\", ...)`. Added #include for SIGTERM/SIGKILL/SIGSEGV/etc. New test worker/testsuite/test_uv_ipc_job_exit_severity.c (4 cases: deliberate SIGTERM, deliberate SIGKILL, genuine crash signals SIGSEGV/SIGABRT/SIGBUS, unrecognized/plain exit codes) white-box #includes uv_ipc.c directly (existing suite convention) and exercises only the pure helper -- no libuv/IPC machinery needed. Two link-only stub functions added (find_worker_by_pid, xcecd_job_remote_start) to satisfy the linker for code paths this test never calls, same stubbing idiom test_resource_sync.c already establishes. Full worker test suite (5 binaries, 25 tests total) green after the fix; real xcecd_worker binary rebuilds cleanly.", + "tags": [ + "worker", + "logging-severity", + "false-error", + "job-kill", + "fixed", + "tdd", + "found-live", + "session-sibling-of-bug-1727", + "session-sibling-of-bug-1728" + ], + "related_bugs": [ + "bug-1727", + "bug-1728" + ], + "occurrences": 1, + "last_seen": "2026-08-28T10:25:14Z" + }, + { + "id": "bug-1730", + "timestamp": "2026-08-28T13:39:36Z", + "error_message": "User-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes remain visible as RUN forever in the scheduler's own state (confirmed both live and via a scheduler session recording's .replay snapshot -- xce_scheduler -i -s showing several 0-12-1787907177-0[N] elements as RUN when the controller's joblist/xcecli already shows them DONE). User's own log correlation: scheduler-side handler.c:414 'sched_handle_sched_update_job updated' fires once per job (the dispatch-time notification) but a second fire for the finish notification never arrives for the affected jobs, while the worker log independently proves those jobs genuinely completed (uv_ipc.c:1042 'Job completed successfully', milliseconds after dispatch -- these are near-instant /bin/true-style array elements).", + "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_UPDATE case), src/controller/sched.c (sched_job_exit_fun/sched_job_start_fun/sched_job_fail_fun), src/controller/job.c (HOOK_DETAIL_JOB_EXIT/START/FAIL fire sites)", + "root_cause": "TWO CONFIRMED FINDINGS from this investigation, one FIXED here, one still OPEN (see bug-1731, its own entry, for the still-open half -- kept separate since the fixed half does not explain the original reported symptom by itself, see below).\n\nFINDING 1 (FIXED by this entry): sched_job_exit_fun()/sched_job_start_fun()/sched_job_fail_fun() (controller/sched.c) -- the controller-side notify handlers wired to route a job exit/start/fail specifically to the scheduler's accounts/allocs-cleaning sched_job_state_change() (via NotifyData_data_schedExitJob etc. -> XCE_SCHED_OP_JSTATE_CHANGE) -- are PERMANENTLY DEAD CODE: each only acts on `situation == HOOK_AFTER_EVENT` (default: break for HOOK_BEFORE_EVENT), but HOOK_DETAIL_JOB_EXIT/JOB_START/JOB_FAIL are ONLY EVER fired as HOOK_BEFORE_EVENT anywhere in the controller (job.c:2515/2640/2744/2749/2783, all inside the durable-append-only record_job_*_fact() receipt-time functions -- confirmed via exhaustive grep, zero HOOK_AFTER_EVENT fires of these three details exist). The actual state mutation + notification for a job finish/start/fail happens later, at deferred-apply time (apply_committed_entry()'s shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH 'group' case, persist.c:4464), which fires a DIFFERENT, generic detail -- HOOK_DETAIL_JOB_UPDATE -- at HOOK_AFTER_EVENT, routing instead through sched_job_update_fun() -> NotifyData_data_schedUpdateJob -> scheduler's sched_handle_sched_update_job() (handler.c:404) -> XCE_SCHED_OP_UPDATE_JOB_FIELDS -> (in the currently-live core_events.c-based driver path) XCE_CORE_EVENT_JOB_UPDATE (core_events.c:933). That handler correctly sets job->state, but -- unlike its sibling XCE_CORE_EVENT_JOB_STATE_CHANGE case right above it in the same file, and unlike scheduler.c's own (also now-dead) sched_job_state_change() -- it never released job->accounts/job->allocs on a terminal transition, so every job finishing via this (the only live) path leaked its resource-allocation/accounting objects for the rest of the scheduler process's life. Almost certainly a regression from the raft-apply-decoupling refactor, which collapsed START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH's deferred-apply hook fire into one generic HOOK_DETAIL_JOB_UPDATE without ever rewiring (or retiring) the scheduler-notification handlers that were built for the old, type-specific hook details.\n\nConfirmed this FINDING alone does NOT explain the user's original reported symptom (job->state itself staying RUN): job->state is set correctly by XCE_CORE_EVENT_JOB_UPDATE's existing code regardless of this leak -- the leak only means job->accounts/allocs are never released, a distinct, confirmed-real defect. Extensive live investigation of the actual state-staying-wrong symptom is tracked separately in bug-1731 (open).", + "fix": "FIXED and TDD-verified (RED->GREEN) for FINDING 1 only. src/scheduler/src/core_events.c's XCE_CORE_EVENT_JOB_UPDATE case: added the identical `if (!JOB_IS_RUN(updated->state)) { free job->accounts via free_job_account(), job->accounts = NULL; free job->allocs via free_JobAlloc_ptr() }` block XCE_CORE_EVENT_JOB_STATE_CHANGE's case already has, right after `job->state = updated->state;`. New test test_job_update_to_terminal_state_frees_accounts_and_allocs (scheduler/testsuite/test_core_events.c): builds a real RUNNING job with a genuine allocation via production make_alloc()/make_host_alloc()/make_job_alloc()/make_job_account() (src/lib/xcealloc.c), pushes a JOB_UPDATE transitioning it to DONE, asserts job->accounts/job->allocs are both NULL afterward. RED confirmed (FAIL: Expected NULL) before the fix; GREEN after. Full scheduler test suite: 52/52 with the new test, only the pre-existing, unrelated symlink-escape-error-message mismatch (sched_admin_request_test) failing before and after -- zero regressions. Real xce_scheduler binary rebuilds cleanly.", + "tags": [ + "scheduler", + "controller", + "job-state", + "dead-code", + "hook-detail-mismatch", + "resource-leak", + "accounts-allocs", + "raft-apply-decoupling-regression", + "fixed", + "tdd", + "found-live", + "session-sibling-of-bug-1731" + ], + "related_bugs": [ + "bug-765", + "bug-1679", + "bug-1731" + ], + "occurrences": 1, + "last_seen": "2026-08-28T13:39:36Z" + }, + { + "id": "bug-1731", + "timestamp": "2026-08-28T13:39:36Z", + "error_message": "STILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a finished job's job->state staying stuck as RUN in the scheduler's own view (live and in a .replay snapshot) despite the controller and the worker both having genuinely completed it -- was NOT conclusively reproduced live despite extensive targeted testing, so its exact trigger mechanism remains unconfirmed.", + "file": "src/controller/persist.c (apply_committed_entry, WAL_JOB_FINISH/START/DISPATCH shared case), src/controller/sched.c (sched_job_update_fun)", + "root_cause": "NOT ISOLATED. Investigation trail: traced the full controller-to-scheduler-to-replay-recording pipeline for a job finish end-to-end via static code reading (every hop -- live apply, hook fire, record-encode in recording_events.c, replay-decode in sim_loop.c, replay-apply -- checked individually and found correctly wired for the ordinary case) and found no defect that would explain a PERMANENT loss by itself. Added temporary diagnostic WARN logging at apply_committed_entry()'s two previously-100%-silent early-return points (job not found in jobmgr; apply_job_update_transition() returning REJECTED_BACKWARD/NOOP_DUPLICATE) and reproduced a burst-array-job workload (matching the user's original scenario -- a 30+ element array of near-instant jobs) against both a single-node and a real 3-node HA deploy of this exact worktree's build. Findings from live reproduction: (a) WAL_JOB_FINISH consistently showed result=APPLIED (hook fires) in every observed instance -- did not catch a live silent-drop for FINISH specifically; (b) DID catch WAL_JOB_START correctly no-op'ing as a harmless duplicate when WAL_JOB_DISPATCH already set RUN moments earlier in the same batch (expected, not a bug); (c) a genuinely concerning, SEPARATE, NOT-YET-DIAGNOSED finding: under a very heavy burst (4 concurrent 50-element arrays on a live 3-node cluster), the xce_scheduler process itself DIED entirely mid-run with zero error/fatal/assert output in its own log -- no backtrace captured yet (a subsequent gdb-attached rerun of a similarly-sized burst did not reproduce the crash within the session's remaining time budget); (d) a separate apparent scheduling-throughput stall under a large PEND backlog (200+ near-instant jobs against a 22-slot worker progressing far slower than expected) -- not distinguished from ordinary fork/exec overhead of spawning hundreds of real OS processes on one worker, so not confirmed as a genuine scheduler defect either. Any of (c)/(d) could plausibly be entangled with, or be a completely independent symptom alongside, the original state-staying-wrong report -- not established either way.\n\nUPDATE (2026-08-29, live gdb-wrapped burst repro): two concrete new findings, neither yet closing this bug's original symptom (job state itself stuck at RUN).\n(1) LIVE-CAPTURED NOTIFY-DROP, SELF-HEALING IN THE OBSERVED CASE: the DEBUG3 diagnostic trace added earlier this session (sched_job_update_fun, sched.c) fired for real under this burst -- e.g. `DIAG sched_job_update_fun: jobid=4618064451492872238 state=0x2 -- c->sched.ep is NULL, notify SKIPPED` during a ~2s scheduler reconnect blip (handle_scheduler_leave then a rejoin then a second, fatal leave at the crash). Across the run: 173 RUN notifies attempted, 178 DONE notifies delivered, 53 SKIPPED (dropped because c->sched.ep was NULL at fire time). The one traced job above still received its terminal DONE notify ~300ms later and self-healed -- but this is a live, reproducible demonstration of the exact drop mechanism this bug has theorized: if a job's TERMINAL notify (not just an intermediate RUN one) were ever dropped in the same way, with no subsequent notify to self-heal it, the scheduler's view would stay stuck at whatever state it last received -- matching the original reported symptom exactly. Not yet caught happening to a terminal notify specifically, so still not a full reproduction, but the mechanism is now demonstrated live rather than theoretical.\n(2) THE SCHEDULER CRASH (open finding (c) above) WAS FINALLY CAPTURED WITH A FULL BACKTRACE -- see bug-1714 for the complete crash detail (SIGSEGV in sched_get_project via a corrupted job_submit_data_t->project_name pointer, core_events.c:221). Tracked as bug-1714 rather than duplicated here since it's a distinct symptom (crash vs. stuck-state) that may or may not share a root cause with this bug's own open half -- both occur under the same heavy concurrent-array-batching conditions, so they remain plausibly entangled, not yet proven either way.\n(3) NO THROUGHPUT STALL observed this run (open finding (d) above) -- DIAG counts climbed steadily and continuously right up to the crash; of 300 submitted array elements only 178 reached DONE before the scheduler died at T+150s, but that is a crash artifact (elements cut off mid-flight), not evidence of an independent stall. Finding (d) remains unconfirmed either way, not further investigated this run.", + "fix": "FIXED (2026-08-29). Root cause (confirmed, not the same theory this entry's prior root_cause investigation was chasing -- see note at the end): apply_committed_entry() (src/controller/persist.c, the shared WAL_JOB_START/FINISH/STOP/RESUME/RUSAGE/CKPT/PREASON/DISPATCH case) fired exactly ONE generic HOOK_DETAIL_JOB_UPDATE at HOOK_AFTER_EVENT for every job-state fact -- correct and still relied on by sched_job_update_fun/apiserver_job_update_fun -- but NEVER the type-specific HOOK_DETAIL_JOB_START/HOOK_DETAIL_JOB_EXIT, even though live handler pairs for both (sched_job_start_fun/sched_job_exit_fun in sched.c, apiserver_job_start_fun/apiserver_job_exit_fun in apiserver.c, all wired via xce_add_hook) have existed the whole time, subscribed at HOOK_AFTER_EVENT. Those handlers were permanently dead code: job.c only ever fires HOOK_DETAIL_JOB_START/JOB_EXIT/JOB_FAIL at HOOK_BEFORE_EVENT (receipt-time, for WAL-append purposes), never AFTER_EVENT -- see persist.h's apply_committed_entry doc comment, 'Deviation from a literal per-type hook fan-out', for the original documented rationale (the old wal_jobstate_t {jobid, state} wire format could not supply the richer job_pidinfo_t/xce_jobfinish_t payloads those handlers require). That blocker is now stale for START and FINISH specifically: bug-1697 (WAL_JOB_START) and bug-1689 (WAL_JOB_FINISH) already enriched this exact wire format with trailing job_pidinfo_t / job_exitinfo_t+finish_time bytes, decoded every apply into j->job.pidinfo / j->job.exit_info -- just never forwarded to the dead handlers. On the apiserver side this meant DefaultHandler.OnJobFinished (src/frontend/apiserver/backend/handler.go) -- itself correct and fully wired via handleNotifyData's NotifyData_data_jobFinishInfo case (clib.go) -- was permanently starved of data: a job could finish for real (worker and controller both correct) and the apiserver's store.Jobs entry would simply never transition off its pre-finish status.\n\nFIX (src/controller/persist.c, apply_committed_entry's shared job-state-fact case): (1) WAL_JOB_FINISH -- inside the existing `result == JOB_TRANSITION_APPLIED` block (the same guard HOOK_DETAIL_JOB_UPDATE already used), also fire HOOK_DETAIL_JOB_EXIT with a stack-local xce_jobfinish_t{jobid=js->jobid, exit_code=(j->job.exit_info ? j->job.exit_info->exitcode : 0)} -- this is the PRIMARY fix. (2) WAL_JOB_START -- a new call-scoped `bool pidinfo_just_decoded` flag, set ONLY where the pre-existing pidinfo decode-and-adopt block (bug-1697) actually decodes a NEW job_pidinfo_t this call (deliberately NOT `j->job.pidinfo != NULL`, which could already be true from an earlier apply and would misfire on an unrelated later transition for the same job); also belt-and-suspenders stamps `decoded->jobid = js->jobid` explicitly right where it is adopted. HOOK_DETAIL_JOB_START now fires whenever pidinfo_just_decoded is true, INDEPENDENT of `result` -- required because bug-1697 already established that every real job's WAL_JOB_START fact applies as a confirmed NOOP_DUPLICATE in production (dispatch_a_job_() sets state=RUN synchronously before WAL_JOB_START ever applies), so gating on result==APPLIED the way JOB_UPDATE is gated would make the new fire dead code forever, live, exactly the class of bug being fixed. The early-return gate right after unlock(j) was widened from a flat `if (result != JOB_TRANSITION_APPLIED) return 0;` to `if (result != JOB_TRANSITION_APPLIED && !pidinfo_just_decoded) return 0;` so a genuinely-just-decoded START pidinfo still reaches the single t_apply_seam_notifying-guarded hook-fire bracket even on a NOOP_DUPLICATE transition; all three fires (new JOB_START, new JOB_EXIT, pre-existing JOB_UPDATE) stay inside that ONE bracket, not duplicated or narrowed. NULL-safety: sched_job_start_fun/sched_job_exit_fun (sched.c) dereference info->jobid unconditionally with no NULL guard (confirmed via codegraph read of both) -- pidinfo_just_decoded's call-scoped design and the belt-and-suspenders jobid stamp exist specifically to make firing with a NULL/stale payload structurally impossible. Confirmed via codegraph/grep that persist_job_hook_fun (the third XCE_CONTROLLER_HOOK_JOB subscriber, registered via ha_register_persist_hooks) does have live HOOK_DETAIL_JOB_START/EXIT cases (unlike the doc comment's characterization of it as 'HOOK_BEFORE_EVENT only for these details') -- but it early-returns immediately on `t_apply_seam_notifying` (Hazard-1 guard, already true for every fire inside apply_committed_entry's bracket) before ever reaching those cases, so no unintended WAL re-append happens; this nuance is a buglog correction to the doc-comment's own framing, not a defect.\n\nTDD evidence: added test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once and test_wjf_apply_committed_entry_reapply_finish_fires_no_hook (test_wal_job_finish_fact.c), test_wjs_apply_committed_entry_fires_job_start_hook_with_pidinfo and test_wjs_apply_committed_entry_old_format_no_job_start_hook (test_wal_job_start_fact.c), using a new detail-aware spy hook per file (distinguishes HOOK_DETAIL_JOB_EXIT/JOB_START from JOB_UPDATE and captures the fired payload, unlike the existing blunt 'any AFTER_EVENT' counters already in those files). RED confirmed on unfixed code (persist.c edits temporarily reverted, test files kept): exactly the 2 new primary-path assertions failed (test_wjs_..._fires_job_start_hook_with_pidinfo: Expected 1 Was 0; test_wjf_..._fires_job_exit_hook_exactly_once: Expected 1 Was 0), plus the pre-existing bug-1561 test_config_repl.c flake pair -- 4 failures total, matching expectations exactly. The two negative-guard tests (old-format-no-pidinfo, reapply-fires-no-hook) already passed RED (correctly -- pre-fix code never fires these hooks at all, so 'must not fire' trivially holds), confirming they are genuine regression guards, not accidentally-tautological.\n\nGREEN evidence: after restoring the fix, four PRE-EXISTING tests broke because their assertions hard-coded the old 'exactly one notification' invariant that this fix deliberately changes (more hooks now fire, by design) -- all four updated in place with corrected expected counts plus an explanatory bug-1731 comment (not weakened, not deleted): test_wjs_apply_committed_entry_sets_pidinfo_but_not_start_time (1 -> 2: JOB_UPDATE + new JOB_START both fire on this genuine PEND->RUN transition with pidinfo), test_wjs_apply_committed_entry_adopts_pidinfo_when_already_run_from_dispatch (0 -> 1: this IS bug-1697's exact NOOP_DUPLICATE production scenario -- the new JOB_START fire is the whole point of the fix here, so asserting 0 would mean re-breaking the bug), test_wjf_apply_committed_entry_sets_finish_time_and_exit_info (1 -> 2: JOB_UPDATE + new JOB_EXIT), test_tari_finish_then_stale_start_preserves_terminal_state_via_dispatch in test_apply_reapply_idempotency.c (2 -> 3 after FINISH, stays 3 after the stale-START no-op -- confirms idempotency: the stale START carries no pidinfo trailing bytes, so pidinfo_just_decoded stays false and the already-guard-rejected transition fires nothing new). Full controller regression after all fixes: 1217 tests, 2 failures -- exactly the known, pre-existing, unrelated bug-1561 flake pair (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config) and nothing else. scheduler/worker/xceadmin builds also completed clean (scheduler build surfaced one unrelated, pre-existing test failure -- test_recordstart_symlink_escape_replies_error, in record-file-path validation, zero relation to job-state hooks or any file this fix touched -- not investigated further, out of scope).\n\nLIVE VERIFICATION (single-node cluster, src/tools/deploy-ha.sh to /tmp/xce-deploy-1731, controller+worker+scheduler+apiserver all built from this fix and started via start-all.sh; two real jobs submitted via xce_test jobsub -u root -q normal -c /bin/true, both ran to completion):\n Scheduler side -- scheduler.log shows, for jobid 0-7-1787994329-448775 (and identically for a second job, 0-14-1787994109-0): `event = schedStartJob` immediately followed by `sched_job_state_change 0-7-1787994329-448775 state 2` (handler.c:388, inside sched_job_state_change, handler.c:381 -- state 2 = JOB_STAT_RUN, the NEW HOOK_DETAIL_JOB_START fire), then later `event = schedExitJob` followed by `sched_job_state_change 0-7-1787994329-448775 state 4` (state 4 = JOB_STAT_EXIT, the NEW HOOK_DETAIL_JOB_EXIT fire) -- with `sched_handle_sched_update_job ... updated` (the pre-existing JOB_UPDATE path) still firing on both sides of it, confirming the new fires are additive, not a replacement. Final `joblist` output confirms the job's state in the scheduler-driven view reached DONE, not stuck at RUN.\n Apiserver side (the part specifically asked to be verified) -- added permanent Info-level logging to DefaultHandler.OnJobFinished (handler.go, mirroring this file's existing zap-field logging convention) since it had none before; rebuilt /tmp/apiserver, restarted it against the live cluster, submitted a fresh job. apiserver.log recorded: `OnJobFinished: job status updated {\\\"jobid\\\": 33400045333567751, \\\"exit_code\\\": 0, \\\"job_status\\\": 8}` -- job_status 8 = models.JOBSTAT[\\\"DONE\\\"], exit_code 0 matching /bin/true's real exit code, jobid matching the submitted job -- direct proof OnJobFinished, previously never invoked in production, now fires correctly end-to-end. All 4 processes (controller, worker, scheduler, apiserver) remained healthy throughout with zero ASAN/crash output in any log.\n\nKept out of scope, as directed: WAL_JOB_FAIL's own dead HOOK_DETAIL_JOB_FAIL handler pair (apiserver_job_fail_fun -> jobFailInfo -> OnJobFinished too) was NOT touched -- it lives in a separate, more complex `case` (dispatches on js->_pad, apply_job_fail_transition vs. apply_job_update_transition) a bit further down persist.c's apply_committed_entry, and is a distinct follow-up fix, not covered here.\n\nFiles touched: src/controller/persist.c (the fix), src/controller/testsuite/test_wal_job_finish_fact.c and test_wal_job_start_fact.c (new regression tests + corrected pre-existing assertions), src/controller/testsuite/test_apply_reapply_idempotency.c (corrected one pre-existing assertion), src/frontend/apiserver/backend/handler.go (permanent OnJobFinished logging, kept -- not reverted, judged a reasonable permanent addition matching this file's existing logging conventions).\n\nCorrection to this entry's own prior root_cause note: the earlier investigation (2026-08-28/29 updates above) was chasing a DIFFERENT theory -- a transient sched.ep==NULL notify-drop window in sched_job_update_fun -- as the suspected mechanism for the original stuck-at-RUN symptom, and left bug-1714 (a scheduler crash found during that chase) as a possibly-entangled open thread. This fix does not address that notify-drop theory or bug-1714 at all; it closes a DIFFERENT, independently confirmed defect (the dead JOB_START/JOB_EXIT hook pair) that was diagnosed and fixed in this session. Both could in principle contribute to real-world stuck-job reports; this entry now tracks the JOB_START/JOB_EXIT hook fix as FIXED, with bug-1714's scheduler-crash thread and any residual sched.ep notify-drop risk left open elsewhere (bug-1714's own entry) rather than reopening speculation here.", + "tags": [ + "scheduler", + "controller", + "job-state", + "notification-loss", + "unresolved", + "unreproduced", + "found-live", + "possible-crash", + "possible-stall", + "session-sibling-of-bug-1730" + ], + "related_bugs": [ + "bug-1714", + "bug-1730" + ], + "occurrences": 2, + "last_seen": "2026-08-29T09:10:00Z" + }, + { + "id": "bug-1732", + "timestamp": "2026-08-29T05:21:00Z", + "error_message": "src/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd xcecd_worker xrun controller_connector_test`), even though deploy-ha.sh requires xcecd_virtual whenever no real systemd manager is present (the common case in sandboxes) -- `/xce-build worker` alone leaves a deploy unable to auto-detect/generate start-worker.sh for the virtual worker.", + "file": "src/tools/xce-build.sh (worker component case)", + "root_cause": "The `worker)` case's explicit meson target list omits `xcecd_virtual`. Confirmed while dispatching a debug-profile build for a bug-1731 live repro: `/xce-build worker debug --incremental` completed successfully but did not produce `xcecd_virtual` in the build dir, requiring a manual `meson compile -C xcecd_virtual` before deploy-ha.sh's own worker-binary auto-detection (Step 3, per the xce-deploy-ha skill) would find it and generate start-worker.sh instead of soft-skipping it.", + "fix": "NOT FIXED -- minor build-tooling gap, not a functional defect in xcecd_virtual itself. Straightforward fix: add `xcecd_virtual` to the worker case's `meson compile` target list in xce-build.sh alongside the existing 4 targets.", + "tags": [ + "build-tooling", + "xce-build.sh", + "worker", + "xcecd_virtual", + "low-priority" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-08-29T05:21:00Z" + }, + { + "id": "bug-1733", + "timestamp": "2026-08-29T06:30:00Z", + "error_message": "AddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517) -- crashed 2 of 3 controllers in a live 3-node HA cluster under a heavy burst-array-job workload (6 concurrent `jobsub -a 50` submissions, 300 near-instant job elements, driving high-frequency WAL-replication-ACK traffic). Found while chasing bug-1714/bug-1731's scheduler crash under ASAN -- the scheduler itself did not crash in this run, but two controllers did, independently, ~7 and ~14 minutes into the run, with byte-for-byte identical ASAN stacks (different PIDs/addresses).", + "file": "src/lib/vtaskmgr.c (vtask_enq, line 517), src/lib/dataq.c (dataq_enq)", + "root_cause": "vtask_enq() (vtaskmgr.c:497-526) calls dataq_enq(&(vtaskmgr->taskq), &(task->link)) at line 514, which PUBLISHES the task onto the shared taskq (under q->lock, released on return) -- immediately making it visible to any idle vcpu worker thread. Only AFTER that publish does line 517 write `task->owner = vtaskmgr;`. A concurrent vcpu thread can dequeue the just-published task, run it to completion (run_task -> the task's own callback, here peer_ctrl_vtask_entry), and vcpu_entry (vtaskmgr.c:106-109) can then call vtask_fini(t)+free(t) -- all BEFORE the producer thread reaches line 517 -- so the producer's write lands on already-freed memory. ASAN's two captured stacks confirm exactly this: thread T6 allocates+enqueues the task (persist_apply_pump_kick -> vtask_alloc/vtask_enq, called via the replication-ACK path repl_ack_decode_and_dispatch -> repl_on_ack -> commit_on_ack -> commit_resolve_pending -> persist_apply_pump_kick), while thread T5 has ALREADY freed the same 48-byte vtask_t region inside vcpu_entry by the time T6's own `task->owner = vtaskmgr` write executes. This is a publish-before-init race -- the task is made visible to concurrent consumers before its own initialization (the `owner` field) completes. Notably, the code immediately above (lines 508-512, `endpoint_ref(ep)`) already gets this right, per an existing bug-1298 comment explicitly calling out 'acquire BEFORE dataq_enq publishes the task to the dispatcher' -- line 517's `owner` assignment simply doesn't follow that same established discipline. Reproduces reliably under this project's burst-array live-HA workload (both controller instances that received the assignment crashed on it during a single ~30-minute run); did not reproduce in the test suite's pool-less fixtures (matches the same class of gap bug-1415 documents for a different vtaskmgr race -- production n_vcpus>1 pool concurrency needed to manifest, not exercised by tests that never call commit_set_pool).", + "fix": "FIXED and TDD-verified (RED attempted, GREEN confirmed). Root cause and fix exactly as diagnosed: vtask_enq() (src/lib/vtaskmgr.c) now sets `task->owner = vtaskmgr;` BEFORE calling dataq_enq(), mirroring the existing bug-1298 endpoint_ref-before-publish discipline immediately above it in the same function; the post-enqueue success block now only calls vtaskmgr_wakeup(vtaskmgr) (the owner write no longer straddles the publish that makes the task visible to concurrent vcpu workers).\n\nNew stress test test_vtask_enq_race_stress (src/lib/testsuite/test_vtask_enq_race.c, wired into libxce_test_src in src/lib/meson.build and testsuite/test.c's extern/TEST() list as test_vtask_enq_race, right after test_vtask_admission): a real vtaskmgr_t pool (8 vcpus) with 16 producer threads each calling vtask_enq_classified(mgr, task, VTASK_LIVENESS) in a tight loop (8000 iterations each, 128,000 tasks total) with true no-op entry callbacks (atomic increment only, matching test_vtask_admission.c's live_task_entry) -- maximizing the odds a consumer vcpu dequeues, runs, and frees a task before the producer's own next instruction lands, exactly the bug-1733 race window.\n\nRED ATTEMPT: this is a genuine data race confined to the two-line window between dataq_enq()'s publish and the owner write, which cannot be forced deterministically from a test without modifying production code (unlike test_vtask_admission.c's fully deterministic cbuf-gated design). Extensive tuning was attempted against the UNMODIFIED vtaskmgr.c before applying the fix -- over a dozen configurations across 60+ manual runs: the default stress shape (6 vcpus/10 producers/4000 iters), bumped concurrency (16v/24p/6000iters, 32v/48p/20000iters), taskset-pinned to 4 cores and to a single core, background CPU-load saturation (22-44 concurrent `yes` processes churning all 22 host cores to force involuntary scheduler preemption right after dataq_enq() returns), and a few-producers/many-idle-vcpus variant (4p/16v/100000iters) meant to keep the dispatcher in a tight dequeue-polling loop. None reproduced the ASAN heap-use-after-free within these bounded attempts, including the official `nix develop ./src --command bash src/tools/xce-build.sh lib asan` run against the unmodified file (the new test itself PASSED cleanly there; the run's only SIGABRT came from a pre-existing, unrelated LeakSanitizer report in test_xcealloc.c -- 456 bytes/29 allocations via strdup in make_alloc/make_host_alloc -- confirmed present byte-for-byte identically both before and after this fix, so not a regression from this change). This is consistent with this entry's own root-cause note that the race needed real multi-node HA production load (WAL-replication-ACK bursts under `jobsub -a 50` burst-array-job load) to manifest, and that it also did not reproduce in prior pool-less test fixtures (same class of gap bug-1415 documents). The original ASAN report captured in this entry (heap-use-after-free WRITE of size 8 at vtask_enq ../lib/vtaskmgr.c:517, freed inside vcpu_entry, matching stacks on 2 of 3 live controllers) remains the authoritative confirmation the bug is real; a stress test that does not happen to hit a hard-to-trigger race in bounded iterations is a weaker regression guard than one that does, but per this fix's own instructions is not a reason to withhold the fix.\n\nGREEN: after the fix, the new stress test passes cleanly and repeatably -- 8 consecutive standalone runs, zero ASAN reports -- at both the isolated and full-suite (`xce-build.sh lib asan --incremental`) level.\n\nFull controller-suite regression (`xce-build.sh controller asan --incremental`, the real consumer of vtask_enq_classified via persist_apply_pump_kick/commit.c): 1213 tests, 2 failures -- both the pre-existing, already-logged bug-1561 timing-collision flake in test_config_repl.c (test_conf_trim_keeps_only_n_versions, test_conf_get_saved_tag_no_config), unrelated to this change. The specific real vtask_enq_classified callers this fix touches indirectly -- test_ha_dispatch_integration.c, test_ha_vep_teardown.c, test_apply_pump.c, test_apply_pump_integration.c, test_commit_admission_pool.c, test_commit_resolve.c -- all pass with zero failures. No regressions.", + "tags": [ + "controller", + "vtaskmgr", + "use-after-free", + "asan-confirmed", + "concurrency", + "high-severity", + "live-crash", + "found-via-bug-1731-repro" + ], + "related_bugs": [ + "bug-1298", + "bug-1415", + "bug-1714", + "bug-1731" + ], + "occurrences": 2, + "last_seen": "2026-08-29T07:23:24Z" } ] } diff --git a/.wolf/memory.md b/.wolf/memory.md index 11260127..f6865597 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -2974,3 +2974,139 @@ | 21:55 | Edited .kiro/specs/scheduler-snapshot-job-allocation/tasks.md | modified up() | ~250 | | 21:57 | Edited src/scheduler/src/snapshot.c | modified load() | ~159 | | 22:12 | Edited .kiro/specs/scheduler-snapshot-job-allocation/tasks.md | inline fix | ~22 | + +## Session: 2026-08-28 23:46 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 23:53 | Edited src/xceadmin/meson-build.sh | expanded (+21 lines) | ~347 | +| 23:54 | Edited src/xceadmin/meson-build.sh | 22→21 lines | ~331 | +| 23:55 | Edited src/xceadmin/meson-build.sh | modified xceadmin() | ~236 | +| 00:07 | Edited src/xceadmin/meson-build.sh | 3→6 lines | ~87 | +| 00:27 | Edited .kiro/specs/admin-config-tool/design.md | inline fix | ~186 | +| 00:28 | Edited .kiro/specs/admin-config-tool/design.md | inline fix | ~25 | +| 00:29 | Edited .kiro/specs/admin-config-tool/design.md | inline fix | ~199 | +| 00:29 | Edited .kiro/specs/admin-config-tool/design.md | 2→2 lines | ~193 | +| 00:29 | Edited .kiro/specs/admin-config-tool/design.md | inline fix | ~114 | +| 00:29 | Edited .kiro/specs/admin-config-tool/design.md | inline fix | ~105 | +| 00:43 | Edited .kiro/specs/admin-config-tool/design.md | 2→2 lines | ~383 | +| 00:43 | Edited .kiro/steering/roadmap.md | "s own premise doesn" → "s address for " | ~351 | +| 00:44 | Edited src/xceadmin/tests/test_e2e_live_ha_push.py | modified UPDATE() | ~994 | +| 00:44 | Edited src/xceadmin/tests/test_e2e_live_ha_push.py | modified UPDATE() | ~253 | +| 00:45 | Edited src/xceadmin/tests/test_e2e_live_ha_push.py | modified test_live_config_push_against_standby_redirects_and_succeeds() | ~998 | +| 00:53 | Edited src/xceadmin/tests/test_e2e_live_ha_push.py | modified _assert_primary_committed() | ~446 | +| 00:53 | Edited src/xceadmin/tests/test_e2e_live_ha_push.py | 7→6 lines | ~102 | +| 00:53 | Edited src/xceadmin/tests/test_e2e_live_ha_push.py | 3→2 lines | ~36 | +| 02:16 | Edited src/scheduler/src/snapshot_load.c | modified if() | ~409 | +| 02:16 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | modified this() | ~310 | +| 02:17 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | modified here() | ~283 | +| 02:18 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | 5→4 lines | ~51 | +| 02:18 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | 16→19 lines | ~286 | +| 02:19 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | 9→13 lines | ~223 | +| 02:19 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | test_sched_account_jobs() → xce_sched_core_load_snapshot() | ~84 | +| 02:20 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | modified 6() | ~1802 | +| 02:21 | Edited src/scheduler/testsuite/test_snapshot_save_load.c | 5→7 lines | ~88 | +| 02:21 | Edited src/scheduler/testsuite/test_sim_startup.c | modified plugin_unregister() | ~255 | +| 02:22 | Edited src/scheduler/testsuite/test_sim_session_ordering.c | modified proof() | ~430 | +| 02:23 | Edited src/scheduler/testsuite/test_sim_session_ordering.c | 4→8 lines | ~119 | +| 02:36 | Edited src/scheduler/testsuite/test_sim_contention_parity.c | expanded (+12 lines) | ~606 | +| 02:41 | Edited src/scheduler/testsuite/test_sim_contention_parity.c | expanded (+13 lines) | ~618 | +| 02:42 | Edited src/scheduler/testsuite/test_sim_contention_parity.c | modified note() | ~164 | +| 02:42 | Edited src/scheduler/testsuite/test_sim_contention_parity.c | 6→6 lines | ~108 | + +## Session: 2026-08-28 02:47 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 03:00 | Created src/scheduler/tools_verify_fixture.c | — | ~1484 | +| 03:00 | Edited src/scheduler/meson.build | expanded (+11 lines) | ~231 | +| 03:01 | Edited src/scheduler/meson.build | expanded (+32 lines) | ~253 | +| 03:01 | Edited src/scheduler/tools_verify_fixture.c | 3→5 lines | ~29 | +| 03:01 | Edited src/scheduler/tools_verify_fixture.c | 2→6 lines | ~58 | +| 03:03 | Edited src/scheduler/tools_verify_fixture.c | 5→6 lines | ~37 | +| 03:03 | Edited src/scheduler/tools_verify_fixture.c | 3→4 lines | ~41 | +| 03:03 | Edited src/scheduler/tools_verify_fixture.c | 3→3 lines | ~39 | +| 03:04 | Edited src/scheduler/src/snapshot_load.c | 1→2 lines | ~30 | +| 03:04 | Edited src/scheduler/src/snapshot_load.c | 2→1 lines | ~8 | +| 03:05 | Edited src/scheduler/meson.build | — | ~0 | +| 03:29 | Edited src/worker/src/uv_ipc.c | 7→8 lines | ~42 | +| 03:29 | Edited src/worker/src/uv_ipc.c | added 1 condition(s) | ~247 | +| 03:29 | Edited src/worker/src/uv_ipc.c | modified if() | ~69 | +| 03:29 | Created src/worker/testsuite/test_uv_ipc_job_exit_severity.c | — | ~706 | +| 03:29 | Edited src/worker/testsuite/meson.build | expanded (+16 lines) | ~273 | +| 03:30 | Edited src/worker/testsuite/test_uv_ipc_job_exit_severity.c | modified with() | ~172 | +| 03:30 | Edited src/worker/src/uv_ipc.c | modified job_exit_log_level() | ~42 | +| 03:30 | Edited src/worker/src/uv_ipc.c | added 1 condition(s) | ~50 | +| 03:41 | Created .kiro/specs/scheduler-config-version-tag/spec.json | — | ~132 | +| 03:42 | Created .kiro/specs/scheduler-config-version-tag/requirements.md | — | ~666 | +| 03:44 | Created .kiro/specs/scheduler-config-version-tag/requirements.md | — | ~2530 | +| 03:44 | Edited .kiro/specs/scheduler-config-version-tag/spec.json | 8→8 lines | ~55 | +| 03:50 | Created .kiro/specs/scheduler-config-version-tag/research.md | — | ~2892 | +| 03:51 | Created .kiro/specs/scheduler-config-version-tag/design.md | — | ~6840 | +| 03:52 | Edited .kiro/specs/scheduler-config-version-tag/spec.json | 12→12 lines | ~74 | +| 03:55 | Created .kiro/specs/scheduler-config-version-tag/tasks.md | — | ~1905 | +| 03:58 | Edited .kiro/specs/scheduler-config-version-tag/tasks.md | 3→2 lines | ~138 | +| 03:58 | Edited .kiro/specs/scheduler-config-version-tag/tasks.md | 2→2 lines | ~5 | +| 03:58 | Edited .kiro/specs/scheduler-config-version-tag/tasks.md | 1.1 → 1 | ~40 | +| 03:58 | Edited .kiro/specs/scheduler-config-version-tag/tasks.md | 1.1 → 1 | ~52 | +| 03:58 | Edited .kiro/specs/scheduler-config-version-tag/tasks.md | inline fix | ~31 | +| 04:00 | Created .kiro/specs/scheduler-config-version-tag/spec.json | — | ~131 | + +## Session: 2026-08-28 04:53 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| + +## Session: 2026-08-28 04:54 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 06:17 | Edited src/controller/persist.c | modified if() | ~262 | +| 06:35 | Edited src/controller/persist.c | reduced (-10 lines) | ~107 | +| 06:37 | Edited src/scheduler/testsuite/test_core_events.c | 3→4 lines | ~23 | +| 06:38 | Edited src/scheduler/testsuite/test_core_events.c | modified test_job_update_to_terminal_state_frees_accounts_and_allocs() | ~709 | +| 06:38 | Edited src/scheduler/testsuite/test_core_events.c | 2→3 lines | ~56 | +| 06:38 | Edited src/scheduler/src/core_events.c | added 3 condition(s) | ~369 | +| 07:26 | Edited src/controller/job.c | 5→9 lines | ~115 | +| 07:28 | Edited src/controller/sched.c | modified sched_job_update_fun() | ~444 | +| 07:46 | Edited src/controller/main.c | added 1 condition(s) | ~184 | +| 07:47 | Edited src/scheduler/src/main.c | added 1 condition(s) | ~189 | + +## Session: 2026-08-29 20:37 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 21:59 | Edited src/controller/job.c | 4→7 lines | ~123 | +| 21:59 | Edited src/controller/sched.c | modified diagnostic() | ~348 | +| 23:41 | Created src/lib/testsuite/test_vtask_enq_race.c | — | ~1427 | +| 23:41 | Edited src/lib/testsuite/test_vtask_enq_race.c | test_vtask_enq_race() → test_vtask_enq_race_stress() | ~13 | +| 23:41 | Edited src/lib/testsuite/test_vtask_enq_race.c | test_vtask_enq_race_suite() → test_vtask_enq_race() | ~21 | +| 23:42 | Edited src/lib/meson.build | 2→3 lines | ~42 | +| 23:42 | Edited src/lib/testsuite/test.c | 2→3 lines | ~30 | +| 23:42 | Edited src/lib/testsuite/test.c | 2→3 lines | ~24 | +| 23:44 | Edited src/lib/testsuite/test_vtask_enq_race.c | 3→3 lines | ~29 | +| 23:47 | Edited src/lib/testsuite/test_vtask_enq_race.c | 3→3 lines | ~29 | +| 23:54 | Edited src/lib/testsuite/test_vtask_enq_race.c | 2→2 lines | ~40 | +| 00:09 | Edited src/lib/testsuite/test_vtask_enq_race.c | 3→3 lines | ~29 | +| 00:14 | Edited src/lib/testsuite/test_vtask_enq_race.c | 3→3 lines | ~29 | +| 00:15 | Edited src/lib/testsuite/test_vtask_enq_race.c | 3→3 lines | ~28 | +| 00:15 | Edited src/lib/testsuite/test_vtask_enq_race.c | 2→2 lines | ~40 | +| 00:17 | Edited src/lib/vtaskmgr.c | modified if() | ~260 | +| 00:59 | Edited src/controller/persist.c | modified if() | ~590 | +| 01:00 | Edited src/controller/persist.c | added 3 condition(s) | ~1170 | +| 01:01 | Edited src/controller/testsuite/test_wal_job_finish_fact.c | added 2 condition(s) | ~395 | +| 01:01 | Edited src/controller/testsuite/test_wal_job_finish_fact.c | modified test_wjf_apply_committed_entry_fires_job_exit_hook_exactly_once() | ~1578 | +| 01:02 | Edited src/controller/testsuite/test_wal_job_finish_fact.c | 3→5 lines | ~103 | +| 01:02 | Edited src/controller/testsuite/test_wal_job_start_fact.c | added 3 condition(s) | ~316 | +| 01:03 | Edited src/controller/testsuite/test_wal_job_start_fact.c | modified test_wjs_apply_committed_entry_fires_job_start_hook_with_pidinfo() | ~1653 | +| 01:03 | Edited src/controller/testsuite/test_wal_job_start_fact.c | 3→5 lines | ~90 | +| 01:03 | Edited src/controller/persist.c | modified if() | ~195 | +| 01:04 | Edited src/controller/persist.c | modified guard() | ~245 | +| 01:24 | Edited src/controller/testsuite/test_wal_job_start_fact.c | expanded (+6 lines) | ~210 | +| 01:25 | Edited src/controller/testsuite/test_wal_job_start_fact.c | expanded (+8 lines) | ~292 | +| 01:25 | Edited src/controller/testsuite/test_wal_job_start_fact.c | 4→8 lines | ~154 | +| 01:25 | Edited src/controller/testsuite/test_wal_job_finish_fact.c | 7→12 lines | ~165 | +| 01:26 | Edited src/controller/testsuite/test_apply_reapply_idempotency.c | notification() → notifications() | ~130 | +| 01:26 | Edited src/controller/testsuite/test_apply_reapply_idempotency.c | 6→8 lines | ~133 | +| 02:04 | Edited src/frontend/apiserver/backend/handler.go | modified OnJobFinished() | ~308 | +| 02:04 | Edited src/frontend/apiserver/backend/handler.go | 5→5 lines | ~51 |