From 76511d356c526e34d2a4ca25f57f385461814243 Mon Sep 17 00:00:00 2001 From: Rongsong Shen Date: Tue, 1 Sep 2026 07:08:39 +0000 Subject: [PATCH] docs: sync OpenWolf state Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Y4M5MJM2zK6nU43da267Ku --- .wolf/anatomy-index.json | 728 +++++++++++++++++++++++++++++++++++++-- .wolf/anatomy.md | 51 ++- .wolf/buglog.json | 93 +++++ .wolf/cerebrum.md | 1 + .wolf/memory.md | 106 ++++++ 5 files changed, 934 insertions(+), 45 deletions(-) diff --git a/.wolf/anatomy-index.json b/.wolf/anatomy-index.json index 860dcb2c..d2dabd9e 100644 --- a/.wolf/anatomy-index.json +++ b/.wolf/anatomy-index.json @@ -1,13 +1,13 @@ { "version": 1, "meta": { - "lastScanned": "2026-08-31T07:27:35.760Z", - "fileCount": 583, + "lastScanned": "2026-09-01T04:31:57.684Z", + "fileCount": 609, "hits": 0, "misses": 0, - "renderedHash": "26a339949bedbbe703d956b089dac35cf48ba98ad08d3456f3f155a3dc6014b7", - "storeUpdatedAt": "2026-08-31T07:27:35.768Z", - "rootHash": "12e99f5286a6e512" + "renderedHash": "ac5b5f6e7d59f258353730a62f37ae07780c5ee1f4f53ca40e6ffc77c894f013", + "storeUpdatedAt": "2026-09-01T04:31:57.700Z", + "rootHash": "41f13be2591204fb" }, "files": { ".agent-shell/transcripts/2026-08-11-11-39-59.md": { @@ -3138,13 +3138,12 @@ }, ".kiro/specs/worker-rejoin-resync/spec.json": { "description": "", - "tokens": 200, - "hash": "6d315d9a2fb820fc", - "size": 700, - "mtimeMs": 1786423400000, - "updatedAt": "2026-08-27T19:00:00.132Z", - "source": "scan", - "importance": 0.21 + "tokens": 234, + "hash": "4879171af821d683", + "size": 816, + "mtimeMs": 1788230617367.6006, + "updatedAt": "2026-09-01T02:43:37.404Z", + "source": "hook" }, ".kiro/specs/xce-test-cli-ux/requirements.md": { "description": "Requirements Document", @@ -6135,11 +6134,11 @@ }, "src/scheduler/testsuite/test_sim_session_ordering.c": { "description": "", - "tokens": 8092, - "hash": "2010da22fa29c474", - "size": 30345, - "mtimeMs": 1787908989473.972, - "updatedAt": "2026-08-28T09:23:09.518Z", + "tokens": 8267, + "hash": "458f9964a224d28a", + "size": 31000, + "mtimeMs": 1788163479007.5847, + "updatedAt": "2026-08-31T08:04:39.048Z", "source": "hook" }, "src/scheduler/testsuite/test_sim_contention_parity.c": { @@ -6252,20 +6251,20 @@ }, "src/scheduler/testsuite/test_core_events.c": { "description": "Declares and", - "tokens": 14476, - "hash": "e73e198fa18f8668", - "size": 54284, - "mtimeMs": 1788146521984.0552, - "updatedAt": "2026-08-31T03:22:02.022Z", + "tokens": 16152, + "hash": "ef3fd846dbd2238c", + "size": 60567, + "mtimeMs": 1788162330447.1611, + "updatedAt": "2026-08-31T07:45:30.494Z", "source": "hook" }, "src/scheduler/src/core_events.c": { "description": "include ", - "tokens": 15175, - "hash": "61bc369088277fa5", - "size": 56904, - "mtimeMs": 1788146214025.4373, - "updatedAt": "2026-08-31T03:16:54.071Z", + "tokens": 15473, + "hash": "b15e16b43042ba42", + "size": 58023, + "mtimeMs": 1788161752878.102, + "updatedAt": "2026-08-31T07:35:52.918Z", "source": "hook" }, "src/controller/job.c": { @@ -6638,11 +6637,11 @@ }, "src/scheduler/testsuite/test_admin_request.c": { "description": "Declares level", - "tokens": 13020, - "hash": "0aca61e7462c9340", - "size": 48824, - "mtimeMs": 1788103082342.1372, - "updatedAt": "2026-08-30T15:18:02.611Z", + "tokens": 13172, + "hash": "6ddb2256e997f95b", + "size": 49395, + "mtimeMs": 1788163535203.614, + "updatedAt": "2026-08-31T08:05:35.237Z", "source": "hook" }, "src/include/xce/types.h": { @@ -6809,11 +6808,11 @@ }, "src/scheduler/testsuite/test_repl_breakpoints_inspection.c": { "description": "Declares and", - "tokens": 20658, - "hash": "d1cc28ebe7a077e6", - "size": 77467, - "mtimeMs": 1788148440614.8572, - "updatedAt": "2026-08-31T03:54:00.657Z", + "tokens": 20801, + "hash": "5691bb0f38075430", + "size": 78002, + "mtimeMs": 1788163543885.636, + "updatedAt": "2026-08-31T08:05:43.937Z", "source": "hook" }, ".kiro/steering/resource-accounting.md": { @@ -6887,6 +6886,661 @@ "mtimeMs": 1788160695621.7214, "updatedAt": "2026-08-31T07:18:15.658Z", "source": "hook" + }, + "src/scheduler/testsuite/test_record_lifecycle.c": { + "description": "Declares char", + "tokens": 12885, + "hash": "bc2a587a947412fe", + "size": 48316, + "mtimeMs": 1788163457579.628, + "updatedAt": "2026-08-31T08:04:17.625Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_alt_events.c": { + "description": "Declares char", + "tokens": 5141, + "hash": "9de156c85623ea97", + "size": 19276, + "mtimeMs": 1788163470841.5977, + "updatedAt": "2026-08-31T08:04:30.890Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_snapshot_reconcile_gap.c": { + "description": "Declares char", + "tokens": 6308, + "hash": "fc3d0b2c3e5115c2", + "size": 23652, + "mtimeMs": 1788163490082.5747, + "updatedAt": "2026-08-31T08:04:50.124Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_core_full_cycle.c": { + "description": "Declares is", + "tokens": 8075, + "hash": "b29457d604318d79", + "size": 30280, + "mtimeMs": 1788163530854.6047, + "updatedAt": "2026-08-31T08:05:30.891Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_repl_equivalence.c": { + "description": "Declares char", + "tokens": 7810, + "hash": "3a5f6c5add4110c9", + "size": 29285, + "mtimeMs": 1788163539560.6245, + "updatedAt": "2026-08-31T08:05:39.596Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_verify.c": { + "description": "Declares mismatch", + "tokens": 15602, + "hash": "35a3398455a4cdf9", + "size": 58505, + "mtimeMs": 1788163548259.6487, + "updatedAt": "2026-08-31T08:05:48.305Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_driver_session_ordering.c": { + "description": "Declares started", + "tokens": 8672, + "hash": "207a8ab9c66d4c55", + "size": 32520, + "mtimeMs": 1788163553099.664, + "updatedAt": "2026-08-31T08:05:53.156Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_repl_execution_control.c": { + "description": "Declares directly", + "tokens": 9054, + "hash": "5f2169ba62c7ff1d", + "size": 33949, + "mtimeMs": 1788163557882.6807, + "updatedAt": "2026-08-31T08:05:57.928Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_driver_decision_cb.c": { + "description": "Declares is", + "tokens": 3835, + "hash": "abd73ff8275469b4", + "size": 14380, + "mtimeMs": 1788163562121.6963, + "updatedAt": "2026-08-31T08:06:02.154Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_repl_snapshot_params.c": { + "description": "Declares char", + "tokens": 8332, + "hash": "3a2d461ef84c2102", + "size": 31242, + "mtimeMs": 1788163566295.713, + "updatedAt": "2026-08-31T08:06:06.330Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_driver_translate.c": { + "description": "Declares field", + "tokens": 9667, + "hash": "c0f565714b4e6174", + "size": 36248, + "mtimeMs": 1788163570412.73, + "updatedAt": "2026-08-31T08:06:10.457Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_record_output.c": { + "description": "Declares char", + "tokens": 6041, + "hash": "1c269f80ccb016f2", + "size": 22651, + "mtimeMs": 1788163574765.7493, + "updatedAt": "2026-08-31T08:06:14.808Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_equivalence.c": { + "description": "Declares BOTH", + "tokens": 8517, + "hash": "a89a90a2111ada20", + "size": 31938, + "mtimeMs": 1788163629024.0742, + "updatedAt": "2026-08-31T08:07:09.065Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_sim_event_loop.c": { + "description": "Declares char", + "tokens": 9504, + "hash": "818d668fbb75430d", + "size": 35638, + "mtimeMs": 1788163633685.1091, + "updatedAt": "2026-08-31T08:07:13.721Z", + "source": "hook" + }, + "src/scheduler/testsuite/test_resval_ownership.c": { + "description": "Declares instead", + "tokens": 5631, + "hash": "ea9c02bf9c42cf86", + "size": 21113, + "mtimeMs": 1788163642640.1794, + "updatedAt": "2026-08-31T08:07:22.676Z", + "source": "hook" + }, + ".kiro/specs/worker-rejoin-resync/requirements.md": { + "description": "Requirements Document", + "tokens": 3631, + "hash": "69bfce8c2ce65f71", + "size": 14540, + "mtimeMs": 1788233275721.5093, + "updatedAt": "2026-09-01T03:27:55.760Z", + "source": "hook" + }, + ".kiro/specs/job-dispatch-commit-batching/spec.json": { + "description": "", + "tokens": 263, + "hash": "920b00de9e90f394", + "size": 919, + "mtimeMs": 1788185280271.2622, + "updatedAt": "2026-08-31T14:08:00.308Z", + "source": "hook" + }, + ".kiro/specs/job-dispatch-commit-batching/requirements.md": { + "description": "Requirements Document", + "tokens": 874, + "hash": "56bbb73ff9f63d09", + "size": 3493, + "mtimeMs": 1788185344106.4456, + "updatedAt": "2026-08-31T14:09:04.154Z", + "source": "hook" + }, + ".kiro/specs/commit-batch-window/spec.json": { + "description": "", + "tokens": 327, + "hash": "825d673757aee8b4", + "size": 1144, + "mtimeMs": 1788185429825.3875, + "updatedAt": "2026-08-31T14:10:29.861Z", + "source": "hook" + }, + ".kiro/specs/commit-batch-window/requirements.md": { + "description": "Requirements Document", + "tokens": 1569, + "hash": "5c8978b23da48cb8", + "size": 6301, + "mtimeMs": 1788185618616.899, + "updatedAt": "2026-08-31T14:13:38.651Z", + "source": "hook" + }, + ".kiro/specs/worker-rejoin-resync/research.md": { + "description": "Research & Design Decisions: worker-rejoin-resync", + "tokens": 5335, + "hash": "81f0c4dd9075072d", + "size": 21468, + "mtimeMs": 1788232268357.4172, + "updatedAt": "2026-09-01T03:11:08.398Z", + "source": "hook" + }, + ".kiro/specs/worker-rejoin-resync/design.md": { + "description": "Design Document: worker-rejoin-resync", + "tokens": 11515, + "hash": "d1250c8112e3d5ae", + "size": 46333, + "mtimeMs": 1788233986582.8726, + "updatedAt": "2026-09-01T03:39:46.644Z", + "source": "hook" + }, + "src/frontend/apiserver/controllers/jobs_test.go": { + "description": "TestPreCheckJobRequest_LostAndFound, TestJobsListFieldValidation_UsesJobInfoNotSubmitRequest, TestPr", + "tokens": 2001, + "hash": "fe8328757f178e16", + "size": 7505, + "mtimeMs": 1788236589849.0354, + "updatedAt": "2026-09-01T04:23:09.938Z", + "source": "hook", + "symbols": [ + { + "name": "resetJobSubmitState", + "kind": "fn", + "startLine": 13, + "endLine": 19, + "tokens": 62 + }, + { + "name": "TestPreCheckJobRequest_LostAndFound", + "kind": "fn", + "startLine": 20, + "endLine": 63, + "tokens": 602 + }, + { + "name": "TestJobsListFieldValidation_UsesJobInfoNotSubmitRequest", + "kind": "fn", + "startLine": 64, + "endLine": 85, + "tokens": 343 + }, + { + "name": "TestPreCheckJobRequest_NormalQueue", + "kind": "fn", + "startLine": 86, + "endLine": 108, + "tokens": 225 + }, + { + "name": "TestPreCheckJobRequest_EmptyQueue", + "kind": "fn", + "startLine": 109, + "endLine": 131, + "tokens": 306 + }, + { + "name": "TestValidateJobControl_HandlesRealJobInfoTypes", + "kind": "fn", + "startLine": 132, + "endLine": 157, + "tokens": 264 + }, + { + "name": "TestValidateJobControl_RejectsNonKillableStatus", + "kind": "fn", + "startLine": 158, + "endLine": 177, + "tokens": 188 + }, + { + "name": "TestFoundJob_MatchesRealArrayIdxType", + "kind": "fn", + "startLine": 178, + "endLine": 195, + "tokens": 117 + } + ] + }, + "src/frontend/apiserver/controllers/jobs.go": { + "description": "Struct: JobController", + "tokens": 30943, + "hash": "a9e0151b1eaea9d0", + "size": 116036, + "mtimeMs": 1788236691520.6968, + "updatedAt": "2026-09-01T04:24:51.605Z", + "source": "hook", + "symbols": [ + { + "name": "JobController", + "kind": "class", + "startLine": 44, + "endLine": 56, + "tokens": 124 + }, + { + "name": "GetJobsInfo", + "kind": "fn", + "startLine": 57, + "endLine": 724, + "tokens": 6553 + }, + { + "name": "GetJobByIDs", + "kind": "fn", + "startLine": 725, + "endLine": 1042, + "tokens": 2976 + }, + { + "name": "extractJobNames", + "kind": "fn", + "startLine": 1043, + "endLine": 1073, + "tokens": 234 + }, + { + "name": "RestartJobByID", + "kind": "fn", + "startLine": 1074, + "endLine": 1374, + "tokens": 2732 + }, + { + "name": "CreateJob", + "kind": "fn", + "startLine": 1375, + "endLine": 1524, + "tokens": 1605 + }, + { + "name": "preCheckJobRestartRequest", + "kind": "fn", + "startLine": 1525, + "endLine": 1532, + "tokens": 60 + }, + { + "name": "preCheckJobRequest", + "kind": "fn", + "startLine": 1533, + "endLine": 1740, + "tokens": 1934 + }, + { + "name": "checkNumProcessors", + "kind": "fn", + "startLine": 1741, + "endLine": 1751, + "tokens": 125 + }, + { + "name": "parseArrayName", + "kind": "fn", + "startLine": 1752, + "endLine": 1814, + "tokens": 584 + }, + { + "name": "createArrayName", + "kind": "fn", + "startLine": 1815, + "endLine": 1971, + "tokens": 1562 + }, + { + "name": "isValidArrayName", + "kind": "fn", + "startLine": 1972, + "endLine": 1983, + "tokens": 121 + }, + { + "name": "isValidJobId", + "kind": "fn", + "startLine": 1984, + "endLine": 1992, + "tokens": 91 + }, + { + "name": "validateAskedHosts", + "kind": "fn", + "startLine": 1993, + "endLine": 2018, + "tokens": 236 + }, + { + "name": "parseTransferFiles", + "kind": "fn", + "startLine": 2019, + "endLine": 2048, + "tokens": 271 + }, + { + "name": "processJobRestartRequest", + "kind": "fn", + "startLine": 2049, + "endLine": 2055, + "tokens": 72 + }, + { + "name": "processJobRequest", + "kind": "fn", + "startLine": 2056, + "endLine": 2207, + "tokens": 1338 + }, + { + "name": "postJobRequest", + "kind": "fn", + "startLine": 2208, + "endLine": 2251, + "tokens": 476 + }, + { + "name": "ControlJobByID", + "kind": "fn", + "startLine": 2252, + "endLine": 2633, + "tokens": 3723 + }, + { + "name": "ControlJob", + "kind": "fn", + "startLine": 2634, + "endLine": 2854, + "tokens": 2038 + }, + { + "name": "preCheckJobControlRequest", + "kind": "fn", + "startLine": 2855, + "endLine": 2921, + "tokens": 548 + }, + { + "name": "getFilters", + "kind": "fn", + "startLine": 2922, + "endLine": 2996, + "tokens": 578 + }, + { + "name": "validateJobControl", + "kind": "fn", + "startLine": 2997, + "endLine": 3092, + "tokens": 1108 + }, + { + "name": "foundJob", + "kind": "fn", + "startLine": 3093, + "endLine": 3116, + "tokens": 202 + }, + { + "name": "processJobData", + "kind": "fn", + "startLine": 3117, + "endLine": 3194, + "tokens": 697 + }, + { + "name": "copyLocalFilesConcurrently", + "kind": "fn", + "startLine": 3195, + "endLine": 3233, + "tokens": 414 + }, + { + "name": "copyRemoteFilesLocally", + "kind": "fn", + "startLine": 3234, + "endLine": 3252, + "tokens": 188 + }, + { + "name": "parseArrayIndexSection", + "kind": "fn", + "startLine": 3253, + "endLine": 3280, + "tokens": 205 + }, + { + "name": "FetchJobList", + "kind": "fn", + "startLine": 3281, + "endLine": 3470, + "tokens": 1561 + }, + { + "name": "FetchJobIdList", + "kind": "fn", + "startLine": 3471, + "endLine": 3560, + "tokens": 622 + } + ] + }, + "src/frontend/apiserver/controllers/host_control_test.go": { + "description": "TestPreCheckHostControlRequest_AllFlag, TestPreCheckHostControlRequest_NotAll, TestPreCheckHostContr", + "tokens": 1579, + "hash": "9a345125acd2dacc", + "size": 5921, + "mtimeMs": 1788237064598.0837, + "updatedAt": "2026-09-01T04:31:04.706Z", + "source": "hook", + "symbols": [ + { + "name": "resetHostControlState", + "kind": "fn", + "startLine": 13, + "endLine": 19, + "tokens": 55 + }, + { + "name": "TestPreCheckHostControlRequest_AllFlag", + "kind": "fn", + "startLine": 20, + "endLine": 38, + "tokens": 142 + }, + { + "name": "TestPreCheckHostControlRequest_NotAll", + "kind": "fn", + "startLine": 39, + "endLine": 57, + "tokens": 153 + }, + { + "name": "TestPreCheckHostControlRequest_NoRace", + "kind": "fn", + "startLine": 58, + "endLine": 104, + "tokens": 328 + }, + { + "name": "TestPreCheckHostControlRequest_InvalidAction", + "kind": "fn", + "startLine": 105, + "endLine": 120, + "tokens": 127 + }, + { + "name": "TestPreCheckHostControlRequest_CommentTooLong", + "kind": "fn", + "startLine": 121, + "endLine": 150, + "tokens": 346 + }, + { + "name": "TestProcessHostData_ComputesSlotUsageFromJobs", + "kind": "fn", + "startLine": 151, + "endLine": 200, + "tokens": 505 + } + ] + }, + "src/frontend/apiserver/controllers/hosts.go": { + "description": "Struct: HostHandler", + "tokens": 5549, + "hash": "f277822603359889", + "size": 20806, + "mtimeMs": 1788237117599.9812, + "updatedAt": "2026-09-01T04:31:57.683Z", + "source": "hook", + "symbols": [ + { + "name": "HostHandler", + "kind": "class", + "startLine": 20, + "endLine": 31, + "tokens": 72 + }, + { + "name": "GetHostsInfo", + "kind": "fn", + "startLine": 32, + "endLine": 198, + "tokens": 1616 + }, + { + "name": "selectFields", + "kind": "fn", + "startLine": 199, + "endLine": 248, + "tokens": 360 + }, + { + "name": "GetHostByHostname", + "kind": "fn", + "startLine": 249, + "endLine": 362, + "tokens": 1142 + }, + { + "name": "preCheckHostRequest", + "kind": "fn", + "startLine": 363, + "endLine": 374, + "tokens": 86 + }, + { + "name": "FilterHostByResReqSelect", + "kind": "fn", + "startLine": 375, + "endLine": 382, + "tokens": 102 + }, + { + "name": "ControlHost", + "kind": "fn", + "startLine": 383, + "endLine": 474, + "tokens": 839 + }, + { + "name": "validateHost", + "kind": "fn", + "startLine": 475, + "endLine": 538, + "tokens": 610 + }, + { + "name": "isHostControlActionAllowed", + "kind": "fn", + "startLine": 539, + "endLine": 550, + "tokens": 85 + }, + { + "name": "preCheckHostControlRequest", + "kind": "fn", + "startLine": 551, + "endLine": 571, + "tokens": 212 + }, + { + "name": "hostSlotUsage", + "kind": "class", + "startLine": 572, + "endLine": 577, + "tokens": 52 + }, + { + "name": "execHostsJSON", + "kind": "class", + "startLine": 578, + "endLine": 588, + "tokens": 124 + }, + { + "name": "computeHostSlotUsage", + "kind": "fn", + "startLine": 589, + "endLine": 616, + "tokens": 201 + }, + { + "name": "processHostData", + "kind": "fn", + "startLine": 617, + "endLine": 658, + "tokens": 368 + } + ] } } } \ No newline at end of file diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index ac87fcf4..ceb4bfad 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,7 +1,7 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-08-31T07:27:35.760Z -> Files: 583 tracked | Anatomy hits: 0 | Misses: 0 +> Auto-maintained by OpenWolf. Last scanned: 2026-09-01T04:31:57.684Z +> Files: 609 tracked | Anatomy hits: 0 | Misses: 0 ## ./ @@ -101,6 +101,11 @@ - `spec.json` (~133 tok) - `tasks.md` — Implementation Plan (~2668 tok) +## .kiro/specs/commit-batch-window/ + +- `requirements.md` — Requirements Document (~1569 tok) +- `spec.json` (~327 tok) + ## .kiro/specs/component-failover-adoption/ - `design.md` — Design Document: component-failover-adoption (~17338 tok) @@ -313,6 +318,11 @@ - `spec.json` (~444 tok) - `tasks.md` — Implementation Plan: job-array-single-wal-record (~15580 tok) +## .kiro/specs/job-dispatch-commit-batching/ + +- `requirements.md` — Requirements Document (~874 tok) +- `spec.json` (~263 tok) + ## .kiro/specs/libxce-failover/ - `brief.md` — Brief: libxce-failover (~2238 tok) @@ -518,7 +528,10 @@ ## .kiro/specs/worker-rejoin-resync/ - `brief.md` — Brief: worker-rejoin-resync (~1394 tok) -- `spec.json` (~200 tok) +- `design.md` — Design Document: worker-rejoin-resync (~11515 tok) +- `requirements.md` — Requirements Document (~3631 tok) +- `research.md` — Research & Design Decisions: worker-rejoin-resync (~5335 tok) +- `spec.json` (~234 tok) ## .kiro/specs/xce-test-cli-ux/ @@ -848,6 +861,13 @@ - `handler.go` — Struct: DefaultHandler (~3107 tok) +## src/frontend/apiserver/controllers/ + +- `host_control_test.go` — TestPreCheckHostControlRequest_AllFlag, TestPreCheckHostControlRequest_NotAll, TestPreCheckHostContr (~1579 tok) +- `hosts.go` — Struct: HostHandler (~5549 tok) +- `jobs_test.go` — TestPreCheckJobRequest_LostAndFound, TestJobsListFieldValidation_UsesJobInfoNotSubmitRequest, TestPr (~2001 tok) +- `jobs.go` — Struct: JobController (~30943 tok) + ## src/include/ - `typemap.h` — if !defined(_TYPEMAP_H_) (~8034 tok) @@ -880,7 +900,7 @@ ## src/scheduler/src/ -- `core_events.c` — include (~15175 tok) +- `core_events.c` — include (~15473 tok) - `driver.c` — include (~9779 tok) - `handler.c` — include (~7039 tok) - `main.c` — include (~5330 tok) @@ -892,18 +912,33 @@ ## src/scheduler/testsuite/ -- `test_admin_request.c` — Declares level (~13020 tok) -- `test_core_events.c` — Declares and (~14476 tok) +- `test_admin_request.c` — Declares level (~13172 tok) +- `test_core_events.c` — Declares and (~16152 tok) +- `test_core_full_cycle.c` — Declares is (~8075 tok) +- `test_driver_decision_cb.c` — Declares is (~3835 tok) +- `test_driver_session_ordering.c` — Declares started (~8672 tok) +- `test_driver_translate.c` — Declares field (~9667 tok) - `test_handlers.c` — define UNIT_TEST 1 (~3218 tok) - `test_job_update.c` — Declares char (~3307 tok) - `test_picker_config_fields.c` — Declares char (~3355 tok) - `test_picker_wiring.c` — Declares is (~5443 tok) -- `test_repl_breakpoints_inspection.c` — Declares and (~20658 tok) +- `test_record_lifecycle.c` — Declares char (~12885 tok) +- `test_repl_breakpoints_inspection.c` — Declares and (~20801 tok) +- `test_repl_equivalence.c` — Declares char (~7810 tok) +- `test_repl_execution_control.c` — Declares directly (~9054 tok) +- `test_repl_snapshot_params.c` — Declares char (~8332 tok) - `test_resource_accounting.c` — Declares char (~8455 tok) +- `test_resval_ownership.c` — Declares instead (~5631 tok) +- `test_sim_alt_events.c` — Declares char (~5141 tok) - `test_sim_contention_parity.c` — Declares calls (~18713 tok) - `test_sim_e2e_original_motivation.c` (~17120 tok) -- `test_sim_session_ordering.c` (~8092 tok) +- `test_sim_equivalence.c` — Declares BOTH (~8517 tok) +- `test_sim_event_loop.c` — Declares char (~9504 tok) +- `test_sim_record_output.c` — Declares char (~6041 tok) +- `test_sim_session_ordering.c` (~8267 tok) +- `test_sim_snapshot_reconcile_gap.c` — Declares char (~6308 tok) - `test_sim_startup.c` — Declares char (~5708 tok) +- `test_sim_verify.c` — Declares mismatch (~15602 tok) - `test_snapshot_save_load.c` (~28712 tok) ## src/schema/ diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 82f018ca..c7abb590 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -20165,6 +20165,71 @@ "occurrences": 1, "last_seen": "2026-08-30T22:15:00Z" }, + { + "id": "bug-1740", + "timestamp": "2026-08-31T22:00:00Z", + "error_message": "The bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage(), resource.c, committed this session) was wired into schedobj.c's sched_apply_job_rusage() -- but that function has exactly ONE caller anywhere in this codebase: scheduler.c's sched_update(), which itself has NO live caller at all (confirmed by exhaustive grep -- superseded by driver.c's xce_sched_driver_translate_pending_events()/xce_core_push_events() mechanism, per driver.c's own comments: \"Replaces sched_update()'s inline handling\"). The REAL, live production path for job rusage -- XCE_CORE_EVENT_JOB_RUSAGE in core_events.c's xce_core_push_events(), the SAME core-level event both the normal-mode driver (driver.c) and the sim/replay driver (sim_loop.c) translate their own rusage events into -- has its OWN separate, duplicate inline implementation of \"store rusage on job\" that never called sched_apply_job_rusage() or the new reconciliation logic at all. Result: the entire reconciliation feature was completely inert in both normal-mode (live cluster) and sim/replay mode -- exactly the context the user's original bug reports (phantom RUN jobs, EXIT/DONE divergence, mem always 0) were observed through -- despite passing all 11 of its own unit tests, which called the reconciliation functions directly, bypassing the real event-dispatch path entirely.", + "root_cause": "Classic 'tested the function in isolation, never proved the real event path reaches it' gap. schedobj.c's sched_apply_job_rusage() and core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler are two independent, duplicate reimplementations of the same 'store rusage on job' bookkeeping (a pre-existing architectural duplication, likely left over from the scheduler-core rewrite that introduced the core/driver split -- sched_update() and its whole XCE_SCHED_OP_* dataq-dispatch mechanism appear to be the OLD, pre-rewrite path, fully superseded by driver.c's XCE_CORE_EVENT_* translation but never deleted). The two implementations had already drifted: they call DIFFERENT picker hooks (on_job_rusage vs plugin_job_rusage_update). Adding new logic to the dead one instead of the live one was an easy mistake to make without first confirming which path is actually reachable in production -- found only by directly grepping for every call site of sched_apply_job_rusage() and sched_update(), not by reading either function's own code or comments, which gave no hint of the duplication.", + "fix": "FIXED (2026-08-31). Added a call to sched_reconcile_job_resource_usage() directly inside core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler (the real, live path), right after its existing job->rusages bookkeeping. schedobj.c's sched_apply_job_rusage() keeps its own copy of the reconciliation call too (harmless dead code, but still exercised by its own existing unit tests, so left in place rather than removed as an unrelated cleanup). Added a genuine end-to-end regression test, test_job_rusage_reconciles_mem_reservation_against_usage (test_core_events.c), that drives the REAL xce_core_push_events() with a dispatched job carrying a real 4MB mem allocation, then a JOB_RUSAGE event claiming 7MB actual usage, and asserts the 3MB overage lands on both the account and the resource pool -- this is the test that would have caught the original wiring gap immediately, since it exercises the actual event dispatch instead of calling the reconciliation function directly. Required white-box #include-ing resource.c into test_core_events.c (previously only core.c/core_events.c) plus two link-satisfying stubs (plugin_prepare_allocation/sched_ctx_get_uncomit_host, matching test_resource_accounting.c's own established pattern).", + "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- live fix site), src/scheduler/src/schedobj.c (sched_apply_job_rusage -- dead-code path, left in place), src/scheduler/src/scheduler.c (sched_update -- unreachable caller), src/scheduler/src/resource.c (sched_reconcile_job_resource_usage/sched_reconcile_host_resource_usage), src/scheduler/src/driver.c (xce_sched_driver_translate_pending_events/xce_core_push_events -- the live path that supersedes sched_update), src/scheduler/testsuite/test_core_events.c (new regression test) [file field reconstructed post-recovery from context in error_message/root_cause/fix -- not the verbatim original string, see recovery note]", + "tags": [ + "scheduler", + "resource-accounting", + "rusage", + "dead-code", + "wiring-gap", + "test-coverage-gap", + "core-events", + "mem", + "duplicate-implementation", + "fixed", + "found-via-self-review" + ], + "occurrences": 1, + "last_seen": "2026-08-31T22:00:00Z" + }, + { + "id": "bug-1741", + "timestamp": "2026-08-31T23:45:00Z", + "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker, build.dbg): after submitting a 50-element array job (jobsub -a 50 -c hostname) immediately following a fresh primary election, `xce_test`'s `joblist` shows array elements [6] through [12] (7 consecutive elements) permanently stuck in state RUN (FinishTime=0) while every other element (0-5, 13-49) correctly reached DONE with real FinishTimes, including elements submitted/dispatched both before and after the stuck range and well after the cluster had fully stabilized (up to ~46s post-submit). The user confirmed via repeated `joblist` checks over several minutes that elements 6-12 never progress -- not a transient lag.", + "file": "src/worker/src/controller_connector.c (xcecd_job_finished), src/lib/client.c (xcecli_notify, xcecli_notify_jobfinish), src/lib/client_internal.h (XCECLI_FF_CALL), src/controller/job.c (on_job_finish, never invoked for the 6 lost reports), src/controller/cluster.c (do_notify/handle_notify)", + "root_cause": "PARTIALLY ROOT-CAUSED via direct log correlation across worker, controller, and scheduler logs from the live incident (not yet independently live-reproduced in isolation this session -- found while investigating the unrelated bug-1698 HA-hang occurrence in the same incident's data, but happened ~19s BEFORE ctrl-1's hang began and is confirmed unrelated to it: ctrl-1 was fully healthy and primary at the time these 7 finish reports were lost). Timeline reconstructed from three independent logs for elements 6-12: xcecd_worker.log shows the xcecd_virtual master daemon spawned one subprocess per array element (its own established per-job-process architecture), each connecting to the master via a local unix-socket IPC (uv_ipc.c) and reporting 'Completed' for all 7 elements within a ~600ms window (16:25:00.943 to 16:25:01.535). ctrl-1.log shows dispatch_a_job_/on_job_start logged correctly for all 7 elements, but job.c's on_job_finish() (traced via 'job finish jobid ... exit code' TRACE, its own first log line) fired for element [6] ONLY -- elements [7] through [12]'s finish reports never reached on_job_finish at all, i.e. the loss is upstream of job.c entirely, not a hook-firing gap (distinct from bug-1731's fix, which addressed apply_committed_entry() firing HOOK_DETAIL_JOB_START/EXIT -- that fix only helps once on_job_finish/WAL_JOB_FINISH already happened; here it never happens for 6 of 7 jobs). scheduler.log's later repeated 'sched_add_job failed ... rc=-17, merging state into existing tracked job' WARNs for exactly elements 6-12 (recurring at 16:25:23, 16:25:41, 16:25:46 across multiple post-failover JOBS_PUSH resyncs from the new primary ctrl-2) confirm the loss is durable, not just delayed: ctrl-2 inherited ctrl-1's own WAL/replicated state, which never recorded a finish for these 6 elements either, so every resync just re-merges the same stale RUN state -- there is no surviving source of truth anywhere in the cluster that ever learned these jobs finished. Traced the worker-side send path for the likely drop mechanism: xcecd_job_finished() (controller_connector.c:1680) -> xcecli_notify_jobfinish() -> xcecli_notify() (client.c:610, explicitly commented 'Fire-and-forget chokepoint') -> XCECLI_FF_CALL (client_internal.h:290-302), whose own doc comment states plainly: 'single-shot, NOT a retry loop... evaluate EXPR exactly ONCE... A lost send rc is surfaced as-is, never retried.' All 7 near-simultaneous finish notifications for this burst go out over the SAME single xcecli_ctx/network connection (one xcecd master daemon process per worker host, matching this codebase's one-connection-per-daemon model) via this explicitly best-effort, non-retrying path -- exactly the kind of rapid same-connection burst most likely to hit whatever failure mode XCECLI_FF_CALL's own comment already acknowledges as a known risk ('A failed reconnect is non-fatal -- the send below is simply best-effort'). This is the SAME XCECLI_FF_CALL mechanism bug-1548 already flagged for a related-but-distinct reliability gap (silently self-healing the transport across ticks, masking a needed rejoin) -- this occurrence is a second, independent real-world manifestation of the same primitive's fire-and-forget design causing silent data loss, this time for job-finish reports specifically. NOT YET CONFIRMED: the exact internal reason xce_notify()'s single-shot send fails/drops 6 of 7 calls in this specific burst (whether it's a genuine transport error being silently discarded because nothing checks xcecd_job_finished()'s return value, some reentrancy/buffer-reuse issue in xce_notify()'s own encode/send path when called back-to-back without waiting for the prior call's completion, or contention on ac->failover.lock combined with a narrow race) -- would need either a live gdb/strace-instrumented repro of this exact burst pattern, or a close read of xce_notify()'s implementation and whether xcecd_job_finished()'s callers check/log its return code at all (not yet done this session). USER-PROMPTED REFINEMENT (2026-08-31, same session): the user hypothesized this might be a stale-primary artifact (worker kept reporting to the old primary and never resent to the new one). That exact framing doesn't fit the millisecond timing (these 7 finishes/sends happened at 16:25:00.9-01.5, before any failover was detected by anyone), but chasing it surfaced an important adjacent fact from the worker daemon's OWN controller-connection log (xcecd.log, root-owned -- required sudo to read, distinct from xcecd_worker.log's per-job-subprocess-only content used for the rest of this investigation): the WORKER's client-side failover logic logged its own first sign of trouble with ctrl-1 at 16:25:12.164 ('xcecli failover: attempt 1 -> localhost:9901') -- 8 SECONDS BEFORE ctrl-1 itself logged anything wrong ('standby[0] silent for 12s ... evicting', 16:25:20 per bug-1698's own timeline), and about 11-19 seconds AFTER the lost job-finish notifies for elements 6-12 were sent (16:25:00.943-01.535). The worker successfully failed over to the new primary at 16:25:22.332 ('rejoin after transparent reconnect succeeded', following attempt 2 -> localhost:9902 at 16:25:22.165). This reframes the likely relationship between this bug and bug-1698: rather than two unrelated defects that happened to occur in the same incident, the timing is consistent with a SINGLE underlying trigger -- ctrl-1's own I/O/connection handling apparently began degrading measurably before ctrl-1 itself noticed (silently failing to process/ack the worker's job-finish sends circa 16:25:00-01, then dropping the worker's connection entirely by ~16:25:12) -- with ctrl-1's own self-diagnosis (the 12s standby-liveness timeout) simply being the slowest of the three symptoms to surface. If that shared-trigger theory holds, this bug's fix should NOT rely solely on 'the worker eventually fails over correctly' (bug-1698's territory, and confirmed here to work correctly once detected, ~10s after the fact) -- XCECLI_FF_CALL's fire-and-forget send has zero resend/replay mechanism, so a notify lost during exactly this kind of covert pre-failover degradation window is gone forever regardless of how well the subsequent failover behaves. Still not proven whether ctrl-1's degradation genuinely started as early as 16:25:00 (matching the lost notifies) or only at 16:25:12 (matching the worker's own first detected symptom) with the 16:25:00-01 notify loss being a coincidentally-timed SEPARATE glitch -- distinguishing these needs the same live gdb/strace capture bug-1698's own next-steps already call for, ideally instrumented to also watch ctrl-1's handling of inbound OPCode_notify requests specifically, not just its outbound peer reconnect attempts.", + "fix": "NOT YET FIXED -- found and root-caused-to-this-depth this session while investigating bug-1698 in the same incident's log data; not yet independently reproduced or fixed. Next steps: (1) confirm whether xcecd_job_finished()'s callers (worker_job_callback / process_exit_callback path in xcecd_worker.c) check and log its return code -- if not, that alone is worth fixing regardless of the deeper cause, so a real send failure is at least loud instead of silent; (2) read xce_notify()'s actual implementation (xceapi.c, shared with every other XCECLI_FF_CALL user) to determine whether it is safe to call back-to-back in a tight loop on the same api_ctx_t without waiting for a prior call's underlying I/O to fully complete, or whether it has an internal single-buffer/reentrancy hazard; (3) attempt a targeted live repro (a single worker host completing 5+ jobs within under a second, e.g. a small array of near-instant jobs against one host with concurrency >= the array size) with the worker daemon under gdb/strace to catch xce_notify()'s return code and underlying syscall result for the calls that get lost; (4) once root-caused, likely fix directions include making job-finish notifications durable/retried (not fire-and-forget) given their consequence (a permanently stuck job holding worker slots/resources forever is a correctness issue, not just a display one -- unlike heartbeats, a job-finish report has no self-healing follow-up message the way bug-1737's DONE-vs-EXIT divergence did), or serializing/queuing bursts of notify calls from the same worker daemon so they cannot race or overwrite each other on the shared connection.", + "tags": [ + "controller", + "worker", + "job-lifecycle", + "job-finish", + "notify", + "xcecli_notify", + "XCECLI_FF_CALL", + "fire-and-forget", + "burst-load", + "array-job", + "phantom-run-job", + "stuck-job", + "data-loss", + "found-live", + "partially-root-caused", + "needs-live-repro", + "needs-owner", + "unfixed", + "related-to-bug-1548", + "distinct-from-bug-1731", + "distinct-from-bug-1698", + "possibly-same-root-cause-as-bug-1698" + ], + "related_bugs": [ + "bug-1548", + "bug-1731", + "bug-1737", + "bug-1738", + "bug-1739", + "bug-1698" + ], + "occurrences": 1, + "last_seen": "2026-08-31T23:45:00Z" + }, { "id": "bug-1742", "timestamp": "2026-09-01T04:10:00Z", @@ -20225,6 +20290,34 @@ ], "occurrences": 1, "last_seen": "2026-09-01T05:10:00Z" + }, + { + "id": "bug-1744", + "timestamp": "2026-09-01T05:45:00Z", + "gitea_issue": 47, + "error_message": "Gitea issue #47 (rusage\u662f\u5426\u505a\u4e86\uff1f): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed resreq string (trailing non-numeric \"a0\" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '\u4e71\u5199\u4e5f\u53ef\u4ee5\u63d0\u4ea4' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent.", + "file": "src/frontend/xcecli/cmd/jobs/submit.go (resReq flag, no value validation, just passed through: jobReq.Resreq = js.resReq), src/frontend/apiserver/controllers/jobs.go:1602-1606 (only length-checks Resreq, no syntax/grammar validation), src/lib/resreq.c (parse_resreq/parse_rusage -- the real PEG-generated grammar parser), src/lib/rusage.peg (rusage_param grammar, mem value must match `number <- [0-9]+('.'[0-9]*)?`, so \"100000000a0\" is a genuine grammar violation), src/scheduler/src/core_events.c:236-245 (parse_resreq call site inside core_build_job()) and :751-760 (JOBS_PUSH caller: on core_build_job()==NULL, WARNs server-side and `continue`s -- job silently dropped, never added to core->jobs)", + "root_cause": "The ONLY resreq validation in the entire submission path is a length check (apiserver/controllers/jobs.go:1602-1606, `len(req.Resreq) > 4094`) -- there is no syntax/grammar validation anywhere client-side or in the apiserver before a job is accepted and persisted. The REAL validator, a PEG-generated grammar parser (parse_resreq()/parse_rusage(), src/lib/resreq.c, built from src/lib/rusage.peg + xce_resreq.peg via meson custom_target), exists but is only ever invoked deep inside the C scheduler at core_build_job() time (core_events.c:236), i.e. asynchronously, well AFTER the apiserver has already accepted the job, returned a submission-success response to the client, and persisted it. When parse_resreq() fails on malformed syntax (confirmed via rusage.peg's `number` rule, `[0-9]+('.'[0-9]*)?` -- \"100000000a0\" cannot match this and leaves \"a0\" unconsumed, a genuine PEG parse failure), core_build_job() logs an ERROR and returns NULL (core_events.c:238-245); its caller in the XCE_CORE_EVENT_JOBS_PUSH handler (core_events.c:756-760) only logs a server-side WARN and `continue`s to the next job in the batch -- the malformed job is silently dropped from the scheduler's core->jobs registry entirely. Net effect is worse than 'garbage accepted': the job is accepted, shows the user a success message, is almost certainly persisted and shown as PEND via `jobs list` (this session did not confirm the exact persisted/listed state live), and then sits PEND FOREVER with zero indication to the user that anything is wrong -- the only trace is a server-side log line the user never sees. This is architecturally the same class of gap as bug-1740 (a real, working validator/reconciler exists but is wired into the wrong -- or in this case, too-late -- point in the pipeline) and matches the maintainer's own comment that front-end validation is the intended design, not yet implemented.", + "fix": "NOT YET FIXED -- investigated and design-confirmed this session; DEPRIORITIZED, deferred to later (explicit user decision, not urgent). Confirmed fix approach: validate resreq synchronously in the apiserver's submit handler (controllers/jobs.go:1602, alongside the existing length check), reusing libxce's existing PEG parser rather than reimplementing the grammar in Go. Specifically parse_rusage() (src/lib/resreq.c:154, declared resreq.h:301 -- `int parse_rusage(xlist_t*, char*, char**)`) needs NO resmgr/resource-manager context, unlike the fuller parse_resreq()/parse_select() (resreq.h:303,305 -- both take a `var resmgr` to validate resource NAMES in select[]/order[] clauses, which the apiserver doesn't currently construct) -- so a resreq-name-agnostic rusage[]-only syntax check is a small, low-risk CGo wrapper, not a larger resmgr-plumbing project. apiserver/backend already has the exact split pattern to follow: a `//go:build cgo` file with the real implementation (e.g. clib.go, which already #includes libxce headers and links -lxce) plus a `//go:build !cgo` stub (stub.go, returns an errNoCGO-style error) for the CGO-disabled test path. Full parse_resreq()/resmgr-backed select[]/order[] validation is explicitly out of scope for now -- would be a separate, larger follow-up if wanted.\n\nSCOPE DECISION (2026-09-01): will be implemented as a direct bugfix when picked up, NOT a kiro spec -- small, contained blast radius (one new CGo wrapper + stub in apiserver/backend, one call site in controllers/jobs.go, matching the size/shape of the already-shipped bug-1742/bug-1743 fixes), no HA/persistence/RBAC involvement, and no open design questions left (exact function, signature, and existing cgo/stub pattern to follow are already nailed down). Only the FULLER resmgr-backed select[]/order[] validation (out of scope here) would warrant at least a /kiro-spec-quick, since it needs new apiserver-side state (a resmgr context kept in sync with resource definitions) -- not needed for this rusage-only fix.", + "tags": [ + "apiserver", + "xcecli", + "job-submission", + "resreq", + "rusage", + "validation-gap", + "silent-failure", + "phantom-pend", + "new", + "found-static-analysis", + "needs-live-repro", + "related-to-bug-1740", + "not-yet-fixed", + "deprioritized", + "deferred" + ], + "occurrences": 1, + "last_seen": "2026-09-01T06:35:00Z" } ] } diff --git a/.wolf/cerebrum.md b/.wolf/cerebrum.md index b54e12cc..dd8df38d 100644 --- a/.wolf/cerebrum.md +++ b/.wolf/cerebrum.md @@ -194,6 +194,7 @@ - **(2026-08-26, bug-1715) The AUTHORITATIVE, complete list of every literal `sched_admin_reply_t.error = "..."` string the real controller/scheduler can send (confirmed by grepping `src/controller/` and `src/scheduler/` for the assignment, then reading each site's surrounding code — not from requirements.md's own worked examples, which under-lists them):** transient/connection-timing (self-resolving, safe to retry): `"scheduler disconnected"` (cluster.c:1062, bug-1693's disconnect path), `"scheduler not connected"` (cluster.c:958, `c->sched.ep==NULL`), `"request timeout"` (cluster.c:1078, `SCHEDADMIN_TIMEOUT_MS` wait expired), `"server busy, try again"` (cluster.c:937, `vtaskmgr_blocking_acquire()` admission-reservation exhaustion — NOT previously documented as transient anywhere, found by reading the admission-gate comment, not the string alone). Permanent/domain (must never retry): `"already recording"` / `"not recording"` (driver.c:390/426, `rc==-EALREADY`) and `validation_error_text()`'s six filename/path-rejection strings (driver.c:298-316: `"invalid record file name"`, `"record directory not found"`, `"record directory path is not a directory"`, `"permission denied accessing record file"`, `"a non-file entry with this name already exists"`, `"record file path too long"`), plus a dynamic (non-literal, can't exact-match) `"cannot write to %s: %s"` I/O-failure message (driver.c:398-404). When classifying an error-reply reason string as retry-safe vs not, read the ERROR-SITE's surrounding code/comments, not just the string's English wording — "server busy, try again" was only correctly classified transient by reading `vtaskmgr_blocking_acquire()`'s admission-gate design comment, not from requirements.md. - **(2026-08-26, xce-test-rc-file task 3.3, bug-1719) `src/tools/xce-build.sh ` with NO `--incremental` does a full `rm -rf "$BUILD_DIR"` + fresh `meson setup` by design (per its own top-of-file comment), then `meson compile` only rebuilds the ONE requested component — so building a second component (e.g. `scheduler`) into an `xce.build` dir that already has a different component's binaries (e.g. `controller`'s `xce_controller`/`xce_test`) silently WIPES the earlier ones.** Always pass `--incremental` when you need a component built alongside another already-built one in the same dir, and never pass an explicit build_dir positional arg unless it matches the script's own default relative-path resolution (`xce.build`, resolved from `src/tools/xce-build.sh`'s own cwd) — passing a repo-root-relative path like `src/xce.build` as the 3rd positional creates a bogus nested `src/src/xce.build` instead (harmless/gitignored via meson's own per-builddir auto `.gitignore: *`, but wastes a build cycle). Rebuilding the wiped component with `--incremental` restores it without disturbing the other. - **(2026-08-26, xce-test-rc-file task 3.3, bug-1720) `xce-test-resources`'s apiserver layer reporting UNREACHABLE for every resource does not by itself mean a propagation regression — check for a stale bind conflict on :8080 first (`lsof -i :8080` / `pgrep -af apiserver`).** An orphaned `/tmp/apiserver` process from an unrelated, long-concluded prior session (its own deploy dir already deleted, 7.5h uptime) was still bound to :8080, so THIS run's fresh apiserver instance failed `listen tcp :8080: bind: address already in use` and never logged anything — surfacing identically to a genuine trace-line-never-appeared UNREACHABLE. `run-resource-verify.sh`'s own trap-based cleanup only reaps processes IT started, not orphans left by an earlier, already-ended unrelated script/session. +- **(2026-08-31, worker-rejoin-resync design phase) Do NOT reach for `ScheduleWakeup` to wait on background subagents launched via the `Agent` tool.** Called it twice in a row ("wait for these 3 research subagents") purely to pass the time — the tool's own description explicitly says not to ("when harness-tracked work finishes, you are re-invoked automatically"). The subagent completion notifications did arrive on their own regardless, so the calls were pure waste; worse, each one plants a recurring `<>` wakeup that keeps firing (with nothing to do) even after the awaited work is long done, until explicitly stopped (`ScheduleWakeup({stop:true})`). Correct behavior when launching parallel `Agent` calls and there's nothing else to do meanwhile: just end the turn (or do other useful reads) — do not call ScheduleWakeup at all unless the user is actually running `/loop` or genuinely polling an external system the harness can't track (a CI run, a deploy). ## Decision Log - HA/consistency model follows Raft + CP strictly: majority ACK is the hard commit barrier for all state-changing writes (reject on timeout, never local-apply-and-degrade); the own-term commit rule (not a 4th "COMMITTING" state) is what gives a new leader safety. diff --git a/.wolf/memory.md b/.wolf/memory.md index 7f22301c..e7e4019d 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -3260,3 +3260,109 @@ | 00:18 | Edited src/scheduler/testsuite/test_resource_accounting.c | 2→3 lines | ~66 | | 00:27 | Edited .kiro/steering/resource-accounting.md | expanded (+8 lines) | ~215 | | 00:27 | Edited .kiro/steering/resource-accounting.md | 5→5 lines | ~46 | +| 00:34 | Session end: 8 writes across 3 files (SKILL.md, CHANGES.md, tasks.md) | 4 reads | ~27378 tok | +| 00:35 | Edited src/scheduler/src/core_events.c | added 1 condition(s) | ~420 | +| 00:43 | Edited src/scheduler/testsuite/test_core_events.c | modified added() | ~419 | +| 00:45 | Edited src/scheduler/testsuite/test_core_events.c | modified with() | ~1447 | +| 00:45 | Edited src/scheduler/testsuite/test_core_events.c | 1→2 lines | ~36 | + +## Session: 2026-08-31 00:48 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 01:04 | Edited src/scheduler/testsuite/test_record_lifecycle.c | modified sched_worker_release_host_ctx() | ~243 | +| 01:04 | Edited src/scheduler/testsuite/test_sim_alt_events.c | modified sched_worker_release_host_ctx() | ~214 | +| 01:04 | Edited src/scheduler/testsuite/test_sim_session_ordering.c | modified sched_worker_release_host_ctx() | ~213 | +| 01:04 | Edited src/scheduler/testsuite/test_sim_snapshot_reconcile_gap.c | modified sched_worker_release_host_ctx() | ~230 | +| 01:05 | Edited src/scheduler/testsuite/test_core_full_cycle.c | modified kill_sched_resval() | ~167 | +| 01:05 | Edited src/scheduler/testsuite/test_admin_request.c | modified kill_sched_resval() | ~167 | +| 01:05 | Edited src/scheduler/testsuite/test_repl_equivalence.c | modified kill_sched_resval() | ~167 | +| 01:05 | Edited src/scheduler/testsuite/test_repl_breakpoints_inspection.c | modified kill_sched_resval() | ~167 | +| 01:05 | Edited src/scheduler/testsuite/test_sim_verify.c | modified kill_sched_resval() | ~167 | +| 01:05 | Edited src/scheduler/testsuite/test_driver_session_ordering.c | modified kill_sched_resval() | ~167 | +| 01:05 | Edited src/scheduler/testsuite/test_repl_execution_control.c | modified kill_sched_resval() | ~167 | +| 01:06 | Edited src/scheduler/testsuite/test_driver_decision_cb.c | modified kill_sched_resval() | ~167 | +| 01:06 | Edited src/scheduler/testsuite/test_repl_snapshot_params.c | modified kill_sched_resval() | ~167 | +| 01:06 | Edited src/scheduler/testsuite/test_driver_translate.c | modified kill_sched_resval() | ~167 | +| 01:06 | Edited src/scheduler/testsuite/test_sim_record_output.c | modified kill_sched_resval() | ~167 | +| 01:07 | Edited src/scheduler/testsuite/test_sim_equivalence.c | modified kill_sched_resval() | ~167 | +| 01:07 | Edited src/scheduler/testsuite/test_sim_event_loop.c | modified kill_sched_resval() | ~167 | +| 01:07 | Edited src/scheduler/testsuite/test_resval_ownership.c | modified sched_reconcile_job_resource_usage() | ~179 | +| 04:09 | Created .kiro/specs/worker-rejoin-resync/requirements.md | — | ~2382 | +| 04:09 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | inline fix | ~60 | +| 04:10 | Edited .kiro/specs/worker-rejoin-resync/spec.json | rejoin() → drop() | ~152 | +| 04:24 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | query() → worker() | ~460 | +| 04:29 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 2→3 lines | ~152 | +| 04:29 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 4→6 lines | ~323 | +| 04:31 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 2→1 lines | ~89 | +| 04:31 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | inline fix | ~67 | +| 04:33 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | expanded (+9 lines) | ~294 | +| 04:34 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 2→3 lines | ~171 | +| 05:58 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~108 | +| 05:58 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~122 | +| 05:59 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 7→7 lines | ~402 | +| 06:03 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~105 | +| 06:03 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~159 | +| 06:04 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 17→17 lines | ~531 | +| 06:04 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~202 | +| 06:06 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | expanded (+9 lines) | ~345 | +| 06:09 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | inline fix | ~155 | +| 07:08 | Created .kiro/specs/job-dispatch-commit-batching/spec.json | — | ~263 | +| 07:08 | Created .kiro/specs/job-dispatch-commit-batching/requirements.md | — | ~594 | +| 07:09 | Edited .kiro/specs/job-dispatch-commit-batching/requirements.md | modified note() | ~384 | +| 07:10 | Created .kiro/specs/commit-batch-window/spec.json | — | ~327 | +| 07:11 | Created .kiro/specs/commit-batch-window/requirements.md | — | ~1037 | +| 07:12 | Edited .kiro/specs/commit-batch-window/requirements.md | modified note() | ~314 | +| 07:13 | Edited .kiro/specs/commit-batch-window/requirements.md | 1→3 lines | ~211 | +| 07:13 | Edited .kiro/specs/commit-batch-window/requirements.md | inline fix | ~227 | +| 19:16 | Session end: 45 writes across 20 files (test_record_lifecycle.c, test_sim_alt_events.c, test_sim_session_ordering.c, test_sim_snapshot_reconcile_gap.c, test_core_full_cycle.c) | 27 reads | ~65920 tok | + +## Session: 2026-09-01 19:16 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 19:25 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~129 | +| 19:25 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | inline fix | ~120 | +| 19:39 | Created .kiro/specs/worker-rejoin-resync/research.md | — | ~5069 | +| 19:42 | Created .kiro/specs/worker-rejoin-resync/design.md | — | ~11060 | +| 19:43 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~76 | +| 19:43 | Edited .kiro/specs/worker-rejoin-resync/design.md | 2→2 lines | ~157 | +| 19:43 | Edited .kiro/specs/worker-rejoin-resync/design.md | 2→3 lines | ~54 | +| 19:43 | Edited .kiro/specs/worker-rejoin-resync/spec.json | 13→13 lines | ~171 | +| 20:08 | Edited .kiro/specs/worker-rejoin-resync/design.md | 2→2 lines | ~322 | +| 20:09 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~452 | +| 20:10 | Edited .kiro/specs/worker-rejoin-resync/design.md | 6→9 lines | ~156 | +| 20:11 | Edited .kiro/specs/worker-rejoin-resync/research.md | expanded (+8 lines) | ~675 | +| 20:14 | Edited .kiro/specs/worker-rejoin-resync/design.md | 6→6 lines | ~71 | +| 20:16 | Edited .kiro/specs/worker-rejoin-resync/design.md | 6→6 lines | ~58 | +| 20:16 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~70 | +| 20:20 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~191 | +| 20:20 | Edited .kiro/specs/worker-rejoin-resync/design.md | 1→2 lines | ~234 | +| 20:26 | Edited .kiro/specs/worker-rejoin-resync/design.md | 3→3 lines | ~40 | +| 20:26 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~20 | +| 20:27 | Edited .kiro/specs/worker-rejoin-resync/requirements.md | 1→2 lines | ~163 | +| 20:28 | Edited .kiro/specs/worker-rejoin-resync/design.md | 4→5 lines | ~126 | +| 20:29 | Edited .kiro/specs/worker-rejoin-resync/design.md | 1→2 lines | ~90 | +| 20:29 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~114 | +| 20:29 | Edited .kiro/specs/worker-rejoin-resync/design.md | 8→9 lines | ~404 | +| 20:30 | Edited .kiro/specs/worker-rejoin-resync/design.md | 1→2 lines | ~108 | +| 20:39 | Edited .kiro/specs/worker-rejoin-resync/design.md | inline fix | ~71 | +| 20:50 | Session end: 26 writes across 4 files (requirements.md, research.md, design.md, spec.json) | 27 reads | ~61793 tok | + +## Session: 2026-09-01 20:50 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------| +| 21:23 | Edited src/frontend/apiserver/controllers/jobs_test.go | modified TestValidateJobControl_HandlesRealJobInfoTypes() | ~781 | +| 21:24 | Edited src/frontend/apiserver/controllers/jobs.go | modified IsControlActionAllowed() | ~876 | +| 21:24 | Edited src/frontend/apiserver/controllers/jobs.go | int() → IntOrInt64() | ~137 | +| 21:31 | Edited src/frontend/apiserver/controllers/host_control_test.go | modified TestProcessHostData_ComputesSlotUsageFromJobs() | ~707 | +| 21:31 | Edited src/frontend/apiserver/controllers/hosts.go | 15→16 lines | ~68 | +| 21:31 | Edited src/frontend/apiserver/controllers/hosts.go | modified computeHostSlotUsage() | ~794 | +| 21:31 | Edited src/frontend/apiserver/controllers/hosts.go | 14→14 lines | ~99 | +| 00:00 | Session end: 7 writes across 4 files (jobs_test.go, jobs.go, host_control_test.go, hosts.go) | 22 reads | ~10689 tok | + +## Session: 2026-09-01 00:00 + +| Time | Action | File(s) | Outcome | ~Tokens | +|------|--------|---------|---------|--------|