diff --git a/.claude/rules/buglog-status.md b/.claude/rules/buglog-status.md new file mode 100644 index 00000000..fbf2bcee --- /dev/null +++ b/.claude/rules/buglog-status.md @@ -0,0 +1,34 @@ +# Buglog status field + +Kept separate from `openwolf.md` because `openwolf update` overwrites that file. +OpenWolf itself ignores this field (search, pre-write hints and the hook-created +entries don't read or set it), so the convention is enforced only by this rule. + +- Every `.wolf/buglog.json` entry carries a `status`, and it is the only source + of truth for whether a bug is open. Never infer it from `fix`: many entries + have a populated `fix` that says "NOT FIXED", "PENDING" or "PARTIAL". +- Values: + - `open`: defect confirmed and still in the code. Intermittent test + failures that do still recur count as `open`, not `cannot-reproduce`. + - `partial`: part fixed; `status_note` says what remains. + - `investigating`: real failure observed, root cause not found yet or + waiting for a reproduction. + - `fixed`: fix committed; cite the commit in `fix` or `status_note`. + - `cannot-reproduce`: reproduction was attempted and failed; record what was + tried in `repro_attempts` (e.g. "0/40 runs, 2026-08-01"). + - `wontfix`: not a bug, accepted limitation, or deferred with no plan. + - `duplicate`: same defect as another entry; set `duplicate_of`. +- Whenever you log a bug or change its state, set `status` and + `status_updated` (YYYY-MM-DD), plus `status_note` unless the status is + `fixed` with the commit already in `fix`. Put them right after `"id"`. +- A `cannot-reproduce` bug that recurs: bump `occurrences`/`last_seen` and set + `status` back to `investigating`. +- Use the `buglog` skill (`scripts/buglog.py`) to add entries and change + status: `add`, `status`, `note`, `recur`, `list`, `triage`. It picks the + next free id and edits only the touched entry's lines. Don't re-serialize + the whole file with `json.dumps`/`jq`; that rewrites ~1,200 unrelated lines. +- Open-bug list: + `jq -r '.bugs[] | select(.status | IN("open","partial","investigating")) | "\(.id) [\(.status)] \(.status_note // .error_message[0:90])"' .wolf/buglog.json` +- Entries with no `status` (hand-written without it, or created by an OpenWolf + hook) need triage: + `jq -r '.bugs[] | select(.status == null) | .id' .wolf/buglog.json` diff --git a/.claude/skills/buglog/SKILL.md b/.claude/skills/buglog/SKILL.md new file mode 100644 index 00000000..99ae855d --- /dev/null +++ b/.claude/skills/buglog/SKILL.md @@ -0,0 +1,64 @@ +--- +name: buglog +description: Log a new bug, change a bug's status (fixed / partial / cannot-reproduce / ...), record a recurrence, or list open bugs in the OpenWolf buglog (.wolf/buglog.json). Use whenever you log or fix a bug, or are asked for the open-bug list. +argument-hint: add | status | note | recur | list [--open] | show | triage +triggers: + - /buglog + - log a bug + - open bug list + - mark bug fixed +--- + +# Buglog Skill + +Every operation goes through `scripts/buglog.py` (run from the repo root). It +edits only the lines of the entry it touches, re-parses the file before +saving, and writes atomically. **Never** rewrite `.wolf/buglog.json` with +`json.dumps`/`jq`/the Write tool: entries mix raw UTF-8 and `\uXXXX` escapes, +so a re-serialize rewrites ~1,200 unrelated lines. Never read the file whole +either (~27K lines); use `list`/`show`. + +Status values and when to use each: `.claude/rules/buglog-status.md`. + +## Before fixing a bug + +Search for prior art first (project rule): `openwolf bug search ""`, +or `python3 scripts/buglog.py list --grep ''`. + +## Commands + +| Task | Command | +|------|---------| +| Log a new bug | `python3 scripts/buglog.py add --error "" --root-cause "" --file [,] --tags scheduler,asan --status open --note ""` → prints the new id | +| Log a bug you already fixed | same, with `--fix ", commit " --status fixed` | +| Mark fixed | `python3 scripts/buglog.py status fixed --note ""` | +| Mark partly fixed | `python3 scripts/buglog.py status partial --note ""` | +| Couldn't reproduce | `python3 scripts/buglog.py status cannot-reproduce --repro "0/40 runs, 2026-09-24" --note ""` | +| Real failure, cause unknown | `python3 scripts/buglog.py status investigating --note ""` | +| Not a bug / accepted | `python3 scripts/buglog.py status wontfix --note ""` | +| Duplicate | `python3 scripts/buglog.py status duplicate --duplicate-of --note ""` | +| Add findings to an entry | `python3 scripts/buglog.py note ""` (appends an `UPDATE (date):` paragraph to `fix`) | +| Seen again | `python3 scripts/buglog.py recur [--note ""]` (bumps occurrences/last_seen; reopens a `fixed`/`wontfix`/`cannot-reproduce` bug) | +| Open-bug list | `python3 scripts/buglog.py list --open` (add `--tag scheduler`, `--grep`, `--since YYYY-MM-DD`) | +| One entry | `python3 scripts/buglog.py show ` | +| Entries missing a status | `python3 scripts/buglog.py triage` | +| Any other single-line field | `python3 scripts/buglog.py set key= ...` | + +`status` requires `--note` for every status except `fixed`, and always +rewrites the note, so a stale note never outlives its status. + +## Rules + +- **Ids**: let `add` pick the id (one past the highest existing). Don't number + entries by hand or by entry count. OpenWolf's post-write hook numbers by + entry count, which collides with existing ids; its auto-detection is turned + off in `.wolf/config.json` for that reason (`openwolf.buglog.auto_detect`). +- **Write the symptom first**: `--error` holds the observed error text or + behavior, not a status banner ("OPEN, NOT FIXED --" goes in `status`). +- **Fixing a bug in code**: after the fix commit, run `status fixed + --note `. If only part is fixed, use `partial` with what remains. +- **Arrays** (`tags`, `related_bugs`, `files`) can only be set at `add` time; + to change them later, edit that entry's lines by hand and re-check with + `python3 -c "import json; json.load(open('.wolf/buglog.json'))"`. +- **Gitea**: status changes reach Gitea on the next `/gitea-sync`, which opens, + closes or reopens issues from `status`. diff --git a/.claude/skills/gitea-sync/SKILL.md b/.claude/skills/gitea-sync/SKILL.md index ca070a5b..b9dc26f3 100644 --- a/.claude/skills/gitea-sync/SKILL.md +++ b/.claude/skills/gitea-sync/SKILL.md @@ -24,6 +24,21 @@ Sync state (issue numbers) is stored in source files for idempotency. /gitea-sync --dry-run # preview without making changes ``` +## Preferred: run the script + +`scripts/gitea-sync.py` implements every step below over the Gitea REST API +and needs no MCP server (token from `GITEA_TOKEN`, or the gitea MCP entry in +`~/.claude.json`). Use it whenever it can reach Gitea; follow the manual MCP +procedure only when it can't: + +```bash +python3 scripts/gitea-sync.py --dry-run --bugs --since 2026-09-01 # preview +python3 scripts/gitea-sync.py --bugs --since 2026-09-01 +python3 scripts/gitea-sync.py --spec +``` + +Keep the script and this procedure in step when changing either. + ## Procedure ### 0. Parse arguments @@ -31,7 +46,10 @@ Sync state (issue numbers) is stored in source files for idempotency. - `--dry-run`: only report what would change, don't call MCP write tools - `--spec `: sync only one spec directory - `--bugs`: sync bugs only (skip specs) -- `--since `: only sync bugs with `last_seen >=` this date +- `--since `: create issues only for bugs with `last_seen >=` this + date; for already-synced bugs (`gitea_issue` set), also mirror those whose + `status_updated >=` this date, so a status change on an old bug still + reaches Gitea without flooding it with old never-synced bugs ### 1. Ensure labels exist @@ -49,6 +67,16 @@ for any missing ones: | `spec` | `3498DB` | Spec-tracked issue | | `bug` | `E74C3C` | Bug-tracked issue | | `bug:fixed` | `27AE60` | Bug resolved | +| `bug:partial` | `F39C12` | Partly fixed, remainder open | +| `bug:investigating` | `E67E22` | Real failure, root cause not found yet | +| `bug:cannot-reproduce` | `95A5A6` | Repro attempted and failed | +| `bug:wontfix` | `7F8C8D` | Not a bug / accepted limitation | +| `bug:duplicate` | `BDC3C7` | Duplicate of another bug | +| `bug:asan` | `8E44AD` | Sanitizer finding | +| `bug:crash` | `C0392B` | Crash (SEGV, double-free, UAF) | +| `bug:race` | `D35400` | Race / TOCTOU | +| `bug:ha` | `2980B9` | HA / Raft / failover | +| `bug:data-loss` | `922B21` | Data loss | | `blocked` | `E74C3C` | Blocked on dependency | ### 2. Sync Specs @@ -95,30 +123,72 @@ For each spec directory in `.kiro/specs/` (or just `--spec `): } ``` - d. **Phase state updates** (for already-synced specs): Check `spec.json` - approvals. If requirements AND design are approved → close the - requirements_design issue via `mcp__gitea__issue_write` method `update` - with `state: "closed"`. If implementation approved → close - implementation and validation issues. If all approved → close milestone - via `mcp__gitea__milestone_write` method `update`. + d. **Phase state updates** (for already-synced specs). `spec.json` + `approvals` only covers requirements/design/tasks — there is no + implementation approval — so use `phase` for the later issues: + + "Done" phases: `implementation-complete`, `implemented`, `superseded` + (no tool distinguishes the first two). + + | Condition | Action | + |-----------|--------| + | `approvals.requirements.approved` AND `approvals.design.approved`, or phase is done | close `requirements_design` | + | phase is done | close `implementation`, `validation` and the milestone | + | phase is not done | reopen `implementation`/`validation`/milestone if closed | + + The same rules give the initial state when the issues are first created. + + Close issues via `mcp__gitea__issue_write` method `update` with + `state: "closed"`, the milestone via `mcp__gitea__milestone_write` + method `update`. ### 3. Sync Bugs -Read `.wolf/buglog.json`. For each entry: +Read `.wolf/buglog.json`. Each entry's `status` field is the source of truth +for open/closed — **never infer it from `fix`**: many entries have a populated +`fix` that says "NOT FIXED", "PENDING" or "PARTIALLY FIXED". -1. **Skip** if: - - Entry has `gitea_issue` field (already synced) AND bug was already fixed - when synced (no state change needed). - - Filter: if `--since ` provided, skip entries with - `last_seen < `. +**Status → Gitea state / label** (`status` values are defined in +`.claude/rules/buglog-status.md`): + +| `status` | Gitea state | Status label | +|----------|-------------|--------------| +| `open` | open | — | +| `partial` | open | `bug:partial` | +| `investigating` | open | `bug:investigating` | +| `fixed` | closed | `bug:fixed` | +| `cannot-reproduce` | closed | `bug:cannot-reproduce` | +| `wontfix` | closed | `bug:wontfix` | +| `duplicate` | closed | `bug:duplicate` | +| missing (hook-created entry) | open | — ; report it in the summary as "needs status" | + +An issue carries exactly one status label. When changing it, remove only the +other `bug:` labels and add the new one; never replace the issue's +whole label set, which would wipe labels a person added. + +**Shared issues**: several entries can point at one `gitea_issue` (e.g. a +user-reported issue whose cause was split into several bugs, like #46). Such +an issue follows its "most open" entry, in this order: `open`, +`investigating`, `partial`, (no status), `cannot-reproduce`, `duplicate`, +`wontfix`, `fixed`. + +For each entry (after the `--since` filter): + +1. **Skip** if it has `gitea_issue` AND `gitea_synced_status == status` + (already mirrored, nothing changed). 2. **Create issue** for entries without `gitea_issue`: Use `mcp__gitea__issue_write` method `create`. - **Title**: `[] ` (max 120 chars) + **Title**: `[] ` (max 120 chars), + after dropping any leading status banner such as "OPEN, NOT FIXED --" or + "PRIORITY: LOW --" (status lives in the state and labels). **Body**: ``` + **Status:** (, if present) + <"Duplicate of " / "Repro attempts: ", if present> + ## Error @@ -126,10 +196,11 @@ Read `.wolf/buglog.json`. For each entry: ## Fix - (or "*Not yet fixed*" if empty) + (if `fix` is an array, join the elements with blank lines; + "*Not yet fixed*" if empty) ## Affected Files - - (split on commas) + - (split on commas), or each element of `files` ## Tags `tag1` `tag2` ... @@ -141,41 +212,62 @@ Read `.wolf/buglog.json`. For each entry: *Synced from `.wolf/buglog.json`* ``` - **Labels**: Start with `["bug"]`. If `fix` is populated, add `bug:fixed`. - Map known tags to labels: `asan`/`ubsan` → create `bug:asan` label, - `SEGV`/`double-free`/`UAF` → `bug:crash`, `race`/`toctou` → `bug:race`, - `ha`/`raft`/`election`/`failover` → `bug:ha`, `data-loss` → `bug:data-loss`. + **Labels**: `["bug"]` + the status label from the table above. Map known + tags to labels: `asan`/`ubsan` → `bug:asan`, `SEGV`/`double-free`/`UAF` → + `bug:crash`, `race`/`toctou` → `bug:race`, `ha`/`raft`/`election`/`failover` + → `bug:ha`, `data-loss` → `bug:data-loss`. - **State**: `"closed"` if `fix` is populated, `"open"` otherwise. + **State**: from the table above. `state: "closed"` may be ignored on + `create` — create as open, then `update` to closed if needed. **Milestone**: None (bugs span specs). -3. **Close fixed bugs**: For entries WITH `gitea_issue` where `fix` is now - populated (was empty before — we track by checking if the Gitea issue is - still open). Use `mcp__gitea__issue_write` method `update` with - `state: "closed"`. +3. **Update changed issues**: for entries WITH `gitea_issue` whose `status` + differs from `gitea_synced_status` (or that have no `gitea_synced_status` + yet), `update` the issue's state and swap its status label per the table + (using the aggregate status for shared issues). + This both closes newly fixed bugs and **reopens** bugs whose status went + back to `open`/`investigating` (e.g. a `cannot-reproduce` bug that recurred). + Post a one-line comment with the new status and `status_note`. -4. **Write `gitea_issue`** back to the buglog entry: add - `"gitea_issue": ` to the entry in `.wolf/buglog.json`. - Read the full file, modify the matching entry, write back with - `json.dumps(data, indent=2)`. + **One-time repair**: issues synced before the `status` field existed were + closed whenever `fix` was non-empty, so entries with `gitea_issue` and no + `gitea_synced_status` whose status is `open`/`partial`/`investigating` are + probably wrongly closed — step 3 reopens them. + +4. **Write sync state back** to the buglog entry immediately after each + issue is created/updated (not once at the end — a failure part-way through + must not lose an issue number, or the next run creates a duplicate issue), with the surgical helper — + never re-serialize the whole file (`json.dumps`/`jq` rewrite ~1,200 + unrelated lines because entries mix raw UTF-8 and `\uXXXX` escapes): + + ```bash + python3 scripts/buglog.py set \ + gitea_issue= 'gitea_synced_status=""' + ``` + + Run it from the repo root. It edits only that entry's lines and refuses to + write anything that no longer parses. For entries without a `status`, + record `gitea_synced_status="unknown"`. ### 4. Report -Print summary: "Specs: N synced, M skipped | Bugs: X created, Y closed, Z skipped" +Print summary: "Specs: N synced, M skipped | Bugs: X created, Y closed, R reopened, Z skipped" +and list any entries with no `status` (hook-created entries) so they can be triaged. ## Important Rules - **Files are ground truth**. Never read Gitea state to update local files. The gitea_issue/milestone_id fields in local files only store sync mappings. - **Idempotent**. Items with existing `gitea_issue` or `gitea` state are skipped - unless their phase/fix state changed. + unless their phase changed (specs) or `status != gitea_synced_status` (bugs). - **Labels**: Check existing labels before creating. Use label IDs (not names) in issue create calls — fetch with `mcp__gitea__label_read` first. - **Rate limiting**: Batch issue creation — create issues sequentially (a few per turn) to stay within API limits. -- **Buglog is large** (~13K lines). Always use `--since` filter for - practical runs (e.g., `--since 2026-07-18`). Without `--since`, sync ALL bugs. +- **Buglog is large** (~1,200 entries, ~27K lines). Always use `--since` filter + for practical runs (e.g., `--since 2026-07-18`). Without `--since`, sync ALL + bugs. Filter with `jq`, don't read the file whole. - **Private repo**: `nexa/xce` is private. MCP auth must have access. ## Gitea MCP Tool Reference diff --git a/.wolf/buglog.json b/.wolf/buglog.json index 07ccb870..d63ab4da 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -3,6 +3,8 @@ "bugs": [ { "id": "bug-002", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-21T07:46:54.198Z", "error_message": "Incorrect value in code", "file": ".claude/skills/kiro-spec-init/SKILL.md", @@ -21,6 +23,8 @@ }, { "id": "bug-004", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-21T07:58:54.673Z", "error_message": "Wrong reference: g_tmpdir should be g_instdir", "file": "src/controller/testsuite/test_config_repl.c", @@ -38,6 +42,8 @@ }, { "id": "bug-005", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-21T08:24:10.738Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -56,6 +62,8 @@ }, { "id": "bug-006", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-21T08:30:20.719Z", "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer", "file": "src/controller/ha.c", @@ -77,6 +85,8 @@ }, { "id": "bug-007", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-21T12:53:13.789Z", "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 \u2014 reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", "file": "src/controller/ha.c", @@ -100,6 +110,8 @@ }, { "id": "bug-008", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-21T13:20:28.953Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -118,6 +130,8 @@ }, { "id": "bug-020", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-25T11:31:55.826Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -136,6 +150,8 @@ }, { "id": "bug-023", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T07:17:31.980Z", "error_message": "Missing guard clause", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -154,6 +170,8 @@ }, { "id": "bug-024", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T07:18:36.066Z", "error_message": "Missing guard clause", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -172,6 +190,8 @@ }, { "id": "bug-026", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T07:26:48.363Z", "error_message": "Wrong return value", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -190,6 +210,8 @@ }, { "id": "bug-027", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T07:27:34.925Z", "error_message": "Wrong condition in logic", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -208,6 +230,8 @@ }, { "id": "bug-029", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T07:29:11.741Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -226,6 +250,8 @@ }, { "id": "bug-032", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T09:07:23.015Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -244,6 +270,8 @@ }, { "id": "bug-034", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T09:23:02.818Z", "error_message": "Missing guard clause", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -262,6 +290,8 @@ }, { "id": "bug-035", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T09:46:12.902Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -280,6 +310,8 @@ }, { "id": "bug-038", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T10:09:32.424Z", "error_message": "Missing guard clause", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -298,6 +330,8 @@ }, { "id": "bug-039", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T10:09:57.711Z", "error_message": "Wrong return value", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -316,6 +350,8 @@ }, { "id": "bug-041", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T10:10:40.149Z", "error_message": "Incorrect value in code", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -334,6 +370,8 @@ }, { "id": "bug-042", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T11:35:44.097Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -352,6 +390,8 @@ }, { "id": "bug-046", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T13:51:32.274Z", "error_message": "Missing error handling in unknown", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -370,6 +410,8 @@ }, { "id": "bug-047", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-26T13:52:49.044Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -388,6 +430,8 @@ }, { "id": "bug-048", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-27T03:02:34.912Z", "error_message": "Missing error handling in unknown", "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", @@ -406,6 +450,8 @@ }, { "id": "bug-059", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-27T07:24:56.724Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-config-bootstrap/brief.md", @@ -424,6 +470,8 @@ }, { "id": "bug-060", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-27T09:24:44.651Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-config-bootstrap/spec.json", @@ -442,6 +490,8 @@ }, { "id": "bug-067", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T03:42:53.810Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -460,6 +510,8 @@ }, { "id": "bug-068", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T04:01:56.031Z", "error_message": "Wrong condition in logic", "file": "src/controller/main.c", @@ -478,6 +530,8 @@ }, { "id": "bug-069", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T04:02:06.925Z", "error_message": "Wrong reference: vcpus should be nres", "file": "src/include/clusterconfig.h", @@ -496,6 +550,8 @@ }, { "id": "bug-071", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T04:02:48.429Z", "error_message": "Incorrect value in code", "file": "src/schema/controller.capnp", @@ -514,6 +570,8 @@ }, { "id": "bug-074", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T05:59:28.525Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -532,6 +590,8 @@ }, { "id": "bug-075", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T07:04:33.895Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -550,6 +610,8 @@ }, { "id": "bug-078", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T07:22:56.418Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -568,6 +630,8 @@ }, { "id": "bug-081", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T07:53:10.883Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -586,6 +650,8 @@ }, { "id": "bug-082", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T08:23:40.040Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -604,6 +670,8 @@ }, { "id": "bug-083", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T08:45:37.921Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -622,6 +690,8 @@ }, { "id": "bug-084", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T08:48:58.609Z", "error_message": "Wrong reference: capn_str_null should be str", "file": "src/controller/config_repl.c", @@ -639,6 +709,8 @@ }, { "id": "bug-086", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T09:13:17.676Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -657,6 +729,8 @@ }, { "id": "bug-087", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T10:13:52.421Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -675,6 +749,8 @@ }, { "id": "bug-091", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T13:53:09.924Z", "error_message": "Wrong reference: in_catch_up should be acked_config_lsn", "file": "src/controller/repl.c", @@ -692,6 +768,8 @@ }, { "id": "bug-092", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T14:11:04.583Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -710,6 +788,8 @@ }, { "id": "bug-094", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T14:47:44.445Z", "error_message": "Wrong reference: ha_state_t should be _Atomic", "file": "src/controller/ha.h", @@ -727,6 +807,8 @@ }, { "id": "bug-095", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T15:04:44.613Z", "error_message": "Wrong reference: socket should be epoll", "file": "src/controller/ha.c", @@ -745,6 +827,8 @@ }, { "id": "bug-096", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-28T15:05:23.129Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -763,6 +847,8 @@ }, { "id": "bug-097", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T02:48:17.371Z", "error_message": "Missing guard clause", "file": "src/lib/request.c", @@ -781,6 +867,8 @@ }, { "id": "bug-098", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T02:50:46.444Z", "error_message": "Wrong reference: XCE_ERR_STANDBY should be E_XCE_STANDBY", "file": "src/lib/request.c", @@ -798,6 +886,8 @@ }, { "id": "bug-099", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T02:51:03.529Z", "error_message": "Wrong reference: XCE_ERR_NOT_READY should be E_XCE_NOT_READY", "file": "src/lib/testsuite/test_libxce_retry.c", @@ -815,6 +905,8 @@ }, { "id": "bug-102", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T03:37:52.117Z", "error_message": "Wrong reference: data should be ctrl_hello_reply_t", "file": "src/include/typemap.h", @@ -832,6 +924,8 @@ }, { "id": "bug-104", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T03:54:23.205Z", "error_message": "Missing error handling in struct", "file": "src/include/typemap.h", @@ -850,6 +944,8 @@ }, { "id": "bug-109", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T06:59:14.045Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -868,6 +964,8 @@ }, { "id": "bug-111", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T07:02:31.287Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -886,6 +984,8 @@ }, { "id": "bug-117", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T13:41:46.841Z", "error_message": "Incorrect value in code", "file": "src/controller/myapp.c", @@ -904,6 +1004,8 @@ }, { "id": "bug-118", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T13:51:42.299Z", "error_message": "Wrong reference: nres should be vcpus", "file": "src/include/clusterconfig.h", @@ -922,6 +1024,8 @@ }, { "id": "bug-119", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-29T16:32:10.285Z", "error_message": "Wrong reference: data should be request_vote_reply_t", "file": "src/include/typemap.h", @@ -939,6 +1043,8 @@ }, { "id": "bug-122", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T00:05:02.052Z", "error_message": "Wrong reference: cluster_init_phase2 should be cluster_register_listener", "file": "src/controller/include/cluster.h", @@ -956,6 +1062,8 @@ }, { "id": "bug-123", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T00:20:27.244Z", "error_message": "Missing guard clause", "file": "src/controller/cluster.c", @@ -974,6 +1082,8 @@ }, { "id": "bug-127", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T01:12:38.572Z", "error_message": "Wrong reference: CTRL_PERF_NHASH should be c", "file": "src/controller/cluster.c", @@ -991,6 +1101,8 @@ }, { "id": "bug-129", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T01:21:20.140Z", "error_message": "Wrong reference: true should be false", "file": ".claude/skills/xce-build/SKILL.md", @@ -1009,6 +1121,8 @@ }, { "id": "bug-130", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T01:28:22.079Z", "error_message": "Wrong reference: 120 should be 300", "file": "src/controller/meson.build", @@ -1027,6 +1141,8 @@ }, { "id": "bug-135", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T06:19:47.181Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/election-iomgr-refactor/design.md", @@ -1045,6 +1161,8 @@ }, { "id": "bug-136", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T06:38:09.325Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/election-iomgr-refactor/design.md", @@ -1063,6 +1181,8 @@ }, { "id": "bug-139", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T06:58:05.115Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/election-iomgr-refactor/design.md", @@ -1081,6 +1201,8 @@ }, { "id": "bug-142", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T07:57:14.052Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1099,6 +1221,8 @@ }, { "id": "bug-143", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T08:16:03.222Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1117,6 +1241,8 @@ }, { "id": "bug-144", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T08:49:04.058Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -1135,6 +1261,8 @@ }, { "id": "bug-145", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T08:49:59.817Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1153,6 +1281,8 @@ }, { "id": "bug-147", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T09:13:19.377Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -1171,6 +1301,8 @@ }, { "id": "bug-149", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T10:31:53.836Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1189,6 +1321,8 @@ }, { "id": "bug-153", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T11:43:01.989Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1207,6 +1341,8 @@ }, { "id": "bug-154", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T11:43:44.829Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/ha.c", @@ -1225,6 +1361,8 @@ }, { "id": "bug-157", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:02:39.009Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_startup.c", @@ -1243,6 +1381,8 @@ }, { "id": "bug-159", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:22:57.013Z", "error_message": "Wrong return value", "file": "src/controller/testsuite/test_ha_startup.c", @@ -1261,6 +1401,8 @@ }, { "id": "bug-160", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:23:18.560Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_ha_startup.c", @@ -1279,6 +1421,8 @@ }, { "id": "bug-161", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:29:45.867Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -1297,6 +1441,8 @@ }, { "id": "bug-162", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:30:10.570Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -1315,6 +1461,8 @@ }, { "id": "bug-163", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:31:02.203Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_failover.c", @@ -1333,6 +1481,8 @@ }, { "id": "bug-164", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:45:22.182Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -1351,6 +1501,8 @@ }, { "id": "bug-166", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T12:50:35.870Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_bootstrap_support.c", @@ -1369,6 +1521,8 @@ }, { "id": "bug-167", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T13:06:57.152Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1387,6 +1541,8 @@ }, { "id": "bug-168", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T13:24:06.024Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1405,6 +1561,8 @@ }, { "id": "bug-169", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T13:42:28.077Z", "error_message": "Wrong reference: third should be second", "file": "src/controller/testsuite/test_ha_repl_ack.c", @@ -1423,6 +1581,8 @@ }, { "id": "bug-170", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T14:35:07.679Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -1441,6 +1601,8 @@ }, { "id": "bug-172", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-30T23:42:12.644Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -1459,6 +1621,8 @@ }, { "id": "bug-176", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T02:09:39.630Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -1477,6 +1641,8 @@ }, { "id": "bug-183", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T03:02:33.311Z", "error_message": "Missing guard clause", "file": "src/lib/request.c", @@ -1495,6 +1661,8 @@ }, { "id": "bug-186", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T04:11:45.534Z", "error_message": "Missing error handling in struct", "file": ".kiro/specs/raft-library/design.md", @@ -1513,6 +1681,8 @@ }, { "id": "bug-188", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T04:19:54.524Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-library/design.md", @@ -1531,6 +1701,8 @@ }, { "id": "bug-190", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T05:07:37.759Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -1549,6 +1721,8 @@ }, { "id": "bug-191", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T05:23:19.394Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -1567,6 +1741,8 @@ }, { "id": "bug-197", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T07:08:23.387Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -1585,6 +1761,8 @@ }, { "id": "bug-206", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T08:40:00.000Z", "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 \u2014 ha->term must be atomically updated to peer term when peer term > ours", "file": "src/controller/ha.c", @@ -1606,6 +1784,8 @@ }, { "id": "bug-207", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T08:47:08.484Z", "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_even_peers_returns_einval", "file": "src/lib/testsuite/test_raft.c", @@ -1623,6 +1803,8 @@ }, { "id": "bug-208", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31", "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite \u2014 build exits with error code", "file": "src/scheduler/testsuite/test.c", @@ -1643,6 +1825,8 @@ }, { "id": "bug-209", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T10:19:47.014Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -1661,6 +1845,8 @@ }, { "id": "bug-211", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T11:29:03.301Z", "error_message": "Wrong reference: 2000 should be 5000", "file": "src/controller/ha.h", @@ -1679,6 +1865,8 @@ }, { "id": "bug-215", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T12:43:45.333Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1697,6 +1885,8 @@ }, { "id": "bug-216", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T12:49:40.554Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-peer-connection-reuse/tasks.md", @@ -1715,6 +1905,8 @@ }, { "id": "bug-218", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T12:58:44.435Z", "error_message": "Wrong reference: Request_body_nodata should be Request_body_nomsg", "file": "src/controller/ha.c", @@ -1732,6 +1924,8 @@ }, { "id": "bug-220", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-05-31T14:20:41.161Z", "error_message": "Wrong reference: vote_reply_msg_handler should be vote_reply_msg_handler_for_test", "file": "src/controller/ha.c", @@ -1749,6 +1943,8 @@ }, { "id": "bug-224", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T02:07:55.176Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1767,6 +1963,8 @@ }, { "id": "bug-229", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T02:49:09.141Z", "error_message": "Wrong operator: && should be !=", "file": "src/controller/ha.c", @@ -1785,6 +1983,8 @@ }, { "id": "bug-233", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T03:07:51.264Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1803,6 +2003,8 @@ }, { "id": "bug-235", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T03:13:14.521Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1821,6 +2023,8 @@ }, { "id": "bug-236", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T03:15:29.129Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1839,6 +2043,8 @@ }, { "id": "bug-238", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T03:23:32.483Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1857,6 +2063,8 @@ }, { "id": "bug-239", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T03:28:46.956Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1875,6 +2083,8 @@ }, { "id": "bug-240", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T04:08:07.950Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1893,6 +2103,8 @@ }, { "id": "bug-241", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T04:13:03.001Z", "error_message": "Wrong reference: test_ha_connect should be test_ha_ctrlhello", "file": "src/controller/testsuite/controller_test.c", @@ -1910,6 +2122,8 @@ }, { "id": "bug-242", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T05:19:12.427Z", "error_message": "Wrong reference: vcpu should be vcpus", "file": "src/tools/deploy-ha.sh", @@ -1928,6 +2142,8 @@ }, { "id": "bug-250", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T07:46:05.128Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -1946,6 +2162,8 @@ }, { "id": "bug-252", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T08:35:29.969Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -1964,6 +2182,8 @@ }, { "id": "bug-255", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T12:02:55.635Z", "error_message": "Wrong condition in logic", "file": "src/controller/cluster.c", @@ -1982,6 +2202,8 @@ }, { "id": "bug-256", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T12:36:28.381Z", "error_message": "Wrong reference: startup_validation should be testsuite_test_ha_liveness", "file": "src/xce.build/build.ninja", @@ -1999,6 +2221,8 @@ }, { "id": "bug-260", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T14:42:41.325Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2017,6 +2241,8 @@ }, { "id": "bug-263", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-01T15:32:59.304Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -2035,6 +2261,8 @@ }, { "id": "bug-265", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-03T08:47:25.835Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -2053,6 +2281,8 @@ }, { "id": "bug-266", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-03T08:47:54.376Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -2071,6 +2301,8 @@ }, { "id": "bug-269", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-03T08:50:58.837Z", "error_message": "Wrong reference: test_am_i_primary should be test_ha_cluster", "file": "src/controller/testsuite/controller_test.c", @@ -2088,6 +2320,8 @@ }, { "id": "bug-279", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T11:54:56.114Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-repl-hardening/design.md", @@ -2106,6 +2340,8 @@ }, { "id": "bug-284", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T14:33:03.464Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/testsuite/test_config_repl.c", @@ -2124,6 +2360,8 @@ }, { "id": "bug-285", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T14:41:35.073Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/cluster.c", @@ -2142,6 +2380,8 @@ }, { "id": "bug-288", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T14:46:44.652Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.h", @@ -2160,6 +2400,8 @@ }, { "id": "bug-292", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T16:15:43.743Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2178,6 +2420,8 @@ }, { "id": "bug-294", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T16:45:25.724Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2196,6 +2440,8 @@ }, { "id": "bug-295", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T16:57:59.213Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_ha_role.c", @@ -2214,6 +2460,8 @@ }, { "id": "bug-296", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T16:58:16.627Z", "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero", "file": "src/controller/testsuite/test_ha_role.c", @@ -2231,6 +2479,8 @@ }, { "id": "bug-297", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T17:08:18.571Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2249,6 +2499,8 @@ }, { "id": "bug-301", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T23:09:43.701Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_config_repl.c", @@ -2267,6 +2519,8 @@ }, { "id": "bug-302", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-04T23:10:44.426Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -2285,6 +2539,8 @@ }, { "id": "bug-308", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T01:19:20.356Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -2303,6 +2559,8 @@ }, { "id": "bug-309", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T01:33:20.146Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2321,6 +2579,8 @@ }, { "id": "bug-310", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T01:34:06.660Z", "error_message": "Missing guard clause", "file": "src/controller/cluster.c", @@ -2339,6 +2599,8 @@ }, { "id": "bug-311", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T02:21:15.033Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2357,6 +2619,8 @@ }, { "id": "bug-312", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T02:47:32.237Z", "error_message": "Wrong reference: uintptr_t should be void", "file": "src/controller/testsuite/test_ha_evaluate_committing.c", @@ -2374,6 +2638,8 @@ }, { "id": "bug-314", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T04:01:58.721Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -2392,6 +2658,8 @@ }, { "id": "bug-315", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T04:52:32.206Z", "error_message": "Wrong reference: 10000 should be 60000", "file": "src/controller/ha.h", @@ -2410,6 +2678,8 @@ }, { "id": "bug-317", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T04:53:55.034Z", "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS", "file": "src/controller/testsuite/test_ha_evaluate_committing.c", @@ -2427,6 +2697,8 @@ }, { "id": "bug-320", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-05T06:50:00Z", "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN", "file": "src/controller/ha.c", @@ -2447,6 +2719,8 @@ }, { "id": "bug-326", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-06T11:06:46.509Z", "error_message": "Wrong reference: DEBUG2 should be INFO", "file": "src/controller/ha.c", @@ -2465,6 +2739,8 @@ }, { "id": "bug-327", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-06T12:11:21.422Z", "error_message": "Incorrect value in code", "file": "src/controller/ha.c", @@ -2483,6 +2759,8 @@ }, { "id": "bug-328", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-06T13:02:49.052Z", "error_message": "Missing guard clause", "file": "src/lib/delay.c", @@ -2501,6 +2779,8 @@ }, { "id": "bug-329", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-08T13:12:54.436Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-library/research.md", @@ -2519,6 +2799,8 @@ }, { "id": "bug-334", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T12:56:11.095Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -2537,6 +2819,8 @@ }, { "id": "bug-335", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T00:00:00Z", "error_message": "UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving resource idx by name", "file": "src/scheduler/src/scheduler.c", @@ -2557,6 +2841,8 @@ }, { "id": "bug-336", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T00:00:00Z", "error_message": "XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST and return value is silently ignored", "file": "src/scheduler/src/scheduler.c", @@ -2578,6 +2864,8 @@ }, { "id": "bug-337", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T00:02:00Z", "error_message": "Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production writer after the liveness refactor", "file": "src/controller/ha.c", @@ -2599,6 +2887,8 @@ }, { "id": "bug-338", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T00:00:00Z", "error_message": "controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0); test_standby_fresh_heartbeat_denies_vote_request FAIL (Expected 0 Was 1, Req 6.2)", "file": "src/controller/ha.c", @@ -2621,6 +2911,8 @@ }, { "id": "bug-339", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T17:30:00Z", "error_message": "After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter an infinite election loop: both only ever raft_cb_become_follower (never become_leader), terms climb in lockstep (4..23), RDHUP -> CTRL_ELECTING -> CTRL_STANDBY repeats forever.", "file": "src/controller/ha.c, src/controller/myapp.c, src/controller/testsuite/test_ha_rdhup_integration.c", @@ -2654,6 +2946,8 @@ }, { "id": "bug-340", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T00:23:21.918Z", "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup \u2192 CTRL_ELECTING \u2192 raft_cb_become_follower \u2192 CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) \u2014 a wedged outbound standby endpoint.", "file": "src/controller/ha.c, src/controller/testsuite/test_ha_liveness.c", @@ -2680,6 +2974,8 @@ }, { "id": "bug-341", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T00:23:52.563Z", "error_message": "Wrong reference: last_heartbeat should be voted_for_term", "file": "src/controller/ha.h", @@ -2697,6 +2993,8 @@ }, { "id": "bug-344", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T00:32:34.054Z", "error_message": "Incorrect value in code", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_bootstrap_progress.md", @@ -2715,6 +3013,8 @@ }, { "id": "bug-349", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T01:39:19.368Z", "error_message": "Missing guard clause", "file": "src/lib/request.c", @@ -2733,6 +3033,8 @@ }, { "id": "bug-350", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T01:39:31.683Z", "error_message": "Missing error handling in unknown", "file": "src/lib/request.c", @@ -2751,6 +3053,8 @@ }, { "id": "bug-351", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-09T00:00:00Z", "error_message": "request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw from incompatible pointer type (xce_peerinfo_t** vs xce_peerinfo_t*)", "file": "src/lib/request.c", @@ -2771,6 +3075,8 @@ }, { "id": "bug-352", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T01:46:28.589Z", "error_message": "ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower (loser path) <- raft_on_vote_reply; election loser crashes on vote-reply processing", "file": "src/controller/ha.c", @@ -2794,6 +3100,8 @@ }, { "id": "bug-353", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T01:46:45.213Z", "error_message": "Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows requestVote/probeConfigTag, returning -E_XCE_ERR_NOT_READY for all other opcodes. If liveness poll fires during election window, any incoming config push is discarded.", "file": "src/controller/ha.c", @@ -2819,6 +3127,8 @@ }, { "id": "bug-358", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T03:55:44.698Z", "error_message": "Wrong condition in logic", "file": "src/controller/config_repl.c", @@ -2837,6 +3147,8 @@ }, { "id": "bug-359", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T04:16:25.052Z", "error_message": "Missing guard clause", "file": "src/scheduler/src/scheduler.c", @@ -2855,6 +3167,8 @@ }, { "id": "bug-361", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T04:47:09.048Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -2873,6 +3187,8 @@ }, { "id": "bug-364", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T06:26:24.340Z", "error_message": "Incorrect value in code", "file": "src/controller/CLAUDE.md", @@ -2891,6 +3207,8 @@ }, { "id": "bug-367", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T07:56:21.347Z", "error_message": "Missing error handling in XCE_TESTING", "file": "src/controller/testsuite/test_repl_unit.c", @@ -2909,6 +3227,8 @@ }, { "id": "bug-368", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T07:57:40.792Z", "error_message": "Wrong reference: WAL_JOB should be WAL_JOB_SUBMIT", "file": "src/controller/testsuite/test_repl_unit.c", @@ -2926,6 +3246,8 @@ }, { "id": "bug-371", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T07:59:24.268Z", "error_message": "Wrong reference: DEBUG3 should be TRACE", "file": "src/controller/repl.c", @@ -2944,6 +3266,8 @@ }, { "id": "bug-373", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T08:45:30.400Z", "error_message": "Wrong reference: request should be ep", "file": "src/controller/cluster.c", @@ -2962,6 +3286,8 @@ }, { "id": "bug-374", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T08:47:48.296Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_ha_role.c", @@ -2980,6 +3306,8 @@ }, { "id": "bug-376", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T09:17:35.218Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -2998,6 +3326,8 @@ }, { "id": "bug-378", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T10:02:00.000Z", "error_message": "AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async", "file": "src/controller/testsuite/test_p2_integration.c", @@ -3018,6 +3348,8 @@ }, { "id": "bug-380", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T10:37:21.924Z", "error_message": "Wrong reference: test_catchup_full_lifecycle_integration should be test_single_node_committed_lsn_unchanged", "file": "src/controller/testsuite/test_p2_integration.c", @@ -3035,6 +3367,8 @@ }, { "id": "bug-382", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T11:54:19.749Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-wal-replication/tasks.md", @@ -3053,6 +3387,8 @@ }, { "id": "bug-384", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T14:13:01.581Z", "error_message": "Function not marked async", "file": ".kiro/steering/tech.md", @@ -3071,6 +3407,8 @@ }, { "id": "bug-386", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T14:43:53.161Z", "error_message": "Function not marked async", "file": "CLAUDE.md", @@ -3089,6 +3427,8 @@ }, { "id": "bug-388", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T14:54:18.503Z", "error_message": "Function not marked async", "file": "src/controller/CLAUDE.md", @@ -3107,6 +3447,8 @@ }, { "id": "bug-389", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T02:46:08.749Z", "error_message": "Wrong reference: signal should be stdio", "file": "src/lib/delay.c", @@ -3125,6 +3467,8 @@ }, { "id": "bug-391", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T03:41:14.912Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -3143,6 +3487,8 @@ }, { "id": "bug-392", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T03:46:02.642Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -3161,6 +3507,8 @@ }, { "id": "bug-394", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T05:09:37.037Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-consistency-invariants/research.md", @@ -3179,6 +3527,8 @@ }, { "id": "bug-396", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-10T22:45:00-07:00", "error_message": "Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch gate rejects them (quorum loss / COMMITTING window): worker never reads replies, never retries, no rejoin resync", "file": "src/worker/src/controller_connector.c, src/lib/xceapi.c, src/controller/cluster.c", @@ -3200,6 +3550,8 @@ }, { "id": "bug-398", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T06:35:26.288Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-consistency-invariants/research.md", @@ -3218,6 +3570,8 @@ }, { "id": "bug-399", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T06:43:14.709Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-consistency-invariants/research.md", @@ -3236,6 +3590,8 @@ }, { "id": "bug-400", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T07:23:53.901Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-consistency-invariants/research.md", @@ -3254,6 +3610,8 @@ }, { "id": "bug-401", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T07:47:34.756Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-consistency-invariants/research.md", @@ -3272,6 +3630,8 @@ }, { "id": "bug-403", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T08:34:47.868Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-consistency-invariants/design.md", @@ -3290,6 +3650,8 @@ }, { "id": "bug-405", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T09:47:16.750Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-single-leader-role/design.md", @@ -3308,6 +3670,8 @@ }, { "id": "bug-406", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T09:49:04.296Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-single-leader-role/tasks.md", @@ -3326,6 +3690,8 @@ }, { "id": "bug-407", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T10:01:15.181Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-barrier-rollout/design.md", @@ -3344,6 +3710,8 @@ }, { "id": "bug-408", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T10:01:39.424Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-convergence-recovery/tasks.md", @@ -3362,6 +3730,8 @@ }, { "id": "bug-410", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T13:30:00Z", "error_message": "test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby", "file": "src/controller/testsuite/test_commit_engine.c", @@ -3381,6 +3751,9 @@ }, { "id": "bug-411", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "95a10ced", "timestamp": "2026-06-11T13:35:00Z", "error_message": "3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013)", "file": "src/controller/testsuite/controller_test.c", @@ -3402,6 +3775,8 @@ }, { "id": "bug-413", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T15:28:35.990Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_engine.c", @@ -3420,6 +3795,8 @@ }, { "id": "bug-420", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T16:19:51.195Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_engine.c", @@ -3438,6 +3815,8 @@ }, { "id": "bug-421", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T16:23:46.118Z", "error_message": "Missing guard clause", "file": "src/controller/repl.c", @@ -3456,6 +3835,8 @@ }, { "id": "bug-423", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T22:39:24.704Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_engine.c", @@ -3474,6 +3855,8 @@ }, { "id": "bug-424", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:08:49.909Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-commit-engine/tasks.md", @@ -3492,6 +3875,8 @@ }, { "id": "bug-425", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:15:25.014Z", "error_message": "Wrong return value", "file": "src/controller/repl.c", @@ -3510,6 +3895,8 @@ }, { "id": "bug-428", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:43:45.216Z", "error_message": "Incorrect value in code", "file": "src/include/xce/cluster_def.h", @@ -3528,6 +3915,8 @@ }, { "id": "bug-429", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:44:16.357Z", "error_message": "Wrong condition in logic", "file": "src/controller/cluster.c", @@ -3546,6 +3935,8 @@ }, { "id": "bug-431", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:45:16.518Z", "error_message": "Incorrect value in code", "file": "src/controller/persist.c", @@ -3564,6 +3955,8 @@ }, { "id": "bug-432", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:45:42.031Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_p2_integration.c", @@ -3582,6 +3975,8 @@ }, { "id": "bug-433", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T23:45:51.649Z", "error_message": "Wrong reference: test_single_node_committed_lsn_unchanged should be test_single_node_committed_lsn_advances", "file": "src/controller/testsuite/test_p2_integration.c", @@ -3599,6 +3994,8 @@ }, { "id": "bug-434", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T00:23:08.815Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_p2_integration.c", @@ -3617,6 +4014,8 @@ }, { "id": "bug-435", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T00:00:00Z", "error_message": "test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by bug-411)", "file": "src/controller/testsuite/test_p2_integration.c", @@ -3638,6 +4037,8 @@ }, { "id": "bug-436", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-11T00:00:00Z", "error_message": "test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected 0 Was -10013 (NOT_READY; baseline failures, masked by bug-411)", "file": "src/controller/testsuite/test_ha_repl_ack.c, src/controller/testsuite/test_ha_rdhup_integration.c", @@ -3659,6 +4060,8 @@ }, { "id": "bug-437", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T00:51:52.673Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-commit-engine/tasks.md", @@ -3677,6 +4080,8 @@ }, { "id": "bug-438", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T03:52:57.142Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -3695,6 +4100,8 @@ }, { "id": "bug-440", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T04:18:19.158Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -3713,6 +4120,8 @@ }, { "id": "bug-442", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T04:50:32.154Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", @@ -3731,6 +4140,8 @@ }, { "id": "bug-443", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T05:14:36.392Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -3749,6 +4160,8 @@ }, { "id": "bug-445", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T09:30:00Z", "error_message": "AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t test_commit_barrier_reply standalone", "file": "src/controller/testsuite/test_commit_barrier_reply.c", @@ -3769,6 +4182,8 @@ }, { "id": "bug-446", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T09:30:00Z", "error_message": "makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture", "file": "src/controller/testsuite/test_commit_barrier_reply.c", @@ -3788,6 +4203,8 @@ }, { "id": "bug-447", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T09:30:00Z", "error_message": "queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1 (key_find found-rc leak)", "file": "src/controller/queue.c", @@ -3811,6 +4228,9 @@ }, { "id": "bug-448", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "job.c job_ctrl normal exit path never dereference(j)", "timestamp": "2026-06-12T10:05:12.464Z", "error_message": "job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446) never calls dereference(j); only the workermgr-miss error path releases it", "file": "src/controller/job.c", @@ -3835,6 +4255,8 @@ }, { "id": "bug-451", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T11:05:00-07:00", "error_message": "va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*", "file": "src/controller/job.c (caller) / src/controller/persist.c (consumer)", @@ -3860,6 +4282,8 @@ }, { "id": "bug-452", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-12T15:03:38.962Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_commit_barrier_reply.c", @@ -3878,6 +4302,8 @@ }, { "id": "bug-457", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T00:18:17.080Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -3896,6 +4322,8 @@ }, { "id": "bug-459", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T00:30:00Z", "error_message": "configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack), denominator n_active with no self-count, and n_active==0 short-circuited to success; upgradeConfig also ACKed the admin tool unconditionally and requested restart even when the majority wait timed out (logged-only result)", "file": "src/controller/repl.c, src/controller/config_repl.c", @@ -3919,6 +4347,8 @@ }, { "id": "bug-461", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T01:11:49.673Z", "error_message": "Wrong condition in logic", "file": "src/controller/config_repl.c", @@ -3937,6 +4367,8 @@ }, { "id": "bug-465", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T01:30:00Z", "error_message": "Config-ACK signal never produced in production: repl_on_config_ack had zero production callers; standby replconfigpushreplymsg silently dropped in repl_ack_decode_and_dispatch (ha.c) and myreq_handler default case (cluster.c) -> every multi-member config upgrade barrier timed out (upgrades bricked)", "file": "src/controller/ha.c", @@ -3958,6 +4390,8 @@ }, { "id": "bug-466", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T01:30:00Z", "error_message": "Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll thread (ha.c raft_cb_become_leader, ha_dispatch_repl_action_for_standby, legacy election path; join.c controller join) -> blocks all controller I/O; the awaited ACK arrives on the blocked thread (structural deadlock)", "file": "src/controller/config_repl.c", @@ -3981,6 +4415,8 @@ }, { "id": "bug-467", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T01:38:08.308Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -3999,6 +4435,8 @@ }, { "id": "bug-468", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T03:56:09.185Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_commit_barrier_reply.c", @@ -4017,6 +4455,8 @@ }, { "id": "bug-469", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T06:33:41.449Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -4035,6 +4475,8 @@ }, { "id": "bug-470", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T06:34:01.929Z", "error_message": "Wrong reference: test_ha_cluster should be test_ha_role_lifecycle", "file": "src/controller/testsuite/controller_test.c", @@ -4052,6 +4494,8 @@ }, { "id": "bug-471", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T06:41:01.532Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_role_lifecycle.c", @@ -4070,6 +4514,8 @@ }, { "id": "bug-473", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T07:32:43.287Z", "error_message": "Wrong return value", "file": "src/controller/persist.c", @@ -4088,6 +4534,8 @@ }, { "id": "bug-476", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T08:38:00.471Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -4121,6 +4569,8 @@ "closed-by-scheduler-sim-live-allocation" ], "id": "bug-477", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13", "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, @@ -4142,6 +4592,8 @@ "closed-by-scheduler-sim-live-allocation" ], "id": "bug-478", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13", "last_seen": "2026-09-19T09:00:00Z", "occurrences": 1, @@ -4149,6 +4601,8 @@ }, { "id": "bug-480", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T08:57:05.687Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_role_lifecycle.c", @@ -4167,6 +4621,8 @@ }, { "id": "bug-482", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T09:36:17.806Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_start_election_async.c", @@ -4185,6 +4641,8 @@ }, { "id": "bug-483", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T09:45:01.531Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_role_lifecycle.c", @@ -4203,6 +4661,8 @@ }, { "id": "bug-484", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T10:16:06.331Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-single-leader-role/tasks.md", @@ -4221,6 +4681,8 @@ }, { "id": "bug-493", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T12:08:04.621Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -4239,6 +4701,8 @@ }, { "id": "bug-494", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T12:10:08.938Z", "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", "file": "src/controller/ha.c", @@ -4256,6 +4720,8 @@ }, { "id": "bug-496", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T13:07:42.791Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_ha_role_lifecycle.c", @@ -4274,6 +4740,8 @@ }, { "id": "bug-497", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T13:10:05.452Z", "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", "file": "src/controller/testsuite/test_ha_connection_reuse.c", @@ -4291,6 +4759,8 @@ }, { "id": "bug-498", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T13:10:16.164Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_ha_connection_reuse.c", @@ -4309,6 +4779,8 @@ }, { "id": "bug-500", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T13:14:11.496Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_ha_startup.c", @@ -4327,6 +4799,8 @@ }, { "id": "bug-501", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T13:15:33.777Z", "error_message": "Wrong reference: test_nonblocking_single_node_election_enters_committing should be test_nonblocking_single_node_election_enters_primary_directly", "file": "src/controller/testsuite/test_ha_startup.c", @@ -4344,6 +4818,8 @@ }, { "id": "bug-503", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T13:30:00.000Z", "error_message": "test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed, expected -XCE_ERR_NOT_READY)", "file": "src/controller/testsuite/test_ha_startup.c", @@ -4365,6 +4841,8 @@ }, { "id": "bug-505", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T14:01:32.553Z", "error_message": "Wrong reference: uint32_t should be ha_config_t", "file": "src/controller/ha.h", @@ -4382,6 +4860,8 @@ }, { "id": "bug-507", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T14:43:54.194Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/ha-async-io.md", @@ -4400,6 +4880,8 @@ }, { "id": "bug-508", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T14:54:21.781Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-deploy-ha/SKILL.md", @@ -4418,6 +4900,8 @@ }, { "id": "bug-509", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-13T15:31:21.740Z", "error_message": "Incorrect value in code", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_consistency_invariants.md", @@ -4436,6 +4920,8 @@ }, { "id": "bug-510", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T00:30:31.077Z", "error_message": "Wrong reference: stdint should be stdbool", "file": "src/controller/include/cluster.h", @@ -4453,6 +4939,8 @@ }, { "id": "bug-512", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T00:34:38.745Z", "error_message": "Wrong reference: xlist_destroy should be xlist_fini", "file": "src/controller/testsuite/test_egress_filter.c", @@ -4470,6 +4958,8 @@ }, { "id": "bug-513", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T00:35:13.711Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -4488,6 +4978,8 @@ }, { "id": "bug-514", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T00:37:31.926Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_egress_filter.c", @@ -4506,6 +4998,8 @@ }, { "id": "bug-516", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T01:46:09.600Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -4524,6 +5018,8 @@ }, { "id": "bug-518", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T01:50:24.429Z", "error_message": "Missing guard clause", "file": "src/controller/sched.c", @@ -4542,6 +5038,8 @@ }, { "id": "bug-519", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T01:52:56.833Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_egress_filter.c", @@ -4560,6 +5058,8 @@ }, { "id": "bug-521", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T03:55:55.380Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_demotion_convergence.c", @@ -4578,6 +5078,8 @@ }, { "id": "bug-523", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T04:21:18.071Z", "error_message": "Missing guard clause", "file": "src/controller/ha_convergence.c", @@ -4596,6 +5098,8 @@ }, { "id": "bug-343", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T04:34:32+00:00", "error_message": "AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart", "file": "src/controller/testsuite/test_demotion_convergence.c", @@ -4619,6 +5123,8 @@ }, { "id": "bug-526", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T04:52:58.813Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-convergence-recovery/tasks.md", @@ -4637,6 +5143,8 @@ }, { "id": "bug-527", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T04:57:22.178Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.h", @@ -4655,6 +5163,8 @@ }, { "id": "bug-532", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T06:12:36.513Z", "error_message": "Missing guard clause", "file": "src/controller/ha_convergence.c", @@ -4673,6 +5183,8 @@ }, { "id": "bug-533", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T06:37:35.904Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_convergence_verification.c", @@ -4691,6 +5203,8 @@ }, { "id": "bug-534", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T06:38:38.792Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_convergence_verification.c", @@ -4709,6 +5223,8 @@ }, { "id": "bug-536", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T07:56:01.819Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_rdhup_integration.c", @@ -4727,6 +5243,8 @@ }, { "id": "bug-537", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T08:02:40.832Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -4745,6 +5263,8 @@ }, { "id": "bug-538", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14", "error_message": "config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config, breaking replication on a live 3-node cluster", "file": "src/controller/ha.c", @@ -4767,6 +5287,8 @@ }, { "id": "bug-542", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T09:29:31.490Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -4785,6 +5307,8 @@ }, { "id": "bug-543", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T09:36:36.967406Z", "error_message": "WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0", "file": "src/controller/config_repl.c", @@ -4808,6 +5332,8 @@ }, { "id": "bug-550", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T11:38:09.614Z", "error_message": "Wrong reference: test_pin_dispatch_gate_degraded_allowlist should be test_empty_standby_fresh_cluster_streams_not_snapshot", "file": "src/controller/testsuite/test_convergence_verification.c", @@ -4825,6 +5351,8 @@ }, { "id": "bug-551", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T11:45:02.411Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -4843,6 +5371,8 @@ }, { "id": "bug-555", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T12:23:52.351Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_repl_apply.c", @@ -4861,6 +5391,8 @@ }, { "id": "bug-556", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T12:25:37.723Z", "error_message": "Function not marked async", "file": "src/include/persist.h", @@ -4879,6 +5411,8 @@ }, { "id": "bug-557", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T12:26:04.137Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -4897,6 +5431,8 @@ }, { "id": "bug-560", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T19:54:55.428Z", "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", @@ -4914,6 +5450,8 @@ }, { "id": "bug-561", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T20:05:48.755409", "error_message": "jobsub intermittent -ETIMEDOUT \"Connection timed out\": commit: WAIT deadline expired without commit (timeout_ms=50)", "file": "src/include/config.h", @@ -4935,6 +5473,8 @@ }, { "id": "bug-562", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-14T23:55:00Z", "error_message": "AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen (lib/controller.capnp.c:13168) on ctrl-2 startup, via xce_reply->xce_encode from OPCode_nop handler (cluster.c:307). Also: job submission succeeds but joblist shows no jobs.", "file": "src/lib/request.c", @@ -4959,6 +5499,8 @@ }, { "id": "bug-563", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-15T09:16:53.679025Z", "error_message": "jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works.", "file": "src/lib/request.c", @@ -4986,6 +5528,8 @@ }, { "id": "bug-570", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-15T11:10:10.553Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/product.md", @@ -5004,6 +5548,8 @@ }, { "id": "bug-572", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-15T11:10:26.312Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/tech.md", @@ -5022,6 +5568,8 @@ }, { "id": "bug-575", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T00:10:49.295Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-prevote/research.md", @@ -5040,6 +5588,8 @@ }, { "id": "bug-576", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T00:18:28.793Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-prevote/research.md", @@ -5058,6 +5608,8 @@ }, { "id": "bug-578", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T03:00:48.818Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-prevote/design.md", @@ -5076,6 +5628,8 @@ }, { "id": "bug-579", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T03:01:07.659Z", "error_message": "Wrong return value", "file": ".kiro/specs/ha-prevote/design.md", @@ -5094,6 +5648,8 @@ }, { "id": "bug-584", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T04:27:50.039Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -5112,6 +5668,8 @@ }, { "id": "bug-585", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T04:36:28.796Z", "error_message": "Missing guard clause", "file": "src/lib/testsuite/test_raft.c", @@ -5130,6 +5688,8 @@ }, { "id": "bug-586", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T04:50:38.855Z", "error_message": "Missing error handling in struct", "file": "src/controller/testsuite/test_ha_vote_gate.c", @@ -5148,6 +5708,8 @@ }, { "id": "bug-591", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T05:27:09.380Z", "error_message": "Wrong reference: test_commit_engine should be test_ha_prevote", "file": "src/controller/testsuite/controller_test.c", @@ -5165,6 +5727,8 @@ }, { "id": "bug-592", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T05:28:01.236Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -5183,6 +5747,8 @@ }, { "id": "bug-593", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T05:44:57.430Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-prevote/tasks.md", @@ -5201,6 +5767,8 @@ }, { "id": "bug-594", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T05:50:25.729Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -5219,6 +5787,8 @@ }, { "id": "bug-595", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T05:50:35.123Z", "error_message": "Wrong reference: test_commit_engine should be test_ha_failover_prevote", "file": "src/controller/testsuite/controller_test.c", @@ -5236,6 +5806,8 @@ }, { "id": "bug-597", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T05:59:43.939Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -5254,6 +5826,8 @@ }, { "id": "bug-598", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T06:33:05.644Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -5272,6 +5846,8 @@ }, { "id": "bug-603", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T08:18:48.567Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -5290,6 +5866,8 @@ }, { "id": "bug-604", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T15:55:00Z", "error_message": "controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap", "file": "src/controller/persist.c", @@ -5314,6 +5892,8 @@ }, { "id": "bug-605", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T16:30:00Z", "error_message": "3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1) loops PreVote LOST every ~11s forever; config-less peers idle as voter-only. REPRODUCED via /xce-deploy-ha + clean restart 2026-06-16.", "file": "src/controller/ha.c (ha_dispatch_gate); test: src/controller/testsuite/test_ha_role.c", @@ -5339,6 +5919,8 @@ }, { "id": "bug-606", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T17:46:00Z", "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win \u2192 CTRL_ELECTING \u2192 become_follower (term bumped) \u2192 ~10s \u2192 repeat, term climbing unbounded (3\u21929\u2192\u221e), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", "file": "src/controller/ha.c (ha_begin_failover, ha_election_stagger_ms, ha_failover_stagger_fn); src/controller/ha.h; test: src/controller/testsuite/test_ha_role.c", @@ -5367,6 +5949,8 @@ }, { "id": "bug-608", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T10:36:00Z", "error_message": "libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument' (EINVAL) indefinitely", "file": "src/lib/request.c", @@ -5389,6 +5973,8 @@ }, { "id": "bug-609", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T10:52:01.800Z", "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call \u2014 transparent failover did not engage", "file": "src/lib/client_failover.c", @@ -5413,6 +5999,8 @@ }, { "id": "bug-610", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T10:52:36.535Z", "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate \u2192 xcecli_retry_next \u2192 xcecli_joblist during live master failover (NULL ep->iomgr deref)", "file": "src/lib/client_failover.c", @@ -5441,6 +6029,8 @@ }, { "id": "bug-613", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T13:15:11.048Z", "error_message": "Function not marked async", "file": ".kiro/specs/libxce-failover/design.md", @@ -5459,6 +6049,8 @@ }, { "id": "bug-614", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T13:18:29.113Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/libxce-failover/design.md", @@ -5477,6 +6069,8 @@ }, { "id": "bug-617", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T23:38:24.000Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/libxce-failover/design.md", @@ -5495,6 +6089,8 @@ }, { "id": "bug-619", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-16T23:40:22.003Z", "error_message": "Wrong reference: address should be hint", "file": ".kiro/specs/libxce-failover/tasks.md", @@ -5513,6 +6109,8 @@ }, { "id": "bug-623", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T00:04:51.775Z", "error_message": "Wrong reference: XCE_MAXHOST_LEN should be XCE_MAXHOSTNAME_LEN", "file": ".kiro/specs/libxce-failover/design.md", @@ -5530,6 +6128,8 @@ }, { "id": "bug-624", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T02:35:29.695Z", "error_message": "Incorrect value in code", "file": "src/lib/meson.build", @@ -5548,6 +6148,8 @@ }, { "id": "bug-626", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T02:48:27.547Z", "error_message": "Missing guard clause", "file": "src/lib/client_failover.c", @@ -5566,6 +6168,8 @@ }, { "id": "bug-629", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T03:18:22.365Z", "error_message": "Missing guard clause", "file": "src/lib/client_failover.c", @@ -5584,6 +6188,8 @@ }, { "id": "bug-631", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T03:31:29.120Z", "error_message": "Missing guard clause", "file": "src/lib/client_failover.c", @@ -5602,6 +6208,8 @@ }, { "id": "bug-633", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T03:40:10.853Z", "error_message": "Missing guard clause", "file": "src/lib/client_failover.c", @@ -5620,6 +6228,8 @@ }, { "id": "bug-635", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T03:51:08.767Z", "error_message": "Wrong return value", "file": "src/lib/client.c", @@ -5638,6 +6248,8 @@ }, { "id": "bug-637", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T03:57:33.232Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/libxce-failover/design.md", @@ -5656,6 +6268,8 @@ }, { "id": "bug-638", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T04:02:05.572Z", "error_message": "Missing guard clause", "file": "src/lib/client_failover.c", @@ -5674,6 +6288,8 @@ }, { "id": "bug-639", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T04:11:59.759Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -5692,6 +6308,8 @@ }, { "id": "bug-640", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T04:12:01.203Z", "error_message": "Wrong reference: test_p1_integration should be test_join_redirect", "file": "src/controller/testsuite/controller_test.c", @@ -5709,6 +6327,8 @@ }, { "id": "bug-641", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T04:19:58.697Z", "error_message": "Wrong reference: test_recover_resets_err_state_to_run should be test_decide_jitter_staggers_many_clients", "file": "src/lib/testsuite/test_libxce_retry.c", @@ -5726,6 +6346,8 @@ }, { "id": "bug-644", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T08:15:04.978Z", "error_message": "Incorrect value in code", "file": "src/lib/meson.build", @@ -5744,6 +6366,8 @@ }, { "id": "bug-646", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T08:21:32.099Z", "error_message": "xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'.", "file": "src/lib/netio.c", @@ -5774,6 +6398,8 @@ }, { "id": "bug-647", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T08:21:56.735Z", "error_message": "Wrong reference: 611 should be 646", "file": "src/lib/netio.c", @@ -5792,6 +6418,8 @@ }, { "id": "bug-648", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T17:30:00Z", "error_message": "Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event.", "file": "src/lib/netio.c", @@ -5821,6 +6449,8 @@ }, { "id": "bug-651", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T10:44:37.350Z", "error_message": "Wrong return value", "file": "src/lib/request.c", @@ -5839,6 +6469,8 @@ }, { "id": "bug-650", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T18:30:00Z", "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket \u2014 the request was never transmitted yet the client waits for a reply forever.", "file": "src/lib/request.c", @@ -5866,6 +6498,8 @@ }, { "id": "bug-654", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T11:21:51.571Z", "error_message": "Wrong return value", "file": "src/lib/testsuite/test_netio_arm_failure.c", @@ -5884,6 +6518,8 @@ }, { "id": "bug-656", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T11:22:17.916Z", "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", "file": "src/lib/testsuite/test_netio_arm_failure.c", @@ -5901,6 +6537,8 @@ }, { "id": "bug-652", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T19:10:00Z", "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", "file": "src/lib/netio.c", @@ -5929,6 +6567,8 @@ }, { "id": "bug-658", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T14:46:54.135Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -5947,6 +6587,8 @@ }, { "id": "bug-659", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T14:47:25.446Z", "error_message": "Wrong condition in logic", "file": "src/controller/cluster.c", @@ -5965,6 +6607,8 @@ }, { "id": "bug-660", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T22:00:00Z", "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", "file": "src/controller/ha.c", @@ -5989,6 +6633,9 @@ }, { "id": "bug-661", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "f139e1c0", "timestamp": "2026-06-17T23:30:00Z", "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", "file": "src/controller/persist.c, src/controller/commit.c, src/controller/repl.c", @@ -6021,6 +6668,8 @@ }, { "id": "bug-662", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T23:34:32.115Z", "error_message": "Function not marked async", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", @@ -6039,6 +6688,8 @@ }, { "id": "bug-664", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-17T23:59:44.577Z", "error_message": "Incorrect value in code", "file": "src/tools/test-commit-durability.sh", @@ -6057,6 +6708,8 @@ }, { "id": "bug-666", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T02:32:07.670Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-commit-durability/brief.md", @@ -6075,6 +6728,8 @@ }, { "id": "bug-667", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T02:35:08.157Z", "error_message": "Function not marked async", "file": ".kiro/steering/roadmap.md", @@ -6093,6 +6748,8 @@ }, { "id": "bug-668", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T03:12:26.259Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/design.md", @@ -6111,6 +6768,8 @@ }, { "id": "bug-671", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T03:15:13.287Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/roadmap.md", @@ -6129,6 +6788,8 @@ }, { "id": "bug-673", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T03:50:26.761Z", "error_message": "Function not marked async", "file": "src/controller/CLAUDE.md", @@ -6147,6 +6808,8 @@ }, { "id": "bug-676", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T04:00:06.872Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/design.md", @@ -6165,6 +6828,8 @@ }, { "id": "bug-677", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T04:39:24.460Z", "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", @@ -6182,6 +6847,8 @@ }, { "id": "bug-678", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T04:50:36.884Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -6200,6 +6867,8 @@ }, { "id": "bug-679", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T04:51:30.430Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -6218,6 +6887,8 @@ }, { "id": "bug-682", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T05:02:07.556Z", "error_message": "Incorrect value in code", "file": "src/tools/test-commit-durability.sh", @@ -6236,6 +6907,8 @@ }, { "id": "bug-684", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T05:28:05.352Z", "error_message": "Function not marked async", "file": "src/tools/test-commit-durability.sh", @@ -6254,6 +6927,8 @@ }, { "id": "bug-686", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T05:43:02.761Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -6272,6 +6947,8 @@ }, { "id": "bug-688", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T05:54:46.908Z", "error_message": "Function not marked async", "file": "src/include/persist.h", @@ -6290,6 +6967,8 @@ }, { "id": "bug-690", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T05:55:33.474Z", "error_message": "Wrong reference: else should be RUN_TEST", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6308,6 +6987,8 @@ }, { "id": "bug-692", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T05:59:17.869Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -6326,6 +7007,8 @@ }, { "id": "bug-693", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:05:00.000Z", "error_message": "durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority", "file": "src/controller/persist.c", @@ -6347,6 +7030,8 @@ }, { "id": "bug-694", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:09:08.298Z", "error_message": "Incorrect value in code", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", @@ -6365,6 +7050,8 @@ }, { "id": "bug-695", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:12:59.924Z", "error_message": "Function not marked async", "file": "src/include/persist.h", @@ -6383,6 +7070,8 @@ }, { "id": "bug-696", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:13:32.891Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -6401,6 +7090,8 @@ }, { "id": "bug-697", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:13:40.659Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -6419,6 +7110,8 @@ }, { "id": "bug-699", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:14:16.158Z", "error_message": "Function not marked async", "file": "src/controller/commit.c", @@ -6437,6 +7130,8 @@ }, { "id": "bug-701", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:15:45.248Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6455,6 +7150,8 @@ }, { "id": "bug-704", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18", "error_message": "controller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0)", "file": "src/controller/testsuite/{test_repl_majority,test_commit_barrier_rollout,test_coldstart_lsn0,test_demotion_convergence}.c", @@ -6480,6 +7177,8 @@ }, { "id": "bug-705", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:33:12.523Z", "error_message": "Function not marked async", "file": "src/include/persist.h", @@ -6498,6 +7197,8 @@ }, { "id": "bug-707", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:34:06.875Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -6516,6 +7217,8 @@ }, { "id": "bug-708", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:34:36.356Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6534,6 +7237,8 @@ }, { "id": "bug-711", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T06:50:49.987Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -6552,6 +7257,8 @@ }, { "id": "bug-714", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T07:00:36.008Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -6570,6 +7277,8 @@ }, { "id": "bug-720", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T07:52:20.595Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6588,6 +7297,8 @@ }, { "id": "bug-721", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T08:04:18.857Z", "error_message": "Missing error handling in enum", "file": "src/include/persist.h", @@ -6606,6 +7317,8 @@ }, { "id": "bug-726", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T08:19:47.036Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -6624,6 +7337,8 @@ }, { "id": "bug-728", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T08:39:54.889Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/design.md", @@ -6642,6 +7357,8 @@ }, { "id": "bug-732", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T08:51:34.635Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_vote_gate.c", @@ -6660,6 +7377,8 @@ }, { "id": "bug-733", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T08:52:12.840Z", "error_message": "Wrong return value", "file": "src/controller/ha.c", @@ -6678,6 +7397,8 @@ }, { "id": "bug-735", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T09:08:09.215Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.c", @@ -6696,6 +7417,8 @@ }, { "id": "bug-736", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T09:09:28.348Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6714,6 +7437,8 @@ }, { "id": "bug-738", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T09:45:40.163Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6732,6 +7457,8 @@ }, { "id": "bug-739", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T09:46:18.019Z", "error_message": "Function not marked async", "file": "src/include/persist.h", @@ -6750,6 +7477,8 @@ }, { "id": "bug-740", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T09:46:46.814Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -6768,6 +7497,8 @@ }, { "id": "bug-741", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T09:49:58.840Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6786,6 +7517,8 @@ }, { "id": "bug-743", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T10:25:02.673Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.h", @@ -6804,6 +7537,8 @@ }, { "id": "bug-745", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T10:26:34.227Z", "error_message": "Wrong operator: != should be ==", "file": "src/controller/ha.c", @@ -6822,6 +7557,8 @@ }, { "id": "bug-746", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T10:30:18.073Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_commit_durability.c", @@ -6840,6 +7577,8 @@ }, { "id": "bug-755", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T10:55:31.898Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/design.md", @@ -6858,6 +7597,8 @@ }, { "id": "bug-759", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T11:21:44.844Z", "error_message": "Incorrect value in code", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", @@ -6876,6 +7617,8 @@ }, { "id": "bug-760", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T13:00:00Z", "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 \u2014 a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", @@ -6902,6 +7645,8 @@ }, { "id": "bug-761", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T13:13:49.968Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", @@ -6920,6 +7665,8 @@ }, { "id": "bug-762", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T13:18:43.638Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-commit-durability/tasks.md", @@ -6938,6 +7685,8 @@ }, { "id": "bug-763", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T14:54:25.478Z", "error_message": "Function not marked async", "file": ".kiro/steering/roadmap.md", @@ -6956,6 +7705,8 @@ }, { "id": "bug-765", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-18T14:30:00Z", "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) \u2192 corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob \u2192 CREATE path)", @@ -6978,6 +7729,8 @@ }, { "id": "bug-766", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T02:55:49.212Z", "error_message": "Wrong return value", "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", @@ -6996,6 +7749,8 @@ }, { "id": "bug-768", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T03:30:28.056Z", "error_message": "Wrong condition in logic", "file": "src/scheduler/src/schedobj.c", @@ -7014,6 +7769,8 @@ }, { "id": "bug-772", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T03:49:23.624Z", "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", @@ -7031,6 +7788,8 @@ }, { "id": "bug-773", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T03:49:39.373Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", @@ -7049,6 +7808,8 @@ }, { "id": "bug-774", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T03:50:12.460Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", @@ -7067,6 +7828,8 @@ }, { "id": "bug-777", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T13:00:00Z", "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", "file": "src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table", @@ -7097,6 +7860,8 @@ }, { "id": "bug-778", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T13:40:00Z", "error_message": "persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss.", "file": "src/controller/persist.c:1336 (apply_wal_entry) + src/controller/ha.c ha_apply_single_entry (~1570-1588)", @@ -7127,6 +7892,8 @@ }, { "id": "bug-781", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T14:02:49.235Z", "error_message": "Wrong reference: hdr should be payload_len", "file": "src/controller/persist.c", @@ -7144,6 +7911,8 @@ }, { "id": "bug-785", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T14:08:12.866Z", "error_message": "Wrong reference: NULL should be type", "file": "src/controller/testsuite/test_ha_role_lifecycle.c", @@ -7162,6 +7931,9 @@ }, { "id": "bug-786", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "fae0bc0f, 86099763", "timestamp": "2026-06-19T15:00:00Z", "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", "file": "src/controller/testsuite/test.c (cmd_jobsub)", @@ -7186,6 +7958,8 @@ }, { "id": "bug-788", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T15:34:41.623Z", "error_message": "Wrong condition in logic", "file": "src/controller/testsuite/test.c", @@ -7204,6 +7978,8 @@ }, { "id": "bug-792", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19T16:13:02.715Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test.c", @@ -7222,6 +7998,8 @@ }, { "id": "bug-795", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19", "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} \u2014 UPSTREAM ha-commit-durability Phase 6.2", @@ -7251,6 +8029,8 @@ }, { "id": "bug-796", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T00:32:43.144Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/spec.json", @@ -7269,6 +8049,8 @@ }, { "id": "bug-799", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T01:05:25.091Z", "error_message": "Incorrect value in code", "file": "src/tools/test-commit-durability.sh", @@ -7287,6 +8069,8 @@ }, { "id": "bug-804", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T03:29:22.678Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/controller-lsid/requirements.md", @@ -7305,6 +8089,8 @@ }, { "id": "bug-805", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T03:29:29.570Z", "error_message": "Wrong return value", "file": ".kiro/specs/controller-lsid/requirements.md", @@ -7323,6 +8109,8 @@ }, { "id": "bug-808", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T04:08:49.684Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/controller-status/design.md", @@ -7341,6 +8129,8 @@ }, { "id": "bug-809", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T04:40:36.648Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -7359,6 +8149,8 @@ }, { "id": "bug-810", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T04:45:48.985Z", "error_message": "Wrong reference: Data should be Text", "file": "src/schema/controller.capnp", @@ -7377,6 +8169,8 @@ }, { "id": "bug-811", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T04:45:49.790Z", "error_message": "Wrong reference: xce_data_t should be char", "file": "src/include/typemap.h", @@ -7394,6 +8188,8 @@ }, { "id": "bug-812", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T05:08:41.030Z", "error_message": "Incorrect value in code", "file": "src/lib/meson.build", @@ -7412,6 +8208,8 @@ }, { "id": "bug-813", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T05:42:31.707Z", "error_message": "Wrong operator: >= should be ==", "file": "src/controller/testsuite/test.c", @@ -7430,6 +8228,8 @@ }, { "id": "bug-817", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-19", "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) \u2014 UPSTREAM, not controller-status", @@ -7453,6 +8253,8 @@ }, { "id": "bug-818", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T06:11:49.084Z", "error_message": "Wrong condition in logic", "file": "src/controller/testsuite/test.c", @@ -7471,6 +8273,8 @@ }, { "id": "bug-820", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T06:16:34.685Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", @@ -7489,6 +8293,8 @@ }, { "id": "bug-821", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T08:14:43.577Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/controller-status/requirements.md", @@ -7507,6 +8313,8 @@ }, { "id": "bug-824", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T09:30:23.040Z", "error_message": "Function not marked async", "file": ".kiro/specs/controller-status/tasks.md", @@ -7525,6 +8333,8 @@ }, { "id": "bug-825", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T09:31:24.076Z", "error_message": "Function not marked async", "file": ".kiro/specs/controller-status/spec.json", @@ -7543,6 +8353,8 @@ }, { "id": "bug-827", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T18:30:00.000Z", "error_message": "AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from cmd_ctrlstatus (test.c:1273) when ctrlstatus failover exhausts and returns E_XCE_NOT_READY (-10013) on a primary-less cluster", "file": "src/lib/utils.c", @@ -7567,6 +8379,9 @@ }, { "id": "bug-828", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "91b9a445, b77d4122", "timestamp": "2026-06-20T10:19:56.678Z", "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", "file": "src/controller/ha.c (vote_reply_dispatch), src/controller/persist.c (persist_wal_write_replicated)", @@ -7604,6 +8419,8 @@ }, { "id": "bug-829", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T11:44:25.496Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -7622,6 +8439,8 @@ }, { "id": "bug-830", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T11:45:26.943Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -7640,6 +8459,8 @@ }, { "id": "bug-831", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T11:54:30.060Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_ha_heartbeat_committed.c", @@ -7658,6 +8479,8 @@ }, { "id": "bug-832", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T11:56:28.240Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -7676,6 +8499,8 @@ }, { "id": "bug-834", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T12:22:39.011Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", @@ -7694,6 +8519,8 @@ }, { "id": "bug-835", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T12:26:15.999Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -7712,6 +8539,8 @@ }, { "id": "bug-836", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T12:31:51.359Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", @@ -7730,6 +8559,8 @@ }, { "id": "bug-837", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T12:32:25.549Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-heartbeat-committed-propagation/spec.json", @@ -7748,6 +8579,8 @@ }, { "id": "bug-840", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:05:49.912Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test.c", @@ -7766,6 +8599,8 @@ }, { "id": "bug-841", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:06:11.393Z", "error_message": "Wrong condition in logic", "file": "src/controller/testsuite/test.c", @@ -7784,6 +8619,8 @@ }, { "id": "bug-843", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:15:35.599Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/roadmap.md", @@ -7802,6 +8639,8 @@ }, { "id": "bug-845", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:30:00.000Z", "error_message": "Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy.", "file": "src/lib/hook.c (xce_add_hook idempotency); root path src/controller/{persist.c persist_start, ha.c ha_register_persist_hooks}", @@ -7828,6 +8667,8 @@ }, { "id": "bug-847", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:37:08.863Z", "error_message": "Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism.", "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) \u2014 capnp multi-segment Data pointers", @@ -7859,6 +8700,8 @@ }, { "id": "bug-848", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:40:02.211Z", "error_message": "On a PRESERVED-DATA restart (no wipe \u2014 user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", "file": "src/controller/cluster.c (cluster_init_phase2, after persist_recover) + src/controller/ha.c (ha_init applied_lsn=0; state report sr->applied_lsn)", @@ -7887,6 +8730,8 @@ }, { "id": "bug-849", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T14:42:09.400Z", "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only \u2014 missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", "file": "src/controller/myapp.c (standby-rejoin path ~317-320) + src/controller/cluster.c cluster_init_phase2 (holds persist_recover)", @@ -7917,6 +8762,8 @@ }, { "id": "bug-850", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T15:14:36.135Z", "error_message": "A rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery.", "file": "src/controller/repl.c repl_add_standby (line ~119) + repl_broadcast in_catch_up gate", @@ -7949,6 +8796,8 @@ }, { "id": "bug-846", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T15:20:00.000Z", "error_message": "After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client.", "file": "src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)", @@ -7972,6 +8821,8 @@ }, { "id": "bug-853", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-20T15:27:41.698Z", "error_message": "Wrong reference: test_wal_batch_entry_without_payload should be test_wal_batch_roundtrip_large_payloads", "file": "src/lib/testsuite/test_wal_wire.c", @@ -7989,6 +8840,8 @@ }, { "id": "bug-861", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T00:20:55.974Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test_persist_recovery.c", @@ -8007,6 +8860,8 @@ }, { "id": "bug-864", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T03:39:07.483Z", "error_message": "Missing guard clause", "file": "src/controller/repl.c", @@ -8025,6 +8880,8 @@ }, { "id": "bug-871", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T04:01:07.089Z", "error_message": "Wrong reference: test_repl_write_rejects_checkpoint should be test_repl_write_rejects_forward_gap", "file": "src/controller/testsuite/test_repl_apply.c", @@ -8042,6 +8899,8 @@ }, { "id": "bug-851", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T13:30:00.000Z", "error_message": "CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn.", "file": "src/controller/persist.c:588/726/1040 (hdr.term=p->term, p->term never set) + src/controller/ha.c (election term = local+1, PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term,lsn) log-match)", @@ -8072,6 +8931,8 @@ }, { "id": "bug-854", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T09:10:00.000Z", "error_message": "Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok.", "file": "src/lib/client.c:403 xcecli_ctrlstatus_primary + src/controller/ctrlstatus.c:104 (standby primaryName=\"\")", @@ -8101,6 +8962,8 @@ }, { "id": "bug-875", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T10:04:37.876Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/requirements.md", @@ -8119,6 +8982,8 @@ }, { "id": "bug-857", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T10:30:00.000Z", "error_message": "Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes pushes config bytes only then restart_requested; each node re-derives state by replaying its WAL under the NEW config, with no snapshot pinning an identical post-upgrade state. Replay is config-dependent (lost_and_found re-homes orphaned jobs), so nodes can diverge; amplified by bug-851 WAL tail divergence.", "file": "src/controller/config_repl.c do_upgrade_config_with_bytes (~1001-1163)", @@ -8147,6 +9012,8 @@ }, { "id": "bug-878", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T10:34:30.593Z", "error_message": "Function not marked async", "file": ".kiro/steering/roadmap.md", @@ -8165,6 +9032,8 @@ }, { "id": "bug-879", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T10:59:29.212Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/TLA-gap-analysis.md", @@ -8183,6 +9052,8 @@ }, { "id": "bug-880", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T12:56:07.740Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/requirements.md", @@ -8201,6 +9072,8 @@ }, { "id": "bug-881", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T13:00:46.210Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/design.md", @@ -8219,6 +9092,8 @@ }, { "id": "bug-883", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T13:05:44.284Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/research.md", @@ -8237,6 +9112,8 @@ }, { "id": "bug-884", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:20:00Z", "error_message": "test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50)", "file": "src/lib/testsuite/test_config.c", @@ -8256,6 +9133,8 @@ }, { "id": "bug-885", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:21:59.689Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -8274,6 +9153,8 @@ }, { "id": "bug-886", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:41:42.176Z", "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_oversize_peers_returns_einval", "file": "src/lib/testsuite/test_raft.c", @@ -8291,6 +9172,8 @@ }, { "id": "bug-887", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:45:29.785Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -8309,6 +9192,8 @@ }, { "id": "bug-888", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:52:45.272Z", "error_message": "Missing guard clause", "file": "src/lib/testsuite/test_raft.c", @@ -8327,6 +9212,8 @@ }, { "id": "bug-889", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:53:14.276Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -8345,6 +9232,8 @@ }, { "id": "bug-890", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T15:59:59.295Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -8363,6 +9252,8 @@ }, { "id": "bug-891", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T16:06:37.923Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -8381,6 +9272,8 @@ }, { "id": "bug-893", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T16:16:47.908Z", "error_message": "Wrong reference: g_te_spy should be g_pe_spy", "file": "src/lib/testsuite/test_raft.c", @@ -8398,6 +9291,8 @@ }, { "id": "bug-894", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T16:20:39.535Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -8416,6 +9311,8 @@ }, { "id": "bug-895", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T16:29:35.778Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -8434,6 +9331,8 @@ }, { "id": "bug-899", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T16:42:30.241Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -8452,6 +9351,8 @@ }, { "id": "bug-900", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T16:51:25.960Z", "error_message": "Missing error handling in unknown", "file": "src/lib/testsuite/test_raft.c", @@ -8470,6 +9371,8 @@ }, { "id": "bug-903", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T17:04:17.830Z", "error_message": "Wrong operator: <= should be >=", "file": "src/lib/testsuite/test_raft.c", @@ -8488,6 +9391,8 @@ }, { "id": "bug-904", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T21:44:26.804Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -8506,6 +9411,8 @@ }, { "id": "bug-905", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T21:49:10.796Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-log-replication/tasks.md", @@ -8524,6 +9431,8 @@ }, { "id": "bug-907", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T23:14:37.837Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft.c", @@ -8542,6 +9451,8 @@ }, { "id": "bug-908", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T23:14:55.547Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -8560,6 +9471,8 @@ }, { "id": "bug-909", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T23:30:04.636Z", "error_message": "Missing guard clause", "file": "src/lib/testsuite/test_raft.c", @@ -8578,6 +9491,8 @@ }, { "id": "bug-911", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-21T23:32:38.225Z", "error_message": "Wrong condition in logic", "file": "src/lib/raft/raft.c", @@ -8596,6 +9511,8 @@ }, { "id": "bug-913", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T00:19:56.768Z", "error_message": "Wrong reference: TEST_ASSERT_NOT_EQUAL_INT should be TEST_ASSERT_NOT_EQUAL", "file": "src/lib/testsuite/test_raft.c", @@ -8613,6 +9530,8 @@ }, { "id": "bug-914", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T01:24:34.541Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/spec.json", @@ -8631,6 +9550,8 @@ }, { "id": "bug-855", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T03:10:06.653962Z", "error_message": "controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch", "file": "src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)", @@ -8657,6 +9578,8 @@ }, { "id": "bug-918", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T03:30:00Z", "error_message": "nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=)", "file": "src/flake.nix", @@ -8680,6 +9603,8 @@ }, { "id": "bug-920", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T03:58:08.538Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -8698,6 +9623,8 @@ }, { "id": "bug-921", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T03:58:19.392Z", "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", "file": "src/controller/testsuite/controller_test.c", @@ -8715,6 +9642,8 @@ }, { "id": "bug-922", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T03:59:43.099Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -8733,6 +9662,8 @@ }, { "id": "bug-923", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T04:01:37.467Z", "error_message": "Wrong condition in logic", "file": "src/controller/persist.c", @@ -8751,6 +9682,8 @@ }, { "id": "bug-924", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22", "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", "file": "environment (/ overlay filesystem)", @@ -8776,6 +9709,8 @@ }, { "id": "bug-925", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T04:42:28.897Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -8794,6 +9729,8 @@ }, { "id": "bug-926", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T04:45:07.018Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -8812,6 +9749,8 @@ }, { "id": "bug-928", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T04:55:58.789Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -8830,6 +9769,8 @@ }, { "id": "bug-929", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T04:57:03.605Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -8848,6 +9789,8 @@ }, { "id": "bug-930", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T04:57:25.141Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -8866,6 +9809,8 @@ }, { "id": "bug-931", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T05:45:09.373Z", "error_message": "Function not marked async", "file": "src/include/repl.h", @@ -8884,6 +9829,8 @@ }, { "id": "bug-933", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T05:47:50.037Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -8902,6 +9849,8 @@ }, { "id": "bug-937", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T06:39:51.351Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -8920,6 +9869,8 @@ }, { "id": "bug-941", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T07:53:06.457Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -8938,6 +9889,8 @@ }, { "id": "bug-942", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T07:54:52.956Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -8956,6 +9909,8 @@ }, { "id": "bug-943", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T07:55:30.714Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -8974,6 +9929,8 @@ }, { "id": "bug-944", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T07:56:08.623Z", "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -8991,6 +9948,8 @@ }, { "id": "bug-945", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T08:09:41.508Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-term-safety/tasks.md", @@ -9009,6 +9968,8 @@ }, { "id": "bug-948", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T08:21:57.062Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.c", @@ -9027,6 +9988,8 @@ }, { "id": "bug-953", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T08:48:45.792Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -9045,6 +10008,8 @@ }, { "id": "bug-957", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T09:08:41.177Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -9063,6 +10028,8 @@ }, { "id": "bug-960", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T09:20:00.000Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_config_repl.c", @@ -9091,6 +10058,8 @@ }, { "id": "bug-959", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T09:31:11.227Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -9109,6 +10078,8 @@ }, { "id": "bug-961", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T09:45:31.779Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -9127,6 +10098,8 @@ }, { "id": "bug-962", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:04:06.960Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -9145,6 +10118,8 @@ }, { "id": "bug-964", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:05:22.007Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -9163,6 +10138,8 @@ }, { "id": "bug-966", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:10:11.668Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -9181,6 +10158,8 @@ }, { "id": "bug-967", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:10:17.811Z", "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", "file": "src/controller/testsuite/test_config_repl.c", @@ -9198,6 +10177,8 @@ }, { "id": "bug-970", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:24:35.905518Z", "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER \u2014 config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", "file": "src/controller/config_repl.c, src/controller/persist.c", @@ -9223,6 +10204,8 @@ }, { "id": "bug-971", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:39:44.072Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_config_repl.c", @@ -9241,6 +10224,8 @@ }, { "id": "bug-972", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T10:40:31.131Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -9259,6 +10244,8 @@ }, { "id": "bug-975", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T11:00:42.661Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -9277,6 +10264,8 @@ }, { "id": "bug-976", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T11:15:00.000Z", "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", "file": "src/controller/config_repl.c, src/controller/ha.c, src/controller/myapp.c", @@ -9304,6 +10293,8 @@ }, { "id": "bug-977", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T11:21:09.742Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -9322,6 +10313,8 @@ }, { "id": "bug-980", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T11:32:35.966Z", "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", "file": "src/controller/testsuite/test_config_repl.c", @@ -9339,6 +10332,8 @@ }, { "id": "bug-981", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T11:35:00Z", "error_message": "ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9.", "file": "src/controller/config_repl.c, src/controller/ha.c", @@ -9363,6 +10358,8 @@ }, { "id": "bug-983", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T11:53:36.582Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -9381,6 +10378,8 @@ }, { "id": "bug-984", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T12:03:15.865Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-term-safety/tasks.md", @@ -9399,6 +10398,8 @@ }, { "id": "bug-985", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T12:06:43.530Z", "error_message": "Wrong return value", "file": "src/controller/testsuite/waldump.c", @@ -9417,6 +10418,8 @@ }, { "id": "bug-986", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T12:08:51.529Z", "error_message": "Incorrect value in code", "file": "src/controller/testsuite/test.c", @@ -9435,6 +10438,8 @@ }, { "id": "bug-987", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T12:09:28.306Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test.c", @@ -9453,6 +10458,8 @@ }, { "id": "bug-990", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T13:24:07.476480", "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster.", "file": "src/controller/repl.c (live send repl_broadcast persist.c:747) + src/controller/ha.c (ha_apply_wal_catchup_entry term!=0 legacy direct-apply)", @@ -9477,6 +10484,8 @@ }, { "id": "bug-991", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T16:30:00.000Z", "error_message": "Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic.", "file": "src/controller/ha.c (ha_apply_term_entry_via_library ~2115; batch ~2205; single-entry ~2225)", @@ -9507,6 +10516,8 @@ }, { "id": "bug-993", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T15:10:39.717Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -9525,6 +10536,8 @@ }, { "id": "bug-998", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T15:27:10.703Z", "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -9542,6 +10555,8 @@ }, { "id": "bug-990-fix", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T16:00:00.000000", "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990.", "file": "src/controller/repl.c (repl_broadcast), src/controller/persist.c (persist_wal_async call site + entry_term_locked), src/controller/config_repl.c (ha_stream_wal_to_standby catch-up), src/controller/ha.c (ha_apply_wal_catchup_entry receive route), src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)", @@ -9569,6 +10584,8 @@ }, { "id": "bug-1000", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T15:44:38.466Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-term-safety/tasks.md", @@ -9587,6 +10604,8 @@ }, { "id": "bug-1001", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T15:44:54.113Z", "error_message": "Missing error handling in errno", "file": ".kiro/specs/ha-term-safety/tasks.md", @@ -9605,6 +10624,8 @@ }, { "id": "bug-1002", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T21:10:31.345Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-term-safety/tasks.md", @@ -9623,6 +10644,8 @@ }, { "id": "bug-1003", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T22:09:23.064Z", "error_message": "Wrong condition in logic", "file": "src/lib/raft/raft.c", @@ -9641,6 +10664,8 @@ }, { "id": "bug-1005", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T22:31:03.315Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -9659,6 +10684,8 @@ }, { "id": "bug-1006", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T22:32:53.722Z", "error_message": "Missing error handling in unknown", "file": "src/controller/ha.c", @@ -9677,6 +10704,8 @@ }, { "id": "bug-1009", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T22:44:50.365Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_config_repl.c", @@ -9695,6 +10724,8 @@ }, { "id": "bug-1013", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-22T00:00:00Z", "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes \u2014 STRICT_TERM live failover gate fails 'committed_lsn>=5'", "file": "src/controller/commit.c", @@ -9723,6 +10754,9 @@ }, { "id": "bug-828-part2", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "b77d4122; upstream gap fixed as bug-661 (f139e1c0)", "timestamp": "2026-06-23T03:05:00.000Z", "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", @@ -9755,6 +10789,8 @@ }, { "id": "bug-1023", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T03:45:00Z", "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", "file": "src/controller/ha.c", @@ -9787,6 +10823,8 @@ }, { "id": "bug-1025", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T04:09:25.647Z", "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", "file": "src/controller/testsuite/clean_failover_preservation.sh", @@ -9804,6 +10842,8 @@ }, { "id": "bug-1028", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23", "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term), config_repl.c (ha_stream_wal_to_standby), persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK).", @@ -9837,6 +10877,8 @@ }, { "id": "bug-1033", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T08:05:43.916Z", "error_message": "Incorrect value in code", "file": "repro854.sh", @@ -9855,6 +10897,8 @@ }, { "id": "bug-1035", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T09:53:37.812Z", "error_message": "Incorrect value in code", "file": "src/lib/xcectx.c", @@ -9873,6 +10917,8 @@ }, { "id": "bug-1036", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T10:44:48.074Z", "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", "file": "src/lib/netio.c", @@ -9890,6 +10936,8 @@ }, { "id": "bug-1037", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T10:48:18.493Z", "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", "file": "src/controller/cluster.c", @@ -9907,6 +10955,8 @@ }, { "id": "bug-1040", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T10:54:14.342Z", "error_message": "Wrong reference: endpoint_ssl_handshake should be handle_ssl_handshake", "file": "src/lib/netio.c", @@ -9924,6 +10974,8 @@ }, { "id": "bug-1045", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T14:08:05.466Z", "error_message": "Missing error handling in unknown", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -9942,6 +10994,8 @@ }, { "id": "bug-1049", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T14:39:01.881Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -9960,6 +11014,8 @@ }, { "id": "bug-1051", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T14:48:18.998Z", "error_message": "Wrong operator: >= should be ==", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -9978,6 +11034,8 @@ }, { "id": "bug-1054", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-23T16:02:42.489Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-test-ha/SKILL.md", @@ -9996,6 +11054,8 @@ }, { "id": "bug-1060", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:05:00.000Z", "error_message": "After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win.", "file": "src/controller/ha.c (raft_cb_become_leader / safe_committed_lsn), src/controller/cluster.c (OPCode_preVoteRequest)", @@ -10022,6 +11082,8 @@ }, { "id": "bug-1066", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T04:36:50.602Z", "error_message": "Incorrect value in code", "file": "src/controller/CLAUDE.md", @@ -10040,6 +11102,8 @@ }, { "id": "bug-1067", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T00:00:00.000Z", "error_message": "Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch)", "file": "src/controller/config_repl.c, src/controller/ha.c", @@ -10066,6 +11130,8 @@ }, { "id": "bug-1070", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T08:18:27.510Z", "error_message": "Missing error handling in unknown", "file": "src/controller/config_repl.c", @@ -10084,6 +11150,8 @@ }, { "id": "bug-1075", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T08:23:04.980Z", "error_message": "Wrong reference: standby_lsn should be leader_term", "file": "src/controller/testsuite/test_coldstart_lsn0.c", @@ -10101,6 +11169,8 @@ }, { "id": "bug-1077", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T17:30:00.000Z", "error_message": "AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c)", "file": "src/lib/xcectx.c", @@ -10130,6 +11200,8 @@ }, { "id": "bug-1078", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T09:18:52.605Z", "error_message": "Wrong reference: handle_reply should be xce_handle_reply", "file": "src/lib/xcectx.c", @@ -10147,6 +11219,8 @@ }, { "id": "bug-1079", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T09:27:59.772Z", "error_message": "Wrong reference: libxce_failover_client_hang should be project_libxce_failover_hang", "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1077_reply_union_type_confusion.md", @@ -10164,6 +11238,8 @@ }, { "id": "bug-1080", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T10:24:07.439Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-sim-conformance/design.md", @@ -10182,6 +11258,8 @@ }, { "id": "bug-1081", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T10:30:45.585Z", "error_message": "Wrong reference: apply should be advanced", "file": ".kiro/specs/raft-sim-conformance/design.md", @@ -10199,6 +11277,8 @@ }, { "id": "bug-1082", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T10:30:56.237Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-sim-conformance/design.md", @@ -10217,6 +11297,8 @@ }, { "id": "bug-1083", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T11:45:52.473Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10235,6 +11317,8 @@ }, { "id": "bug-1085", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:00:01.924Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10253,6 +11337,8 @@ }, { "id": "bug-1086", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:13:27.532Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10271,6 +11357,8 @@ }, { "id": "bug-1087", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:22:22.094Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10289,6 +11377,8 @@ }, { "id": "bug-1088", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:30:35.436Z", "error_message": "Wrong reference: test_sim_check_election_safety_fail_on_two_leaders should be test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", "file": "src/lib/testsuite/test_raft.c", @@ -10306,6 +11396,8 @@ }, { "id": "bug-1089", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:38:40.360Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10324,6 +11416,8 @@ }, { "id": "bug-1090", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T12:47:03.843Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10342,6 +11436,8 @@ }, { "id": "bug-1091", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T13:04:27.057Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10360,6 +11456,8 @@ }, { "id": "bug-1094", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T13:22:05.858Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10378,6 +11476,8 @@ }, { "id": "bug-1095", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24", "error_message": "Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a duplicated VOTE_RSP grant double-counts a single peer's vote", "file": "src/lib/raft/raft.c", @@ -10406,6 +11506,8 @@ }, { "id": "bug-1096", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24", "error_message": "raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it", "file": "src/lib/raft/raft.c", @@ -10433,6 +11535,8 @@ }, { "id": "bug-1099", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T14:08:44.250Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10451,6 +11555,8 @@ }, { "id": "bug-1104", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T14:28:33.169Z", "error_message": "Wrong condition in logic", "file": "src/lib/raft/raft_sim.c", @@ -10469,6 +11575,8 @@ }, { "id": "bug-1108", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T14:38:06.432Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10487,6 +11595,8 @@ }, { "id": "bug-1110", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T14:51:30.855Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10505,6 +11615,8 @@ }, { "id": "bug-1111", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T15:13:15.333Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10523,6 +11635,9 @@ }, { "id": "bug-1112", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "b1f289d2", "timestamp": "2026-06-24T15:15:03.882281Z", "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 \u2014 lib test binary fails to link", "file": "src/lib/raft/raft_sim.c", @@ -10543,6 +11658,8 @@ }, { "id": "bug-1113", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T15:16:52.683Z", "error_message": "Missing error handling in unknown", "file": "src/lib/raft/raft_sim.c", @@ -10561,6 +11678,8 @@ }, { "id": "bug-1115", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T15:18:59.248Z", "error_message": "Wrong operator: != should be ==", "file": "src/lib/raft/raft_sim.c", @@ -10579,6 +11698,8 @@ }, { "id": "bug-1116", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T15:25:17.886Z", "error_message": "Missing guard clause", "file": "src/lib/raft/raft_sim.c", @@ -10597,6 +11718,8 @@ }, { "id": "bug-1117", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T15:32:34.085Z", "error_message": "Wrong condition in logic", "file": "src/lib/raft/raft.c", @@ -10615,6 +11738,8 @@ }, { "id": "bug-1120", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T16:09:04.749Z", "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_1_fault_switch_mechanism", "file": "src/lib/testsuite/test_raft.c", @@ -10632,6 +11757,8 @@ }, { "id": "bug-1123", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T16:22:58.427Z", "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_2_negative_both_directions", "file": "src/lib/testsuite/test_raft.c", @@ -10649,6 +11776,8 @@ }, { "id": "bug-1125", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T16:42:32.261Z", "error_message": "Wrong reference: test_sim_4_2_step_liveness_and_coverage should be test_sim_8_1_export_trace_named_and_gated", "file": "src/lib/testsuite/test_raft.c", @@ -10666,6 +11795,8 @@ }, { "id": "bug-1126", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-24T16:43:39.427Z", "error_message": "Wrong condition in logic", "file": "src/lib/raft/raft_sim.c", @@ -10684,6 +11815,8 @@ }, { "id": "bug-1127", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T03:17:45.530Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_demotion_convergence.c", @@ -10702,6 +11835,8 @@ }, { "id": "bug-1128", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T03:20:00.000Z", "error_message": "test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY", "file": "src/controller/testsuite/test_demotion_convergence.c", @@ -10736,6 +11871,8 @@ }, { "id": "bug-1133", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T07:12:01.309Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-log-replication/spec.json", @@ -10754,6 +11891,8 @@ }, { "id": "bug-1134", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T08:18:43.915Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", @@ -10772,6 +11911,8 @@ }, { "id": "bug-1135", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T09:26:49.441Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", @@ -10790,6 +11931,8 @@ }, { "id": "bug-1136", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T09:27:00.312Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-apply-decoupling/brief.md", @@ -10808,6 +11951,8 @@ }, { "id": "bug-1138", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T11:36:52.942Z", "error_message": "Function not marked async", "file": ".kiro/steering/roadmap.md", @@ -10826,6 +11971,8 @@ }, { "id": "bug-1139", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T14:02:51.225Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/research.md", @@ -10844,6 +11991,8 @@ }, { "id": "bug-1141", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T15:12:16.193Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-drive-async/research.md", @@ -10862,6 +12011,8 @@ }, { "id": "bug-1142", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T15:13:00.445Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/roadmap.md", @@ -10880,6 +12031,8 @@ }, { "id": "bug-1143", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T23:37:38.579Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/raft-drive-async/design.md", @@ -10898,6 +12051,8 @@ }, { "id": "bug-1144", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T23:37:49.697Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/design.md", @@ -10916,6 +12071,8 @@ }, { "id": "bug-1145", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-25T23:38:08.746Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-drive-async/design.md", @@ -10934,6 +12091,8 @@ }, { "id": "bug-1147", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T03:46:38.904419+00:00", "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor \u2192 lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", "file": "src/controller/persist.c:1374 (trim_wal_segments)", @@ -10963,6 +12122,8 @@ }, { "id": "bug-1148", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T03:46:51.038Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/design.md", @@ -10981,6 +12142,8 @@ }, { "id": "bug-1149", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T03:48:48.222Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/roadmap.md", @@ -10999,6 +12162,8 @@ }, { "id": "bug-1151", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:31:53.879Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/design.md", @@ -11017,6 +12182,8 @@ }, { "id": "bug-1152", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:32:20.960Z", "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", "file": ".kiro/specs/raft-drive-async/design.md", @@ -11034,6 +12201,8 @@ }, { "id": "bug-1153", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:33:10.286Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/roadmap.md", @@ -11052,6 +12221,8 @@ }, { "id": "bug-1154", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:37:18.052Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -11070,6 +12241,8 @@ }, { "id": "bug-1155", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:47:49.937Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -11088,6 +12261,8 @@ }, { "id": "bug-1156", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:48:34.246Z", "error_message": "Function not marked async", "file": "src/include/config.h", @@ -11106,6 +12281,8 @@ }, { "id": "bug-1157", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:48:50.005Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -11124,6 +12301,8 @@ }, { "id": "bug-1158", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:49:04.568Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11142,6 +12321,8 @@ }, { "id": "bug-1159", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:49:44.821Z", "error_message": "Function not marked async", "file": "src/controller/cluster.c", @@ -11160,6 +12341,8 @@ }, { "id": "bug-1160", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T04:57:57.877Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -11178,6 +12361,8 @@ }, { "id": "bug-1161", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T05:02:36.720Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -11196,6 +12381,8 @@ }, { "id": "bug-1162", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T05:06:30.148Z", "error_message": "Function not marked async", "file": "src/include/xce/cluster_def.h", @@ -11214,6 +12401,8 @@ }, { "id": "bug-1163", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T05:06:42.680Z", "error_message": "Wrong condition in logic", "file": "src/controller/cluster.c", @@ -11232,6 +12421,8 @@ }, { "id": "bug-1164", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T05:06:53.129Z", "error_message": "Function not marked async", "file": "src/controller/cluster.c", @@ -11250,6 +12441,8 @@ }, { "id": "bug-1165", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T06:51:00.808Z", "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", "file": "src/lib/testsuite/test.c", @@ -11267,6 +12460,8 @@ }, { "id": "bug-1166", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T06:51:57.649Z", "error_message": "Function not marked async", "file": "src/lib/netio.c", @@ -11285,6 +12480,8 @@ }, { "id": "bug-1168", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T06:52:21.608Z", "error_message": "Function not marked async", "file": "src/include/netio.h", @@ -11303,6 +12500,8 @@ }, { "id": "bug-1169", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:16:36.377Z", "error_message": "Function not marked async", "file": "src/include/vtaskmgr.h", @@ -11321,6 +12520,8 @@ }, { "id": "bug-1171", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:18:02.307Z", "error_message": "Function not marked async", "file": "src/lib/vtaskmgr.c", @@ -11339,6 +12540,8 @@ }, { "id": "bug-1172", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:19:14.857Z", "error_message": "Function not marked async", "file": "src/controller/include/commit.h", @@ -11357,6 +12560,8 @@ }, { "id": "bug-1173", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:19:45.059Z", "error_message": "Function not marked async", "file": "src/controller/commit.c", @@ -11375,6 +12580,8 @@ }, { "id": "bug-1174", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:20:07.233Z", "error_message": "Wrong operator: >= should be !=", "file": "src/controller/commit.c", @@ -11393,6 +12600,8 @@ }, { "id": "bug-1175", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:20:41.448Z", "error_message": "Function not marked async", "file": "src/controller/cluster.c", @@ -11411,6 +12620,8 @@ }, { "id": "bug-1177", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:47:19.731Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -11429,6 +12640,8 @@ }, { "id": "bug-1178", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:48:00.209Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11447,6 +12660,8 @@ }, { "id": "bug-1179", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:51:01.491Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11465,6 +12680,8 @@ }, { "id": "bug-1180", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:51:10.764Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11483,6 +12700,8 @@ }, { "id": "bug-1183", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T07:57:35.484Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11501,6 +12720,8 @@ }, { "id": "bug-1184", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T08:12:52.624Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11519,6 +12740,8 @@ }, { "id": "bug-1185", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T08:13:19.511Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11537,6 +12760,8 @@ }, { "id": "bug-1186", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T08:15:44.315Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -11555,6 +12780,8 @@ }, { "id": "bug-1188", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:20:33.560Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -11573,6 +12800,8 @@ }, { "id": "bug-1189", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:21:12.319Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11591,6 +12820,8 @@ }, { "id": "bug-1190", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:24:21.473Z", "error_message": "Function not marked async", "file": "src/controller/cluster.c", @@ -11609,6 +12840,8 @@ }, { "id": "bug-1191", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:27:00.557Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11627,6 +12860,8 @@ }, { "id": "bug-1192", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:27:14.358Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11645,6 +12880,8 @@ }, { "id": "bug-1193", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:56:43.461Z", "error_message": "Function not marked async", "file": "src/include/vtaskmgr.h", @@ -11663,6 +12900,8 @@ }, { "id": "bug-1194", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:57:18.280Z", "error_message": "Function not marked async", "file": "src/lib/vtaskmgr.c", @@ -11681,6 +12920,8 @@ }, { "id": "bug-1195", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T10:57:44.750Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11699,6 +12940,8 @@ }, { "id": "bug-1196", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T11:00:15.213Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11717,6 +12960,8 @@ }, { "id": "bug-1197", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T00:00:00.000Z", "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF \u2014 raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", "file": "src/controller/ha.c", @@ -11745,6 +12990,8 @@ }, { "id": "bug-1199", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T00:00:00.000Z", "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", "file": "src/lib/vtaskmgr.c", @@ -11768,6 +13015,8 @@ }, { "id": "bug-1198", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T00:00:00.000Z", "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline \u2014 a blocking op under the never-gated liveness reservation breaks deadlock-freedom", "file": "src/controller/ha.c", @@ -11792,6 +13041,8 @@ }, { "id": "bug-1202", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T11:33:28.328Z", "error_message": "Missing error handling in task", "file": "src/controller/testsuite/test_e2e_checkpoint.c", @@ -11810,6 +13061,8 @@ }, { "id": "bug-1204", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T11:35:37.597Z", "error_message": "Wrong reference: file_exists_for_trim_test should be seg_file_exists", "file": "src/controller/testsuite/test_e2e_checkpoint.c", @@ -11827,6 +13080,8 @@ }, { "id": "bug-1205", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T11:39:53.357Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -11845,6 +13100,8 @@ }, { "id": "bug-1206", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T15:16:40.996Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11863,6 +13120,8 @@ }, { "id": "bug-1207", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T15:28:19.885Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -11881,6 +13140,8 @@ }, { "id": "bug-1210", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26", "error_message": "raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted ON (=1); header doc also said default ON", "file": "src/controller/ha.c, src/controller/ha.h", @@ -11909,6 +13170,8 @@ }, { "id": "bug-1211", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T15:49:46.434Z", "error_message": "Function not marked async", "file": "src/controller/ha.h", @@ -11927,6 +13190,8 @@ }, { "id": "bug-1212", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T15:50:27.084Z", "error_message": "Function not marked async", "file": "src/controller/ha.c", @@ -11945,6 +13210,8 @@ }, { "id": "bug-1213", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T15:51:29.747Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -11963,6 +13230,8 @@ }, { "id": "bug-1214", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T15:54:05.463Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -11981,6 +13250,8 @@ }, { "id": "bug-1216", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T16:54:57.384Z", "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", "file": "src/include/netio.h", @@ -11998,6 +13269,8 @@ }, { "id": "bug-1217", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T16:55:15.332Z", "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", "file": "src/lib/netio.c", @@ -12015,6 +13288,8 @@ }, { "id": "bug-1218", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T16:55:48.690Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -12033,6 +13308,8 @@ }, { "id": "bug-1222", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26T17:44:25.670Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -12051,6 +13328,8 @@ }, { "id": "bug-1224", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26", "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently \u2014 controller_unit_test never completes (timeout-killed)", "file": "src/controller/testsuite/test_ha_dispatch_integration.c", @@ -12076,6 +13355,8 @@ }, { "id": "bug-1225", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-26", "error_message": "/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 \"cluster did not reconverge to 3-of-3 after restarting ctrl-2\"", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh (wait_for_full_convergence)", @@ -12104,6 +13385,8 @@ }, { "id": "bug-1226", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T02:53:39.273Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -12122,6 +13405,8 @@ }, { "id": "bug-1227", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T03:04:05.891Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -12140,6 +13425,8 @@ }, { "id": "bug-1228", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T04:13:35.943Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/design.md", @@ -12158,6 +13445,8 @@ }, { "id": "bug-1233", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T04:59:57.805Z", "error_message": "Missing error handling in unknown", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -12176,6 +13465,8 @@ }, { "id": "bug-1234", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T05:00:06.726Z", "error_message": "Function not marked async", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -12194,6 +13485,8 @@ }, { "id": "bug-1236", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T09:27:45.080Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/design.md", @@ -12212,6 +13505,8 @@ }, { "id": "bug-1237", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T09:28:07.944Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/raft-ha-core-safety/design.md", @@ -12230,6 +13525,8 @@ }, { "id": "bug-1243", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:02:56.820Z", "error_message": "Wrong operator: == should be <<", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -12248,6 +13545,8 @@ }, { "id": "bug-1244", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:04:31.979Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -12266,6 +13565,8 @@ }, { "id": "bug-1246", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:07:37.692Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-test-ha/SKILL.md", @@ -12284,6 +13585,8 @@ }, { "id": "bug-1247", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:25:38.366Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -12302,6 +13605,8 @@ }, { "id": "bug-1249", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:45:00.686Z", "error_message": "Missing error handling in unknown", "file": "src/controller/persist.c", @@ -12320,6 +13625,8 @@ }, { "id": "bug-1250", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:45:34.063Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -12338,6 +13645,8 @@ }, { "id": "bug-1252", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T10:47:06.628Z", "error_message": "Wrong condition in logic", "file": "src/controller/persist.c", @@ -12356,6 +13665,8 @@ }, { "id": "bug-1254", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T11:14:27.133Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_ha_core_safety.c", @@ -12374,6 +13685,8 @@ }, { "id": "bug-1255", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-27T11:28:45.351Z", "error_message": "Missing guard clause", "file": "src/controller/config_repl.c", @@ -12392,6 +13705,8 @@ }, { "id": "bug-1264", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-29T04:13:02.187Z", "error_message": "Incorrect value in code", "file": "src/controller/meson.build", @@ -12410,6 +13725,8 @@ }, { "id": "bug-1266", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-29T04:25:39.111Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_partition.c", @@ -12428,6 +13745,8 @@ }, { "id": "bug-1269", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T01:47:07.635Z", "error_message": "Missing await", "file": "src/controller/ha.c", @@ -12446,6 +13765,8 @@ }, { "id": "bug-1271", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T02:25:32.055Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12464,6 +13785,8 @@ }, { "id": "bug-1272", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T02:25:00Z", "error_message": "raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills", "file": "src/controller/ha.c (election/PreVote path)", @@ -12497,6 +13820,8 @@ }, { "id": "bug-1273", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T02:41:41.017Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_ha_core_safety.c", @@ -12515,6 +13840,8 @@ }, { "id": "bug-1275", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T02:55:23.808Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12533,6 +13860,8 @@ }, { "id": "bug-1276", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T02:55:37.693Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12551,6 +13880,8 @@ }, { "id": "bug-1281", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T03:25:00Z", "error_message": "Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall.", "file": "src/controller (restart/recovery/election path \u2014 exact site unknown, no backtrace)", @@ -12583,6 +13914,8 @@ }, { "id": "bug-1279", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T03:33:06.273Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12601,6 +13934,8 @@ }, { "id": "bug-1280", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T03:57:32.784Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12619,6 +13954,8 @@ }, { "id": "bug-1282", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T04:17:08.329Z", "error_message": "ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV.", "file": "src/lib/dataq.c:65 (dataq_deq holds q->lock across match cb) + src/lib/vtaskmgr.c:255 (vtask_eligible) + src/lib/netio.c:1001 (endpoint_is_task_blocked takes ep->lock) + src/lib/netio.c:672 (io_worker holds ep->lock across vtask_enq->dataq_enq)", @@ -12649,6 +13986,8 @@ }, { "id": "bug-1283", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T04:17:58.579Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12667,6 +14006,8 @@ }, { "id": "bug-1285", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T08:24:48.860Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12685,6 +14026,8 @@ }, { "id": "bug-1286", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T05:30:00Z", "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart \u2014 Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", "file": "src/controller/cluster.c:251 (myreq_handler) + src/controller/ha.c:3194 (ha_dispatch_gate)", @@ -12713,6 +14056,8 @@ }, { "id": "bug-1290", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T09:15:12.861Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12731,6 +14076,8 @@ }, { "id": "bug-1292", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T09:33:08.918Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12749,6 +14096,8 @@ }, { "id": "bug-1295", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T13:19:42.706Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -12767,6 +14116,8 @@ }, { "id": "bug-1296", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T13:21:00.000Z", "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP\u2192PreVote\u2192timeout\u2192repeat livelock \u2014 zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait \u2014 chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request \u2192 mutual timeout \u2192 RDHUP fires again \u2192 repeat forever.", "file": "src/controller/ha.c", @@ -12794,6 +14145,8 @@ }, { "id": "bug-1297", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T13:28:01.170Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -12812,6 +14165,8 @@ }, { "id": "bug-1298", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T13:36:00-07:00", "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) \u2014 endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3\u21924, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", "file": "src/lib/netio.c / src/lib/vtaskmgr.c / src/controller/cluster.c", @@ -12842,6 +14197,8 @@ }, { "id": "bug-1299", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T13:39:09.194Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12860,6 +14217,8 @@ }, { "id": "bug-1304", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T14:24:36.874Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-ha-core-safety/research.md", @@ -12878,6 +14237,8 @@ }, { "id": "bug-1306", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T14:31:00-07:00", "error_message": "6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean.", "file": "src/lib/hook.c (xce_run_hooks); job.c:603 gate is correct", @@ -12911,6 +14272,8 @@ }, { "id": "bug-1307", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T15:05:00.000Z", "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN \u2014 silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix \u2192 Leader Completeness violation.", "file": "src/controller/ha.c", @@ -12940,6 +14303,8 @@ }, { "id": "bug-1308", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T21:22:32.991Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -12958,6 +14323,8 @@ }, { "id": "bug-1310", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-06-30T18:40:00Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -12985,6 +14352,8 @@ }, { "id": "bug-1312", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T00:29:56.872Z", "error_message": "Wrong reference: test_job_hook_start_writes_wal_entry should be test_job_hook_new_is_notify_only", "file": "src/controller/testsuite/test_persist_hooks.c", @@ -13002,6 +14371,8 @@ }, { "id": "bug-1313", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T00:42:13.591Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -13020,6 +14391,8 @@ }, { "id": "bug-1315", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T01:26:38.341Z", "error_message": "Wrong reference: unistd should be time", "file": "src/controller/testsuite/test_ha_rdhup_integration.c", @@ -13038,6 +14411,8 @@ }, { "id": "bug-1319", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T04:04:13.051Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-entity-snapshots/design.md", @@ -13056,6 +14431,8 @@ }, { "id": "bug-1320", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T04:47:19.662Z", "error_message": "Wrong reference: test_ha_partition should be test_persist_snapshot_entities", "file": "src/controller/testsuite/controller_test.c", @@ -13073,6 +14450,8 @@ }, { "id": "bug-1326", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T06:17:52.049Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_persist_snapshot_entities.c", @@ -13091,6 +14470,8 @@ }, { "id": "bug-1327", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T06:18:39.877Z", "error_message": "Wrong reference: pthread should be fcntl", "file": "src/controller/testsuite/test_persist_snapshot_entities.c", @@ -13109,6 +14490,8 @@ }, { "id": "bug-1328", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T06:33:25.117Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_persist_snapshot_entities.c", @@ -13127,6 +14510,8 @@ }, { "id": "bug-1330", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T07:24:22.541Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_ha_liveness.c", @@ -13145,6 +14530,8 @@ }, { "id": "bug-1331", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T07:46:42.046Z", "error_message": "Wrong return value", "file": "src/controller/persist.c", @@ -13163,6 +14550,8 @@ }, { "id": "bug-1340", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T09:55:17.967Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -13181,6 +14570,8 @@ }, { "id": "bug-1343", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T13:27:42.519Z", "error_message": "Missing error handling in every", "file": "src/controller/testsuite/test_ha_core_safety.c", @@ -13199,6 +14590,8 @@ }, { "id": "bug-1348", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T14:10:48.987Z", "error_message": "Wrong reference: test_pre_vote_catch_up_denies should be test_pre_vote_catch_up_grants_when_primary_unreachable", "file": "src/controller/testsuite/test_ha_vote_gate.c", @@ -13216,6 +14609,8 @@ }, { "id": "bug-1349", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-01T14:24:21.310Z", "error_message": "Wrong condition in logic", "file": "src/controller/ha.c", @@ -13234,6 +14629,8 @@ }, { "id": "bug-1355", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T03:03:21.985Z", "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", @@ -13251,6 +14648,8 @@ }, { "id": "bug-1357", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T03:03:51.842Z", "error_message": "Function not marked async", "file": "src/controller/main.c", @@ -13269,6 +14668,8 @@ }, { "id": "bug-1362", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T03:50:14.650Z", "error_message": "Wrong reference: persist_config_t should be uint32_t", "file": "src/include/persist.h", @@ -13286,6 +14687,8 @@ }, { "id": "bug-1363", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T03:50:42.583Z", "error_message": "Wrong reference: WAL_SEGMENT_SIZE_BYTES should be p", "file": "src/controller/persist.c", @@ -13303,6 +14706,8 @@ }, { "id": "bug-1364", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T03:52:09.933Z", "error_message": "Wrong reference: myconf_t should be uint32_t", "file": "src/include/myconf.h", @@ -13320,6 +14725,8 @@ }, { "id": "bug-1369", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T03:57:52.302Z", "error_message": "Incorrect value in code", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -13338,6 +14745,8 @@ }, { "id": "bug-1370", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T04:10:10.430Z", "error_message": "Function not marked async", "file": ".kiro/specs/raft-ha-core-safety/tasks.md", @@ -13356,6 +14765,8 @@ }, { "id": "bug-1372", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T04:39:29.527Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_config_repl.c", @@ -13374,6 +14785,8 @@ }, { "id": "bug-1376", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T05:10:00.000Z", "error_message": "raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout.", "file": "src/controller/ha.c", @@ -13396,6 +14809,8 @@ }, { "id": "bug-1380", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T07:16:35.420Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -13414,6 +14829,8 @@ }, { "id": "bug-1381", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T00:35:00.000Z", "error_message": "raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has a pre-existing endpoint-lifetime race, independent of the g_drive_async_configpush_offload flag -- it grabs raw endpoint_t* pointers for every registered standby under c->ha.lock, releases the lock, then spawns one pthread per standby (push_thread) that reads/writes those endpoints unprotected for up to CONF_PUSH_TIMEOUT_MS. If a standby RDHUP-disconnects during that window, the io_worker's teardown path (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) can free the endpoint while a push_thread is still using it -- a UAF. This is NOT specific to the lane offload: the transport fan-out spawns its own independent OS threads regardless of which caller (inline drive-thread or lane worker) invokes it, so the documented 'runs inline on io_worker so teardown and push are serialized on one thread' rationale for keeping the flag OFF does not actually hold -- the race exists in the inline path today too.", "file": "src/controller/config_repl.c", @@ -13436,6 +14853,8 @@ }, { "id": "bug-1382", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T07:23:50.910Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -13454,6 +14873,8 @@ }, { "id": "bug-1383", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T07:31:28.737Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_config_repl.c", @@ -13472,6 +14893,8 @@ }, { "id": "bug-1384", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T07:37:33.427Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -13490,6 +14913,8 @@ }, { "id": "bug-1386", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T08:15:00.000Z", "error_message": "/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install (\"HA: snapshot loaded/legacy\" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip.", "file": "src/controller/config_repl.c", @@ -13512,6 +14937,8 @@ }, { "id": "bug-1388", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T08:19:04.015Z", "error_message": "Missing guard clause", "file": "src/controller/testsuite/test_config_repl.c", @@ -13530,6 +14957,8 @@ }, { "id": "bug-1389", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T09:05:00.000Z", "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times \u2014 the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT \u2014 the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) \u2014 nondeterministic, not a fixed structural bug.", "file": "src/lib/request.c (suspected: xce_snapshot_chunk_encode_raw capnp Data encode) or src/controller/ha.c (ha_handle_snapshot_chunk per-chunk append)", @@ -13553,6 +14982,8 @@ }, { "id": "bug-1393", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T12:05:51.415Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -13571,6 +15002,8 @@ }, { "id": "bug-1395", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T13:25:19.081Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/raft-drive-async/tasks.md", @@ -13589,6 +15022,10 @@ }, { "id": "bug-1396", + "status": "cannot-reproduce", + "status_updated": "2026-09-24", + "status_note": "extra committed job (20 vs 19) never reproduced after 9318f997 harness hardening", + "repro_attempts": "2 fresh repro attempts x3 rounds (2026-07-09), 0 reproduced", "timestamp": "2026-07-02T14:00:00.000Z", "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle \u2014 the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) \u2014 an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", "file": "src/tools/test-commit-durability.sh (harness) or controller job-recovery path (unconfirmed which)", @@ -13614,6 +15051,8 @@ }, { "id": "bug-1397", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T13:57:56.610Z", "error_message": "Function not marked async", "file": ".kiro/specs/ha-commit-durability/tasks.md", @@ -13632,6 +15071,8 @@ }, { "id": "bug-1398", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T13:58:18.885Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-commit-durability/tasks.md", @@ -13650,6 +15091,8 @@ }, { "id": "bug-1399", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T14:03:20.814Z", "error_message": "Incorrect value in code", "file": "src/controller/CLAUDE.md", @@ -13668,6 +15111,8 @@ }, { "id": "bug-1400", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T14:20:00.000Z", "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` \u2014 positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", "file": "src/controller/testsuite/test.c (cmd_jobsub, ~line 1748)", @@ -13689,6 +15134,8 @@ }, { "id": "bug-1401", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T14:25:00.000Z", "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) \u2014 observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", "file": "src/lib/controller.capnp.c (decode_CtrlStatus_ptr, generated codec) or its caller in request.c/xcecli", @@ -13708,6 +15155,8 @@ }, { "id": "bug-1402", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-02T14:21:35.080Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/controller-status/tasks.md", @@ -13726,6 +15175,8 @@ }, { "id": "bug-1405", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-03T00:08:25.320Z", "error_message": "Wrong reference: test_xcecli_ctrlstatus_null_ctx_typed_failure should be test_xce_free_ctrlstatus_frees_the_struct_itself", "file": "src/lib/testsuite/test_ctrlstatus.c", @@ -13743,6 +15194,8 @@ }, { "id": "bug-1406", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-03T08:00:00.000Z", "error_message": "controller_unit_test SIGSEGV (full run, no --run filter): Thread 1 crashes in SSL_CTX_free() called from iomgr_fini() during test_ha_vep_teardown.c:271 (test_vep_teardown_runs_after_inflight), which runs at controller_test.c main() TEST(test_ha_vep_teardown). Kills the whole binary before any later-registered TEST() can run, so a bare `controller_unit_test` invocation with no --run filter never reaches a final summary and the exit code is 139 (SIGSEGV), not a Unity pass/fail count.", "file": "src/controller/testsuite/test_ha_vep_teardown.c", @@ -13764,6 +15217,8 @@ }, { "id": "bug-1407", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-03T08:00:00.000Z", "error_message": "controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence FAILS (\"Expected TRUE Was FALSE\") when running --run test_config_repl (RUN_TEST site: test_config_repl.c:4008). NOTE: Unity's own printed file:line prefix for this failure is misleading/stale (shows test_repl_unit.c:1543 or controller_test.c:1543 depending on which other suites ran earlier in the same process) -- Unity.TestFile is apparently not re-set per test_xxx(void) group in this harness, only Unity.CurrentTestLineNumber is accurate. The true source is test_config_repl.c.", "file": "src/controller/testsuite/test_config_repl.c", @@ -13788,6 +15243,8 @@ }, { "id": "bug-1408", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-03T21:00:00.000Z", "error_message": "raft-apply-decoupling task 3.1 round-1 handle_jobsubmit(): when an array job submission has >1 element and ONE element fails post-commit construction (create_job()/makeJob_() returns NULL, e.g. a jobid collision that slipped past jobid_range_exists() pre-check TOCTOU, or a plain calloc failure), the function returned the construction error IMMEDIATELY after the per-element construct loop, before ever building the AFTER_EVENT notify jlist or firing HOOK_DETAIL_JOB_NEW -- even for SIBLING array elements that constructed successfully and are already durably committed on a majority.", "file": "src/controller/job.c", @@ -13812,6 +15269,8 @@ }, { "id": "bug-1409", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T17:00:00.000Z", "error_message": "test_demotion_convergence hung/spun (18.5M+ log lines, timeout kill) once commit.c's commit_resolve_pending unconditionally called the new persist_apply_pump_kick", "file": "src/controller/persist.c", @@ -13834,6 +15293,8 @@ }, { "id": "bug-1410", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T17:00:00.000Z", "error_message": "test_apply_pump's spy hook recorded jobid=0 for HOOK_DETAIL_JOB_UPDATE/HOOK_DETAIL_JOB_NEW instead of the real jobid", "file": "src/controller/testsuite/test_apply_pump.c", @@ -13854,6 +15315,8 @@ }, { "id": "bug-1411", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T17:00:00.000Z", "error_message": "AddressSanitizer global-buffer-overflow in dataq_enq when test_apply_pump.c called persist_submit_job_fact with a real 3-member repl setup", "file": "src/controller/testsuite/test_apply_pump.c", @@ -13875,6 +15338,8 @@ }, { "id": "bug-1412", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T17:00:00.000Z", "error_message": "test_apply_pump.c failed to compile: 'unknown type name RETAIN', stray bytes, __gnuc_va_list errors cascading from an unrelated location", "file": "src/controller/testsuite/test_apply_pump.c", @@ -13895,6 +15360,8 @@ }, { "id": "bug-1413", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T17:00:00.000Z", "error_message": "test_apply_pump's burst-ordering test hung the NEXT test (longjmp-past-teardown) after a wrong off-by-one assertion (expected lsn_before+3, actual watermark is lsn_before+2 for 3 appended entries) failed", "file": "src/controller/testsuite/test_apply_pump.c", @@ -13918,6 +15385,8 @@ }, { "id": "bug-1414", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T18:00:00.000Z", "error_message": "raft-apply-decoupling task 6.1 round 1 REJECTED: FINISH (LSN N) then a POST_EXEC-stage FAIL (LSN N+1) for the same job, with FAIL received before FINISH applies, corrupts the job to RUN|EXIT|PERR (0x106) instead of the correct DONE|PERR (0x108) once both apply in LSN order -- DONE is silently lost and a stray RUN bit appears.", "file": "src/controller/job.c, src/controller/persist.c, src/include/wal_types.h, src/include/persist.h, src/controller/include/cluster.h", @@ -13944,6 +15413,8 @@ }, { "id": "bug-1415", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-04T19:30:00.000Z", "error_message": "raft-apply-decoupling task 6.2 review finding (NOT reproduced under the test suite's pool-less fixtures, NOT fixed): a real vtaskmgr pool (n_vcpus > 1) can run the commit continuation's dispatch (job_commit_resolve_adapter -> jobsubmit_commit_continuation -> persist_register_pending_apply) and the apply pump's kick (persist_apply_pump_kick -> apply_pump_drain -> persist_apply_pump_run) on two DIFFERENT worker threads with no completion-order guarantee, even though commit_resolve_pending() enqueues them in that order. If the apply-pump task happens to finish first, persist_take_pending_apply() finds nothing registered yet for that lsn (0, correct no-op per its own contract), applies the entry, and advances last_applied_lsn past it -- then when the continuation task finally runs and calls persist_register_pending_apply() for that now-already-applied lsn, the registration is permanently orphaned: the walk only ever revisits [last_applied_lsn+1..committed_lsn], so nothing will ever call persist_take_pending_apply() for that lsn again. Net effect: the job is correctly constructed and visible in c->jobmgr, but the client that submitted it never receives a reply -- silently, forever.", "file": "src/controller/commit.c (commit_resolve_pending, commit_resolve/dispatch_continuation), src/controller/persist.c (persist_apply_pump_kick, persist_apply_pump_run)", @@ -13969,6 +15440,8 @@ }, { "id": "bug-1416", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-05T06:15:00.000Z", "error_message": "[xce-test-ha] FAIL: factstream round 1: cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- root cause: persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=72 new_job_count=23); persist_recover_loud_fence_if_unrecoverable: UNRECOVERABLE, staying alive/non-promotable", "file": "src/controller/persist.c (persist_verify_apply_correctness / persist_recover, cluster.c cluster_init_phase2)", @@ -13998,6 +15471,8 @@ }, { "id": "bug-1417", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-07T09:30:00.000Z", "error_message": "[xce-test-ha --chaotic] FAIL round 4/10: election stall after killing ctrl-1 \u2014 ctrl-2 PreVote wins but real election times out despite ctrl-3 granting RequestVote", "file": "src/controller/ha.c", @@ -14025,6 +15500,8 @@ }, { "id": "bug-1419", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-07T14:00:00.000Z", "error_message": "[xce-test-ha --chaotic] genuine, reproducible election-liveness stall (not the bug-1418 harness process-stacking artifact, which was a separate, now-fixed issue): after 2+ overlapping kills, the surviving 2-node set can permanently deadlock -- neither node can ever become primary again for the rest of the run.", "file": "src/controller/persist.c (apply-correctness verification) + src/controller/ha.c (PreVote freshness gate) + WAL duplicate-submit mechanism (unlocated)", @@ -14055,6 +15532,8 @@ }, { "id": "bug-1420", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-08T04:00:00.000Z", "error_message": "xce_test CLI harness generates colliding jobids across separate process invocations within the same wall-clock second, surfacing as a spurious APPLY_DIVERGENT fence in chaotic HA soak (misattributed at first to bug-1419's controller-side dedup logic).", "file": "src/controller/testsuite/test.c", @@ -14076,6 +15555,8 @@ }, { "id": "bug-1421", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-08T06:55:15.000Z", "error_message": "Sequential HA soak regression: no primary elected at term>8 by round 6 (initially suspected to stem from the bug-1417 fix); root-caused 2026-07-09 as the primary never discovering a follower's higher term via AppendEntries-reject responses", "file": "src/controller/ha.c", @@ -14101,6 +15582,8 @@ }, { "id": "bug-1422", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-09T10:14:14.000Z", "error_message": "40-round sequential HA soak FAIL at round 36: verify_no_loss reported '0/111 jobs confirmed present' right after a fresh election (new primary ctrl-1, term=40) \u2014 xce_test's expect-job got E_XCE_STANDBY (10014) from a fresh client session whose own wait-primary had JUST confirmed ctrl-1 as serving 45ms earlier", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -14124,6 +15607,9 @@ }, { "id": "bug-1418", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "9b173c9a + bug-1421 + raft-apply-decoupling", "timestamp": "2026-07-09T10:30:00.000Z", "error_message": "Final resolution pass over bug-1418 (all 4 prior occurrences): the two contributing factors from the 2026-07-07T18:20 analysis (term desync + apply-divergence promotion-gate liveness deadlock) are both now resolved, not by the originally-proposed gate relaxation.", "file": "src/controller/ha.c", @@ -14151,6 +15637,8 @@ }, { "id": "bug-1423", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-09T13:58:16.000Z", "error_message": "30-round sequential HA soak FAIL at round 5: post-failover submit did not get 3 acks after 5 retry attempts (15s+). Root-caused as a genuine permanent election deadlock, not a client-side false positive: ctrl-3 won PreVote and entered CTRL_ELECTING (real election, term=10) with ctrl-1 dead and ctrl-2 alive, then never showed another HA state-machine event for 2+ minutes (until the harness killed the cluster on FAIL) \u2014 stuck in CTRL_ELECTING forever. Meanwhile ctrl-2 endlessly retried its own PreVote every ~15-17s, correctly denied every time by ctrl-3 on freshness grounds, but that denial never unstuck ctrl-3's already-in-flight real election.", "file": "src/controller/ha.c", @@ -14177,6 +15665,8 @@ }, { "id": "bug-1424", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-09T17:00:00.000Z", "error_message": "PreVote twin of bug-1423: raft_cb_send_pre_vote_request's connect/monitor failure branches (a dead/unreachable peer) never registered a reply with the raft library's PreVote tally, in EITHER test or production mode (worse than the real-vote path, which at least handled test mode).", "file": "src/controller/ha.c", @@ -14198,6 +15688,9 @@ }, { "id": "bug-1425", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "8a6c339a", "timestamp": "2026-07-09T21:15:00.000Z", "error_message": "Intermittent (~2 of 9 attempts) client hang in the synchronous jobsub (COMMIT_MODE_WAIT) path shortly after a fresh single-node election.", "file": "src/controller/persist.c or src/controller/commit.c (unrooted)", @@ -14223,6 +15716,8 @@ }, { "id": "bug-1426", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-09T21:15:00.000Z", "error_message": "trim_wal_segments's retention floor has no awareness of the local apply pump's own lag \u2014 a reachable scenario since apply-lag-behind-commit is raft-apply-decoupling's entire premise.", "file": "src/controller/persist.c (trim_wal_segments, apply_pump_read_entry cache)", @@ -14243,6 +15738,8 @@ }, { "id": "bug-1427", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-09T21:15:00.000Z", "error_message": "APPLY_DIVERGENT's entity-count correctness check (persist_verify_apply_correctness) covers job entities only \u2014 worker entities can diverge (worker_from_capnp()/restore_worker() genuinely creates a new worker on a WAL_WORKER_CTRL/HOOK_DETAIL_WORKER_JOIN entry when absent) but this is currently invisible to the check.", "file": "src/controller/persist.c (persist_verify_apply_correctness, count_jobmgr_entries)", @@ -14263,6 +15760,8 @@ }, { "id": "bug-1428", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-09T21:15:00.000Z", "error_message": "Narrow TOCTOU residual in persist_register_pending_apply() (task 6.2, part of bug-1415's fix) \u2014 not reliably exercised by any current test.", "file": "src/controller/persist.c (persist_register_pending_apply)", @@ -14285,6 +15784,8 @@ }, { "id": "bug-1429", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-10", "error_message": "controller_unit_test suite TIMEOUT (300s, killed by SIGTERM) during /xce-build controller run immediately after uncommitted edits to persist.c, ha_convergence.c, config_repl.c (delaymgr-inline blocking I/O moved to VTASK_LIVE_BLOCK vtasks with inline-fallback-on-failure). Compile succeeded with zero errors (only pre-existing warnings). meson test summary: Ok:0 Fail:0 Timeout:1. Last test output before the hang was test_demotion_replay_yields_committed_only (test_commit_barrier_reply.c) printing overlayfs-SKIP, followed by a burst of HA FSM / persist_apply_pump_run log lines ending in \"HA quorum lost: connected=0 required=1\", then total silence for the remaining ~294s until SIGTERM. No ASAN/crash report -- looks like a genuine hang/deadlock, not a crash.", "file": "src/controller/testsuite/test_commit_barrier_rollout.c, test_demotion_convergence.c, test_convergence_verification.c, test_egress_filter.c, test_apply_pump.c, test_apply_pump_integration.c, test_job_array_fence.c, test_ha_role_lifecycle.c, test_job_array_replay_fence.c, test_commit_durability.c, test_commit_engine.c, test_p2_integration.c (test-fixture teardown ordering, 21 sites across 12 files); src/controller/commit.c (commit_engine_fini, unchanged -- confirms the documented contract); src/controller/persist.c (fsync_worker, unchanged -- confirms the race window)", @@ -14325,6 +15826,8 @@ }, { "id": "bug-1430", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-10T07:12:00Z", "error_message": "AddressSanitizer: heap-use-after-free on address ... WRITE of size 4 ... #0 ha_prevote_cancel_round_timeout ../controller/ha.c:6712 #1 ha_start_prevote_async ../controller/ha.c:6889 #2 ha_failover_stagger_fn ../controller/ha.c:7259 -- crashed 2 of 3 nodes during a live chaotic HA soak (10-round overlapping-kill mode)", "file": "src/controller/ha.c", @@ -14349,6 +15852,8 @@ }, { "id": "bug-1431", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-10T10:02:00Z", "error_message": "chaotic HA soak: committed_job_loss=5-8 across 10 rounds (VERDICT result=FAIL) after raft-apply-decoupling task 16 gated standby apply strictly on commit; committed_divergence stayed 0 and all jobs eventually reappeared at final settlement -- a visibility-latency gap, not true data loss", "file": "src/controller/ha.c, src/controller/persist.c, src/include/persist.h", @@ -14372,6 +15877,8 @@ }, { "id": "bug-1432", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-10T10:15:00Z", "error_message": "chaotic HA soak still FAIL (committed_job_loss=8) after bug-1431 force-flush fix; jobs above the conservative 2-voter safe_committed_lsn floor stayed invisible via expect-job for ~3.1s post-promotion, matching the primary heartbeat interval (3000ms) almost exactly", "file": "src/controller/ha.c", @@ -14395,6 +15902,9 @@ }, { "id": "bug-1433", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "fdatasync removed (40f510fb); pread in persist_apply_pump_run_sync still on become-leader path (ha.c)", "timestamp": "2026-07-10T10:59:00Z", "error_message": "controller-invariant-audit (2026-07-10, /xce-audit-invariants) async-I/O-thread-role dimension: bug-1431 round-2 force-flush fix (persist_promotion_advance_committed calling flusher_flush_once, a blocking fdatasync) is reachable via the real multi-node winning-vote-reply dispatch path on a VTASK_LIVENESS-classified vtask worker -- a task class the design documents as never admission-gated / never expected to block. This is the same bug class as bug-1198 (HA_REPL_SEND_SNAPSHOT persist_checkpoint on VTASK_LIVENESS), which was previously closed by offloading the blocking work off that worker class entirely.", "file": "src/controller/persist.c, src/controller/ha.c", @@ -14421,6 +15931,8 @@ }, { "id": "bug-1434", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-10T10:56:00Z", "error_message": "controller-invariant-audit (2026-07-10, /xce-audit-invariants) Raft TLA+ checklist dimension: persist_promotion_advance_committed and ha_standby_advance_committed_and_apply can race on persist_t.committed_lsn during the promotion window before role flips to CTRL_PRIMARY, via an unlocked load-then-conditional-store pattern in both functions -- a losing racer could clobber a larger value with a smaller one, regressing committed_lsn (violates the documented commitIndex-monotonic invariant).", "file": "src/controller/persist.c, src/controller/ha.c", @@ -14444,6 +15956,8 @@ }, { "id": "bug-1435", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-12T04:45:15Z", "error_message": "40-round sequential /xce-test-ha FAIL at round 36/40: verify_no_loss reported 0/111 jobs confirmed after 3 attempts; xce_joblist client call returned E_XCE_STANDBY (10014) against freshly-elected primary ctrl-1 (term=41) moments after wait-primary succeeded on the same session", "file": "src/controller/ha.c (raft_state_t entry points), src/controller/ha.h (new raft_lock field), src/controller/cluster.c (raft_on_vote_request/raft_on_pre_vote_request call sites)", @@ -14477,6 +15991,8 @@ }, { "id": "bug-1436", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-12T05:50:00Z", "error_message": "40-round sequential /xce-test-ha FAIL at round 36/40: xce_joblist: Connection reset by peer, immediately after wait-primary confirmed ctrl-2 serving in 46ms. RE-OPENED 2026-07-12 ~14:00-16:00Z after a live re-test (fresh isolated 40-round run, no concurrent soak) reproduced the IDENTICAL failure with ctrl-2 confirmed NEVER killed (same PID throughout, continuous RAFT/heartbeat log activity, still answering other clients after the failure) -- disproving the process-collision theory as the (sole) root cause. ROOT-CAUSED FOR REAL via live capture (tcpdump unavailable in sandbox -- no CAP_NET_RAW; used direct -vvv log correlation on a live reproduction instead) 2026-07-12 ~16:00-16:15Z. FIXED.", "file": "src/lib/client_internal.h (new XCECLI_FAILOVER_CALL_COUNT macro), src/lib/client.c (xcecli_reslist/xcecli_workerlist/xcecli_joblist/xcecli_resval/xcecli_jobinfo switched to it). .claude/skills/xce-test-ha/run-ha-soak.sh's node_pattern()/run_pattern() workdir-scoping fix (prior pass) remains a legitimate, independent improvement -- KEPT, not reverted.", @@ -14513,6 +16029,9 @@ }, { "id": "bug-1437", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "LSan leaks in test_jobj_hardening (makeInstance 240B + 7B strdup), no fix/suppression", "timestamp": "2026-07-13T00:55:00Z", "error_message": "Verification run of lib build + xcelib_test --run test_jobj_hardening (ASAN_OPTIONS=detect_leaks=1) after user changed 4x .datakiller=destroy -> .datakiller=dereference in src/lib/testsuite/test_jobj_hardening.c (to fix a leak in the kvmgr-of-job_t test fixture, since job_t.destroy is NULL in RTTI typeinfo and teardown only runs via job_dereference): Unity reports 5/5 PASS, 0 Failures, 0 Ignored, but LeakSanitizer STILL reports two leaks totaling 247 bytes and the process exits 1 (no halt_on_error/abort_on_error set, so no SIGABRT with plain detect_leaks=1 -- differs from the meson full-suite run which sets abort_on_error=1 and gets killed by signal 6).", "file": "src/lib/testsuite/test_jobj_hardening.c", @@ -14542,6 +16061,8 @@ }, { "id": "bug-1438", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-14T09:36:00Z", "error_message": "test_job_array_submit_appends_exactly_one_wal_entry_and_one_continuation: Expected 11 Was 5 (WAL type mismatch); subsequent test hung all 3 threads on futex_do_wait", "file": "src/controller/testsuite/test_commit_barrier_rollout.c", @@ -14563,6 +16084,9 @@ }, { "id": "bug-1439", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "654c6e8c, c1c6c9a2", "timestamp": "2026-07-14T13:15:00Z", "error_message": "lib/controller.capnp.h:4170:43: error: unknown type name 'LimitCon'; lib/controller.capnp.h:1: error: unterminated #ifndef \u2014 during ninja compile of lib/libxce.a.p/utils.c.o, cascading into ~440 bogus errors (glibc math.h storage-class errors, utils.h ectx_t redefinition) for the rest of that one translation unit", "file": "src/xce.build/lib/controller.capnp.h (generated by [5/190] Generating lib/controllercapnp.[ch] custom command)", @@ -14590,6 +16114,9 @@ }, { "id": "bug-1440", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "b7b90e9d", "timestamp": "2026-07-15T09:35:00Z", "error_message": "use of cgo in test wire_dispatch_test.go not supported / FAIL apiserver/backend [setup failed]", "file": "src/frontend/apiserver/backend/wire_dispatch_test.go", @@ -14613,6 +16140,9 @@ }, { "id": "bug-1441", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "wire_dispatch_support.go direct Go call", "timestamp": "2026-07-15T09:40:00Z", "error_message": "backend/wire_dispatch_support.go:154:8: could not determine what C.go_push_handler refers to / FAIL apiserver/backend [build failed]", "file": "src/frontend/apiserver/backend/wire_dispatch_support.go", @@ -14637,6 +16167,8 @@ }, { "id": "bug-1442", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-15T09:50:00Z", "error_message": "AddressSanitizer: attempting double-free on 0x...1b0 in thread T0 -- fires immediately on TestWireDispatch_OnePushProducesNRecordsAndLogsOnce, aborts the whole backend.test binary before it can print PASS/FAIL, so TestWireDispatch_RedeliveryCreatesNoNewRecordsAndLogsNothing never runs at all", "file": "src/frontend/apiserver/backend/wire_dispatch_support.go", @@ -14664,6 +16196,8 @@ }, { "id": "bug-1443", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-16T02:05:00Z", "error_message": "OPEN, NOT A CRASH -- discovered as a design-time/test-proven gap, not a runtime failure report. restore_job_array()'s final bulk-insert pass into c->jobmgr can itself fail partway through (e.g. realloc() exhaustion growing c->jobmgr's backing table). Because the underlying xkvmgr_type store has no key_del, an already-inserted prefix [0, k) cannot be rolled back and is left permanently, genuinely visible in c->jobmgr -- individually addressable (status/cancel/completion all work via direct jobid lookup) -- while the function honestly returns a negative errno (never silently claims success). Critically, that prefix is never included in out_new_jobs, so HOOK_DETAIL_JOB_NEW never fires for it, not on this attempt or on any later successful retry of the same LSN (the per-index idempotent-skip logic on retry treats the already-inserted prefix as 'already exists', excluding it from the retry's own out_new_jobs too). Practical consequence: these array elements exist and are queryable forever, but never reach the scheduler's or API-server's push and so are never actually scheduled to run -- a silent, permanent no-op job, not merely a bookkeeping gap. requirements-coverage validation (kiro-validate-impl, 2026-07-16) confirmed this is a literal violation of Requirement 2.3 ('a standby shall never observe a single array WAL record in a partially-applied form') and Requirement 9.4 ('shall not construct a partial/divergent set of elements'), and its shadow extends to 9.5 (self-heal, since the orphaned prefix never gets a fresh chance to hook-fire even once the retry succeeds) and 9.6 (the capstone fault-injection test deliberately targets the fully-atomic construction-phase seam instead of this bulk-insert seam, precisely because asserting 'zero partial elements' against this seam would be dishonest).", "file": "src/controller/job.c (restore_job_array(), bulk-insert phase)", @@ -14692,6 +16226,8 @@ }, { "id": "bug-1444", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-16T02:48:18Z", "error_message": "persist_recover: APPLY_DIVERGENT -- materialized state does not match the committed log tip (expected last_applied_lsn=5 new_job_count=3); a hidden apply defect is present (ha-commit-durability recover_verdict=RECOVER_CLEAN) / persist_recover: UNRECOVERABLE -- post-replay apply-correctness verification is DIVERGENT ... Staying ALIVE and PERMANENTLY non-promotable (refusing to serve divergent state) and RETRYING catch-up indefinitely -- reproduced live during job-array-single-wal-record task 6.1's HA soak scenario the FIRST time a node is killed and restarted after ANY WAL_JOB_ARRAY_SUBMIT record has ever been committed to its local WAL (e.g. after a 2-element array submission).", "file": "src/controller/persist.c (replay_wal(), ~line 3902-3904; consumed by persist_recover()/persist_verify_apply_correctness(), ~line 3960-3991)", @@ -14722,6 +16258,8 @@ }, { "id": "bug-1445", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-17T09:47:00Z", "error_message": "A node's very first promotion-eligibility evaluation in a fresh process runs before that node's own first persist_recover()/replay_wal() call, so the Live Apply Resource Fence (job-array-single-wal-record tasks 4.1-4.3, extended to boot replay by task 7.2) is invisible to that one specific check. A node that would otherwise correctly fence itself non-promotable on a resource-exhaustion apply failure during its first-ever boot replay can instead pass its first promotion check looking fully eligible, because the fence flag hasn't been set yet at the point that first gate runs.", "file": "src/controller/ha.c", @@ -14747,6 +16285,8 @@ }, { "id": "bug-1446", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-18T00:47:00Z", "error_message": "/kiro-validate-impl ha-commit-durability: test-commit-durability.sh --deterministic reported 3/3 FAIL ('DURABILITY GAP -- primary acked with 0 fdatasync in-window'), i.e. a false RED that looked like a regression of the spec's core invariant on the current build.", "file": "src/tools/test-commit-durability.sh (det_probe_once / date usage) -- test harness only, NOT a product file", @@ -14770,6 +16310,8 @@ }, { "id": "bug-1447", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-18T01:05:02Z", "error_message": "/kiro-validate-impl raft-drive-async: 10-round SEQUENTIAL HA soak (the fully-supported single-failover lifecycle, hard-fail-on-any-loss) failed at round 9/10 -- 'FAIL: round 9: post-failover submit did not get 3 acks' (0/3 acked across 5 retry attempts). Root traced to ctrl-3 (a standby) crashing: AddressSanitizer DEADLYSIGNAL SEGV on address 0x40 (near-null, READ) in openssl's ssl3_read_bytes <- tls_get_message_header <- state_machine <- handle_ssl_handshake (src/lib/netio.c:548) <- io_handler (netio.c:588) <- io_worker (netio.c:633), thread T2 (an io_worker thread). With ctrl-1 already killed (the round's own kill target) and ctrl-3 now also dead, no 2-of-3 quorum remained for ctrl-2 (freshly elected PRIMARY, term=11) to commit writes -- explaining the observed 0/3-acks symptom as a downstream consequence, not the primary defect.", "file": "src/lib/netio.c (handle_ssl_handshake / io_handler / io_worker -- crash site, shared iomgr epoll dispatch); src/controller/ha.c (ha_connect_prep_peer/ha_connect_finish_peer/reconnect_lane_fn, raft-drive-async Task 2.1/2.2 reconnect-offload -- suspected trigger); src/controller/cluster.c (handle_connection / endpoint_accept -- inbound accept path, alternate suspect)", @@ -14807,6 +16349,8 @@ }, { "id": "bug-1448", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-18T01:54:00Z", "error_message": "/kiro-validate-impl ha-heartbeat-committed-propagation: the xce-build skill's own documented pattern (\"dispatch via a plain Agent-tool call, run_in_background: false, wait for it\") produced an INCOMPLETE build -- ninja log ends with 'ninja: build stopped: interrupted by user.' at 85/192 targets, roughly 14 minutes after launch with zero nix/meson/ninja processes still running and no error in the log. The synchronous Agent call returned to the caller almost immediately (~38s) with text claiming 'The build is running in the background... I'll wait for it to complete' -- i.e. the subagent itself spawned an internal Bash run_in_background job and then ended its own turn, rather than blocking until the command finished.", "file": "src/tools/xce-build.sh (skill invocation pattern, not the script itself); .claude/skills/xce-build/SKILL.md's 'Always dispatch the build via the Agent tool' instruction", @@ -14833,6 +16377,8 @@ }, { "id": "bug-1449", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-18T09:56:00.000Z", "error_message": "[xce-test-ha] FAIL: round 1: no new primary elected after kill (no serving primary at term>2) -- hit once during ha-elect-ctx-lifecycle kiro-validate-impl soak, did NOT reproduce on immediate retry (10/10 clean)", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh (sequential mode round-1 kill timing) / src/controller/ha.c ha_on_primary_ctrl_fd_rdhup (ha-config-bootstrap Req 2.8 voter-only guard, working as designed)", @@ -14859,6 +16405,9 @@ }, { "id": "bug-1450", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "slow-lane offload exists (f88e4ef4) but flag defaults OFF (ha_convergence.c)", "timestamp": "2026-07-18", "error_message": "xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_demotion_convergence_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: it truncates the WAL (persist_truncate_wal_above, file I/O under p->lock) then calls ha_self_restart (tlog_exit synchronous log flush + execv) -- all inline on the delaymgr thread, the SAME single thread that drives heartbeats and election timeouts cluster-wide for the whole cluster.", "file": "src/controller/ha_convergence.c, src/controller/ha.h", @@ -14887,6 +16436,9 @@ }, { "id": "bug-1451", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "slow-lane offload exists (f88e4ef4) but flag defaults OFF (config_repl.c)", "timestamp": "2026-07-18", "error_message": "xce-audit-invariants controller pass (dim: async I/O thread-role) found ha_catchup_continuation_fn scheduled directly as a delaymgr task body with no vtask/slow-lane hop: catchup_stream_range -> ha_stream_wal_to_standby does blocking WAL file reads + socket writes to a standby directly on the delaymgr thread. Flagged as the MORE URGENT of the two violations -- catch-up can stream a variable, potentially large amount of WAL data, so it can block delaymgr (and therefore heartbeats to every OTHER peer) for an unbounded, unpredictable time, a real spurious-election risk.", "file": "src/controller/config_repl.c, src/controller/ha.h", @@ -14916,6 +16468,8 @@ }, { "id": "bug-1452", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-18T00:00:00Z", "error_message": "AddressSanitizer: SEGV on unknown address 0x000000000038 in commit_advance_locked (controller/commit.c:208), called from commit_on_durable_advance (commit.c:341) <- fsync_worker (persist.c:399), a background ASAN thread started by persist_start (persist.c:723) inside test_persist_truncate.c's make_fresh_persist / test_truncate_above_resolves_pending_continuations_as_failed. Reproduced by temporarily flipping BOTH g_drive_async_demotion_offload and g_drive_async_catchup_offload (bug-1450/1451) from their default 0 to 1 and running the standalone controller_unit_test binary directly (976-test suite) -- crashed immediately, before completing the suite. Confirmed absent with both flags at their committed default of 0: the same binary/build (just reverted) completed cleanly, 976 Tests 0 Failures 0 Ignored, twice.", "file": "src/controller/commit.c (crash site, commit_advance_locked); src/controller/persist.c (fsync_worker background thread, apply_committed_entry/restore_job/makeJob chain also appears in a related non-crashing stack from the same investigation); src/controller/ha_convergence.c + config_repl.c (g_drive_async_demotion_offload / g_drive_async_catchup_offload -- the flags whose flip triggers this)", @@ -14952,6 +16506,8 @@ }, { "id": "bug-1453", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-20T00:00:00.000Z", "error_message": "Unnecessary CA certificate plumbing across C daemons and Go CLI \u2014 ca_cert_path fields, --ca-cert flags, and CACert config are dead or misleading; TLS verification is SSL_VERIFY_NONE everywhere", "file": "multiple \u2014 see root_cause", @@ -14976,6 +16532,9 @@ }, { "id": "bug-1454", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "94f50ce6", "timestamp": "2026-07-20T00:00:00.000Z", "error_message": "Scheduler --peers correctly wired for initial controller discovery; minor issues: nmasters=1 (no EndpointRotator) and leaked ctx on HA path", "file": "src/scheduler/src/scheduler.c", @@ -14996,6 +16555,8 @@ }, { "id": "bug-1455", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-20T00:00:00.000Z", "error_message": "Scheduler --peers silently ignored: sched_init() memset(sched,0,...) wipes sched->cluster_cfg that main.c populated via sched_set_cluster_cfg() just before calling sched_init(), so the HA candidate loop's candidates_count check always reads 0 and falls back to the hardcoded localhost:9990 single-host path regardless of --peers.", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/main.c, src/scheduler/include/scheduler.h", @@ -15018,6 +16579,8 @@ }, { "id": "bug-1456", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-20T00:00:00.000Z", "error_message": "Dangling pointer risk in sched_init() HA candidate loop: a later refactor (following bug-1454) changed the loop to build a per-iteration stack-local xce_client_config_t candidate_config and pass &candidate_config into xcecli_ctx_init().", "file": "src/scheduler/src/scheduler.c", @@ -15042,6 +16605,8 @@ }, { "id": "bug-1457", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-20T22:56:00Z", "error_message": "xce_controller with no --peers flag exits with -EINVAL from cluster_open_listener instead of starting as single-node primary", "file": "src/controller/myapp.c", @@ -15066,6 +16631,8 @@ }, { "id": "bug-1458", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-22T03:22:00Z", "error_message": "tools/xce-build.sh: line 134: pip: command not found (inside nix develop ./src, running the new `xceadmin` build case, exit 127)", "file": "src/nixSupport/xce.nix", @@ -15088,6 +16655,8 @@ }, { "id": "bug-1459", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-22T03:45:00Z", "error_message": "pip install -e . inside `nix develop ./src` fails: \"error: externally-managed-environment ... This command has been disabled as it tries to modify the immutable /nix/store filesystem\" (PEP 668)", "file": "src/tools/xce-build.sh", @@ -15114,6 +16683,8 @@ }, { "id": "bug-1460", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-22T03:52:00Z", "error_message": "Under `xce-build.sh xceadmin release` (pinned, non-editable install), the installed xcecommon package in site-packages has no schema_files/ directory at all \u2014 controller.capnp/c.capnp/JSON schemas are silently dropped from the wheel", "file": "src/xceadmin/pyproject.toml", @@ -15141,6 +16712,8 @@ }, { "id": "bug-1461", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-22T02:00:00Z", "error_message": "pycapnp / capn-c: setting UpgradeConfigRequest.sha256 or .bytes raises capnp.lib.capnp.KjException: 'expected reader.type == STRUCT [6 == 10]; Value type mismatch' -- and the generated C accessors (UpgradeConfigRequest_get_sha256 etc.) were typed Data_ptr instead of a real byte blob", "file": "src/schema/controller.capnp", @@ -15167,6 +16740,8 @@ }, { "id": "bug-1462", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-23T03:04:50.000Z", "error_message": "ctrl-3 conf/ empty after HA failover deploy at /tmp/xce-ha-deploy -- config never replicated, but WAL/jobs replicated fine", "file": "src/controller/ha.c", @@ -15190,6 +16765,8 @@ }, { "id": "bug-1463", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-23T06:37:00.000Z", "error_message": "xcecli build fails: ../common/request/request.go:143:32: undefined: tls / :144:13: undefined: loadRootCAs -- FAILED: [code=1] frontend/xcecli/xcecli", "file": "src/frontend/xcecli/common/request/request.go, src/frontend/xcecli/cmd/jobs/peek.go", @@ -15214,6 +16791,8 @@ }, { "id": "bug-1464", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-23T09:50:00Z", "error_message": "deploy-ha.sh FAIL: Error: xceadmin binary not found -- run 'xce-build.sh xceadmin' or 'nix build .#xcecore' first (after controller had just been rebuilt and xceadmin had been built successfully earlier in the same session)", "file": "src/tools/xce-build.sh", @@ -15238,6 +16817,8 @@ }, { "id": "bug-1465", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-24T08:00:00Z", "error_message": "test_snapent_checkpoint_job_content_roundtrip / test_snapent_checkpoint_concurrent_mutation_not_torn / test_snapent_checkpoint_queue_status_roundtrip: FAIL: checkpoint file must parse as a well-formed entity-bearing format_version=1 payload", "file": "src/controller/testsuite/test_persist_snapshot_entities.c", @@ -15260,6 +16841,8 @@ }, { "id": "bug-1466", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-24T15:36:56.329888Z", "error_message": "test_config_repl.c: test_committed_marker_survives_checkpoint_trim and test_commit_advance_retry_clears_push_before_commit_fence flaked/failed (persist_checkpoint returned -EBUSY; new_mlsn <= committed_at_old) after wiring config-upgrade-snapshot task 5's WAL_CONFIG_MARKER apply case", "file": "src/controller/testsuite/test_config_repl.c", @@ -15285,6 +16868,8 @@ }, { "id": "bug-1467", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-25T08:55:00Z", "error_message": "Reviewer rejected config-upgrade-snapshot task 8.4: configcompare_paths() (configcompare.c/.h) was a correct, fully-tested library function but was not reachable from anywhere outside the unit test binary -- unlike its siblings walcompare/snapverify/snapdiff, it was never registered as an xce_test CLI subcommand in test.c. `echo help | xce_test` confirmed configcompare was missing from the command list.", "file": "src/controller/testsuite/test.c", @@ -15307,6 +16892,8 @@ }, { "id": "bug-1468", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-25T00:00:00Z", "error_message": "OP_upgradeConfig sent by any network client is permanently rejected -- handle_upgrade_config() was unreachable via the wire dispatch path, exercised only by unit tests calling it directly. Independently diagnosed by the admin-config-tool spec's design.md (Boundary Commitments / Risks) as a blocking controller defect out of that spec's scope; re-discovered here while building config-upgrade-snapshot task 8.6's live OP_upgradeConfig client (no prior client existed to send it and hit the gate).", "file": "src/controller/cluster.c", @@ -15330,6 +16917,9 @@ }, { "id": "bug-1469", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "reorder fixed (959b0cda); queue added by upgrade still needs restart (build_queues only at boot)", "timestamp": "2026-07-25T13:05:00Z", "error_message": "config-upgrade-snapshot task 8.6's live soak (--config-upgrade, run-ha-soak.sh) FAILED on its first real run: after a config upgrade committed successfully and genuinely interleaved with a 3-round chaotic kill/restart sequence (all 6 jobs survived, upgrade confirmed committed), the post-convergence materialized-state identity check failed -- snapdiff reported ctrl-1's latest checkpoint (snap-000000000000000b.capnp, LSN 11) as 51569 bytes vs ctrl-2/ctrl-3's identical 51621-byte file. This is a live, empirical reproduction of bug-857's originally-hypothesized failure mode (config upgrade leaves members with divergent materialized state), which bug-857's own buglog entry explicitly flagged as 'NOT empirically reproduced yet' and task 8.5 could only address at the unit level (test_bug857_regression.c, a DIFFERENT and already-fixed mechanism -- schedule_checkpoint's WAL-threshold trigger racing the marker boundary). This is a third, distinct mechanism, first observed here.", "file": "src/controller/config_repl.c do_upgrade_config_with_bytes (~1611-1770)", @@ -15362,6 +16952,9 @@ }, { "id": "bug-1470", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "later full controller suites pass", "timestamp": "2026-07-25T14:25:00Z", "error_message": "controller_unit_test: test_commit_advance_retry_clears_push_before_commit_fence (test_config_repl.c:1854) FAILS (\"Expected TRUE Was FALSE\") DETERMINISTICALLY (4/4 runs: 1x via full xce-build.sh controller clean build + meson test, 3x direct re-invocation of the built controller_unit_test binary with no rebuild) on the bug-1469 fix commit (persist.c +91/test_config_repl.c +144, uncommitted worktree diff at time of this build). controller_unit_test overall: 1025 Tests 1 Failures 0 Ignored / meson summary Ok:2 Fail:1.", "file": "src/controller/testsuite/test_config_repl.c", @@ -15389,6 +16982,8 @@ }, { "id": "bug-1471", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-25T14:45:00Z", "error_message": "config-upgrade-snapshot task 8.6 live --config-upgrade soak: after avoiding bug-1469's queue-creation gap (switched the upgrade-target config change from ADDING a queue to updating an existing queue's description text -- see bug-1469's updated entry), the materialized-state identity check (snapdiff of each member's latest checkpoint file, all at the same LSN) still fails, but now with a MUCH smaller divergence: exactly ONE byte differs (offset 42879/51161 bytes total) between ctrl-1 and ctrl-2/ctrl-3 (ctrl-2 and ctrl-3 are byte-IDENTICAL to each other). ctrl-1 has 0x50 at that offset; ctrl-2/ctrl-3 have 0x00.", "file": "src/controller/persist.c collect_host_frames / capnp_encode_host_admin_state (~lines 141-162, 2028)", @@ -15422,6 +17017,8 @@ }, { "id": "bug-1472", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-25T16:35:00Z", "error_message": "Host admin-control status changes (hostctrl -c / hostctrl -o, i.e. HOST_CLOSE/HOST_OPEN via worker_ctrl()) never propagate to standby members' in-memory worker state once the target host has already been joined -- discovered while root-causing bug-1471's single-byte checkpoint divergence, but the defect itself is general and has nothing to do with checkpoints, config upgrades, or the config-upgrade-snapshot spec.", "file": "src/controller/worker.c worker_from_capnp (~827-856); src/controller/persist.c case WAL_WORKER_CTRL apply path (~3724-3786)", @@ -15450,6 +17047,8 @@ }, { "id": "bug-1473", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-25T19:12:00Z", "error_message": "test_second_config_marker_supersedes_first_still_installing_on_member (task 8.8) FAILED on first run: Expected 0 Was 1 -- the fence must clear once materialization for the latest marker (B) completes.", "file": "src/controller/testsuite/test_config_repl.c (test-only; no production code involved)", @@ -15473,6 +17072,9 @@ }, { "id": "bug-1474", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "slot hardening c0b0c7d1; spurious duplicate reconnect (angle a) re-observed live 2026-09-16", "timestamp": "2026-07-25T20:50:00Z", "error_message": "config-upgrade-snapshot task 8.9's new live --config-upgrade-wal-trim soak (run-ha-soak.sh, run_config_upgrade_wal_trim_round): the marker-triggered checkpoint's WAL-segment trim (persist_checkpoint_marker_triggered -> trim_wal_segments -> repl_trim_lsn, bug-1147's guard) strands a genuinely mid-catch-up standby's still-needed WAL segment -- but NOT because bug-1147's guard logic itself is wrong. Reproduced identically on 2/2 independent live 3-node runs: a killed, still-down standby's disconnect-guard (last_disconnected_lsn) is silently DESTROYED before the trim ever runs, because an UNRELATED, already-connected peer's redundant reconnect attempt reuses the dead standby's just-freed replicator slot.", "file": "src/controller/ha.c (primary_heartbeat_fn's 'reconnect to any peer missing from standby_endpoints' loop, ~9219-9253; on_standby_connected, ~8658-8708; raft_cb_become_leader's vote-ep promotion, ~1888-1921); src/controller/repl.c (repl_add_standby's first-empty-slot allocation, ~85-133, and its memset(s,0,...) which wipes last_disconnected_lsn on reuse)", @@ -15516,6 +17118,8 @@ }, { "id": "bug-1472-remainder", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-25T20:15:00Z", "error_message": "Follow-up split out of bug-1472: HOOK_DETAIL_WORKER_DEAD entries applied via WAL_WORKER_CTRL against an already-existing worker still return -EEXIST (no-op) -- a dead-worker transition detected by one member (mark_worker_unavail(), worker.c:661-679) still never replicates its state to other members (standbys, or this member's own WAL replay), unconditionally. NOT a regression: this was already true before bug-1472's investigation; bug-1472's fix deliberately scoped itself to the byte-precise-confirmed HOOK_DETAIL_WORKER_UPDATE case only and left DEAD as -EEXIST/no-op, unchanged.", "file": "src/controller/worker.c worker_from_capnp (~827-895, post-bug-1472-fix); mark_worker_unavail (~661-679); src/schema/controller.capnp struct Worker (~136-144)", @@ -15545,6 +17149,8 @@ }, { "id": "bug-1475", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-26T04:36:00Z", "error_message": "Live config-upgrade-snapshot task 8.6 soak (`run-ha-soak.sh --config-upgrade`), run to verify bug-1472's fix, FAILED \u2014 but NOT at bug-1472's own assertions (materialized-state checkpoint byte-identity and active-configuration identity both PASSED cleanly across all 3 members, confirming bug-1472's fix holds). Failure occurred one step later, inside the Requirement 6.4 host-admin-status-rotation check's own `force_elect_node(\"ctrl-1\")` maneuver: after 3 rounds of overlapping chaotic kills + a live config upgrade all completed and the cluster reconverged 3-of-3, `force_elect_node` killed the spoiler (ctrl-3) and tried to submit a job batch on the surviving {ctrl-2 primary, ctrl-1 standby} pair \u2014 5 consecutive attempts all got 0/3 acked (\"waiting for write-quorum\"), and the round hard-failed: `[xce-test-ha] FAIL: force-elect ctrl-1: submit on {ctrl-2,ctrl-1} failed`. The Req 6.4 workerlist/admin-status assertion itself was never reached.", "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby, ha_consume_append_entries, ha_evaluate_repl_action \u2014 HA_REPL_STREAM_WAL decision/dispatch loop); src/controller/config_repl.c (ha_stream_wal_to_standby catch-up streamer)", @@ -15580,6 +17186,8 @@ }, { "id": "bug-1476", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-26T04:44:00Z", "error_message": "Live config-upgrade-snapshot task 8.9 re-verification soak (run-ha-soak.sh --config-upgrade-wal-trim, run to confirm bug-1474's repl.c fix), deploy /home/claude/tmp/xce-ha-waltrim3: ctrl-3 ABORTed via AddressSanitizer heap-use-after-free during the round's live config-upgrade submission, well before TARGET(ctrl-1) was ever restarted. The round's own bug-1474 WAL-trim assertions (floor segment preserved, below-floor segments trimmed, active segment present) all PASSED cleanly before this crash was noticed -- ctrl-3's death was only discovered because it, combined with a second unrelated issue (bug-1475), prevented the round's final '3-of-3 reconvergence' assertion from ever succeeding.", "file": "src/controller/config_repl.c (load_primary_config ~line 656-666 free_ClusterConfig/free(myconf->cluster_config) call site; config_activate_committed ~line 360; handle_repl_config_push ~line 1280); lib/controller.capnp.c (decode_ClusterConfig_ptr ~line 17074 calloc site, decode_Resource_list ~line 4057 crash site)", @@ -15615,6 +17223,8 @@ }, { "id": "bug-1477", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-28T08:25:00Z", "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade), run on top of bug-1475's already-fixed and already-built-in commit ce8df990 (confirmed via binary mtime 08:16:27 postdating ce8df990 04:48:53Z and b08e392e 08:13:54Z, plus `strings` matching TERM_INDEX_UNSET/persist_durable_floor), FAILED: \"ctrl-3 has no on-disk checkpoint after the upgrade\". ctrl-1/ctrl-2 both have data/snap/snap-000000000000000b.capnp (51161 bytes, LSN 11 hex = the WAL_CONFIG_MARKER's own lsn); ctrl-3's data/snap/ is completely empty. ctrl-3's log shows a PERMANENT loop (repeats every ~3s/heartbeat for the entire ~3-minute run, never progressing): a bare heartbeat probe at prevIdx=11 succeeds (ae_already_have, match=11), but every walCatchupEntry at prevIdx=12/13/14 is rejected forever with ae_reject_logmismatch. Unlike bug-1475's own signature, this is NOT a transient race that self-heals once durable_lsn catches up -- it never recovers for the rest of the run.", "file": "src/controller/myapp.c (~line 388-425, the \"leader already known -- skip own election\" fast path and its single guarded cluster_init_phase2 retry, bug-849 lineage); src/controller/ha.c (raft_cb_become_follower ~line 2000-2012, raft_cb_become_leader ~line 1772-1783, ha_elect_evaluate_and_signal ~line 5609-5629/5699-5710 -- all three unconditionally set ha->phase2_done=1 after calling cluster_init_phase2(c) without checking its return value); src/controller/cluster.c (cluster_init_phase2 ~line 1342-1598, the sole call site of commit_engine_init/persist_recover/ha_sync_raft_log_from_recovery); src/controller/persist.c (~line 796-799, on_durable_advance documented as registered ONLY by commit_engine_init)", @@ -15652,6 +17262,8 @@ }, { "id": "bug-1478", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-28T09:00:00Z", "error_message": "config-upgrade-snapshot task 8.9 live soak (run-ha-soak.sh --config-upgrade-wal-trim ~/tmp/xce-ha-waltrim): bug-1474's floor-preservation fix confirmed working (floor segment seg-5 correctly preserved, all below-floor segments correctly trimmed) but the round still FAILED: \"cluster did not reconverge to 3-of-3 after restarting ctrl-1 -- the preserved floor segment did not actually suffice for real incremental catch-up\". ctrl-1 (TARGET, killed+restarted 3x) settles into a PERMANENT, zero-progress loop: its own raft AppendEntries exchanges cleanly reach match=22 with zero ae_reject events (heartbeat probe at prevIdx=22,n=0 -> success=1 match=22, repeating every ~3s forever), while ctrl-2 (primary) simultaneously and forever logs 'HA repl: standby[0] lsn=21 < committed_lsn=22 -> stream WAL batches' / 'ha_stream_wal_to_standby: streamed WAL [21+1..22] to standby' every heartbeat cycle with zero effect -- an off-by-one disagreement (21 vs 22) between the two sides that never reconciles.", "file": "src/controller/ha.c (repl_ack_decode_and_dispatch's heartbeat/stateReport reply branch, ~6379-6388 pre-fix -- the `target_lsn = max(committed_lsn, last_written)` computation feeding ha_dispatch_repl_action_for_standby)", @@ -15692,6 +17304,8 @@ }, { "id": "bug-1479", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-28T13:25:00Z", "error_message": "Live config-upgrade-snapshot task 8.6 soak (run-ha-soak.sh --config-upgrade ~/tmp/xce-ha-configupgrade2), run on top of bug-1477's fix (checkpoint-identity, config-identity, and host-admin-status-survival assertions ALL PASSED for the first time), FAILED later in the force-elect-rotation step: 5 consecutive job-submit attempts on {ctrl-1 (primary), ctrl-2 (standby)} all got 0/3 acked (\"waiting for write-quorum\"), then hard-failed. Investigated the ORIGINAL hypothesis (a leaked VTASK_LIVE_BLOCK/commit-admission slot from an earlier burst of 17 commit timeouts, lsn=15..31, rc=-110) and REFUTED it: commit_resolve()/wait_release_locked() correctly release the admission slot on every timeout (confirmed 17 'async continuation deadline expired' + 17 'commit failed/timed out' WARN pairs, 1:1, in ctrl-1.log) \u2014 no admission-pool leak exists. The REAL mechanism is unrelated to admission control entirely: a permanent Raft AppendEntries log-conflict wedge on standby ctrl-2, compounded by a NACK-handling gap that never triggers catch-up.", "file": "src/controller/ha.c (repl_ack_decode_and_dispatch's WAL-ACK branch, ~6456-6644 -- the `!success` NACK path at ~6631-6642 only checks for a higher term and WARNs \"ignoring\", never calls repl_on_ack/repl_start_catchup/any retry); src/controller/repl.c (repl_broadcast ~295-380, the live single-entry send path -- fire-and-forget, no nextIndex backoff-and-retry on rejection; repl_on_ack ~250-271, the ONLY site that decrements pending_ack_count, never invoked on a NACK)", @@ -15746,6 +17360,8 @@ }, { "id": "bug-1480", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-29T07:05:00Z", "error_message": "raft-adapter-conformance systematic audit finding (not yet fixed): handle_repl_config_push (config_repl.c:1261-1337) has 4 reject paths (malformed body, missing bytes/sha256, sha256 mismatch, conf_save_config failure) that all return without ever sending a ReplConfigPushReply. Only the success path (1326-1335) sends {accepted=1}; there is no {accepted=0} anywhere in this function, despite its own doc-comment implying a reject reply exists. cluster.c dispatch case (652-655) has no fallback error-reply either.", "file": "src/controller/config_repl.c (handle_repl_config_push, ~1261-1337)", @@ -15772,6 +17388,8 @@ }, { "id": "bug-1481", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-29T07:05:00Z", "error_message": "raft-adapter-conformance systematic audit finding (not yet fixed): vote_ep_err_handler (ha.c:7564-7608), the shared on_err handler wired onto both real-vote and PreVote endpoints (ha.c:688-689, 827-828), handles post-connect endpoint failures (RDHUP/SSL error/remote close arriving after connect+monitor succeeded but before a reply) by calling the legacy handle_request_vote_reply directly instead of routing through vote_reply_dispatch / pre_vote_reply_dispatch (the production tally path fixed by bug-1423/1424 for connect-time and monitor-time failures only).", "file": "src/controller/ha.c (vote_ep_err_handler, ~7564-7608; correct sibling sites at ~693-726 and ~830-855)", @@ -15800,6 +17418,8 @@ }, { "id": "bug-1482", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-29T10:43:00Z", "error_message": "AddressSanitizer: heap-use-after-free in free_packet ../lib/netio.c:437, called from test_config_push_reply_higher_term_adopts_and_steps_down (test_config_repl.c:3718)", "file": "src/controller/testsuite/test_config_repl.c; doc comment in src/controller/cluster.c (myreq_handler_for_test)", @@ -15826,6 +17446,8 @@ }, { "id": "bug-1483", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-29T10:20:00Z", "error_message": "ha_arm_heartbeat_timeout_vtask (ha.c ~8805-8848), when called with c==NULL (unit-test mode driving a PreVote round to LOST without stubbing ha_prevote_rearm_liveness_hook), callocs a liveness_poll_ctx_t and stores it into the static s_liveness_ctx, but the guard skips scheduling anything to poll or free it -- the context leaks/dangles permanently in that code path.", "file": "src/controller/ha.c (ha_arm_heartbeat_timeout_vtask, ~8805-8848)", @@ -15851,6 +17473,8 @@ }, { "id": "bug-1484", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-29T21:10:00Z", "error_message": "No test anywhere in src/controller/testsuite/ drives the real production dispatch path for either term-discovery vector bug-1421 introduced (heartbeat-reply state-report term adoption, ha.c:6690-6711; WAL-NACK reply_term adoption inside repl_ack_decode_and_dispatch, ha.c:6805-6814) and asserts raft_on_higher_term_observed actually fires. The look-alike test_intake_higher_term_heartbeat_adopts_term (test_term_safety_adapter.c:1232) drives a different code path (ha_consume_append_entries -> raft_on_append_entries_request library-level UpdateTerm, an AE-request vector, not the AE-reply/state-report fallback bug-1421 added). test_persist_recovery.c (cited by design.md line 146 as covering this) has zero term-adoption content -- pure recovery mechanics. test_config_repl.c only covers the NEW config-push term vector added by this feature (Req 7.4), not the pre-existing heartbeat/WAL-NACK vectors.", "file": "src/controller/ha.c (~6690-6711 heartbeat-reply vector, ~6805-6814 WAL-NACK vector); no test file drives either", @@ -15881,6 +17505,10 @@ }, { "id": "bug-1485", + "status": "cannot-reproduce", + "status_updated": "2026-09-24", + "status_note": "stderr capture added a5560256, never re-seen", + "repro_attempts": "0/40 repro attempts (2026-08-01)", "timestamp": "2026-07-30T14:35:00Z", "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path flaky (near-100% fail rate across ~25 live runs during admin-config-tool task 6.3 round-2 remediation): TimeoutError: controller never opened localhost:: [Errno 111] Connection refused, raised from a fresh wait_for_port re-probe of the elected primary's own address performed AFTER _wait_for_primary (CTRL_PRIMARY marker) + a new _wait_for_standbys (both other nodes show CTRL_STANDBY) + a 3s settle sleep all already succeeded.", "file": "src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture); root cause is in src/controller (exact site not yet isolated -- C-side investigation out of this task's boundary)", @@ -15909,6 +17537,8 @@ }, { "id": "bug-1486", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-31T03:48:00Z", "error_message": "Backgrounded `xce-build.sh all` run went silently stale mid-flight ([360/362], no ninja FAILED/error line, no completion marker, log just stopped growing at 03:38:35) with no matching build process left in `ps aux`", "file": "src/tools/xce-build.sh", @@ -15934,6 +17564,8 @@ }, { "id": "bug-1487", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-07-31T04:50:00Z", "error_message": "30-round chaotic HA soak (run-ha-soak.sh --chaotic 30): committed_job_loss=1 at round 25/30 (VERDICT result=FAIL). expect-job reported job 0-6-1785473261-216067 not found on newly-elected primary ctrl-2 (term=27) immediately after the round-25 kill of ctrl-1 (term=26). committed_divergence stayed 0 for the entire run; the job (and all others) was confirmed present on every subsequent verify (rounds 26-30) and the final post-run sweep of all 93 tracked jobs.", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -15960,6 +17592,8 @@ }, { "id": "bug-1489", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-03T04:00:00Z", "error_message": "gcc: 'unknown type name xce_sched_core_t; did you mean xce_sched_ctx_t?' at core.h:92 (xce_sched_core_init prototype) when compiling scheduler.h consumers (handler.c, main.c, etc).", "file": "src/scheduler/include/core.h", @@ -15980,6 +17614,8 @@ }, { "id": "bug-1490", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-03T04:00:00Z", "error_message": "ld: undefined reference to 'reset_sched_obj' when linking xce_scheduler (referenced from plugins/limit.c's limit_job_election()).", "file": "src/scheduler/src/schedobj.c", @@ -16002,6 +17638,8 @@ }, { "id": "bug-1491", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-03T06:33:00Z", "error_message": "AddressSanitizer: SEGV on unknown address 0x000000000000 in dataq_deq_head (lib/dataq.c:15), called from dataq_fini <- xce_sched_core_fini (core.c:126), when a test called xce_sched_core_init() then xce_sched_core_fini() back-to-back on a fresh xce_sched_core_t.", "file": "src/scheduler/testsuite/test_core_jbuf_capacity_pin.c", @@ -16027,6 +17665,8 @@ }, { "id": "bug-1492", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-04T01:04:00Z", "error_message": "plugin_end_session() checks plugin->begin_session_fn != NULL but calls plugin->end_session_fn(plugin) -- a guard/call field mismatch", "file": "src/scheduler/src/plugin.c", @@ -16050,6 +17690,8 @@ }, { "id": "bug-1493", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-04T12:35:00Z", "error_message": "gcc: 'expected declaration specifiers before sched_snapshot_data_t' cascading into ~20 errors across typemap.h/controller.capnp.h/controller.capnp.c when building lib/controller.capnp.c (scheduler-core task 7.3 SnapshotData C struct).", "file": "src/include/typemap.h", @@ -16071,6 +17713,8 @@ }, { "id": "bug-1494", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-04T00:00:00Z", "error_message": "clang-format -i produced bizarre whole-file +1-space indentation drift (starting right after the top #include block, persisting to EOF) in a newly-written test_snapshot_save_load.c, with kill_sched_job() alone expanded to multiple lines while identical sibling one-liners stayed collapsed.", "file": "src/scheduler/testsuite/test_snapshot_save_load.c", @@ -16093,6 +17737,8 @@ }, { "id": "bug-1495", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-04T00:00:00Z", "error_message": "sched_get_user()/sched_get_project() (core.c) always returned NULL for any user/project name not already present in core->users/core->projects, even though both are documented and relied upon (core_events.c JOBS_PUSH, schedobj.c make_sched_job, and task 8.1's own snapshot_load.c) as get-or-create. Surfaced as job->user/job->project being silently NULL for every job referencing a brand-new user/project, discovered while writing test_snapshot_save_load.c's cross-core equivalence test (RED phase: TEST_ASSERT_NOT_NULL(sched_get_project(&core_a, \"proj1\")) failed immediately after a successful JOBS_PUSH).", "file": "src/scheduler/src/core.c", @@ -16117,6 +17763,8 @@ }, { "id": "bug-1496", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-04T00:00:00Z", "error_message": "xce_sched_core_load_snapshot() never restored core->prng_seed from the snapshot -- a freshly loaded core silently kept xce_sched_core_init()'s default (0) instead of the value xce_sched_core_save_snapshot() had captured. Caught by test_snapshot_save_load.c's field-level assertion: TEST_ASSERT_EQUAL_UINT64(0xDEADBEEFULL, core_b.prng_seed) failed with Expected 3735928559 Was 0 after a successful save+load round trip.", "file": "src/scheduler/src/snapshot_load.c", @@ -16137,6 +17785,8 @@ }, { "id": "bug-1497", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-04T00:00:00Z", "error_message": "ASan SEGV inside dup_val() -> makeString() -> hash_string() while building test_snapshot_save_load.c's cross-core equivalence test fixtures.", "file": "src/scheduler/testsuite/test_snapshot_save_load.c", @@ -16159,6 +17809,8 @@ }, { "id": "bug-1498", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-05T00:00:00Z", "error_message": "test_toctou_hardlink_planted_after_validation_rejected FAILED: Expected Non-NULL -- fgets() on the outside 'victim' file returned NULL (file was 0 bytes) after the TOCTOU regression test ran, even though sched_record_start()/the reproduced open-hardening sequence both correctly REJECTED the hard-link swap and returned an error.", "file": "src/scheduler/src/recording.c", @@ -16182,6 +17834,8 @@ }, { "id": "bug-1499", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-05T00:00:00Z", "error_message": "sched_driver_session_ordering_test TIMEOUT (30s, killed by SIGTERM) after adding task 9.2's recording-write hooks to xce_sched_driver_run_normal_mode_round() -- picker thread, allocator thread, and the main round all blocked forever.", "file": "src/scheduler/src/driver.c", @@ -16204,6 +17858,8 @@ }, { "id": "bug-1500", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-06T00:00:00Z", "error_message": "test_run_processes_multiple_sessions_in_order_and_no_longer_returns_enosys FAIL: Expected 5000 Was 743938186628040 (a raw-looking CLOCK_MONOTONIC nanosecond value, not the fixture's own beginRound).", "file": "src/scheduler/testsuite/test_sim_event_loop.c", @@ -16226,6 +17882,8 @@ }, { "id": "bug-1501", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T00:00:00Z", "error_message": "gcc: 'expected =, ,, ;, asm or __attribute__ before is' / 'unknown type name is' / 'missing terminating character' cascade starting at sim_verify.c:59, plus 'implicit declaration of function sim_verify_str_eq' further down.", "file": "src/scheduler/src/sim_verify.c", @@ -16250,6 +17908,8 @@ }, { "id": "bug-1502", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T00:00:00Z", "error_message": "Task 11.5 review round 1 REJECTED: previous implementer's self-reported RED-phase evidence (git stash of 4 'behavioral production files' -> 3 tests failing under meson test) did not reproduce -- reviewer got a hard ninja build failure instead (test_sim_event_loop.c:467 calling sim_process_replay_event_message with 5 args against a stashed-back 4-arg sim_loop.c declaration).", "file": "src/scheduler/src/sim_loop.c", @@ -16272,6 +17932,8 @@ }, { "id": "bug-1503", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T03:11:00Z", "error_message": "AddressSanitizer: attempting free on address which was not malloc()-ed, inside free_QueueConf() <- free_QueueConf_ptr() <- release_translated_payload() (driver.c) <- xce_sched_driver_translate_pending_events(). Crashing pointer was 56 bytes inside a 61-byte region allocated by makeString()->makeInstance().", "file": "src/scheduler/testsuite/test_sim_equivalence.c", @@ -16294,6 +17956,8 @@ }, { "id": "bug-1504", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T08:00:00Z", "error_message": "sched_sim_startup_test SIGSEGV in write_SnapshotData (lib/controller.capnp.c) -- deterministic crash inside libCapnC_Runtime's read_ptr/capn_setp, discovered during scheduler-core task 12.1 review remediation (full clean rebuild needed for a libedit dependency swap).", "file": "src/schema/controller.capnp", @@ -16320,6 +17984,8 @@ }, { "id": "bug-1505", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T08:10:00Z", "error_message": "controller_unit_test SIGSEGV in write_WalCatchupEntry (lib/controller.capnp.c), test_wal_catchup_append_term_fields_round_trip -- discovered incidentally while chasing bug-1504; NOT fixed, logged for its own follow-up.", "file": "src/schema/controller.capnp", @@ -16348,6 +18014,8 @@ }, { "id": "bug-1506", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T09:05:00Z", "error_message": "sched_repl_breakpoints_inspection_test: repl_dispatch(sim, \"break JOB_STATE_CHANGE jobid=12345\") stored NULL for the breakpoint's token/filter key/value -- TEST_ASSERT_EQUAL_STRING(\"jobid\", ...) failed with 'Expected jobid Was NULL'.", "file": "src/scheduler/testsuite/test_repl_breakpoints_inspection.c", @@ -16371,6 +18039,8 @@ }, { "id": "bug-1507", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-07T09:10:00Z", "error_message": "sched_repl_breakpoints_inspection_test: 'free(): invalid pointer' SIGABRT inside repl_breakpoint_free(), reached from repl_cmd_break()'s malformed-filter rejection path (\"break JOB_PUSH nofiltermark\"). gdb backtrace: malloc_printerr <- repl_breakpoint_free (repl_break.c:375) <- repl_cmd_break <- repl_dispatch <- test_break_malformed_filter_rejected.", "file": "src/scheduler/src/repl_break.c", @@ -16395,6 +18065,9 @@ }, { "id": "bug-1508", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "worker fetchfile opens any path: no job-ownership/path-traversal check (controller_connector.c)", "timestamp": "2026-08-08T00:00:00Z", "error_message": "No user report / no test failure -- discovered incidentally during doc-cleanup fact-check audit of src/docs/job_peek.md against current code.", "file": "src/worker/src/xcecd_worker.c (handle_fetchfile, jobs-peek fetchfile feature)", @@ -16416,6 +18089,8 @@ }, { "id": "bug-1509", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T07:34:55.093767+00:00", "error_message": "extralib flake dependency pulls external repo for trivial linked-list code used only by autoscaler scaler.h", "file": "src/flake.nix,src/nixSupport/xce.nix,src/autoscaler/scaler/include/scaler.h", @@ -16435,6 +18110,8 @@ }, { "id": "bug-1510", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T13:07:36.020Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/design.md", @@ -16453,6 +18130,8 @@ }, { "id": "bug-1511", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T13:09:14.910Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/design.md", @@ -16471,6 +18150,8 @@ }, { "id": "bug-1512", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T13:10:14.229Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/requirements.md", @@ -16489,6 +18170,8 @@ }, { "id": "bug-1513", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T13:47:21.938Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -16507,6 +18190,8 @@ }, { "id": "bug-1514", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T18:46:00.000Z", "error_message": "kiro-validate-design Critical Issue 1: design.md claims sched_job_picker/sched_allocator are 'unaffected' by a blocked xcecli_jobdispatch call, violating Req 3 AC1", "file": ".kiro/specs/component-failover-adoption/design.md, tasks.md, research.md", @@ -16527,6 +18212,8 @@ }, { "id": "bug-1515", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-10T20:20:00.000Z", "error_message": "/goal-driven deep re-review of component-failover-adoption requirements.md+design.md found 5 additional distinct defects after bug-1514's fix, several introduced by bug-1514's own Req 7 fix", "file": ".kiro/specs/component-failover-adoption/requirements.md, design.md, tasks.md, research.md", @@ -16553,6 +18240,8 @@ }, { "id": "bug-1516", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T11:35:00.157Z", "error_message": "Significant refactor of ", "file": "src/scheduler/testsuite/meson.build", @@ -16571,6 +18260,8 @@ }, { "id": "bug-1517", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T11:35:50.895Z", "error_message": "Wrong reference: true should be false", "file": "src/scheduler/src/scheduler.c", @@ -16589,6 +18280,8 @@ }, { "id": "bug-1518", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T12:22:54.348Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", @@ -16607,6 +18300,9 @@ }, { "id": "bug-1519", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "b6a2f5ec (bug-1756)", "timestamp": "2026-08-11T12:49:17.815Z", "error_message": "==ERROR: AddressSanitizer: Joining already joined thread, aborting.", "file": "src/worker/src/main.c (xcecd daemon startup)", @@ -16630,6 +18326,8 @@ }, { "id": "bug-1520", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T13:05:01.694Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", @@ -16648,6 +18346,8 @@ }, { "id": "bug-1521", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T13:14:08.399Z", "error_message": "Wrong reference: test_sched_init_no_peers_default_also_fails_cleanly should be test_all_candidates_unreachable_log_names_every_candidate_host_port", "file": "src/scheduler/testsuite/test_scheduler_connect.c", @@ -16666,6 +18366,8 @@ }, { "id": "bug-1522", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T13:14:51.964Z", "error_message": "Significant refactor of ", "file": "src/scheduler/testsuite/test_scheduler_connect.c", @@ -16684,6 +18386,8 @@ }, { "id": "bug-1523", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T14:34:18.839Z", "error_message": "Significant refactor of ", "file": "src/scheduler/testsuite/test_scheduler_decision_redelivery.c", @@ -16702,6 +18406,8 @@ }, { "id": "bug-1524", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T14:37:43.571Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", @@ -16720,6 +18426,8 @@ }, { "id": "bug-1525", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T23:39:27.643Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/scheduler.c", @@ -16738,6 +18446,8 @@ }, { "id": "bug-1526", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-11T23:49:56.443Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -16756,6 +18466,8 @@ }, { "id": "bug-1527", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T02:41:37.052Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", @@ -16774,6 +18486,8 @@ }, { "id": "bug-1528", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T02:41:56.690Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/config.go", @@ -16792,6 +18506,8 @@ }, { "id": "bug-1529", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T03:09:17.720Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/candidates.go", @@ -16810,6 +18526,8 @@ }, { "id": "bug-1530", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T04:33:27.866Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -16828,6 +18546,8 @@ }, { "id": "bug-1531", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T07:05:08.345Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", @@ -16846,6 +18566,8 @@ }, { "id": "bug-1532", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T00:20:00.000Z", "error_message": "apiserver clib.go tryConnect(): use-after-free on ctx->conf->masters[i] (host address string) on every post-initial-connect failover rotation attempt", "file": "src/frontend/apiserver/backend/clib.go", @@ -16867,6 +18589,8 @@ }, { "id": "bug-1533", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T07:46:01.677Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -16885,6 +18609,8 @@ }, { "id": "bug-1534", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T08:45:46.229Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/reconnect_test.go", @@ -16903,6 +18629,8 @@ }, { "id": "bug-1535", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T09:15:53.632Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -16921,6 +18649,8 @@ }, { "id": "bug-1536", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T09:25:02.926Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/heartbeat.go", @@ -16939,6 +18669,8 @@ }, { "id": "bug-1537", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T09:25:33.041Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", @@ -16957,6 +18689,8 @@ }, { "id": "bug-1538", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T09:26:08.038Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/heartbeat_test.go", @@ -16975,6 +18709,8 @@ }, { "id": "bug-1539", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T09:58:02.265Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -16993,6 +18729,8 @@ }, { "id": "bug-1540", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T10:16:08.309Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", @@ -17011,6 +18749,8 @@ }, { "id": "bug-1541", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T10:17:17.099Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/reconnect.go", @@ -17029,6 +18769,8 @@ }, { "id": "bug-1542", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T10:18:42.192Z", "error_message": "health_rejoin_test.go referenced C.E_XCE_NOT_READY directly (via -int(C.E_XCE_NOT_READY)), which requires import \"C\" in a _test.go file -- illegal per golang/go#13938, would have failed to compile", "file": "src/frontend/apiserver/backend/health_rejoin_test.go", @@ -17048,6 +18790,8 @@ }, { "id": "bug-1543", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T10:20:03.350Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/reconnect_test.go", @@ -17066,6 +18810,8 @@ }, { "id": "bug-1544", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T10:51:41.466Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17084,6 +18830,8 @@ }, { "id": "bug-1545", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T12:21:30.458Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17102,6 +18850,8 @@ }, { "id": "bug-1546", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T12:42:35.605Z", "error_message": "Significant refactor of ", "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", @@ -17120,6 +18870,8 @@ }, { "id": "bug-1547", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T12:48:36.145Z", "error_message": "Incorrect value in code", "file": "../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-scheduler-framework/93f96538-d154-48c8-b92b-c8f9342a87cf/scratchpad/phase2_failover.sh", @@ -17138,6 +18890,8 @@ }, { "id": "bug-1548", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T12:59:31Z", "error_message": "apiserver never re-registers (application-level xcecli_join with HostRole_apiserver) with the new primary after a live controller failover, even though its transport/write-call connectivity clearly recovers (job/host-control POSTs keep succeeding); observed for 10+ minutes and 20+ default-30s heartbeat ticks with zero occurrences of reconnect.go's own 'checkAndPerformRejoin: rejoin after transparent reconnect succeeded/failed' log lines in apiserver_out.log, across 3 separate write calls (2 job submits + 1 host control) issued after the primary was kill -9'd. TASK 5.2 UPDATE (2026-08-12, second independent live E2E reproduction, cleaner isolation): ran component-failover-adoption task 5.2's own scenario (apiserver-only, no worker/scheduler, 3-node controller cluster at localhost:19991-19993, apiserver -d /home/claude/tmp/xce-e2e-task52/apiserver) specifically to isolate the heartbeat-only idle-period path task 5.1's write-traffic-muddied run could not cleanly separate. Confirmed real election: ctrl-1 (primary, term=2) kill -9'd at 13:33:31Z; ctrl-2 promoted primary at term=3 within 9s (13:33:40Z, raft_cb_become_leader). Zero write requests issued for the full outage window: apiserver_out.log is BYTE-IDENTICAL (23 lines) from before the kill through 268s / ~9 heartbeat ticks after it (well past the task's 2-minute/3-4-tick minimum) -- zero occurrences of 'checkAndPerformRejoin'/'rejoin'/'needsRejoin'/'xcecli_join' anywhere. Diagnostic control issued only AFTER the observation window (13:39:52Z, ~6.5 min post-kill): POST /xce/v1/jobs/create succeeded with a real controller-assigned job ID (73930919093731328) and server-generated success message -- proving the transport had already reached the NEW primary (ctrl-2) -- yet this write call ALSO produced zero rejoin log lines, full apiserver_out.log grep for the same 4 terms still empty after it. This is strictly stronger evidence than task 5.1's: it directly demonstrates the write path's own XCECLI_FAILOVER_CALL/xcecli_invoke_rejoin also did not fire even when explicitly exercised, consistent with the transport having already been silently repaired by an earlier heartbeat tick's XCECLI_FF_CALL before the write call ever needed to retry -- exactly bug-1548's leading hypothesis, not merely 'consistent with' it as task 5.1 could only say. Corroborating bug-1549 downstream symptom also reproduced: GET /xce/v1/jobs immediately after the successful submit returned {\"count\":0,\"rows\":[]} -- the just-submitted job never reached apiserver's local store, consistent with apiserver never having actually re-issued xcecli_join(HostRole_apiserver) against ctrl-2. ctrl-2.log has no apiserver-join lines either direction (join.c logs only rejections, per task 5.1's prior finding -- reconfirmed, not independently informative here). Task 5.2's own observable-completion criterion ('the scenario passes without requiring any client-triggered write request during the outage window' -- i.e. apiserver rejoins and resumes push notifications via the heartbeat-triggered path alone) was NOT met: no rejoin evidence appeared at any point in the heartbeat-only window. Evidence preserved at /home/claude/tmp/xce-e2e-task52/ (apiserver/logs/apiserver_out.log, logs/ctrl-{1,2,3}.log, logon_response.json, diagnostic_job_submit_response.json, diagnostic_joblist_response.json, KILL_TIMESTAMP_UTC.txt, DIAGNOSTIC_CONTROL_START_UTC.txt, DIAGNOSTIC_CONTROL_SUBMIT_UTC.txt).", "file": "src/frontend/apiserver/backend/reconnect.go, heartbeat.go, clib.go (component-failover-adoption task 4.3/4.4's checkAndPerformRejoin/needsRejoin mechanism)", @@ -17164,6 +18918,8 @@ }, { "id": "bug-1549", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T12:59:31Z", "error_message": "Jobs submitted via apiserver REST API (POST /xce/v1/jobs/create) never appear in apiserver's own job store (GET /xce/v1/jobs / GET /xce/v1/jobs/:id returns empty), even though the job is confirmed genuinely committed on the controller (verified directly via xce_test's `joblist` showing both jobs in PEND state). Reproduced for 2 jobs (one submitted at the moment of a primary kill, one submitted >1 minute after a clean new-primary failover) -- so this is NOT specific to the failover window, appears to be a general gap. By contrast, worker/host-list push (apiserver_push_workers/apiserver_push_workerlist, controller/apiserver.c) DOES work: GET /xce/v1/hosts correctly showed the live-joined worker with real data.", "file": "src/controller/apiserver.c (apiserver_job_new_fun / apiserver_push_joblist hook), src/frontend/apiserver/backend/clib.go (go_push_handler / handleDataset)", @@ -17189,6 +18945,8 @@ }, { "id": "bug-1550", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-12T13:22:40.852Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17207,6 +18965,8 @@ }, { "id": "bug-1551", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T02:38:03.922Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17225,6 +18985,8 @@ }, { "id": "bug-1552", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T03:02:32.434Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17243,6 +19005,8 @@ }, { "id": "bug-872", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:20:20.800Z", "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", @@ -17261,6 +19025,8 @@ }, { "id": "bug-873", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:20:42.831Z", "error_message": "Significant refactor of ", "file": "src/xceadmin/tests/test_mapping.py", @@ -17279,6 +19045,8 @@ }, { "id": "bug-874", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:25:51.662Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_worker_join.c", @@ -17297,6 +19065,8 @@ }, { "id": "bug-1633", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:27:23.041Z", "error_message": "Significant refactor of ", "file": "src/controller/worker.c", @@ -17315,6 +19085,9 @@ }, { "id": "bug-1553", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "intermittent (~1 in 3) controller_unit_test SIGSEGV; not isolated, re-seen 5bf123cc", "timestamp": "2026-08-13T07:35:00Z", "error_message": "controller_unit_test (combined 1095-test Unity binary) segfaults intermittently (~1 in 3 full runs observed) somewhere between test_drive_async_config.c and the next testsuite file (test_configcompare.c/test_slowlane.c area) -- no assertion failure, no ASAN report captured before the crash, process just dies (exit 139/SIGSEGV)", "file": "src/controller/testsuite/ (combined binary, crash site not yet isolated -- occurs after test_drive_async_config.c's liveness_reserved_* tests)", @@ -17336,6 +19109,8 @@ }, { "id": "bug-1554", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:36:00Z", "error_message": "workerlist / resval show maxswap=1024 and maxtmp=2048 for every host regardless of actual hardware, unless overridden by a per-host config resource entry", "file": "src/worker/src/xcecd.c", @@ -17355,6 +19130,8 @@ }, { "id": "bug-1634", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T08:06:37.286Z", "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test.c", @@ -17373,6 +19150,8 @@ }, { "id": "bug-1555", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:40:00Z", "error_message": "controller log spammed with '[ERROR] update_resval: Success' / 'update_resval: Resource temporarily unavailable' -- misleading, the real failure reason is never logged", "file": "src/controller/worker.c", @@ -17390,6 +19169,8 @@ }, { "id": "bug-1556", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T07:41:00Z", "error_message": "Every host using the documented hosts.yaml default (no max_slot, meaning 'auto-detect from CPU count') gets 0 scheduling slots -- permanently blocks the scheduler from placing any job on that host", "file": "src/xceadmin/src/xcecommon/mapping.py", @@ -17409,6 +19190,8 @@ }, { "id": "bug-1635", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T08:16:40.941Z", "error_message": "Significant refactor of ", "file": "src/schema/controller.capnp", @@ -17427,6 +19210,8 @@ }, { "id": "bug-882", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T08:18:53.065Z", "error_message": "Significant refactor of ", "file": "src/controller/opget.c", @@ -17445,6 +19230,8 @@ }, { "id": "bug-1636", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T08:19:26.708Z", "error_message": "Incorrect value in code", "file": "src/configs/schema/resources.schema.json", @@ -17463,6 +19250,8 @@ }, { "id": "bug-1637", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T08:20:21.520Z", "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", @@ -17481,6 +19270,8 @@ }, { "id": "bug-1638", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T08:22:12.974Z", "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test.c", @@ -17499,6 +19290,8 @@ }, { "id": "bug-1639", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:22:19.958Z", "error_message": "Significant refactor of ", "file": "src/worker/src/xcecd.c", @@ -17517,6 +19310,8 @@ }, { "id": "bug-1640", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:22:43.708Z", "error_message": "Incorrect value in code", "file": "src/configs/resources.yaml", @@ -17535,6 +19330,8 @@ }, { "id": "bug-1641", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:23:07.016Z", "error_message": "Significant refactor of ", "file": "src/configs/resources.yaml", @@ -17553,6 +19350,8 @@ }, { "id": "bug-1642", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:23:47.981Z", "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test.c", @@ -17571,6 +19370,8 @@ }, { "id": "bug-1643", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:25:25.166Z", "error_message": "Significant refactor of ", "file": "src/configs/schema/hosts.schema.json", @@ -17589,6 +19390,8 @@ }, { "id": "bug-1644", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:25:49.927Z", "error_message": "Significant refactor of ", "file": "src/configs/schema/queues.schema.json", @@ -17607,6 +19410,8 @@ }, { "id": "bug-892", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T09:26:55.689Z", "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", @@ -17625,6 +19430,8 @@ }, { "id": "bug-1645", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T10:45:55.441Z", "error_message": "Significant refactor of ", "file": "src/tools/deploy-ha.sh", @@ -17643,6 +19450,8 @@ }, { "id": "bug-1646", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T10:52:01.122Z", "error_message": "Incorrect value in code", "file": "src/tools/deploy-ha.sh", @@ -17661,6 +19470,8 @@ }, { "id": "bug-1557", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T12:00:00Z", "error_message": "xce_test's `worker` -> `join` command defaults to maxmem=64 -- discovered while writing/verifying detailed `help join` text", "file": "src/controller/testsuite/test.c (cmd_join_worker, hostattrs_t attrs initializer)", @@ -17682,6 +19493,8 @@ }, { "id": "bug-896", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T13:08:50.126Z", "error_message": "Significant refactor of ", "file": "src/xceadmin/src/xcecommon/mapping.py", @@ -17700,6 +19513,8 @@ }, { "id": "bug-897", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-13T13:11:27.503Z", "error_message": "Wrong reference: write_output should be dump_cluster_config", "file": "src/xceadmin/src/xceadmin/cli.py", @@ -17718,6 +19533,8 @@ }, { "id": "bug-1558", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T11:20:00Z", "error_message": "make test-backend fails standalone: backend/reconnect.go:6:10: fatal error: client.h: No such file or directory -- FAIL apiserver/backend [build failed], make: *** [Makefile:33: test-backend] Error 1. Hit during /kiro-validate-impl component-failover-adoption full-suite validation.", "file": "src/frontend/Makefile (test-backend target), src/frontend/apiserver/meson.build", @@ -17745,6 +19562,8 @@ }, { "id": "bug-1559", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T13:10:00Z", "error_message": "xce-build.sh scheduler's RUN_TESTS block never builds or runs the 4 new component-failover-adoption task-3.x scheduler test binaries (sched_scheduler_connect_test, sched_scheduler_backpressure_test, sched_scheduler_decision_publish_retry_test, sched_scheduler_decision_redelivery_test) -- it only builds/runs 'xce_scheduler sched_test'. They only got compiled in this session as a side effect of a separate worker build and had to be run directly, bypassing the canonical script.", "file": "src/tools/xce-build.sh, src/scheduler/testsuite/meson.build", @@ -17769,6 +19588,8 @@ }, { "id": "bug-1647", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T13:02:22.188Z", "error_message": "Significant refactor of ", "file": "src/frontend/apiserver/backend/clib.go", @@ -17787,6 +19608,8 @@ }, { "id": "bug-901", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T13:05:02.816Z", "error_message": "Significant refactor of ", "file": "src/frontend/Makefile", @@ -17805,6 +19628,8 @@ }, { "id": "bug-902", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T13:06:10.183Z", "error_message": "Wrong reference: 039 should be 1548", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17823,6 +19648,8 @@ }, { "id": "bug-1648", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T13:07:00.267Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17841,6 +19668,8 @@ }, { "id": "bug-1649", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T13:07:21.491Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/spec.json", @@ -17859,6 +19688,8 @@ }, { "id": "bug-1650", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T14:20:32.090Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/component-failover-adoption/tasks.md", @@ -17877,6 +19708,8 @@ }, { "id": "bug-906", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T14:20:51.010Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/component-failover-adoption/spec.json", @@ -17895,6 +19728,8 @@ }, { "id": "bug-1651", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T14:21:07.376Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/component-failover-adoption/spec.json", @@ -17913,6 +19748,8 @@ }, { "id": "bug-1560", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-15T22:54:35Z", "error_message": "test_recordstart_symlink_escape_replies_error (src/scheduler/testsuite/test_admin_request.c:850) FAILED: Expected 'invalid record file name' Was 'permission denied accessing record file'. Surfaced only now because bug-1559's fix made xce-build.sh's scheduler case run the full `meson test --suite scheduler` for the first time -- this test was never reached by the old sched_test-only invocation.", "file": "src/scheduler/testsuite/test_admin_request.c, src/scheduler/src/recording.c", @@ -17938,6 +19775,8 @@ }, { "id": "bug-1652", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T03:42:14.435Z", "error_message": "Missing error handling in unknown", "file": "reports/aug08-aug15-2026.md", @@ -17956,6 +19795,8 @@ }, { "id": "bug-910", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T03:57:07.719Z", "error_message": "Incorrect value in code", "file": "reports/aug08-aug15-2026.md", @@ -17974,6 +19815,8 @@ }, { "id": "bug-1653", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T04:29:20.063Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/config-upgrade-snapshot/spec.json", @@ -17992,6 +19835,8 @@ }, { "id": "bug-1654", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T04:51:52.141Z", "error_message": "Significant refactor of ", "file": "src/controller/testsuite/test_commit_barrier_reply.c", @@ -18010,6 +19855,8 @@ }, { "id": "bug-1655", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T04:54:26.304Z", "error_message": "Missing guard clause", "file": "src/controller/persist.c", @@ -18028,6 +19875,9 @@ }, { "id": "bug-1561", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "a02d4777", "timestamp": "2026-08-16T00:00:00.000Z", "error_message": "controller_unit_test: test_conf_trim_keeps_only_n_versions and test_conf_get_saved_tag_no_config (test_config_repl.c) FAIL during an unrelated ASAN controller build/test run (bug-451 fix session)", "file": "src/controller/testsuite/test_config_repl.c", @@ -18055,6 +19905,8 @@ }, { "id": "bug-912", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T05:04:02.838Z", "error_message": "Significant refactor of ", "file": "src/controller/persist.c", @@ -18073,6 +19925,8 @@ }, { "id": "bug-1656", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T05:22:23.386Z", "error_message": "Missing error handling in unknown", "file": "src/controller/testsuite/test_e2e_checkpoint.c", @@ -18091,6 +19945,8 @@ }, { "id": "bug-1657", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T05:25:35.899Z", "error_message": "Significant refactor of ", "file": "src/controller/persist.c", @@ -18109,6 +19965,8 @@ }, { "id": "bug-915", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T06:54:02.070Z", "error_message": "Missing guard clause", "file": "src/controller/ha.c", @@ -18127,6 +19985,8 @@ }, { "id": "bug-916", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T07:03:34.559Z", "error_message": "Significant refactor of ", "file": "src/controller/worker.c", @@ -18145,6 +20005,8 @@ }, { "id": "bug-917", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T09:54:06.127Z", "error_message": "Significant refactor of ", "file": "src/controller/worker.c", @@ -18163,6 +20025,8 @@ }, { "id": "bug-1658", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T11:12:53.822Z", "error_message": "Missing error handling in unknown", "file": ".kiro/specs/admin-config-tool/tasks.md", @@ -18181,6 +20045,8 @@ }, { "id": "bug-919", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T12:12:34.395Z", "error_message": "Incorrect value in code", "file": ".kiro/specs/ha-config-bootstrap/spec.json", @@ -18199,6 +20065,8 @@ }, { "id": "bug-1659", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-16T12:12:57.617Z", "error_message": "Significant refactor of ", "file": ".kiro/specs/resval-id-decoupling/spec.json", @@ -18217,6 +20085,8 @@ }, { "id": "bug-1562", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T03:40:00.000Z", "error_message": "3-node HA bootstrap never elects a primary: ctrl-1 (only node with local config) goes completely silent ~22s after boot -- no crash, no log output, cluster.log shows no further activity even after ctrl-2/ctrl-3 join minutes later. Downstream symptom: worker gets 'SSL_do_handshake: Broken pipe' in a loop and ctrl-2/ctrl-3 log repeated 'incorrect worker idx -1' ERRORs from handle_endpoint_error (harmless noise -- join.c's handle_join_ stamps priv->role=HostRole_worker before the ha_no_primary_known guard can reject the join, so u.idx is still -1 when the rejected connection is torn down).", "file": "src/controller/ha.c (ha_on_primary_ctrl_fd_rdhup, 5.10b debounce branch)", @@ -18241,6 +20111,8 @@ }, { "id": "bug-1660", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T03:45:49.314Z", "error_message": "Significant refactor of ", "file": "src/controller/ha.c", @@ -18259,6 +20131,8 @@ }, { "id": "bug-1661", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T04:59:39.812Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_term_safety_adapter.c", @@ -18277,6 +20151,8 @@ }, { "id": "bug-1662", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T05:01:33.795Z", "error_message": "Function not marked async", "file": "src/controller/persist.c", @@ -18295,6 +20171,8 @@ }, { "id": "bug-1563", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T04:59:00.000Z", "error_message": "40-round chaotic /xce-test-ha soak: ctrl-2's own log shows 309,671 'persist: flusher fdatasync FAILED (errno=22 Invalid argument) -- durable_lsn held at 163 (tip=164 NOT made durable)' ERROR lines, 706 of them landing within a single 705ms window (04:52:54.000-04:52:54.705) on the freshly-promoted primary -- a sustained sync failure the flusher never recovered from or backed off on for the rest of the run. The soak's own verdict still reports PASS (committed_divergence=0, committed_job_loss=0) because the other two standbys' acks alone satisfy majority and the bug-924 fdatasync-error exclusion swallows the count -- but this specific pattern (a stuck LSN, EINVAL not the documented EIO, hundreds of retries per second) is a distinct, real defect from the known overlayfs sandbox quirk, not just noise.", "file": "src/controller/persist.c (fsync_worker)", @@ -18320,6 +20198,8 @@ }, { "id": "bug-1663", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T06:41:48.625Z", "error_message": "Significant refactor of ", "file": "src/tools/deploy-ha.sh", @@ -18338,6 +20218,8 @@ }, { "id": "bug-927", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T06:44:51.034Z", "error_message": "Significant refactor of ", "file": ".claude/skills/xce-deploy-ha/SKILL.md", @@ -18356,6 +20238,8 @@ }, { "id": "bug-1664", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T06:51:13.785Z", "error_message": "Significant refactor of ", "file": ".claude/skills/xce-build/SKILL.md", @@ -18374,6 +20258,8 @@ }, { "id": "bug-1665", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T06:52:21.795Z", "error_message": "Significant refactor of ", "file": "src/xceadmin/pyproject.toml", @@ -18392,6 +20278,8 @@ }, { "id": "bug-1666", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T07:00:05.291Z", "error_message": "Significant refactor of ", "file": "src/tools/deploy-ha.sh", @@ -18410,6 +20298,8 @@ }, { "id": "bug-1667", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T07:00:21.077Z", "error_message": "Incorrect value in code", "file": "src/tools/deploy-ha.sh", @@ -18428,6 +20318,8 @@ }, { "id": "bug-932", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T07:15:26.246Z", "error_message": "Function not marked async", "file": "src/controller/testsuite/test_commit_durability.c", @@ -18446,6 +20338,8 @@ }, { "id": "bug-1668", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T07:15:58.937Z", "error_message": "Wrong reference: fire_count should be g_durable_advance_fire_count", "file": "src/controller/testsuite/test_commit_durability.c", @@ -18464,6 +20358,8 @@ }, { "id": "bug-1564", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T07:15:00.000Z", "error_message": "controller_unit_test: test_durable_majority_commit_end_to_end FAIL: Expected 0 Was -110 (commit_submit(WAIT) must return 0 after durable-majority) -- deterministic 3/3 direct runs of the binary, not a flake. Surfaced while live-testing an unrelated /xce-deploy-ha enhancement, right after rebuilding controller on top of bug-1563's fix.", "file": "src/controller/persist.c (persist_flush)", @@ -18488,6 +20384,8 @@ }, { "id": "bug-1565", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T07:16:00.000Z", "error_message": "src/tools/xce-build.sh's setup_build() does an unconditional `rm -rf \"$BUILD_DIR\"` + fresh `meson setup` for EVERY component case (controller, worker, xceadmin, etc), not just when actually needed -- running `/xce-build xceadmin` (or any single-component target) after `/xce-build controller` in the SAME build dir silently deletes the already-built xce_controller/xce_test/etc binaries, leaving only that one component's fresh output. Discovered live while enhancing /xce-deploy-ha: a working xce_controller from an earlier turn vanished after an unrelated `/xce-build xceadmin` run in the same xce.build dir, breaking a live-deploy test with 'No such file or directory' for the controller binary.", "file": "src/tools/xce-build.sh (setup_build)", @@ -18508,6 +20406,8 @@ }, { "id": "bug-936", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:02:07.121Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/persistence.md", @@ -18526,6 +20426,8 @@ }, { "id": "bug-1669", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:03:27.006Z", "error_message": "Significant refactor of ", "file": ".kiro/steering/build-deployment.md", @@ -18544,6 +20446,8 @@ }, { "id": "bug-938", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:18:40.345Z", "error_message": "Significant refactor of ", "file": ".claude/skills/xce-build/SKILL.md", @@ -18562,6 +20466,8 @@ }, { "id": "bug-939", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:19:13.328Z", "error_message": "Wrong operator: && should be <<", "file": ".claude/skills/xce-build/SKILL.md", @@ -18580,6 +20486,8 @@ }, { "id": "bug-940", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:19:30.812Z", "error_message": "Incorrect value in code", "file": ".kiro/steering/build-deployment.md", @@ -18598,6 +20506,8 @@ }, { "id": "bug-1670", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:24:49.130Z", "error_message": "Incorrect value in code", "file": "src/xceadmin/tests/test_cli.py", @@ -18616,6 +20526,8 @@ }, { "id": "bug-1566", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T08:30:00.000Z", "error_message": "User-reported: `./scripts/xceadmin scheduler record status` (from a /xce-deploy-ha deployment) failed with 'Error: cannot connect to localhost:9991: [Errno 111] Connection refused' -- the deployment's actual --peers were localhost:9901/9902/9903, not 9991.", "file": "src/xceadmin/src/xceadmin/cli.py, src/tools/deploy-ha.sh", @@ -18637,6 +20549,8 @@ }, { "id": "bug-1567", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T09:00:00Z", "error_message": "xce_scheduler -i -s : WARN 'failed to resolve job picker plugin (rc=-2)' on load; all pre-existing jobs stay PEND forever, nothing ever dispatches in simulation/interactive mode", "file": "src/scheduler/src/sim.c, src/scheduler/src/sim_loop.c", @@ -18659,6 +20573,8 @@ }, { "id": "bug-1568", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T09:00:00Z", "error_message": "xce_scheduler -i -s : 'info worker ' shows total=0.00 available=0.00 for every resource, even though the live cluster (xce_test resval/workerlist) reports real non-zero values for the same worker -- CONFIRMED ROOT CAUSE (superseding this entry's earlier 'architecture gap' theory, which was wrong -- corrected after the user pointed to worker.c's real mark_host_res()/update_val_res() write path)", "file": "src/scheduler/src/repl_info.c, src/scheduler/src/snapshot_wire.c, src/scheduler/src/snapshot_load.c, src/controller/sched.c", @@ -18685,6 +20601,8 @@ }, { "id": "bug-1671", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T09:14:31.784Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/sim_loop.c", @@ -18703,6 +20621,8 @@ }, { "id": "bug-946", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T09:14:35.731Z", "error_message": "Wrong reference: sim_register_default_picker should be sim_bootstrap_picker", "file": "src/scheduler/src/sim_loop.c", @@ -18721,6 +20641,8 @@ }, { "id": "bug-947", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T10:14:50.237Z", "error_message": "Wrong reference: test_job_rusage_applies_state_when_hook_is_null should be test_workers_push_twice_updates_existing_worker_in_place", "file": "src/scheduler/testsuite/test_core_events.c", @@ -18739,6 +20661,8 @@ }, { "id": "bug-1672", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T10:16:21.240Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/core_events.c", @@ -18757,6 +20681,8 @@ }, { "id": "bug-949", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T10:31:46.356Z", "error_message": "Significant refactor of ", "file": "src/scheduler/src/repl_info.c", @@ -18775,6 +20701,8 @@ }, { "id": "bug-1569", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T10:50:00Z", "error_message": "TWO related, still-open problems on a live normal-mode (non-sim) deployment, found while verifying bug-1567/bug-1568's fixes actually let a real job dispatch: (A) 'info worker'/xceadmin-recorded snapshot still shows 0.00 for 'mem'/'r1m'/'r15m'/'r15s'/'ut'/'pg'/'ls'/'it'/'swp'/'io'/'tmp' (the 11 periodic resource_sync 'load index' metrics) even though xce_test's live 'resval -n mem -m pc13v' / 'resval -n r15s -m pc13v' show correct non-zero values on the SAME worker at the SAME time -- hostattrs-derived resources (slots/maxcpus/maxmem/maxswap/maxtmp/cpuf/ndisks, all fixed correctly by bug-1568) display fine; only the periodic/dynamic ones are still wrong. (B) A submitted job (single or array, queue 'xce', resreq 'type == any', worker 'pc13v' with 22 real slots available) never leaves PEND -- xce_sched_driver_run_normal_mode_round logs 'decision = (nil)' on EVERY round, forever (60+ consecutive rounds observed), meaning the scheduler never produces even a PEND decision, let alone DISPATCH, for any job.", "root_cause": "(A) FULLY ROOT-CAUSED AND FIXED. The periodic resval update path is DIFFERENT from the join-time hostattrs push bug-1568 fixed: src/controller/opupd.c's do_update_resvals() applies a worker's periodic report locally using the CORRECT connection-resolved worker `w`, but then re-fires HOOK_DETAIL_RESOURCE_UPDATE with the wire `resvals` list's OWN unmodified `.host` field, forwarded unchanged by src/controller/sched.c's sched_resource_hook_fun() -> notify() -> scheduler's handler.c/driver.c/core_events.c XCE_CORE_EVENT_RESVALS_DELTA case, which does `sched_get_worker(core, rv->host)`. The true source of the wrong `.host` was WORKER-side, not xcecd_virtual-specific: src/worker/src/resource_sync.c's collect_load_indices() tagged every periodic report with gethostname() (the raw OS/container hostname -- 'xce' in this sandbox) instead of xcecd_ctx.virtual_name (the worker's actual join identity from -n/--name, e.g. 'pc13v'), which is what every OTHER worker-keyed lookup (mark_host_res()/sched_get_worker()) uses. FIXED (this session): collect_load_indices() now tags .host with xcecd_ctx.virtual_name, falling back to gethostname() only when virtual_name is unset. IMPORTANT CORRECTION to this entry's earlier note: after applying this fix, 'info worker'/'resval' via the INTERACTIVE REPL (`xce_scheduler -i -s `) appeared to still show 0.00 for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp -- this was a TEST-METHODOLOGY ARTIFACT, not a residual bug: the REPL starts PAUSED at the initial snapshot and requires an explicit `step`/`continue` command to advance the simulated clock past the recorded resvalsDelta ReplayEvent before querying state; querying immediately after load (as this investigation's own test scripts did, e.g. `echo -e \"info worker pc13v\\nquit\"` with no step/continue first) only ever shows the pre-periodic-report baseline. Live-confirmed end-to-end via -vvv scheduler.log (schedUpdateResvals notify arrives, zero 'unknown worker'/'unknown resource' WARNs, xceadmin scheduler dump shows the resvalsDelta event correctly captured with all 11 metrics @pc13v) AND via `step N` in the REPL (mem=16786444288.00, r1m=0.64, r15m=0.99, r15s=0.64, ut=2.08, pg=463.20, it=3341.19, tmp=65732259840.00 -- all correct; ls/swp/io legitimately 0 on this idle sandbox). (B) FULLY ROOT-CAUSED AND FIXED -- THREE independent causes stacked on the same symptom, each one masking the next (each was only reachable once the prior one was fixed), all live-confirmed via gdb backtraces on real SIGSEGVs: B1 (legacy xce_sched_fcfs picker active) was RULED OUT via the main.c diagnostic INFO log -- confirmed 'active job picker plugin: builtin_fcfs'. B2 (resource-starvation) was RULED OUT -- dynamic resvals confirmed correctly flowing into live core state (thread A's fix), yet decisions stayed nil. The TRUE causes: (B-i) scheduler.c's sched_init_queues() -- the scheduler's ONE-TIME synchronous initial-config-push handler (handler.c's scheduler_req_handler, Request_body_configpushmsg) -- only called the pre-scheduler-core-rewrite plugin_queue_update() fan-out, which reaches a plugin's OLD job_elect_fn-era `.hooks.queue_hook` only; builtin_fcfs has no `.hooks`, only `picker_iface`, so on_queue_update() was NEVER called for any of the cluster's initial queues. builtin_fcfs's own internal per-queue candidate registry (fcfs_plugin_data_t.queues) stayed permanently empty from process start, so next_job() found zero candidates every session forever -- exactly 'decision=(nil) forever', regardless of resources/readiness. Every OTHER queue-registration path (core_events.c's core_upsert_queue(), used by QUEUE_UPDATE and a later CONFIG_PUSH config upgrade) already fired on_queue_update() correctly -- this was the one remaining call site still skipping it. (B-ii) Once B-i was fixed and a job first reached the picker, core_events.c's core_build_job() was found to never construct job->schedobj -- a DELIBERATE, DOCUMENTED omission from the scheduler-core rewrite ('no picker hook reads job->schedobj... belongs to the allocation path (scheduler-allocation spec), out of this task's scope') whose assumption was wrong: scheduler.c's sched_allocator() thread (plugin_job_election() + sched_make_allocation()) dereferences job->schedobj directly with no NULL check, and no 'scheduler-allocation spec' construction step exists anywhere in the live codebase -- every job built via JOBS_PUSH has had schedobj==NULL since the rewrite, reproduced live as a SIGSEGV in plugins/limit.c's limit_job_election() (obj->configver dereference on NULL obj), gdb-confirmed. (B-iii) Once B-i and B-ii were both fixed, a job reached genuine allocation (sched_make_allocation returned 1, a real DISPATCH) for the first time ever, and crashed a THIRD time inside resource.c's put_resvals() (gdb: NULL deref on xvals[0]) during post-dispatch resource-charge accounting -- an off-by-one: `if (idxs[i] <= 0) continue;` incorrectly treated resource index 0 ('slots', almost always the first resource resmgr registers) as an invalid sentinel, so the 'create a zero-value entry if missing' step was skipped for it, leaving xvals[0] NULL for the unconditional `xvals[i]->u.fvalue = vals[i]` write two lines later. get_resvals() (its read-side sibling in the same file) has no such guard, confirming this was a bug, not intentional design.", @@ -18806,6 +20734,8 @@ }, { "id": "bug-1570", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T23:45:00Z", "error_message": "resource-value-verification task 2.1 round-1 REJECTED: `resupdate -n mem -v 5 -v 8` silently overwrote to 8 (last-write-wins) instead of rejecting the duplicate -v and keeping the first value 5", "file": "src/controller/testsuite/test.c", @@ -18831,6 +20761,8 @@ }, { "id": "bug-1571", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T00:00:00.000Z", "error_message": "apiserver logs \"Backend not configured -- running in stub mode\" and never calls RegisterToBackend, even though the generated app.yaml backend.peers list is fully populated with a valid HA candidate list.", "file": "src/tools/deploy-ha.sh", @@ -18852,6 +20784,9 @@ }, { "id": "bug-1572", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "c9acd502 (bug-1585)", "timestamp": "2026-08-18T01:30:00Z", "error_message": "xce_scheduler silently exits (no ASAN report captured -- backgrounded via setsid nohup ... >/dev/null 2>&1) a few seconds after a worker's SECOND periodic resource_sync report for the same resource(s); first report processes fine, second triggers the crash with zero \"resval applied\" TRACE lines logged for that batch and \"handle_scheduler_leave\" observed on the controller ~1s later.", "file": "src/scheduler/src/driver.c, src/scheduler/src/core_events.c", @@ -18878,6 +20813,8 @@ }, { "id": "bug-1573", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-18T01:20:00Z", "error_message": "resource-value-verification task 4.1 round-1 REJECTED by independent review: the RESVALS_DELTA TRACE logging in xce_core_push_events() assumed any non-string Value_u_which tag was Value_u_fvalue and read rv->val->u.fvalue unconditionally in the else branch -- Value_u_bvalue (and ivalue32/uvalue32/ivalue64/uvalue64) are real, reachable tags at this exact path (worker.c's mark_host_res() sets Value_u_bvalue for any XCE_VAL_BOOLEAN resource, routed through the same update_val_res()/resupdate RPC) that would read the wrong C union member.", "file": "src/scheduler/src/core_events.c", @@ -18903,6 +20840,9 @@ }, { "id": "bug-1574", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test resupdate string/type crash: set_resval() lacks svalue case, vals[] uninitialized (src/harness/test.c)", "timestamp": "2026-08-17T18:55:00Z", "error_message": "xce_test segfaults (SIGSEGV, exit 139) immediately on 'resupdate -n model -v ' (and by the same code path, -n type) run in the worker context against a live controller -- no output printed before the crash", "file": "src/controller/testsuite/test.c", @@ -18930,6 +20870,8 @@ }, { "id": "bug-1575", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-18T03:15:00Z", "error_message": "run-resource-verify.sh's trap-based cleanup() (EXIT/INT/TERM) non-deterministically left the scheduler and/or apiserver process running after the script was interrupted mid-deployment via an external SIGTERM -- confirmed processes still alive (e.g. `xce_scheduler --peers ... --logfile .../scheduler.log`, `/tmp/apiserver -d .../apiserver`) seconds after the orchestrator's own `wait` returned with exit code 143. The SAME cleanup() code, reached via a normal fail()/exit-1 path (no external signal involved) inside the same single foreground invocation, worked perfectly every time (verified zero leftover processes).", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", @@ -18956,6 +20898,8 @@ }, { "id": "bug-1576", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-18T03:15:00Z", "error_message": "Multiple interactive diagnostic Bash calls (`pgrep -af \"\"`, `pkill -9 -f -- \"\"`, `ps -eo pid,args | grep \"\"`) unexpectedly matched -- and in one case SIGKILLed -- the CURRENTLY-RUNNING Bash tool call's own wrapper shell process, not just the intended target process(es). One instance: `pkill -9 -f -- \"rrv_dummy_marker_TEST1|...\"` killed the invoking Bash call itself mid-script, truncating all output after the pkill line with no further commands executing (looked like a mysterious 'exit code 1, script just stops' failure).", "file": "N/A (diagnostic/interactive Bash tool usage, not a repo source file)", @@ -18979,6 +20923,8 @@ }, { "id": "bug-1577", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-17T00:00:00Z", "error_message": "No explicit error -- discovered via live testing task 6.1 (resource-value-verification): xce-deploy-ha's deploy-ha.sh Step 3 opt-in-by-presence worker branch (predates this feature) generates start-worker.sh whenever a worker binary (xcecd_virtual) happens to exist alongside the controller build, and generated start-all.sh always launches it. run-resource-verify.sh's own deployment (task 5.1+6.1) unconditionally hit this since a full `/xce-build all` leaves xcecd_virtual built. Result: a REAL worker daemon ('worker-1') ran for the whole duration of every xce-test-resources run, silently violating Requirement 2.1's design intent ('no real worker daemon process required') and periodically re-reporting its own resources (~every 15s) -- a live risk of triggering the still-open bug-1572 (scheduler double-free/crash on a resource's 2nd periodic report) if a run happened to last that long.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", @@ -19001,6 +20947,8 @@ }, { "id": "bug-1578", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-18T04:45:00Z", "error_message": "Controller (xce_controller) reproducibly exits via SIGSEGV (si_addr=NULL, si_code=1/SEGV_MAPERR) within ~1s of the last resupdate follow-up verification reply completing, when the synthetic worker then disconnects ('quit'). ROOT-CAUSED via live repro (2/2) with an LD_PRELOAD signal-handler shim (gdb reliably masked the crash for both ASAN and debug/-O0 builds -- ptrace overhead perturbs timing enough to avoid the window; ~20s under gdb = no crash, every time). Resolved backtrace (addr2line against the debug-symbol'd ASAN binary): encode_NameValue <- encode_NameValue_list <- encode_Worker <- capnp_encode_worker (persist.c) <- persist_worker_hook_fun <- xce_run_hooks <- mark_worker_unavail <- handle_worker_leave <- handle_endpoint_error (cluster.c) <- io_worker_dispatch_batch <- io_worker (netio.c).", "file": "src/controller/sched.c (sched_worker_change(), ~line 548-553) -- crash SITE is persist.c/controller.capnp.c (capnp_encode_worker/encode_Worker/encode_NameValue_list/encode_NameValue) but the DEFECT is in sched.c", @@ -19043,6 +20991,9 @@ }, { "id": "bug-1579", + "status": "wontfix", + "status_updated": "2026-09-24", + "status_note": "not a bug: intended host-attrs reset on disconnect; script fixed b673b7c9", "timestamp": "2026-08-18T09:00:00Z", "error_message": "run-resource-verify.sh's new controller-side check (task 6.2, query_controller_resvals()) reported a live MISMATCH for the default resource set's 'cpuf' resource: 'controller check: cpuf -> MISMATCH: injected=9.5 observed=0.000000', even though inject_resources()'s own resupdate call reported 'cpuf=9.5 -> accepted' moments earlier in the same run.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh (not a bug in this file's new logic -- see root_cause); src/controller/worker.c (add_hostattrs_resvals(), apply_worker_dead_mutation()) is the production code responsible for the behavior being (correctly) observed", @@ -19074,6 +21025,9 @@ }, { "id": "bug-1580", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "intermittent controller_unit_test \"2 Failures\" with all cases PASS; not investigated", "timestamp": "2026-08-18T10:39:52Z", "error_message": "meson test 'xce:controller / controller test' (controller_unit_test) reports '1104 Tests 2 Failures 0 Ignored' / exit status 2, but grep of the full raw testlog for ':FAIL:' or a per-test [FAIL] marker finds ZERO matches -- every individual Unity test case visibly printed PASS. Reproduced deterministically 2/2 times (once via meson test, once via a direct standalone re-run of the binary with ASAN_OPTIONS=detect_leaks=0), unlike bug-1553's documented ~1-in-3 intermittent SIGSEGV (exit 139) signature -- this is exit 2 with no crash indication, a distinct manifestation.", "file": "src/controller/testsuite/ (combined controller_unit_test binary; discovered running xce-build.sh all for the resource-value-verification bug-1579 follow-up fix -- zero C files touched this session, confirmed via git status)", @@ -19097,6 +21051,8 @@ }, { "id": "bug-1581", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-18T12:10:00Z", "error_message": "aggregate_verdict() synthetic self-test scenario (c) [one layer UNREACHABLE] reported unreachable=2 instead of 1, and the apiserver layer of an accepted+reachable record showed api_ok=2.5/api_obs=(empty) after IFS=$'\\t' read -- fields silently shifted by one position.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", @@ -19117,6 +21073,8 @@ }, { "id": "bug-1582", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-18T13:00:00Z", "error_message": "run-resource-verify.sh task 7.1 round-1 review REJECTED: independent reviewer killed the live scheduler process mid-run (after deployment confirmed up, before its trace-log poll completed) and got VERDICT line `unreachable=0 mismatches=1` plus FAILURE line `status=MISMATCH ... observed=` -- should have been `unreachable=1 mismatches=0` and `status=UNREACHABLE ... observed=`.", "file": ".claude/skills/xce-test-resources/run-resource-verify.sh", @@ -19143,6 +21101,9 @@ }, { "id": "bug-1583", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "apiserver stores resource names the controller rejected (handler.go Put, opupd.c hook list unfiltered)", "timestamp": "2026-08-18T13:35:00.000Z", "error_message": "Live task 8.1 run (resource-value-verification, comprehensive e2e validation): injecting resupdate -n totally-fake-resource-xyz -v 1 (a deliberately-unregistered resource name) alongside 9 genuinely-registered names -- the controller correctly rejects it (resval_typeresolve/update_resval's key_find(c->resmgr,...) returns -ENOENT, never written to w->host.resvals, confirmed empty via a direct resval query) and the scheduler correctly discards it (core_events.c's RESVALS_DELTA handler logs 'xce_core_push_events RESVALS_DELTA: unknown resource totally-fake-resource-xyz, skipping' at WARN and never applies it) -- but the apiserver's OnResourceLocationUpdated (src/frontend/apiserver/backend/handler.go:92) has NO such validation: it logged 'resource-location stored {\"resource\": \"totally-fake-resource-xyz\", \"host\": \"resverify-worker\", \"value\": 1}' and unconditionally called store.ResourceLocations.Put() for it, live-confirmed in apiserver.log.", "file": "src/frontend/apiserver/backend/handler.go", @@ -19165,6 +21126,8 @@ }, { "id": "bug-1584", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T02:47:00Z", "error_message": "User-reported: `xceadmin scheduler record start/status/stop` always shows 'started at:' near 1970-01-01 (e.g. '1970-01-04T20:11:26') instead of the real start time, even though 'file'/'events'/'recording' fields are correct.", "file": "src/scheduler/src/core_clock.c, src/scheduler/include/core.h, src/scheduler/src/core.c, src/scheduler/src/scheduler.c, src/scheduler/testsuite/test_clock.c", @@ -19190,6 +21153,8 @@ }, { "id": "bug-1585", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T02:47:00Z", "error_message": "User-reported, live `xce_scheduler -i -s /tmp/x1.replay` REPL session against a real .replay file recorded from a live normal-mode deployment: `info summary`/`info worker all` show '0 jobs (0 pending, 0 running)' immediately after load, even though the scheduler log and xce_test confirm jobs were actually submitted/present during that recording. Startup also logs a WARN: `xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load`.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_RESVALS_FULL/RESVALS_DELTA handlers -- root cause), src/scheduler/src/main.c (--record-snapshot-interval CLI wiring), src/scheduler/src/repl.c (banner clarification), src/scheduler/testsuite/test_resval_ownership.c (new regression test), src/scheduler/testsuite/test_core_events.c/test_core_full_cycle.c/test_driver_decision_cb.c/test_driver_session_ordering.c/test_driver_translate.c (dup_val link stub + stale-comment/fixture fix)", @@ -19225,6 +21190,9 @@ }, { "id": "bug-1586", + "status": "wontfix", + "status_updated": "2026-09-24", + "status_note": "not a bug: resval getters mode-agnostic, live values confirmed non-zero", "timestamp": "2026-08-19T02:47:00Z", "error_message": "User-reported, same live REPL session: `info worker all` shows `total=0.00 available=0.00` for mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io (the periodic/dynamic resval metrics) on a worker that should have real non-zero values. User's own hypothesis: this worker was running in 'virtual host mode' (xcecd_virtual, a synthetic/mocked worker used for sandbox testing, not a real xcecd against real host hardware) which may never populate these metrics with real readings; wants to validate against a real (non-virtual) xcecd worker instead, but notes the sandbox/container this session runs in needs a fix to make `sudo` work before that's possible (real xcecd's periodic resource collection -- disk/load/mem readings -- may need elevated privileges not currently available in this container).", "file": "src/worker/src/resource_sync.c (collect_load_indices), src/worker/src/node_information.c (the 10+1 node_info_get_* getters), src/worker/meson.build (xcecd vs xcecd_virtual target definitions)", @@ -19254,6 +21222,9 @@ }, { "id": "bug-1587", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "c9acd502", "timestamp": "2026-08-19T02:47:00Z", "error_message": "User feedback (UX, not a functional defect): the REPL's startup line `[sim] loaded: 0 jobs, snapshot @ ts=331886899213871` prints the snapshot timestamp as a raw nanosecond integer -- would be more readable as a human-readable date/time.", "file": "src/scheduler/src/repl.c:183", @@ -19278,6 +21249,8 @@ }, { "id": "bug-1588", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T02:47:00Z", "error_message": "User feedback (UX/observability, not a functional defect), from logs/ctrl-1.log: job IDs printed in controller log lines are large, non-sequential-looking 64-bit numbers (e.g. `4631574334564990976`, `4631574334564990977`, `4631574334633148416`) that are hard to visually scan/compare/diff -- would be better rendered in a more human-readable form. Affected call sites (src/controller/job.c): `dispatch_a_job_()` (job.c:841, 'dispatch_a_job_ jobid '), `on_job_start()` (job.c:1662, 'on_job_start jobid '), `on_job_finish()` (job.c:1766, 'on_job_finish job finish jobid exit code '). Likely other job.c/controller log call sites print the same raw jobid and would benefit from the same treatment (not yet enumerated).", "file": "src/controller/job.c (dispatch_a_job_ ~841, on_job_start ~1662, on_job_finish ~1766) -- likely more sites, not yet grepped exhaustively; PLUS (open, not yet fixed) src/worker/src/{xcecd_dbus_manager,xcecd,main,xcecd_worker,controller_connector,uv_ipc,xcecd_cgroups,xcecd_worker_utils}.c and src/controller/testsuite/test.c -- see fix field for full enumeration", @@ -19302,6 +21275,8 @@ }, { "id": "bug-1589", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T03:20:00Z", "error_message": "User-reported, live 3-node deployment log analysis: a 5-element job array (base_jobid=0-7-1787111880-0, array_size=5) was fully expanded and all 5 elements were dispatched by the controller (job.c's dispatch_a_job_ fires for [0]-[4], confirmed in logs/ctrl-1.log) and elected by the scheduler's limit plugin (logs/scheduler.log's limit_job_election fires for [0]-[4]), but only element [0] ever produced an on_job_start log line (job.c:1673) in either log file -- elements [1]-[4] were dispatched but never reported as started. User's own hypothesis: may be related to the worker running in 'virtual host mode' (xcecd_virtual) rather than a real xcecd worker.", "file": "src/worker/src/xcecd_dbus_manager.c (dbus_workers_update() -- THE fix), src/worker/src/main.c (confirms master_job_status_callback/ipc_master path is UNREACHABLE for xcecd_virtual -- and for real xcecd too), src/worker/testsuite/test_dbus_workers_update.c (new regression test), src/controller/job.c (on_job_start/dispatch_a_job_ -- read/traced, confirmed innocent, not touched)", @@ -19329,6 +21304,8 @@ }, { "id": "bug-1590", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T03:40:00Z", "error_message": "User-reported: `(sim) info worker all` in the scheduler's interactive REPL shows `resource type: total=0.00 available=0.00` and `resource model: total=0.00 available=0.00` -- but `type`/`model` are STRING-valued resources (e.g. OS type, hardware model name), not numeric, so 0.00 is meaningless. User also notes boolean-typed resources exist and asked to verify resource-value type-correct formatting broadly, across both REPL output and log lines, not just this one command.", "file": "src/scheduler/src/repl_info.c (repl_info_worker_one ~287-335, repl_info_resource_one ~410-496); src/scheduler/src/core_events.c (~838-861, TRACE 'resval applied'); src/controller/worker.c (~107-119); src/controller/dataex.c (~86-90)", @@ -19354,6 +21331,8 @@ }, { "id": "bug-1591", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T03:45:00Z", "error_message": "User-reported: `(sim) info queue all` in the scheduler's interactive REPL prints raw packed jobids instead of the human-readable form, e.g. `[sim] queued job: jobid=4645085138445664257 priority=0` instead of the 'cluster-apiserver-timestamp-sequence' form (e.g. '3-1-1755590313-976') already used elsewhere in this codebase (bug-1588 fixed the same class of issue in src/controller/job.c's TRACE lines).", "file": "src/scheduler/src/repl_info.c (queued job: ~230, assigned job: ~356), src/scheduler/src/repl_break.c (~665), src/scheduler/src/sim_loop.c (~379,383,387), src/scheduler/src/sim_verify.c (~293,325,335,346,373,380,389), src/scheduler/src/driver.c (~652), src/scheduler/src/core_events.c (~872,907,979), src/scheduler/src/recording_session.c (~173) -- full raw-jobid-print inventory for the scheduler tree, not yet fixed", @@ -19378,6 +21357,8 @@ }, { "id": "bug-1592", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T07:38:00Z", "error_message": "grep/tail: Permission denied on $TOP/logs/xcecd.log after starting the real (production) xcecd worker via sudo", "file": "src/worker/src/main.c", @@ -19400,6 +21381,8 @@ }, { "id": "bug-1593", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T08:00:00Z", "error_message": "User-reported (live REPL testing): `info resource all` and `info resource ` aggregate every worker's per-host resource values (total=sum(available), available=sum) for EVERY resource, including host-based ones like cpus/mem -- a fabricated, meaningless number for a per-host consumable already correctly shown by `info worker `. User asked that `info resource` instead show resource metadata and be restricted to globally-shared resources only.", "file": "src/scheduler/src/repl_info.c (repl_info_resource_one, repl_info_resource)", @@ -19423,6 +21406,8 @@ }, { "id": "bug-1594", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T08:00:00Z", "error_message": "User-reported: in the scheduler's --interactive REPL, `continue` with no breakpoints set runs the simulation to completion and the process then exits -- the operator never gets a chance to inspect the final simulated state (info job/queue/worker/resource/summary). Related: the simulator should never terminate except via an explicit `quit`.", "file": "src/scheduler/src/sim_loop.c (sched_sim_driver_run), src/scheduler/src/repl.c (repl_dispatch, new xce_repl_on_replay_complete), src/scheduler/include/repl.h, src/scheduler/src/repl_internal.h", @@ -19445,6 +21430,8 @@ }, { "id": "bug-1595", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T08:00:00Z", "error_message": "User-reported: in each `step`, it would be better to show the decisions/events that occur in the current session in the sequence they actually occur -- currently only decisions are printed, input events (JOB_PUSH, JOB_STATE_CHANGE, WORKERS_PUSH, etc.) are completely silent unless a breakpoint happens to match one.", "file": "src/scheduler/src/repl_break.c (xce_repl_on_event)", @@ -19465,6 +21452,8 @@ }, { "id": "bug-1596", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T08:00:00Z", "error_message": "User-reported: `info job \"0-15-1787129769-0[0]\"` (jobid copied from `info jobs`' own output, wrapped in double quotes out of habit) returns \"[sim] info job: no such job 0 (0-0-0-0)\" even though the job is genuinely present -- the identical jobid without quotes works.", "file": "src/scheduler/src/repl.c (repl_split_tokens)", @@ -19487,6 +21476,9 @@ }, { "id": "bug-1597", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "obsolete: src/frontend/Makefile removed (6fabdba9)", "timestamp": "2026-08-19T00:00:00Z", "error_message": "backend/reconnect.go:6:10: fatal error: client.h: No such file or directory (compilation terminated) -> make: *** [Makefile:33: build-server] Error 1", "file": "src/frontend/Makefile (build-server target)", @@ -19511,6 +21503,8 @@ }, { "id": "bug-1598", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-19T00:00:00Z", "error_message": "AddressSanitizer: stack-use-after-return on address inside `sim` (xce_sched_sim_driver_t) at cbuf_get, thread T2 (sim_picker_thread_fn), reading test_run_deactivates_tlog_virtual_clock_logging_on_completion's stack frame after it returned", "file": "src/scheduler/testsuite/test_sim_event_loop.c", @@ -19533,6 +21527,9 @@ }, { "id": "bug-1599", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "sim_loop.c comment reworded", "timestamp": "2026-08-20T02:50:00Z", "error_message": "sched_no_bare_printf_test / test_scheduler_src_has_no_bare_printf_calls FAIL: '.../scheduler/src/sim_loop.c calls bare printf() -- xce_scheduler link-wraps printf ...' -- even though sim_print_decision() in sim_loop.c already correctly uses fprintf(stdout, ...) for all 3 PEND/DISPATCH/PREEMPT lines (verified via grep: lines 398/402/406 all fprintf(stdout, ...), zero bare printf( calls in actual code).", "file": "src/scheduler/testsuite/test_no_bare_printf.c (new regression test) flags src/scheduler/src/sim_loop.c", @@ -19555,6 +21552,9 @@ }, { "id": "bug-1600", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "ddee884c", "timestamp": "2026-08-19T21:05:00Z", "error_message": "xce_scheduler crashed with free(): invalid pointer / ASAN bad-free during controller failover retry (3x config-push retry loop); 6 crash-loop occurrences on 2026-08-17, 2 more on 2026-08-19 (core dumps confirmed via gdb bt full and coredumpctl, crash site consistently makeGroup() at src/lib/group.c:234)", "file": "src/lib/group.c", @@ -19584,6 +21584,8 @@ }, { "id": "bug-1601", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T07:26:00Z", "error_message": "User request: info resource/info worker did not distinguish consumable from non-consumable resources -- repl_info_worker_one() showed a fabricated total=/available= pair for EVERY numeric per-worker resource regardless of XCE_RESF_CONSUMABLE, and repl_info_resource_one() never reported a used/in-use amount for a CONSUMABLE GLOBAL/SHARED resource at all; the resource definition line was also missing ordtype.", "file": "src/scheduler/src/repl_info.c", @@ -19609,6 +21611,8 @@ }, { "id": "bug-1602", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T08:30:00Z", "error_message": "User request: scheduler simulation mode should show event time and session when events occur, and loading a replay file should show its total session count up front.", "file": "src/scheduler/src/repl_break.c, src/scheduler/src/sim.c", @@ -19633,6 +21637,8 @@ }, { "id": "bug-1603", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T09:20:00Z", "error_message": "build-time-versioning task 2.2 REJECTED by reviewer: `nix build .#xcecore_debug` (no flags, the exact documented/task-Observable command) failed at evaluation with `error: attribute 'currentTime' missing`; also the raw epoch-seconds string didn't satisfy Requirement 1.2's absolute-UTC-form requirement.", "file": "src/flake.nix, src/nixSupport/default.nix, src/nixSupport/xce.nix, src/nixSupport/xcecli.nix", @@ -19656,6 +21662,9 @@ }, { "id": "bug-1604", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "concurrent xce-build.sh runs race on rm -rf of shared BUILD_DIR (no lock)", "timestamp": "2026-08-20T10:24:39Z", "error_message": "rm: cannot remove 'xce.build/controller/controller_unit_test.p': Directory not empty (xce-build.sh setup_build()'s rm -rf \\$BUILD_DIR step aborted the script under set -euo pipefail before meson setup ran)", "file": "src/tools/xce-build.sh", @@ -19682,6 +21691,10 @@ }, { "id": "bug-1605", + "status": "cannot-reproduce", + "status_updated": "2026-09-24", + "status_note": "transient build failure, passed on immediate retry; likely bug-1604 collision", + "repro_attempts": "1 retry, clean", "timestamp": "2026-08-20T10:27:00.000Z", "error_message": "controller ASAN build: two failures in the same ninja run -- (1) controller/config_repl.c:1921:28: error: 'RequestReply_data_probeconfigtagreplymsg' undeclared (first use in this function); did you mean 'RequestReply_data_upgradeconfigreplymsg' -- and (2) lib/controller.capnp.c:25013:1: fatal error: opening dependency file lib/libxce.a.p/meson-generated_.._controller.capnp.c.o.d: No such file or directory, compilation terminated; ninja: build stopped: subcommand failed.", "file": "src/lib/controller.capnp.c (generated), src/controller/config_repl.c (consumer)", @@ -19711,6 +21724,8 @@ }, { "id": "bug-1606", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T16:00:00Z", "error_message": "grep: invalid option -- 'P' (BusyBox usage dump) when a standalone .sh script (run via `bash script.sh`, not typed as an inline Bash-tool command) calls `grep -oP '...\\K...'`", "file": "src/tools/test-version-startup-consistency.sh", @@ -19733,6 +21748,8 @@ }, { "id": "bug-1607", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T16:15:00Z", "error_message": "`git status` showed src/frontend/apiserver/go.sum modified (+65 lines) after a direct `go build ./cmd/main.go` invocation that never touched go.mod and produced a successful, working binary -- no warning or error printed by the build itself", "file": "src/frontend/apiserver/go.sum", @@ -19754,6 +21771,8 @@ }, { "id": "bug-1608", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T15:57:00Z", "error_message": "`xce-build.sh apiserver ` (meson -> go_build custom_target) hung indefinitely (8-10+ minutes, zero progress) building xcecli's Go dependencies; no error, no timeout, just stuck", "file": "src/tools/go_build", @@ -19779,6 +21798,8 @@ }, { "id": "bug-1609", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T21:10:00Z", "error_message": "User-reported (live REPL session, xce_scheduler -i -s ): after 'continue' with zero breakpoints set, output is just as verbose as single-stepping -- every decoded event still prints a '[sim] t=... session=...' + '[sim] event: ...' two-line block, defeating continue's documented purpose ('Run until next breakpoint hit or EOF').", "file": "src/scheduler/src/repl_break.c", @@ -19804,6 +21825,8 @@ }, { "id": "bug-1610", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-20T21:10:00Z", "error_message": "User-reported (same live REPL session): under 'continue', an unchanged PEND decision for the same jobid prints an identical '[sim] PEND jobid=...' line every scheduling round it remains pending -- e.g. jobs [22]-[29] (8 jobs) print their 8 byte-identical PEND lines 5+ times in a row across consecutive rounds with zero new information each time.", "file": "src/scheduler/src/sim_loop.c", @@ -19827,6 +21850,9 @@ }, { "id": "bug-1611", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "e2bb98c8, 984a6869", "timestamp": "2026-08-21T15:10:00Z", "error_message": "Discovered analyzing a real 108-session replay (xce_scheduler -i -s scheduler-replay-20260820-070103.replay, continue to EOF): sim_print_decision() (sim_loop.c) never printed a single DISPATCH or PREEMPT line across the entire replay -- only PEND -- even though 22 of the 30 jobs pushed at round 32 clearly transitioned to running (confirmed by ongoing JOB_UPDATE/JOB_RUSAGE ticks for jobids [0]-[21] every few rounds through round 107). An operator watching only decision output has zero confirmation those 22 jobs ever dispatched; they just silently stop appearing in PEND.", "file": "src/scheduler/src/sim_loop.c", @@ -19854,6 +21880,8 @@ }, { "id": "bug-1612", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T00:00:00Z", "error_message": "Adversarial review REJECTED dirty-content-fingerprint task 4.3 (src/frontend/Makefile DIRTY_FINGERPRINT): when `git diff HEAD` failed but the following `git ls-files | sort | xargs cat` pipe still succeeded, the computation silently proceeded and emitted a fabricated non-empty fingerprint instead of degrading to empty.", "file": "src/frontend/Makefile", @@ -19876,6 +21904,9 @@ }, { "id": "bug-1613", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "e2bb98c8 (scheduler-sim-decision-parity)", "timestamp": "2026-08-21T07:25:00Z", "error_message": "User-reported (xce_scheduler -i -s , 'info worker '): every CONSUMABLE resource's used= value reads 0.00 for a worker regardless of how many jobs are actually dispatched and running on it, e.g. 'resource slots: value=22.00 used=0.00' and 'resource mem: value=17286225920.00 used=0.00' with jobs clearly active (confirmed via ongoing JOB_RUSAGE ticks in the same session). Confirmed real, not a display bug.", "file": "src/scheduler/src/sim_loop.c", @@ -19902,6 +21933,8 @@ }, { "id": "bug-1614", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T08:50:00Z", "error_message": "pytest FAILED xceadmin/repo-mirror/src/xceadmin/tests/test_cli.py::test_cli_version_flag_includes_build_info_fields -- AssertionError: assert '2bf62a64-dirty' in 'xceadmin, version 0.1.0 (2bf62a64-857bd5b5, 2026-08-21T08:49:15Z)\\n' (1 failed, 266 passed, 3 skipped) -- blocks meson's xceadmin-build.stamp custom_target, so no wheel is produced.", "file": "src/xceadmin/tests/test_cli.py", @@ -19927,6 +21960,8 @@ }, { "id": "bug-1615", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T09:37:18Z", "error_message": "src/tools/test-dirty-fingerprint-distinguishability.sh: line 344: iso_src: unbound variable", "file": "src/tools/test-dirty-fingerprint-distinguishability.sh", @@ -19949,6 +21984,8 @@ }, { "id": "bug-1616", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T00:00:00Z", "error_message": "src/tools/test-dirty-fingerprint-nix-production-path.sh: line 191: flag: unbound variable", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", @@ -19974,6 +22011,8 @@ }, { "id": "bug-1617", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T00:00:00Z", "error_message": "mktemp: Invalid argument", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", @@ -19996,6 +22035,8 @@ }, { "id": "bug-1618", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T00:00:00Z", "error_message": "error: pathspec 'flake.nix' did not match any file(s) known to git", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", @@ -20017,6 +22058,8 @@ }, { "id": "bug-1619", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T00:00:00Z", "error_message": "FAIL: this script itself contains the string '--impure' -- the Observable requires no --impure flag anywhere.", "file": "src/tools/test-dirty-fingerprint-nix-production-path.sh", @@ -20039,6 +22082,8 @@ }, { "id": "bug-1620", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-21T15:56:00Z", "error_message": "ERROR: Unknown option: \"dirty_fingerprint\".", "file": "src/tools/xce-build.sh (setup_build, --incremental path)", @@ -20063,6 +22108,8 @@ }, { "id": "bug-1621", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-22T08:12:00Z", "error_message": "LeakSanitizer (run with ASAN_OPTIONS=detect_leaks=1, overriding this test suite's own detect_leaks=0 convention): 'Direct leak of 8 byte(s) ... #1 make_host_account ../lib/xcealloc.c:119 #2 make_job_account ../lib/xcealloc.c:188' -- an xce_host_account_t's `host` field (strdup'd in make_host_account(), src/lib/xcealloc.c:119) is never freed.", "file": "src/lib/misc.c (free_host_account)", @@ -20089,6 +22136,8 @@ }, { "id": "bug-1622", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-22T09:40:00Z", "error_message": "TEST_ASSERT_EQUAL_INT(0, rc) failed: Expected 0 Was 1, in test_sim_round_boundary_reconciliation.c calling get_resvals(&(w->resInUse), 1, &residx, got)", "file": "src/scheduler/testsuite/test_sim_round_boundary_reconciliation.c", @@ -20110,6 +22159,8 @@ }, { "id": "bug-1623", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-22T12:40:00Z", "error_message": "Test-authoring bug while building scheduler-sim-decision-parity task 4.3 (test_sim_e2e_original_motivation.c): initial version asserted resInUse checkpoints of 17.00/18.00/20.00/12.00 (derived from the RECORDING driver own in-memory resInUse arithmetic) but the actual replay-side info worker output showed only 12.00 (ANCHOR own charge) at every checkpoint -- the first three assertions failed.", "file": "src/scheduler/testsuite/test_sim_e2e_original_motivation.c", @@ -20134,6 +22185,8 @@ }, { "id": "bug-1624", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-22T19:53:00Z", "error_message": "None yet observed at runtime -- found via code trace while answering an architectural question about job RUN-state boundaries and over/under-commit safety.", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/core_session.c, src/scheduler/src/resource.c", @@ -20160,6 +22213,8 @@ }, { "id": "bug-1625", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-23T05:20:00Z", "error_message": "None yet observed at runtime -- found via code trace while designing scheduler-dispatch-commit. Referenced throughout that spec's requirements.md/research.md/design.md by this id; being logged now after noticing it was never actually appended to buglog.json.", "file": "src/scheduler/src/handler.c", @@ -20183,6 +22238,9 @@ }, { "id": "bug-1626", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "dispatch path takes write lock; job_ctrl still mutates job under read lock (job.c)", "timestamp": "2026-08-23T05:20:00Z", "error_message": "None yet observed at runtime -- found via code trace during /kiro-validate-design review of scheduler-dispatch-commit.", "file": "src/controller/job.c", @@ -20207,6 +22265,8 @@ }, { "id": "bug-1627", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-23T05:35:00Z", "error_message": "None yet observed at runtime -- found via code trace during round 3/4 of /kiro-validate-design review of scheduler-dispatch-commit.", "file": "src/controller/job.c", @@ -20230,6 +22290,8 @@ }, { "id": "bug-1628", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-23T08:03:59", "error_message": "date: invalid date '-6 days' / date: invalid date '2026-06-29 +5 days' -- relative date arithmetic silently yields EMPTY shell variables, and the empty values then flow into `git log --since=\"\" --until=\"\"`, which git treats as NO time filter (unbounded history) instead of erroring.", "file": ".claude/skills/weekly_report/SKILL.md", @@ -20257,6 +22319,8 @@ }, { "id": "bug-1629", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-23T08:12:20", "error_message": "AddressSanitizer SEGV in zarray_get_ex (lib/zarray.c:332) when a test-fixture xce_res_account_t.residx was never resolved from -1", "file": "src/scheduler/testsuite/test_scheduler_dispatch_commit.c", @@ -20279,6 +22343,8 @@ }, { "id": "bug-1630", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-23T08:35:55", "error_message": "buglog.json integrity: 39 duplicate bug IDs present (bug-875..bug-948). The Stop hook's duplicate-ID validation flags this every turn. bug-908, for example, is BOTH a 2026-06-21 raft_sim.c missing-guard-clause entry AND a 2026-08-16 config-upgrade-snapshot/spec.json refactor note -- two unrelated bugs sharing one id.", "file": ".wolf/buglog.json", @@ -20302,6 +22368,9 @@ }, { "id": "bug-1631", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "reconcile path fixed cfe66dde; dispatch_a_job_ retry branches keep allocs", "timestamp": "2026-08-23T19:30:00Z", "error_message": "New test test_rpc_failure_bound_resets_after_intervening_success (test_dispatch_fail_retry.c) fails: 'Req 4.1 (bug-1631, NOT YET FIXED): a retryable dispatch failure must return the job to PEND -- fails here only because this job previously had a REAL successful dispatch (step 2)'. Expected JOB_STAT_PEND, got JOB_STAT_RUN (and separately, job.allocs stays non-NULL instead of being released).", "file": "src/controller/job.c", @@ -20332,6 +22401,9 @@ }, { "id": "bug-1632", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "test_sim_verify.c stack-local sim read by picker thread (ASAN stack-use-after-return)", "timestamp": "2026-08-23T21:03:00.000Z", "error_message": "sched_sim_verify_test: AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T2 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of run_sim_and_capture_stdout's `sim` (xce_sched_sim_driver_t) after that frame returned/was reused. Fired once during a full `/xce-build scheduler` run (scheduler-dispatch-commit task 5.4 regression pass): Ok:50 Fail:2 instead of the documented Ok:51 Fail:1 baseline, extra failure = 'scheduler sim verify test'. Did NOT reproduce on: 3 standalone direct re-runs of sched_sim_verify_test (16 Tests 0 Failures each time) or a second full `meson test --suite scheduler` re-run against the same build (back to Ok:51 Fail:1, only the known test_admin_request.c symlink-escape failure) -- confirmed non-deterministic/order-dependent, not caused by scheduler-dispatch-commit.", "file": "src/scheduler/testsuite/test_sim_verify.c", @@ -20359,6 +22431,8 @@ }, { "id": "bug-1673", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T13:40:00Z", "error_message": "Gitea issue #40 (\"job \u8c03\u5ea6\u4e0d\u6b63\u786e\"): `xcecli hosts list` shows MAX_SLOT=0 for a dynamically-joined worker even though the host's real NCPUS (e.g. 16 or 22) is correctly reported and scheduling correctly dispatches up to that many concurrent jobs. Reproduced live on branch scheduler-framework: single-node deploy, worker joined as 'worker-1', `xcecli hosts list` -> NCPUS=22 MAX_SLOT=0. Submitted 9 single-slot sleep jobs -- all 9 eventually dispatched and ran (no hard cap observed under this test), so the reporter's secondary claim of a hard ~8-job concurrency ceiling did NOT reproduce here (most likely a mid-dispatch snapshot, dispatch is not instantaneous across all pending jobs on this branch; a queue/usergroup slot limit in limits.yaml, e.g. ug1_normal_slots=8, remains a plausible alternate explanation worth asking the original reporter about). Only the MAX_SLOT=0 display value itself is a confirmed, reproducible bug.", "file": "src/frontend/apiserver/backend/handler.go, src/configs/hosts.yaml", @@ -20385,6 +22459,8 @@ }, { "id": "bug-1674", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T13:40:00Z", "error_message": "Gitea issue #41 (\"job output timestamp is incorrect\"): reporter's transcript shows `xcecli jobs list -l ` succeeding but omitting submit/start/done timestamps. Reproduced live on branch scheduler-framework, but the actual failure is more severe than reported: `xcecli jobs list -l ` (and bare `xcecli jobs list -l`) fail OUTRIGHT with `query request failed: [invalid parameters: unsupported query field: pending_reason]` -- no output is produced at all, not even a partial listing missing timestamps. Likely a regression introduced after the original issue was filed (this branch is under active scheduler-framework development).", "file": "src/frontend/apiserver/controllers/jobs.go:553, src/frontend/apiserver/controllers/jobs.go:901", @@ -20408,6 +22484,8 @@ }, { "id": "bug-1675", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T13:40:00Z", "error_message": "Gitea issue #42 (\"\u521a\u542f\u52a8\u7684\u65f6\u5019\u67e5\u8be2\u4f5c\u4e1a\u62a5\u7684\u9519\u8bef\u4e0d\u592a\u53cb\u597d\"): `xcecli jobs list` run in the narrow window right after the apiserver (re)starts, before its listener is up, prints a raw `get the request failed: [connection refused]` instead of a user-friendly message. Reproduced live on branch scheduler-framework: killed and relaunched the apiserver, hammered `xcecli jobs list` in a tight loop from the instant of relaunch -- attempts 1-13 all returned the exact reported text; the listener came up by attempt 14.", "file": "src/frontend/xcecli/common/request/request.go:97-98, src/frontend/xcecli/common/request/request.go:185-188", @@ -20431,6 +22509,8 @@ }, { "id": "bug-1676", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T13:40:00Z", "error_message": "Gitea issue #44 (\"FROM_HOST is -\"): `xcecli jobs list`'s FROM_HOST column always shows `-` instead of the host the job was submitted from. Reproduced live on branch scheduler-framework: submitted a job, checked `xcecli jobs list` ~1 second later (job still PEND) -- FROM_HOST blank; confirmed via the apiserver's own debug log showing `\"from_host\":\"\"` on every job regardless of state (PEND/RUN/DONE), across every job submitted in the session.", "file": "src/frontend/apiserver/controllers/jobs.go:1483, src/frontend/apiserver/backend/handler.go:18-20, src/schema/controller.capnp", @@ -20455,6 +22535,8 @@ }, { "id": "bug-1677", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T13:40:00Z", "error_message": "scripts/gitea-sync.py's sync_bugs() would crash (or silently no-op) on any real invocation: `buglog = json.loads(BUGLOG_PATH.read_text())` (line 726) then `for i, bug in enumerate(buglog):` (line 731) treats buglog.json as a bare list, but its actual top-level structure is `{\"version\": 1, \"bugs\": [...]}` -- a dict. `enumerate()` over a dict yields its keys (\"version\", \"bugs\") as `bug`, and the very next line's `bug.get(\"id\", ...)` would raise AttributeError on the string \"version\". Discovered incidentally while checking whether to set the `gitea_issue` cross-reference field on newly-logged bugs (bug-1673..bug-1676) during a Gitea-issue verification pass -- not run live, found via static read of the sync script against the current buglog.json wrapper format.", "file": "scripts/gitea-sync.py", @@ -20476,6 +22558,8 @@ }, { "id": "bug-1678", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T13:55:00Z", "error_message": "Gitea issue #43 (\"\u91cd\u542fdaemon\u540e\u4f5c\u4e1a\u8c03\u5ea6/\u4fe1\u606f\u4e0d\u5bf9\"): after restarting ALL daemons (controller, scheduler, worker, apiserver), previously-RUN jobs come back with STATUS=RUN correctly recovered but EXEC_HOST permanently blank (\"-\"), even though the underlying job is genuinely still running. CONFIRMED LIVE on branch scheduler-framework, corrected finding: an earlier verification pass on this same issue (2026-08-24, same session) restarted only controller+scheduler+worker and left the apiserver running, and did NOT reproduce the symptom -- that was a false negative, caused by the still-running apiserver simply continuing to display its stale-but-correct in-memory job cache from before the restart, never refreshed. Repeating the test with the apiserver ALSO restarted (matching the original report's \"restart all daemons\" exactly) reproduces the bug immediately and durably: 3 RUN jobs with EXEC_HOST=1*worker-1 before the restart; killed all 4 daemons; ran start-all.sh; EXEC_HOST was blank on every poll (15 polls over 30s, first poll ~6s after apiserver came back up) while STATUS stayed correctly RUN throughout. Controller log corroborates the mechanism: ctrl-1.log's recovery trace shows apply_job_update_transition logging 'duplicate UPDATE to state 0x2 -- no-op' for every job during WAL replay -- consistent with the WAL_JOB_DISPATCH fact applying SILENTLY first (PEND->RUN, no log line on the APPLIED path), immediately followed by each job's WAL_JOB_START fact (worker-confirmed start, also state=RUN) correctly detecting the now-redundant transition and logging the no-op. Neither replay step ever touches job.allocs.", "file": "src/controller/job.c:1369-1370, src/controller/persist.c:3708-3757, src/controller/job.c:288-351 (restore_job), src/controller/persist.c:6690-6713 (persist_submit_job_fact)", @@ -20506,6 +22590,8 @@ }, { "id": "bug-1679", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T03:15:00Z", "error_message": "A job's live dispatch/start/finish transitions are never pushed to the apiserver -- xcecli jobs list shows a job stuck at PEND forever (or, in one observed case, disappearing from the default query entirely) even though the controller's own WAL and in-memory state correctly progress the job all the way through RUN to EXIT/DONE. Discovered as a side effect while live-verifying bug-1678 (restart-recovery of job.allocs) -- this is a SEPARATE, unrelated, more severe bug: it reproduces with NO restart involved at all, on a single, freshly-submitted job, every time. Confirmed reproducible on the STOCK branch (git-stashed all of bug-1678's changes, rebuilt, redeployed fresh) -- NOT a regression introduced by bug-1678 or any other fix landed this session (bug-1673/1674/1675/1676/1677 touch only Go/apiserver code, a Python script, or job_submit_data_t's from_host field + jobj.c's dedup identity check -- none touch the state-transition/hook-firing logic implicated here).", "file": "src/controller/job.c (dispatch_a_job_, on_job_start, and likely other direct-mutation call sites), src/controller/persist.c (apply_committed_entry's WAL_JOB_START/.../DISPATCH shared case, apply_job_update_transition)", @@ -20534,6 +22620,8 @@ }, { "id": "bug-1680", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T03:50:00Z", "error_message": "`xcecli jobs list -a` (\"list run,done,pending,suspended jobs\" -- i.e. show every job regardless of status) returns \"no job found\" even when jobs genuinely exist and are correctly stored with a real status. Discovered as a side effect of live-verifying bug-1679: two jobs that correctly showed PEND->RUN->DONE via the default query and via `-d` (done-only) both come back empty under `-a` specifically. Not a data-loss bug (the jobs ARE correctly stored and queryable through other flags -- `-d`, the default active-jobs view -- this is specifically `-a`'s own query construction that's broken) but a real, reproducible CLI defect: `-a` is documented as the way to see literally everything, and it shows nothing.", "file": "src/frontend/xcecli/filters/job_filter.go:58-60, src/frontend/apiserver/utils/types.go:262-322, src/frontend/apiserver/store/query.go:70-73", @@ -20559,6 +22647,8 @@ }, { "id": "bug-1681", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T21:30:00Z", "error_message": "xce-audit-invariants scheduler-driver-equivalence audit finding (not user-reported, no live symptom today): driver.c's normal-mode round (xce_sched_driver_run_normal_mode_round, driver.c:750,802) calls plugin_begin_session(sched)/plugin_end_session(sched) -- iterating every registered plugin and invoking its begin_session_fn/end_session_fn hook, called 'real, load-bearing per-session bookkeeping for EVERY registered plugin' per driver.c's own comment (lines 688-693). sim_loop.c has zero equivalent call anywhere (grep-confirmed across sim*.c/repl*.c). This is structurally the same shape of gap as bug-1613's original root cause (a driver.c-only per-session call with no sim_loop.c mirror), but under the codebase's only currently-supported configuration it is inert: sim mode's sim_bootstrap_picker() (sim_picker.c:60-70) only ever registers builtin_fcfs, which explicitly leaves begin_session_fn/end_session_fn NULL (plugins/builtin_fcfs.c:973-981), and the one plugin that does set a hook (legacy fcfs, plugins/fcfs.c:352, end_session_fn=fcfs_end_session, only sets privdata->new_session=1, no reach into resInUse/sharedInUse/job->accounts) is unreachable in sim mode either way (sim mode refuses any picker name other than builtin_fcfs).", "file": "src/scheduler/src/sim_loop.c", @@ -20583,6 +22673,8 @@ }, { "id": "bug-1682", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-24T21:40:00Z", "error_message": "Found while fixing bug-1681 (source read of plugin_end_session(), src/scheduler/src/plugin.c): the per-plugin iteration guard checks the WRONG field. plugin_end_session()'s loop tests `if (plugin->begin_session_fn != NULL) { plugin->end_session_fn(plugin); }` -- guards on begin_session_fn but calls end_session_fn. Pre-existing, not introduced by bug-1681's fix (untouched by that diff). Previously documented (not fixed) in testsuite/test_driver_session_ordering.c's own file header comment as a known 'CONCERN surfaced during this test's construction' -- flagged there for the reviewer, deliberately left unfixed as out of that task's scope. Confirmed still present in current code.", "file": "src/scheduler/src/plugin.c", @@ -20607,6 +22699,8 @@ }, { "id": "bug-1683", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T00:05:00Z", "error_message": "xceadmin pytest failure: test_send_frame_writes_exact_known_good_byte_sequence (src/xceadmin/tests/test_wire.py:200). Test sends a 4-byte payload b'xce!' via wire.send_frame() over a real TLS socket to a raw capture handler that reads exactly 12 bytes (8-byte length header + 4-byte payload) directly off the wire, bypassing this module's own recv_frame() decoder. Expected the captured bytes to be struct.pack('=Q', 4) + b'xce!' (header correctly encodes length=4, followed by the literal payload bytes). Actual captured bytes: the 8-byte header is correct (b'\\x04\\x00\\x00\\x00\\x00\\x00\\x00\\x00', i.e. length=4 little-endian native uint64_t), but the trailing 4 bytes are all zero (b'\\x00\\x00\\x00\\x00') instead of b'xce!' -- the real payload bytes never reached the wire. 1 failed, 266 passed, 3 skipped -- isolated to this one test.", "file": "src/xceadmin/src/xcecommon/wire.py", @@ -20631,6 +22725,8 @@ }, { "id": "bug-1684", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T00:50:00Z", "error_message": "xce-audit-invariants controller audit (dimension: error handling) finding: raft_cb_persist() (src/controller/ha.c:2257) always returns 0 to the raft library's on_persist callback contract regardless of whether the term/voted_for durability write to disk actually succeeded -- a disk I/O failure (full disk, permission error, transient I/O error) could let a node grant a vote it has no durable record of, opening a split-vote path if that node then restarts and forgets it voted (Election Safety: 'a server grants at most one vote per term' via durably-persisted votedFor).", "file": "src/controller/ha.c", @@ -20652,6 +22748,8 @@ }, { "id": "bug-1685", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T15:47:00Z", "error_message": "User-reported live crash: xce_scheduler SIGABRT via glibc malloc_printerr (heap corruption detected during a malloc/free bookkeeping check -- 'double free or corruption'-class abort, not a clean segfault). Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv). Backtrace: malloc_printerr -> kill_limit (src/lib/xcelimit.c:181) <- limit_update_config (src/scheduler/src/plugins/limit.c:76) <- plugin_config_update (src/scheduler/src/plugin.c:246) <- sched_init_config (src/scheduler/src/scheduler.c:936) <- scheduler_req_handler (src/scheduler/src/handler.c:687) <- vtask_entry (src/lib/xcectx.c:172) <- run_task/vcpu_entry (src/lib/vtaskmgr.c). limit_update_config's own locals at the crash frame: rc=-17 (-EEXIST), nlimits=7, i=0 -- crashes on (or near) the FIRST limit in a 7-limit config update.", "file": "src/scheduler/src/plugins/limit.c", @@ -20674,6 +22772,8 @@ }, { "id": "bug-1686", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T16:24:00Z", "error_message": "User-reported live crash (build.dbg, commit b040dfc0 -- AFTER the bug-1685 double-free fix landed, so this is a distinct, separate crash): xce_scheduler SIGSEGV in fcfs_cursor_enter_queue (src/scheduler/src/plugins/builtin_fcfs.c:673), called from builtin_fcfs_next_job (builtin_fcfs.c:810) <- sched_job_picker (scheduler.c:118) <- picker thread. Crash line 673 is the NULL branch of a ternary expression. queue_item=0x7db7d0000000 -- a suspiciously round address (large power-of-2-aligned), possibly a corrupted/uninitialized pointer rather than a normal heap allocation. Core captured from a live 3-node HA deployment run (--peers ctrl-1/ctrl-2/ctrl-3, -vvv).", "file": "src/scheduler/src/plugins/builtin_fcfs.c", @@ -20696,6 +22796,8 @@ }, { "id": "bug-1687", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T17:30:00Z", "error_message": "xce-audit-invariants finding: WAL_JOB_START silently drops job fields on a controller restart replay -- the same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), but for the worker-confirmed-start fact. apply_job_start_transition() (job.c) sets job->job.pidinfo (real pids/pgids/exdata a worker reported) and job->job.start_time, but the WAL_JOB_START fact written by record_job_start_fact() only carried wal_jobstate_t{jobid, state} -- pidinfo was never in the log at all, and start_time was re-fabricated with time(NULL) at every apply (both boot replay and standby catch-up) instead of carrying the true worker-reported value. Confirmed live-relevant: user reported StartTime always showing 0 in `xcecli joblist` for both RUN and DONE jobs on a live deployment -- deferred as a distinct follow-up investigation (that deployment's controller predates this fix; whether StartTime=0 specifically, rather than the pre-fix fabricated-nonzero time(NULL), indicates a SEPARATE live-apply-pump gap has not yet been root-caused).", "file": "src/controller/job.c, src/controller/persist.c, src/include/persist.h, src/include/wal_types.h", @@ -20727,6 +22829,8 @@ }, { "id": "bug-1688", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T17:40:00Z", "error_message": "User-reported issue in the scheduler's interactive simulator REPL (`xce_scheduler -i`/`--sim`): `info worker all` and `info worker ` (e.g. `info worker pc13v`) initially print full per-resource detail (`[sim] resource slots: value=22.00 used=0.00`, `resource mem: value=...`, etc -- ~19 resource lines including slots/mem/r1m/r15m/r15s/ut/pg/ls/it/swp/io/tmp/type/model/cpuf/maxcpus/maxmem/maxswap/maxtmp/ndisks). After stepping the simulation forward through several sessions (`step 5` repeated, crossing RESVALS_DELTA, JOB_PUSH (2 batches, 40+80 jobs), JOB_UPDATE, and WORKERS_PUSH events across sessions 2 through 30), the SAME commands later in the SAME REPL session regress to printing ONLY the header line: `[sim] worker pc13v: status=online` -- zero resource lines, even though the worker still exists and is still 'online' (not removed/dead). Binary under test: `./build.dbg/controller/xce_controller --version` reports `xce v0.0.1-bc3f76b4-5ff97622 built 2026-08-25T08:56:58Z` (a build.dbg build -- note the user ran the CONTROLLER binary's --version, though the symptom itself is in the SCHEDULER's sim REPL; worth confirming which binary actually serves `xce_scheduler -i`/`--sim` and its own version once investigated). A real replay file reproducing the sequence is preserved at /tmp/scheduler-replay-20260825-090251.replay (~1MB, captured 2026-08-25 09:24 UTC) for a future investigation to replay directly instead of needing to reproduce from scratch.", "file": "src/scheduler/src/repl_info.c (repl_info_worker_one(), ~line 309-388, the `info worker` print path), src/scheduler/src/core_events.c (core_apply_worker_resvals(), ~line 478, the WORKERS_PUSH resval-merge path), src/lib/zarray.c (zarray_put_ex(), ~line 267-303, count-vs-slot bookkeeping)", @@ -20753,6 +22857,8 @@ }, { "id": "bug-1689", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T18:10:00Z", "error_message": "xce-audit-invariants finding: WAL_JOB_FINISH silently drops job fields on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs) and bug-1687 (WAL_JOB_START/job.pidinfo+start_time), but for the worker-confirmed-finish fact. apply_job_finish_transition() (job.c) sets job->job.exit_info->exitcode and job->job.finish_time/run_time, but the WAL_JOB_FINISH fact written by record_job_finish_fact() only carried wal_jobstate_t{jobid, state} -- exit_info was never in the log at all (and, worse, apply_job_finish_transition() itself is effectively dead code on the real production apply path -- both apply_committed_entry()'s and apply_wal_entry()'s WAL_JOB_FINISH handling route through apply_job_update_transition() (state-only) instead, per raft-apply-decoupling's deferred-apply design, so exit_info was NEVER populated through the real commit-apply pipeline, restart or not, prior to this fix), and finish_time was re-fabricated with time(NULL) at every apply_committed_entry() apply (and, on apply_wal_entry()'s standby catch-up seam, never touched at all -- an even larger gap than WAL_JOB_START's pre-1687 state) instead of carrying the true worker-reported receipt time.", "file": "src/controller/job.c (apply_job_finish_transition ~L2292, record_job_finish_fact ~L2371, restore_job_finish_state new), src/controller/persist.c (apply_committed_entry's shared job-fact group ~L3890-4080, apply_wal_entry's WAL_JOB_FINISH case, persist_submit_job_finish_fact new, decode_jobexitinfo new), src/include/persist.h, src/include/wal_types.h, src/controller/testsuite/test_wal_job_finish_fact.c (new)", @@ -20784,6 +22890,8 @@ }, { "id": "bug-1690", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T11:00:00Z", "error_message": "xce-audit-invariants finding: WAL_JOB_RUSAGE silently drops job.run_usage (client-reported per-host CPU/mem usage) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time). update_job_rusages() (job.c) sets job->job.run_usage (xce_job_runusage_t*, an accumulated per-host struct) via update_host_runusage() (misc.c), but the WAL_JOB_RUSAGE fact persist_job_hook_fun (persist.c) wrote only carried wal_jobstate_t{jobid, state} -- run_usage was never in the log at all, so a restart replay left it permanently NULL even though a client actively reported CPU/mem usage. Unlike the three precedents, xce_job_runusage_t had NO existing Cap'n Proto mapping at all (confirmed: grep for xce_job_runusage_t in controller.capnp matched nothing; a similar-looking but unrelated type, xce_job_rusages_t/JobRusages/SchedJobRusages -- the CLIENT's wire update-request shape {jobid, pids[], pgids[], usages[]} -- already existed and is NOT interchangeable), so this fix required a genuine schema addition, not a reuse.", "file": "src/schema/controller.capnp (new HostRunusage/JobRunusage structs), src/controller/persist.c (capnp_encode_jobrunusage/decode_jobrunusage ~L327-388, persist_job_hook_fun's HOOK_DETAIL_JOB_RUSAGE_UPDATE case ~L7440-7500, apply_committed_entry's WAL_JOB_RUSAGE branch ~L4116-4144, apply_wal_entry's WAL_JOB_RUSAGE case ~L3712-3740), src/controller/job.c (merge_job_runusage new ~L893-1000, restore_job_rusage_state new), src/include/wal_types.h (WAL_JOB_RUSAGE doc comment), src/controller/testsuite/test_wal_job_rusage_fact.c (new, 8 tests)", @@ -20817,6 +22925,8 @@ }, { "id": "bug-1691", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T12:00:00Z", "error_message": "xce-audit-invariants finding: WAL_JOB_CKPT silently drops job.ckptinfo (the worker/scheduler-reported checkpoint info string) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), and bug-1690 (WAL_JOB_RUSAGE/job.run_usage). on_job_ckpt() (job.c) sets job->job.ckptinfo via makeString(), but the WAL_JOB_CKPT fact -- purely hook-driven via persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT arm (on_job_ckpt() itself never calls a persist_submit_job_*_fact() function directly, unlike DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored ckptinfo -- it stayed NULL after restart. job->job.ckptinfo (xce/types.h) is a plain NUL-terminated char*, NOT the richer xce_ckptinfo_t/CheckpointInfo capnp struct used elsewhere for a different purpose (that struct is on_job_ckpt()'s OWN incoming-report parameter type).", "file": "src/controller/job.c:2999 (on_job_ckpt), src/controller/job.c:1046 (restore_job_ckpt_state, new), src/controller/persist.c:354 (decode_ckptinfo, new), src/controller/persist.c:7579 (persist_job_hook_fun's HOOK_DETAIL_JOB_CKPT case, new), src/controller/persist.c:3757 (apply_wal_entry's WAL_JOB_CKPT case, split out), src/controller/persist.c:4147 (apply_committed_entry's WAL_JOB_CKPT unconditional-adopt block), src/include/wal_types.h (WAL_JOB_CKPT doc comment), src/controller/testsuite/test_wal_job_ckpt_fact.c (new)", @@ -20851,6 +22961,8 @@ }, { "id": "bug-1692", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T13:10:00Z", "error_message": "xce-audit-invariants finding: WAL_JOB_PREASON silently drops job.pend_reasons (the scheduler-reported host/resource-based pending-job explanation) on a controller restart replay -- same bug class as bug-1678 (WAL_JOB_DISPATCH/job.allocs), bug-1687 (WAL_JOB_START/job.pidinfo+start_time), bug-1689 (WAL_JOB_FINISH/job.exit_info+finish_time), bug-1690 (WAL_JOB_RUSAGE/job.run_usage), and bug-1691 (WAL_JOB_CKPT/job.ckptinfo). set_preason()/do_clr_preason() (job.c) set/clear job->pend_reasons (xce_job_preason_t*: main_reason plus xce_host_reason_t**/xce_res_reason_t** nested arrays), but the WAL_JOB_PREASON fact -- purely hook-driven via persist_job_hook_fun's `default:` arm (shared with STOP/RESUME/UPDATE, same class as bug-1690/1691's hook-driven precedent, NOT a direct persist_submit_job_*_fact() call site like DISPATCH/START/FINISH) -- only ever carried the shared wal_jobstate_t{jobid, state} payload. Neither apply seam restored pend_reasons after a restart. This was the LAST of the five field-parity gaps from the original xce-audit-invariants pass -- closes that invariant class.", "file": "src/controller/persist.c:206 (capnp_encode_jobpreason, new), src/controller/persist.c:290 (decode_jobpreason, new), src/controller/persist.c:78 (persist_job_hook_fun local vars pend_buf/pend_len/pend_present), src/controller/persist.c (persist_job_hook_fun's new HOOK_DETAIL_SETPREASON/HOOK_DETAIL_CLRPREASON case, split out of default arm), src/controller/persist.c (persist_job_hook_fun's shared-tail pend_present append block), src/controller/persist.c (apply_wal_entry's WAL_JOB_PREASON case, split out of the WAL_JOB_FAIL/STOP/RESUME/UPDATE shared group), src/controller/persist.c (apply_committed_entry's WAL_JOB_PREASON unconditional-adopt block, added after the WAL_JOB_CKPT block, before the JOB_TRANSITION_APPLIED gate), src/controller/job.c (restore_job_preason_state, new, added after restore_job_ckpt_state), src/include/wal_types.h (WAL_JOB_PREASON=10 doc comment, new), src/controller/testsuite/test_wal_job_preason_fact.c (new, 11 tests), src/controller/meson.build, src/controller/testsuite/controller_test.c (test wiring)", @@ -20887,6 +22999,8 @@ }, { "id": "bug-1693", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T14:05:00Z", "error_message": "Invariant audit (Decision Log fidelity dimension) found two violations of the recorded Decision Log entry \"No client reply-wait timeout as a fix for a hung request -- fail loud at the real transport/source instead of masking with a deadline backstop\": (1) OPCode_schedAdmin's forward-and-block-for-reply (cluster.c) blocks up to SCHEDADMIN_TIMEOUT_MS=5000ms via pthread_cond_timedwait, but handle_scheduler_leave() (sched.c) -- the actual scheduler-disconnect handler -- only ever nulled c->sched.ep; it never signalled the pending waiter, so a scheduler disconnect mid-wait could only ever be observed at the full 5s deadline. (2) on_fetchfile's identical proxy pattern (job.c, 10s pthread_cond_timedwait) had the same gap: handle_worker_leave() (worker.c) only ever nulled w->ep, never woke a pending fetchfile waiter proxied to that worker.", "file": "src/controller/cluster.c, src/controller/sched.c, src/controller/job.c, src/controller/worker.c, src/controller/include/cluster.h", @@ -20921,6 +23035,8 @@ }, { "id": "bug-1694", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T00:00:00.000Z", "error_message": "xce-audit-invariants finding (dimension 1, async I/O thread-role): checkpoint_task_fn (src/controller/persist.c) was the last delaymgr-triggered path still calling its full blocking body (persist_write_checkpoint_marker's pwrite+fdatasync, write_snapshot()'s unbounded open/write walk over every job/queue/host frame held under apply_pump_lock, trim_wal_segments, rotate_snapshots' unlink) directly on the delaymgr timer thread -- the same single thread that drives heartbeats and election timeouts cluster-wide (ha-async-io.md thread-role contract: delaymgr must only trigger work, never block). Triggered from the WAL-append hot path via schedule_checkpoint()/delay_task() whenever wal_record_count crosses max_wal_records. Every sibling long-blocking-I/O path (snapshot send, config-push, reconnect) had already been offloaded by raft-drive-async; checkpoint was the one documented remaining gap.", "file": "src/controller/persist.c, src/include/persist.h, src/controller/testsuite/test_persist_checkpoint_offload.c, src/controller/testsuite/controller_test.c, src/controller/meson.build", @@ -20948,6 +23064,8 @@ }, { "id": "bug-1695", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T15:05:00Z", "error_message": "Found via the mandatory live HA chaotic soak (xce-audit-invariants controller campaign, Step 4) -- NOT caused by any of the 9 fixes that campaign made (bug-1684/1687/1689/1690/1691/1692/1693/1694), all of which are in src/controller/{ha,persist,job,cluster,sched,worker}.c; this bug is in myapp.c, untouched by any of them. 10-round --chaotic soak: every single round after the first shows 'NO new primary within window (possible election stall)'; ctrl-1's own log shows, on EVERY restart after the first boot, 'myapp_run: leader already known after startup delay (voted_for=ctrl-1, voted_term=1, raft_term=1, primary_fd=-1) -- skipping own election' -- primary_fd=-1 proves there is no live connection to any primary, yet the node skips holding its own election anyway. ctrl-2/ctrl-3 are simultaneously stuck in a permanent ~3s retry loop ('cluster_init_phase2: cluster_config not loaded yet', rc=-22, the bug-1477 self-rescheduling retry firing forever without ever succeeding). The cluster never elects a new, live, functioning primary after the very first election of the cluster's lifetime.", "file": "src/controller/myapp.c (myapp_run's bootstrap check, call site only); src/controller/startup_validation.c (myapp_leader_already_known, the extracted+fixed decision function -- placed here, not myapp.c, so it links into controller_unit_test); src/controller/testsuite/test_ha_startup.c (7 new regression tests)", @@ -20979,6 +23097,8 @@ }, { "id": "bug-1696", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T16:20:00Z", "error_message": "Chaotic HA soak (post-bug-1695-fix) reports committed_job_loss on every round; waldump on ctrl-1's live data dir shows the ENTIRE on-disk WAL truncated to a single NOOP entry (entries: 1 submits: 0) despite 21+ jobs previously verified committed; no snapshot exists; log shows ha_compute_safe_committed_lsn=0 immediately before raft_cb_become_leader promotes ctrl-1 to CTRL_PRIMARY at term=2 on its SECOND boot.", "file": "src/controller/ha.c", @@ -21015,6 +23135,8 @@ }, { "id": "bug-1697", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-25T20:22:00Z", "error_message": "Job start_time is always 0 for every job, live, no restart needed (observed via xce_test REPL joblist during an HA soak). run_time (finish_time - start_time) is consequently also wrong for every finished job.", "file": "src/controller/job.c, src/controller/persist.c", @@ -21050,6 +23172,8 @@ }, { "id": "bug-1698", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T07:20:00Z", "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster, build.dbg): after ./scripts/start-all.sh reported 'Controller cluster healthy (primary elected)', ctrl-1's log showed it perpetually retrying ha_connect_finish_peer() against both peer[1] (localhost:9902) and peer[2] (localhost:9903) every ~3s for 8+ minutes straight (120+ attempts, INFO-level 'connecting outbound to peer[N]' logged each time -- the SUCCESS-path log line per ha_connect_finish_peer's own code, not a failure), never once completing a working peer connection. Meanwhile ctrl-2/ctrl-3 were actively, healthily replicating with each other (real AppendEntries traffic, LSN steadily advancing) -- they formed a working 2-of-3 quorum excluding ctrl-1 entirely. `xceadmin scheduler record status` failed with 'no primary available: timed out contacting localhost:9901' (ctrl-1's own client port) even though `xce_test`'s REPL joblist worked fine against the real 2-node quorum.", "file": "src/controller/ha.c (ha_connect_finish_peer, reconnect_lane_fn, ha_connect_to_peer)", @@ -21081,6 +23205,8 @@ }, { "id": "bug-1699", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T07:35:00Z", "error_message": "ctrl-1 segfaulted (SIGSEGV, core dumped -- confirmed via bash's own accurate job-control message, not a misread) within ~1.5s of a genuinely fresh boot, immediately after processing its first client 'ctrlstatus' probe request, during a fresh 10-round --chaotic run-ha-soak.sh --chaotic invocation (build.dbg, no ASAN) launched to verify bug-1696. Driver reported 'FAIL: no primary elected within ~90s' as a consequence (ctrl-1 never came back; ctrl-2 was subsequently Killed by cleanup).", "file": "src/controller/ha.c or src/controller/cluster.c (exact function not identified -- crash occurred somewhere in the ctrlstatus request-handling path shortly after raft_cb_become_follower/first_boot=1)", @@ -21110,6 +23236,8 @@ }, { "id": "bug-1700", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T16:35:00Z", "error_message": "User-reported: `xceadmin scheduler record status` (and by the same mechanism, `record start`/`record stop`) fails with `Error: controller returned status -10014` whenever the `--controller`/default address happens to point at a controller that is currently a STANDBY, not the primary. User's expectation: 'xceadmin/libxce should find the correct primary to talk to it' instead of surfacing the raw rejection.", "file": "src/controller/cluster.c (myreq_handler, ~line 406-414)", @@ -21143,6 +23271,8 @@ }, { "id": "bug-1701", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T00:00:00Z", "error_message": "Discovered while implementing/live-verifying bug-1700's fix (myreq_handler's generic dispatch-gate rejection now calls fill_redirect_reply() on -XCE_ERR_STANDBY, cluster.c ~410-433): the fix has ZERO effect in a real deployment. `xceadmin scheduler record status --controller ` against a live 3-node cluster still fails with 'Error: controller returned status -10014' identically before and after the fix, with no 'redirected: ...' hop ever printed.", "file": "src/controller/ha.h:161-162 (ha_state_t.primary_host/primary_port), src/controller/join.c:27-31 (fill_redirect_reply), src/controller/ctrlstatus.c:38 (pre-existing doc comment already naming this)", @@ -21173,6 +23303,9 @@ }, { "id": "bug-1702", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test jobrun -m host strncpy into 256B stack buffer unbounded (src/harness/test.c)", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_jobrun's `-m :` parsing copies the host-name substring into a fixed 256-byte stack buffer with an unbounded strncpy (length = bytes before ':' in the user-supplied argument, never clamped to sizeof(buf)). A -m argument with a hostname longer than 256 bytes before the ':' overflows the stack buffer.", "file": "src/controller/testsuite/test.c:2130-2136 (cmd_jobrun)", @@ -21198,6 +23331,9 @@ }, { "id": "bug-1703", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test jobsub -n min strncpy into 64B stack buffer unbounded (src/harness/test.c)", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_jobsub's `-n ,` parsing copies the min-value substring into a fixed 64-byte stack buffer with an unbounded strncpy (length = bytes before ',' in the user-supplied argument, never clamped to sizeof(buf)). A -n argument with 64+ characters before the comma overflows the stack buffer.", "file": "src/controller/testsuite/test.c:2246-2254 (cmd_jobsub)", @@ -21223,6 +23359,8 @@ }, { "id": "bug-1704", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_qctrl declares `int action;` with no initializer and only assigns it inside the -a/-i/-o/-c getopt cases; there is no post-loop check that one of them was given before `queuectrl.action = action` is sent over the wire. Running `qctrl -q normal` with no action flag (syntactically valid per USAGE_QCTRL, which lists all four as optional) transmits indeterminate stack memory as the queue action.", "file": "src/controller/testsuite/test.c:2663 (declaration), :2668-2688 (getopt loop), :2695 (queuectrl.action = action)", @@ -21244,6 +23382,9 @@ }, { "id": "bug-1705", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test jobsub -x exdata overwritten by build_job_exdata and leaked", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_jobsub's `-x ` flag is documented (USAGE_JOBSUB) as overriding the built-in PATH-only exdata JSON, but build_job_exdata()/build_job_envs() unconditionally overwrite submit.start_info->exdata with the built-in PATH JSON regardless of whether -x already set a value -- the exact opposite of the documented behavior. The original -x value is also leaked (never freed before being replaced).", "file": "src/controller/testsuite/test.c:2243-2245 (-x parsing), :2181-2195 (build_job_envs unconditional overwrite), :2336 (unconditional call site), USAGE_JOBSUB at :484-485", @@ -21268,6 +23409,10 @@ }, { "id": "bug-1706", + "status": "duplicate", + "status_updated": "2026-09-24", + "status_note": "same defect as bug-1574 (set_resval() has no string case)", + "duplicate_of": "bug-1574", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_resupdate's own documented example (`resupdate -n mylabel -v foo -t string`) and any string-typed resource (e.g. `type`, `model`) corrupts the submitted resource value: set_resval() has no `case Value_u_svalue:` in its switch, so for a string-typed resource, vals[idx].name is left as uninitialized stack memory and the wire type stays the calloc'd default (ivalue32) instead of the string the user typed.", "file": "src/controller/testsuite/test.c:3323-3364 (set_resval, missing Value_u_svalue case), :3483 (vals[] never zeroed), :363-364/375 (USAGE_RESUPDATE's own -t string example)", @@ -21294,6 +23439,9 @@ }, { "id": "bug-1707", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test resupdate trailing -n without -v silently dropped", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_resupdate: `resupdate -n slots -v 4 -n mem` silently drops the trailing `mem` resource with zero diagnostic (the final pending -n/-v pair is only flushed if a value was seen), yet the command still prints \"resource update ok!\" as if everything the user typed succeeded. A mid-list missing -v does print a warning; only the trailing one is silent.", "file": "src/controller/testsuite/test.c:3551-3561 (trailing-pair flush, comment admits the drop), :3502-3507 (mid-list warning that trailing case lacks), :3567 (unconditional success message)", @@ -21318,6 +23466,9 @@ }, { "id": "bug-1708", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test jobkill -s non-numeric -> signal 0, still reports success", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_jobkill's `-s ` is parsed via unchecked atoi(); a non-numeric value (e.g. a plausible typo of the symbolic name `SIGKILL` instead of `9`) silently becomes signo=0 (POSIX null signal, delivers nothing), yet the command still prints \"Job signaled\" as if it succeeded.", "file": "src/controller/testsuite/test.c:2572-2574 (signo = atoi(optarg)), :2601 (unconditional success message)", @@ -21340,6 +23491,9 @@ }, { "id": "bug-1709", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test jobinfo returns 0 on malformed/missing jobid", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_jobinfo always returns 0 (success) even when its -j jobid is malformed (parse_jobid's return code is discarded), when no -j is given at all, or when xcecli_jobinfo fails for every -j given -- breaking the tool's own scripted-mode exit-code-verdict contract for this command specifically.", "file": "src/controller/testsuite/test.c:2029-2058 (cmd_jobinfo)", @@ -21363,6 +23517,9 @@ }, { "id": "bug-1710", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test hello -c <=0 reports vacuous success", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_hello with -c 0, a negative count, or a non-numeric value (all yield count<=0 from unchecked atoi()) sends zero hellos (the while(count>0) loop body never runs) but still returns the still-zero-initialized rc, reporting success even though nothing was sent.", "file": "src/controller/testsuite/test.c:1341 (rc=0 init), :1351 (count=atoi(optarg), unchecked), :1367 (while count>0 loop), :1373 (return rc)", @@ -21385,6 +23542,9 @@ }, { "id": "bug-1711", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "a8154acb", "timestamp": "2026-08-26T09:32:18Z", "error_message": "xce_test: several command-specific typos and output-formatting inconsistencies found during a /cli-ux command-by-command audit: worker_status_str returns the misspelled \"UNKOWN\" for an unrecognized worker status bit (visible in every workerlist row hitting that path); cmd_help prints the malformed \"No commands matchs `%s'\" on an unknown help topic; cmd_runtasks's default launch_type for a single task is misspelled \"sequentail\" (display/data-only, not behavior-altering -- worker only special-cases \"openmpi\"/\"mpich\"); cmd_reslist right-justifies its Name/Description text columns (inconsistent with cmd_workerlist's left-justified equivalent columns) and prints the Flags column as unlabeled %x hex (ambiguous vs decimal, unlike this file's own \"0x%x\"-prefixed state_raw precedent); cmd_joblist prints SubmitTime/StartTime/FinishTime as raw Unix epoch seconds instead of using this file's own ctrlstatus_format_time_ms-style formatting, and labels a hidden mintasks,maxtasks pair as a single \"NTasks\" column; cmd_joblist's job_state_str has no raw-state fallback for an ambiguous RUN+EXIT state the way print_jobinfo already guards against two functions away; getconf's User Groups/Host Groups/Configs sections always print their header followed by nothing (print_grouplist/print_configlist are no-op stubs), indistinguishable from a genuinely empty cluster.", "file": "src/controller/testsuite/test.c:1435 (UNKOWN), :1069 (matchs), :2939 (sequentail), :1401/1404 (reslist formatting), :1454/1473 (workerlist formatting for contrast), :1551-1572 (joblist raw epoch + NTasks label), :1486 (job_state_str), :1972-1992 (print_jobinfo's raw-state fallback for contrast), :3632/3634 (print_grouplist/print_configlist stubs), :3659-3666 (getconf headers)", @@ -21407,6 +23567,9 @@ }, { "id": "bug-1712", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test resupdate duplicate -v gives misleading message", "timestamp": "2026-08-26T09:32:18Z", "error_message": "cmd_resupdate: a duplicate -v for the same pending resource (`resupdate -n slots -v 4 -v 8`) prints the misleading message \"ignore res value `%s' as no res name found\" even though a resource name WAS found -- the real problem is a value is already pending for it, not a missing name.", "file": "src/controller/testsuite/test.c:3525-3531 (case 'v' else-branch)", @@ -21432,6 +23595,8 @@ }, { "id": "bug-1713", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T17:15:00Z", "error_message": "User-reported, live 3-node cluster, right after bug-1700/bug-1701's redirect fix landed and was live-tested: `xceadmin scheduler record status --controller ` now correctly redirects ('redirected: localhost:9903 -> localhost:9901'), but the SAME command against the SAME redirect target intermittently fails with `Error: controller returned status -10013` (sometimes succeeds with real recording status, sometimes fails on an immediate repeat) instead of always succeeding.", "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request), possibly xceadmin/discovery.py (follow_redirects)", @@ -21463,6 +23628,9 @@ }, { "id": "bug-1714", + "status": "investigating", + "status_updated": "2026-09-24", + "status_note": "intermittent scheduler segfault under load; crash site not found, d5ec1eb3 mitigation only", "timestamp": "2026-08-26T18:35:00Z", "error_message": "User-reported live deployment: xce_scheduler process died and restarted unexpectedly (PID changed from 3964918 to 4137745 in scheduler.log with a ~79s gap, no manual restart by the user) during a live 3-node cluster test session; xceadmin briefly reported 'Error: scheduler disconnected' (the already-correct bug-1693 disconnect-detection path in cluster.c, working as designed -- not itself a bug) then recovered on the next attempt once the scheduler rejoined. User confirmed via `sudo dmesg -T` a REAL kernel-level segfault: `xce_scheduler[2815660]: segfault at 30 ip 0000601569e1fb35 sp 00007db7db1f6e00 error 4 in xce_scheduler[23b35,601569e01000+de000] likely on CPU 5 (core 16, socket 0)` -- though this specific dmesg entry is dated Aug 25 19:21:39, a DIFFERENT, earlier occurrence than today's specific restart (Aug 26 ~18:00-18:02), not necessarily the direct cause of THIS restart. No core dump exists for xce_scheduler anywhere in /var/lib/systemd/coredump/ (only xce_controller and python entries present, despite the scheduler segfault being kernel-confirmed) -- a separate, notable gap in this environment's crash-capture coverage for the scheduler process specifically.", "file": "src/scheduler/ (exact crash site unknown -- no core dump, no backtrace available for either occurrence)", @@ -21513,6 +23681,8 @@ }, { "id": "bug-1715", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T18:45:00Z", "error_message": "User-reported, live 3-node cluster: `xceadmin scheduler record status --controller localhost:9901` failed once with `Error: scheduler disconnected`, then succeeded on the very next identical invocation moments later. Root-caused to a full cluster redeploy (user confirmed: 'This is a full rebuilt and re-deployed cluster') restarting all 5 processes (3 controllers + worker + scheduler) together -- PIDs across ctrl-1/ctrl-2/ctrl-3/scheduler/worker logs are all tightly clustered (4137667/4137684/4137696/4137745/4138046), confirming a coordinated restart, not an isolated scheduler crash. The request happened to land in the brief, normal window between the old scheduler disconnecting and the new one reconnecting.", "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)", @@ -21541,6 +23711,8 @@ }, { "id": "bug-1716", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T14:25:00Z", "error_message": "test_rcfile_load_directory_path_warns_and_returns_0:FAIL: Expected TRUE Was FALSE (stderr capture was empty -- no warning printed) when rcfile_load() opened a directory path", "file": "src/controller/testsuite/rcfile.c", @@ -21564,6 +23736,8 @@ }, { "id": "bug-1717", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T00:00:00Z", "error_message": "ASAN heap-use-after-free abort in controller_unit_test: commit_resolve_pending (commit.c:807) racing fsync_worker (persist.c:867) against tap_teardown (test_apply_pump.c:243)", "file": "src/controller/commit.c", @@ -21589,6 +23763,8 @@ }, { "id": "bug-1718", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T16:23:00Z", "error_message": "A python3 pty-driver embedded via a heredoc on a command that ALSO receives piped stdin silently discards the piped input", "file": "src/controller/testsuite/test_rcfile_integration.sh", @@ -21611,6 +23787,8 @@ }, { "id": "bug-1719", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T17:14:00Z", "error_message": "resource-value-verification skill's deployment failed: 'controller binary missing' immediately after a scheduler-only xce-build.sh run that had just succeeded", "file": "src/tools/xce-build.sh", @@ -21633,6 +23811,8 @@ }, { "id": "bug-1720", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-26T17:17:00Z", "error_message": "xce-test-resources run-resource-verify.sh: apiserver layer UNREACHABLE for all 9 injected resources; apiserver.log showed 'API Server fails to start: listen tcp :8080: bind: address already in use'", "file": "environment (stale orphaned process, not a code defect)", @@ -21655,6 +23835,9 @@ }, { "id": "bug-1721", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "clean_failover_preservation.sh default ha/data dir no longer exists", "timestamp": "2026-08-26T00:00:00Z", "error_message": "clean_failover_preservation.sh FAIL: cluster.bin missing under /ha/data (deploy prerequisite)", "file": "src/controller/testsuite/clean_failover_preservation.sh", @@ -21676,6 +23859,8 @@ }, { "id": "bug-1722", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-27T00:00:00Z", "error_message": "AddressSanitizer heap-buffer-overflow in make_job_account() (src/lib/xcealloc.c:188), WRITE of size 8 -- found while verifying viability of a proposed scheduler snapshot-allocation-capture fix, not user-reported.", "file": "src/lib/xcealloc.c", @@ -21700,6 +23885,8 @@ }, { "id": "bug-1723", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-27T00:00:00Z", "error_message": "LeakSanitizer (xcelib_test's ASAN default detect_leaks=1, per bug-1437's established precedent that this is a separate pass/fail axis from Unity PASS/FAIL) aborts the whole xcelib_test binary at exit with SIGABRT even on a completely clean run (all Unity tests, including the new scheduler-snapshot-job-allocation task 1.1 dup_job_alloc() suite, report 0 Failures) -- 456 byte(s) leaked in 29 allocation(s), confirmed present identically at HEAD before task 1.1's changes (baseline A/B comparison: stashed task 1.1's 3 changed files, rebuilt, reran standalone binary -- same 456 bytes/29 allocations, same two leak sources, same SIGABRT).", "file": "src/lib/testsuite/test_xcealloc.c", @@ -21726,6 +23913,8 @@ }, { "id": "bug-1724", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-28T00:00:00Z", "error_message": "3 xceadmin live E2E tests (test_e2e_live_push.py::test_live_config_push_against_real_single_node_controller, test_e2e_live_ha_push.py::test_live_config_push_against_primary_reaches_success_path, test_e2e_live_ha_push.py::test_live_config_push_against_standby_returns_standby_rejection_not_redirect) always SKIPPED under the meson-driven build path (`/xce-build xceadmin`, `nix build .#xcecore`) with 'no xce_controller binary found under xce.build/build.release/build.dbg', regardless of whether a real xce_controller/xceadmin binary actually existed anywhere in the checkout.", "file": "src/xceadmin/meson-build.sh", @@ -21750,6 +23939,8 @@ }, { "id": "bug-1725", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-28T00:00:00Z", "error_message": "After bug-1724's fix made test_live_config_push_against_standby_returns_standby_rejection_not_redirect actually execute (previously always skipped), it FAILED for real: pushing config to a standby now redirects to the primary and succeeds, contradicting the test's own hard-coded 'standby flatly rejects, no redirect' assertions. Separately, the rewritten success-path test's reused _assert_primary_committed() helper intermittently failed its 4th assertion ('config activated from committed marker tag=...' not found in the primary's log immediately after the push CLI returned success).", "file": "src/xceadmin/tests/test_e2e_live_ha_push.py", @@ -21781,6 +23972,10 @@ }, { "id": "bug-1726", + "status": "cannot-reproduce", + "status_updated": "2026-09-24", + "status_note": "single occurrence, no surviving logs", + "repro_attempts": "5 immediate retries, 0 reproduced", "timestamp": "2026-08-28T08:15:00Z", "error_message": "test_e2e_live_ha_push.py::test_live_config_push_against_standby_redirects_and_succeeds ERRORed at live_ha_cluster fixture setup (single occurrence, user-run via `nix develop ./src` + direct pytest against the built build.dbg mirror, not via any Claude-driven build): _wait_for_primary(nodes, 30.0) raised TimeoutError -- none of the 3 freshly-spawned xce_controller processes ever printed the CTRL_PRIMARY election-complete marker within the 30s timeout (normal election completes ~10.2s per this file's own docstring, keyed to HA_DEFAULT_HEARTBEAT_TIMEOUT_MS=10000).", "file": "src/xceadmin/tests/test_e2e_live_ha_push.py (live_ha_cluster fixture, _wait_for_primary)", @@ -21808,6 +24003,8 @@ }, { "id": "bug-1727", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-30T22:00:00Z", "error_message": "User-reported (live REPL, real cluster-captured replay /tmp/scheduler-replay-20260828-085356.replay, 40 sessions, 60 jobs): after `xce_scheduler -i -s ` + `step`, `info worker all` shows `resource slots: value=44.00 used=22.00` for worker pc13v, but the worker's real installed capacity is 22 slots (confirmed by the same worker's own `resource maxcpus: value=22.00` line) and all 22 running jobs are 1-slot-each, so the correct display is `value=22.00 used=22.00` (fully packed, 0 free) -- `used=22.00` is correct, `value=44.00` is double the true installed capacity.", "file": "src/scheduler/src/repl_info.c (repl_info_worker_one, ~line 281-390)", @@ -21836,6 +24033,9 @@ }, { "id": "bug-1728", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "spurious \"incorrect worker idx -1\" ERROR after worker-join commit timeout (cluster.c)", "timestamp": "2026-08-28T10:25:14Z", "error_message": "Controller logs `ERROR [handle_endpoint_error] incorrect worker idx -1` in all 3 nodes of a live HA cluster (ctrl-1.log, ctrl-2.log, ctrl-3.log), always immediately after the same two-line sequence: `WARN [commit_submit] commit: WAIT deadline expired without commit (lsn=X committed_lsn=Y writable_lsn=1 timeout_ms=500)` then `WARN [handle_worker_join] handle_worker_join commit barrier failed for pc13v, rc -110`. All three occurrences in the sampled log correlate exactly with an in-progress PreVote/failover window (liveness poll timeout -> primary_ctrl_fd RDHUP -> PreVote begin, immediately preceding each ERROR).", "file": "src/controller/cluster.c (handle_endpoint_error, ~line 52-77); src/controller/worker.c or join.c (handle_worker_join, commit-barrier caller -- not yet located)", @@ -21862,6 +24062,8 @@ }, { "id": "bug-1729", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-28T10:25:14Z", "error_message": "xcecd_worker.log logs `[ERROR] Job exited with code 137: Killed` for an ordinary, deliberate job kill/cancel (exit code 137 = 128+SIGKILL) -- user-reported as misleading, since 'kill job' is a normal admin operation, not a failure.", "file": "src/worker/src/uv_ipc.c (ipc_worker_send_status, JOB_EXITED branch)", @@ -21887,6 +24089,8 @@ }, { "id": "bug-1730", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-28T13:39:36Z", "error_message": "User-reported (live 3-node HA cluster, real workload): a finished job's array elements sometimes remain visible as RUN forever in the scheduler's own state (confirmed both live and via a scheduler session recording's .replay snapshot -- xce_scheduler -i -s showing several 0-12-1787907177-0[N] elements as RUN when the controller's joblist/xcecli already shows them DONE). User's own log correlation: scheduler-side handler.c:414 'sched_handle_sched_update_job updated' fires once per job (the dispatch-time notification) but a second fire for the finish notification never arrives for the affected jobs, while the worker log independently proves those jobs genuinely completed (uv_ipc.c:1042 'Job completed successfully', milliseconds after dispatch -- these are near-instant /bin/true-style array elements).", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_UPDATE case), src/controller/sched.c (sched_job_exit_fun/sched_job_start_fun/sched_job_fail_fun), src/controller/job.c (HOOK_DETAIL_JOB_EXIT/START/FAIL fire sites)", @@ -21916,6 +24120,8 @@ }, { "id": "bug-1731", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-28T13:39:36Z", "error_message": "STILL OPEN half of the bug-1730 investigation: the user's originally-reported symptom itself -- a finished job's job->state staying stuck as RUN in the scheduler's own view (live and in a .replay snapshot) despite the controller and the worker both having genuinely completed it -- was NOT conclusively reproduced live despite extensive targeted testing, so its exact trigger mechanism remains unconfirmed.", "file": "src/controller/persist.c (apply_committed_entry, WAL_JOB_FINISH/START/DISPATCH shared case), src/controller/sched.c (sched_job_update_fun)", @@ -21944,6 +24150,8 @@ }, { "id": "bug-1732", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-29T05:21:00Z", "error_message": "src/tools/xce-build.sh's `worker` component case never builds `xcecd_virtual` (only `xcecd xcecd_worker xrun controller_connector_test`), even though deploy-ha.sh requires xcecd_virtual whenever no real systemd manager is present (the common case in sandboxes) -- `/xce-build worker` alone leaves a deploy unable to auto-detect/generate start-worker.sh for the virtual worker.", "file": "src/tools/xce-build.sh (worker component case)", @@ -21964,6 +24172,8 @@ }, { "id": "bug-1733", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-29T06:30:00Z", "error_message": "AddressSanitizer: heap-use-after-free WRITE of size 8 at vtask_enq (src/lib/vtaskmgr.c:517) -- crashed 2 of 3 controllers in a live 3-node HA cluster under a heavy burst-array-job workload (6 concurrent `jobsub -a 50` submissions, 300 near-instant job elements, driving high-frequency WAL-replication-ACK traffic). Found while chasing bug-1714/bug-1731's scheduler crash under ASAN -- the scheduler itself did not crash in this run, but two controllers did, independently, ~7 and ~14 minutes into the run, with byte-for-byte identical ASAN stacks (different PIDs/addresses).", "file": "src/lib/vtaskmgr.c (vtask_enq, line 517), src/lib/dataq.c (dataq_enq)", @@ -21992,6 +24202,8 @@ }, { "id": "bug-1734", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-30T10:15:00Z", "error_message": "User-reported live deployment: scheduler's view of a job array's state (`[sim] jobs` / xce_scheduler -i -s) diverges sharply from the controller's own joblist for the SAME array, well after every element genuinely finished. Array 0-4-1788084587-0 (50 elements, command `hostname`): controller's `joblist` shows ALL 50 elements as DONE with real, monotonically-increasing StartTime/FinishTime timestamps (1788084587..1788084602) -- the array is genuinely, fully complete. The scheduler's own concurrent view of the identical array shows a scrambled mix: [0-4]=RUN, [5-16]=DONE, [17]=EXIT, [18-19]=DONE, [20]=EXIT, [21]=DONE, [22-24]=EXIT, [25]=DONE, [26-29]=EXIT, [30-46]=RUN, [47-49]=PEND. Two things about this pattern are themselves suspicious, not just 'the scheduler is behind': (a) elements [30-46] show RUN and [47-49] show PEND, i.e. the scheduler believes 20 elements haven't even STARTED yet, even though the controller's timestamps show the whole array finished by 1788084602 -- this is not a small lag, the scheduler's view is drastically behind reality for over a third of the array; (b) EXIT appears interspersed with DONE for a uniform array of a single reliably-successful command (`hostname`, not `/bin/false` or anything failure-prone) -- elements 17/20/22-24/26-29 show EXIT while their immediate neighbors show DONE, which is not what a simple 'scheduler hasn't caught up yet' lag would produce (a lagging-but-otherwise-correct view should show a clean PEND/RUN prefix behind a DONE suffix, not DONE and EXIT interleaved for what should be uniform outcomes). CORRELATED evidence from the SAME run: xce_scheduler's own log shows 10 consecutive WARN lines at the very same timeframe -- `xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), discarding duplicate/orphaned job` for jobid 4631575359520768000 through 4631575359520768009 -- which decode (JobID bit layout: tag/cluster/apiserver/timestamp/array_idx) to EXACTLY array elements 0-4-1788084587-0[0] through [9] of this SAME array. This WARN is emitted by the bug-1714-adjacent fix landed earlier this session (commit 38437e57, 'fix(scheduler): don't hand an orphaned duplicate job to the picker') -- confirms live, in this exact reproduction, that this array's first 10 elements were pushed to the scheduler TWICE (the TOCTOU dedup race in sched_on_job_array()/handler.c that commit documented as theoretically possible is firing for real here), and the second copy is now being correctly discarded rather than corrupting state or crashing (that part of the fix is working as designed). User also separately requested (logged here as part of the same report, lower priority, not a correctness bug): xce_core_push_events's WARN line above prints the raw numeric jobid (`4631575359520768000`) instead of this project's human-readable fmt_jobid() form (`0-4-1788084587-0[0]`) that every other job-related log line in this codebase uses -- a logging-usability gap in that specific WARN, worth a one-line fix.", "file": "src/scheduler/src/handler.c (sched_on_job_array's dedup-race, already documented as a known TOCTOU race by commit 38437e57), src/scheduler/src/core_events.c (xce_core_push_events, WARN log site + the JOBS_PUSH duplicate-discard path from commit 38437e57), src/scheduler/src/driver.c / src/controller/sched.c / src/controller/persist.c (the job-update notify pipeline this session's bug-1731/bug-1714 investigations already covers)", @@ -22025,6 +24237,9 @@ }, { "id": "bug-1735", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "d8a37fa2", "timestamp": "2026-08-30T10:30:00Z", "error_message": "PRIORITY: HIGH -- affects data the scheduler needs for resource-usage-aware scheduling decisions (see priority note below). Split 2026-08-30 from a broader initial report into this (core usage: memory + CPU time) and bug-1736 (display-only statistics: swap, task count, peak/efficiency percentages) at user request, since the two have very different urgency and the broader one should not block this one.\n\nUser-reported: job memory usage always shows 0 in the scheduler's simulation record file (.replay), despite the worker daemon presumably measuring real resource usage. User asked to check whether the worker even reports this data at all -- CONFIRMED: it does, but the value never survives to the controller due to a plain resource-name string mismatch, plus a second, independent gap in how the controller aggregates whatever it does receive.", "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages, line ~1418-1436 -- the worker-side cgroup-based measurement and resname assignment), src/controller/misc.c (update_host_runusage, line ~128-138 -- the controller-side parser), src/controller/job.c (update_job_rusages, line ~3200 -- pre-existing 'FIXME update other fields of job rusage later' comment), src/scheduler/include/sched/types.h:115 (xce_sched_job_t.rusages -- confirmed the scheduler stores/forwards the RAW xce_job_rusages_t wire report unmodified, does not itself re-parse or aggregate it), src/scheduler/src/recording_events.c:142 (wire.body.job_rusage = ev->u.job_rusage.rusages -- confirms the .replay file records whatever raw resname/value pairs it received, unchanged)", @@ -22061,6 +24276,9 @@ }, { "id": "bug-1736", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "swap/tasks/peak_cpu_time fixed 5c2c6664; peak_cpu_duration/efficiency still not computed", "timestamp": "2026-08-30T10:45:00Z", "error_message": "PRIORITY: LOW -- display/reporting only, not consumed by any scheduling logic. Split 2026-08-30 from bug-1735 at user request: the 'statistics-only' half of that finding -- resource usage fields with no scheduling relevance, purely for operator-facing job accounting/reporting (bjobs -l/bhist-style views), as opposed to bug-1735's memory+CPU-time core, which is a prerequisite for future scheduling decisions. Worker (collect_job_rusages(), xcecd_cgroups.c) already measures and sends 'memory_swap' and 'tasks_current' per job, but the controller has NO field to receive them at all (xce_host_runusage_t only has cpu_time/mem_usage -- no swap or task-count field exists) -- these two are silently dropped entirely, not just misnamed. Separately, xce_job_runusage_t's peak/efficiency fields (peak_cpu_time, peak_cpu_duration, mem_efficiency, avg_cpu_efficiency, peak_cpu_efficiency) have no computation logic anywhere in the codebase (grepped) -- nothing populates them, ever, regardless of bug-1735's fix.", "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages -- already sends memory_swap/tasks_current, no controller-side field to receive them), src/include/xce/types.h (xce_host_runusage_t -- would need new fields for swap/task-count; xce_job_runusage_t -- the peak/efficiency fields with no producer), src/controller/misc.c (update_host_runusage -- would need new parse branches), src/controller/job.c (update_job_rusages -- same open FIXME as bug-1735, this is the other half of what it would need to cover)", @@ -22094,6 +24312,8 @@ }, { "id": "bug-1737", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-30T22:05:00Z", "error_message": "User-reported (live 3-node HA cluster + worker, 50-element array job `sleep 100 && hostname`, xce_scheduler -i -s vs controller joblist): job 0-8-1788143749-0[38] shows EXIT in the scheduler's replay (`info jobs`) but DONE in the controller's joblist -- confirmed a REAL state divergence (JOB_STAT_EXIT=0x04 and JOB_STAT_DONE=0x08 are genuinely different bits, src/include/xce/types.h), not cosmetic.", "file": "src/controller/sched.c (sched_job_exit_fun, renamed sched_job_finish_fun), src/scheduler/src/handler.c (sched_handle_notify's NotifyData_data_schedExitJob case), src/schema/controller.capnp (schedExitJob field), src/include/typemap.h (xce_notify_data_t's sched_exit_job union member), src/lib/helper.c (xce_notify_data_ptr)", @@ -22121,6 +24341,8 @@ }, { "id": "bug-1738", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-30T22:10:00Z", "error_message": "Found investigating Symptom 4 (EXIT vs DONE divergence) alongside bug-1737: a SEPARATE, genuinely different mechanism that can ALSO clobber a correct terminal job state on the scheduler side, independent of the schedExitJob exit_code gap. Not separately live-reproduced this session (no post_exec-stage job failure was exercised against the live cluster) -- confirmed by direct code reading with an exact, unambiguous mechanism.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_STATE_CHANGE case, and the JOBS_PUSH duplicate-jobid merge bug-1734 added), src/controller/sched.c (sched_job_fail_fun, comment only)", @@ -22144,6 +24366,8 @@ }, { "id": "bug-1739", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-30T22:15:00Z", "error_message": "User-reported (live 3-node HA cluster + worker, 50-element array job): `info summary` showed 30 elements of an array marked RUN in `info jobs`, but `info worker pc13v` only listed 22 'assigned job:' lines (matching the worker's real 22-core capacity) -- 8 elements showed job-level state RUN with NO corresponding worker/slot-assignment record at all, a 'phantom RUN' exceeding real hardware capacity.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge path)", @@ -22167,6 +24391,8 @@ }, { "id": "bug-1740", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-08-31T22:00:00Z", "error_message": "The bug-1735/resource-accounting.md reservation-vs-usage reconciliation feature (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage(), resource.c, committed this session) was wired into schedobj.c's sched_apply_job_rusage() -- but that function has exactly ONE caller anywhere in this codebase: scheduler.c's sched_update(), which itself has NO live caller at all (confirmed by exhaustive grep -- superseded by driver.c's xce_sched_driver_translate_pending_events()/xce_core_push_events() mechanism, per driver.c's own comments: \"Replaces sched_update()'s inline handling\"). The REAL, live production path for job rusage -- XCE_CORE_EVENT_JOB_RUSAGE in core_events.c's xce_core_push_events(), the SAME core-level event both the normal-mode driver (driver.c) and the sim/replay driver (sim_loop.c) translate their own rusage events into -- has its OWN separate, duplicate inline implementation of \"store rusage on job\" that never called sched_apply_job_rusage() or the new reconciliation logic at all. Result: the entire reconciliation feature was completely inert in both normal-mode (live cluster) and sim/replay mode -- exactly the context the user's original bug reports (phantom RUN jobs, EXIT/DONE divergence, mem always 0) were observed through -- despite passing all 11 of its own unit tests, which called the reconciliation functions directly, bypassing the real event-dispatch path entirely.", "root_cause": "Classic 'tested the function in isolation, never proved the real event path reaches it' gap. schedobj.c's sched_apply_job_rusage() and core_events.c's XCE_CORE_EVENT_JOB_RUSAGE handler are two independent, duplicate reimplementations of the same 'store rusage on job' bookkeeping (a pre-existing architectural duplication, likely left over from the scheduler-core rewrite that introduced the core/driver split -- sched_update() and its whole XCE_SCHED_OP_* dataq-dispatch mechanism appear to be the OLD, pre-rewrite path, fully superseded by driver.c's XCE_CORE_EVENT_* translation but never deleted). The two implementations had already drifted: they call DIFFERENT picker hooks (on_job_rusage vs plugin_job_rusage_update). Adding new logic to the dead one instead of the live one was an easy mistake to make without first confirming which path is actually reachable in production -- found only by directly grepping for every call site of sched_apply_job_rusage() and sched_update(), not by reading either function's own code or comments, which gave no hint of the duplication.", @@ -22190,6 +24416,9 @@ }, { "id": "bug-1741", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "rejoin resync recovers stuck jobs (165fc158); fire-and-forget finish notify still not retried", "timestamp": "2026-08-31T23:45:00Z", "error_message": "User-reported live deployment (/tmp/xce-ha-deploy, 3-node HA cluster + xcecd_virtual worker, build.dbg): after submitting a 50-element array job (jobsub -a 50 -c hostname) immediately following a fresh primary election, `xce_test`'s `joblist` shows array elements [6] through [12] (7 consecutive elements) permanently stuck in state RUN (FinishTime=0) while every other element (0-5, 13-49) correctly reached DONE with real FinishTimes, including elements submitted/dispatched both before and after the stuck range and well after the cluster had fully stabilized (up to ~46s post-submit). The user confirmed via repeated `joblist` checks over several minutes that elements 6-12 never progress -- not a transient lag.", "file": "src/worker/src/controller_connector.c (xcecd_job_finished), src/lib/client.c (xcecli_notify, xcecli_notify_jobfinish), src/lib/client_internal.h (XCECLI_FF_CALL), src/controller/job.c (on_job_finish, never invoked for the 6 lost reports), src/controller/cluster.c (do_notify/handle_notify)", @@ -22234,6 +24463,8 @@ }, { "id": "bug-1742", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T04:10:00Z", "gitea_issue": 45, "error_message": "Gitea issue #45: `xcecli jobs kill ` fails for every job-control action (kill/stop/resume/checkpoint) with `request to /xce/v1/jobs//control failed: []` -- an empty error body, not a normal ReturnError JSON payload. Reproduced by static analysis of the full CLI->apiserver call path (job ID conversion verified correct by hand: MakeJobID(1,0,1788186846,0,1) == 73932643848159233, matching the URL in the report, so the numeric-vs-string ID mismatch in the bug report is a red herring).", @@ -22260,6 +24491,8 @@ }, { "id": "bug-1743", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T04:10:00Z", "gitea_issue": 46, "related_bugs": [ @@ -22303,6 +24536,9 @@ }, { "id": "bug-1744", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "malformed resreq accepted at submit then silently dropped (controllers/jobs.go length-only check)", "timestamp": "2026-09-01T05:45:00Z", "gitea_issue": 47, "error_message": "Gitea issue #47 (rusage\u662f\u5426\u505a\u4e86\uff1f): `xcecli jobs submit -R \"rusage[mem=100000000a0]\"` -- a malformed resreq string (trailing non-numeric \"a0\" on the mem value) -- is silently ACCEPTED and returns a normal 'Job <...> is submitted' success message. Reporter's original comment: '\u4e71\u5199\u4e5f\u53ef\u4ee5\u63d0\u4ea4' (garbage input can be submitted too). Maintainer (rshen) already commented on the issue: this is a front-end (apiserver+cli) validation gap by design intent.", @@ -22333,6 +24569,8 @@ }, { "id": "bug-1745", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T00:20:00Z", "error_message": "kiro-validate-design worker-rejoin-resync (design phase, spec never implemented yet): initial pass found 2 critical issues in design.md, self-correction during the fix pass found 2 more. None are runtime bugs (no code exists for this spec yet) -- this entry records design-document defects caught before implementation, same category as bug-1514.", "file": ".kiro/specs/worker-rejoin-resync/design.md", @@ -22361,6 +24599,8 @@ }, { "id": "bug-1746", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T00:00:00Z", "error_message": "worker-rejoin-resync task 2.2 implementation: persist_submit_job_dispatch_fact(p, jobid, state, ts, alloc=NULL) crashes with a NULL-pointer SEGV instead of producing the \"no allocation\" WAL_JOB_DISPATCH payload design.md (Data Models / Out-of-Boundary section) and bug-1745's own design-review fix claimed was already \"a tested NULL-alloc path\" reusable verbatim. Confirmed via AddressSanitizer: SEGV on unknown address 0x0 in encode_JobAlloc (lib/controller.capnp.c:9805), called from capnp_encode_joballoc (controller/persist.c:235), called from persist_submit_job_dispatch_fact (controller/persist.c:7856), called from the new record_job_reconcile_assign_fact (controller/job.c) -- reproduced twice, deterministically, both via the full meson controller-test suite and via an isolated -t test_worker_resync run.", "file": "src/controller/persist.c", @@ -22388,6 +24628,9 @@ }, { "id": "bug-1747", + "status": "partial", + "status_updated": "2026-09-24", + "status_note": "FINISH half fixed eec2596c; failed-job finish_time still apply-time time(NULL) (job.c)", "timestamp": "2026-09-01T00:00:00Z", "error_message": "Pre-existing, systemic gap predating worker-rejoin-resync entirely, discovered while implementing task 3.2 (resync_apply_report, resync.c): WAL_JOB_FAIL carries no fail-time/finish-time slot in its wire payload at ANY layer, and apply_job_fail_transition() (job.c) unconditionally stamps job.finish_time = time(NULL) at APPLY time (not even receipt time) for EVERY fail-reporting caller in this codebase -- not just reconciliation. This means Requirement 4.2 ('the Controller shall record the finish time the Worker reported ... rather than the time the Controller processed the reconciliation') is currently unsatisfiable for a reconciled finished-FAILURE outcome via any existing path. UPDATE (2026-09-01, coordinator-directed re-scope): the sibling FINISH-success half of this gap is FIXED -- see 'fix' field below; only the FAIL half remains open.", "file": "src/controller/job.c", @@ -22423,6 +24666,8 @@ }, { "id": "bug-1748", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T02:00:00Z", "error_message": "worker-rejoin-resync task 3.2 (resync_apply_running_entry, resync.c), round-2 independent review: the round-1 fix for the PEND/unassigned-job regression (bug-1747-adjacent, not itself in bug-1747) introduced a NEW Requirement 4.4 violation. The round-1 guard `if (!resync_job_is_terminal(job->job.state) && resync_job_assigned_to_host(job, w->host.name)) { no-op } else { record_job_reconcile_assign_fact(...) }` short-circuits the `&&` to false whenever job->job.state IS terminal, falling into the `else` branch and calling record_job_reconcile_assign_fact() on an ALREADY-TERMINAL (DONE/EXIT) job -- which has no backward-transition guard of its own and unconditionally sets job->job.state = JOB_STAT_RUN, clobbering a terminal job back to RUN. Confirmed live and, worse, CERTIFIED AS CORRECT by the round-1 implementation's own test: test_apply_report_all_five_outcomes_updates_job_table seeded j_extra at JOB_STAT_DONE with a comment claiming this 'satisfies Req 4.8's trigger,' then asserted j_extra->job.state == JOB_STAT_RUN afterward and the suite reported PASS -- the test baked the bug in as an expected outcome instead of catching it.", "file": "src/controller/resync.c", @@ -22452,6 +24697,8 @@ }, { "id": "bug-1750", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T09:00:00Z", "error_message": "worker-rejoin-resync task 4.3, round-1 review: the resync_pending gate was checked as checking_connection()'s very first statement (an entry guard), returning early before xcecli_hello() or the needs_rejoin-triggered rejoin_call() could run. xcecd_resync_handle_query_page() sets resync_pending = true on receiving a job-state query's first page and only clears it when called again with a later page. If the worker's connection to the controller drops after a non-final page of a multi-page OPCode_jobStateQuery arrives (a query spanning more non-terminal jobs than fit in one page -- explicitly in scope per Requirement 8.1/8.3), no further page ever arrives, so xcecd_resync_handle_query_page() is never invoked again and resync_pending is stuck true forever -- permanently disabling checking_connection()'s entire body, including the worker's ONLY connection-recovery mechanism (xcecli_hello() / needs_rejoin-triggered rejoin). Directly violated Requirement 6.1 (the worker must be able to disconnect-and-rejoin during an incomplete reconciliation round) and regressed the intent of the pre-existing invariant test test_checking_connection_never_permanently_disables_worker, though none of round 1's own new tests drove the multi-page-then-disconnect sequence so it passed the full suite undetected.", "file": "src/worker/src/controller_connector.c", @@ -22480,6 +24727,8 @@ }, { "id": "bug-1751", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T09:00:00Z", "error_message": "worker-rejoin-resync task 5 (scheduler candidate-filter exclusion for XCE_WORKER_RECONCILING): design.md's Components/Architecture/Testing Strategy sections all cite scheduler.c:1211 (sched_reset_worker_(), the IS_WORKER_UNAVAIL skip inside the legacy sched_reset_workers()/sched_begin_session() path) as 'the existing candidate-host filter' to extend with the new reconciling-bit disjunct. Investigating before implementing (grep for every IS_WORKER_UNAVAIL skip-style check across src/scheduler/src/*.c) found sched_begin_session() has NO live callers anywhere in the current driver.c/sim_loop.c production paths -- both call xce_core_begin_session() (core_session.c, added by the scheduler-core refactor) directly instead, which runs core_reset_workers() (core_session.c:102), an independently-maintained mirror of sched_reset_worker_() with the IDENTICAL IS_WORKER_UNAVAIL-only check, per that function's own header comment ('mirrors scheduler.c's sched_reset_workers/sched_reset_worker_'). Extending only scheduler.c:1211 as design.md literally specifies would leave the task's own Observable Completion Criterion (a RECONCILING host is never selected as a dispatch candidate while the flag is set) factually false on both real drivers, since neither ever executes the code path being changed. bug-1688's own investigation (2026-08-26) independently confirms core_reset_workers()/xce_core_begin_session() as the actively-traced, live production path. No test in src/scheduler/testsuite exercises sched_reset_worker_()'s or sched_begin_session()'s IS_WORKER_UNAVAIL behavior either (design.md's Testing Strategy claim of an 'existing IS_WORKER_UNAVAIL candidate-filter test' to extend does not correspond to any test that actually exists).", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/core_session.c", @@ -22514,6 +24763,8 @@ }, { "id": "bug-1752", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T10:00:00Z", "error_message": "worker-rejoin-resync task 6.1 (worker-side config-delivery gap): tasks.md's own task text says to have 'the worker's existing config-receive handling call xcecd_finished_job_retention_set_window_ms() with the received value' -- premised on some existing per-field config-receive dispatch already existing at the worker that just needed one more field wired in. Investigating before implementing (reading src/worker/src/controller_connector.c's myreq_handler dispatch switch and src/lib/xcectx.c's vtask_entry, which decodes every inbound request before myreq_handler ever sees it) found this premise false: myreq_handler's switch had NO case for OPCode_push at all (only OPCode_startjob/runtasks/jobcontrol/fetchfile/jobStateQuery existed) -- a config push (Request_body_configpushmsg, the body_which xce_pushconfig_req()/worker_push_config() send) fell straight through to `default: break;` and was silently discarded in its entirety, for EVERY field including the pre-existing clustername, not just the not-yet-added workerResyncWindowMs. xcecd_finished_job_retention_set_window_ms()'s own doc comment (xcecd_worker_utils.h, task 4.1) had already independently found and documented the same gap ('the worker daemon has no runtime delivery path for this ClusterConfig field today... worker_push_config() only ever pushes hosttype/hostmodel/resource confvars via collect_confvars()... this setter is the seam a later task wires the real operator-configured value into once that delivery path exists') -- task 6.1 is that later task, but its own task text still assumed the delivery path itself already existed when it did not.", "file": "src/worker/src/controller_connector.c", @@ -22541,6 +24792,8 @@ }, { "id": "bug-1753", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-01T20:30:00Z", "error_message": "worker-rejoin-resync task 6.2 (controller-side OPCode_jobStateReport dispatch): design.md's Error Handling section only anticipated the STANDBY-role case for this opcode pair (\"verify... whether OPCode_jobStateQuery/OPCode_jobStateReport need an explicit redirect-reply path... Expected answer: no... but this must be confirmed, not assumed, given bug-1700's documented history of this exact class of gap recurring across two other opcodes\"). Auditing ha_dispatch_gate() (ha.c) in full per that instruction found the STANDBY case was indeed already correctly handled automatically (OPCode_jobStateReport, left absent from the CTRL_STANDBY allow-list, falls to the generic default: return -XCE_ERR_STANDBY branch, and cluster.c's existing bug-1700 post-gate block already fills the redirect hint for any such rejection with zero opcode-specific code needed) -- but the same audit surfaced a DIFFERENT, unanticipated gap in the SAME function: a CTRL_PRIMARY controller that has temporarily lost write quorum rejects OPCode_jobStateReport outright with -XCE_ERR_NOT_READY, because the opcode was absent from ha_dispatch_gate's quorum-loss \"fact-bearing ops\" allowlist (OPCode_notify/OPCode_update/OPCode_setpreason/OPCode_clrpreason, ha.c ~line 4686-4696) that already exists specifically to prevent this exact class of loss (bug-396) for other worker/scheduler fact reports.", "root_cause": "OPCode_jobStateReport is sent fire-and-forget (xce_job_state_report_req(), request.c/xce_request()) with no redelivery on rejection -- structurally identical to OPCode_notify, the opcode bug-396's fix (ha-commit-barrier-rollout) originally added this allowlist to protect. Since worker-rejoin-resync's two new opcodes were added to controller.capnp's OPCode enum without also being added to ha_dispatch_gate's quorum-loss allowlist, a job-state reconciliation report arriving at a primary during a transient write-quorum-loss window (e.g. mid-failover, the exact scenario this whole feature exists to recover from) would be rejected outright and never retried by the worker -- reproducing bug-396's own loss mode for precisely the facts worker-rejoin-resync (bug-1741) exists to recover.", @@ -22573,6 +24826,9 @@ }, { "id": "bug-1754", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "failed job exit code never persisted: apply_job_fail_transition never sets exit_info (job.c)", "timestamp": "2026-09-01T13:00:00Z", "error_message": "Discovered while building worker-rejoin-resync task 7.1's full round-trip integration test (test_full_rejoin_reconciliation_round_trip, test_worker_resync.c): a worker-reported exit code for a finished-FAILURE outcome is never durably recorded by ANY existing path in this codebase, reconciliation or ordinary. record_job_fail_fact()'s WAL_JOB_FAIL payload (job.c, persist_submit_job_fact_ex(..., WAL_JOB_FAIL, jobid, 0, WAL_JOB_FAIL_STAGE_FLAG|stage)) carries no exit-code slot at all, and apply_job_fail_transition() (job.c:2831-2853) never touches job.job.exit_info for the FAIL path -- it stays NULL unconditionally for every job failure. The new round-trip test asserts this honestly (TEST_ASSERT_NULL on job.job.exit_info with an inline comment) rather than encoding a wrong 'correct' expectation.", "file": "src/controller/job.c", @@ -22601,6 +24857,8 @@ }, { "id": "bug-1755", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T00:30:00Z", "error_message": "Discovered while building worker-rejoin-resync task 7.4's large job-count pagination test (test_large_job_count_pagination_correctness, test_worker_resync.c): the controller's real production OPCode_jobStateQuery send site, worker_send_resync_query() (src/controller/worker.c:730), calls xce_job_state_query_req(ep, c->myhost, &query, NULL, NULL, 0) with a hardcoded pktlimit of 0. xce_job_state_query_req()'s own pktlimit<=0 convention (request.c:1822-1824, \"if (pktlimit <= 0) { pktlimit = nrows; }\") means this ALWAYS sends the entire query as exactly ONE wire page, however many non-terminal jobs the controller queries a worker about -- there is no bound at all. This is asymmetric with the worker's own real report-send site (controller_connector.c's handle_jobstatequery -> XCECD_RESYNC_REPORT_PKTLIMIT == 100), which DOES genuinely cap and paginate OPCode_jobStateReport in production. The query-direction pagination mechanism (xce_job_state_query_req()'s own do/while page loop, task 1.4) is fully implemented and correct -- task 7.4's own test calls it directly with an explicit page size and proves it genuinely spans many wire pages with no drops/duplicates -- but it is effectively dead code in production today because its only call site never supplies a bound.", "file": "src/controller/worker.c", @@ -22629,6 +24887,8 @@ }, { "id": "bug-1756", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T02:40:00Z", "error_message": "worker-rejoin-resync task 4.4 (Requirement 3.8, resource_sync.c): AddressSanitizer abort \"Joining already joined thread, aborting.\" while building the task's own new resync_pending-gate tests for resource_sync_thread(). Discovered live: the new tests are the first callers ever to exercise resource_sync_create()/resource_sync_start()/resource_sync_stop()/resource_sync_destroy() as a full lifecycle from a test binary (the two pre-existing tests only ever called collect_load_indices() directly). Each test calls resource_sync_stop(sync) then resource_sync_destroy(sync) back-to-back -- the exact same sequence xcecd.c's own real shutdown path (xcecd_cleanup(), src/worker/src/xcecd.c:580-582) already uses in production.", "file": "src/worker/src/resource_sync.c", @@ -22657,6 +24917,8 @@ }, { "id": "bug-1757", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T02:45:00Z", "error_message": "worker-rejoin-resync task 4.4 (resource_sync.c): after fixing bug-1756's double-join abort, the same new resync_pending-gate tests still failed deterministically -- \"No load indices to report\" logged, and both test_resource_sync_thread_resumes_load_indices_send_once_resync_clears and test_resource_sync_thread_sends_load_indices_normally_when_not_pending failed asserting the load-indices send call count stayed 0. Both tests run AFTER an earlier test in the same binary has already completed a full resource_sync_create()/start()/stop()/destroy() cycle.", "file": "src/worker/src/resource_sync.c", @@ -22681,6 +24943,8 @@ }, { "id": "bug-1758", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T03:23:00Z", "error_message": "scheduler suite (meson test --suite scheduler): sched_core_events_test FAILs deterministically -- test_core_events.c:1364:test_job_rusage_reconciles_mem_reservation_against_usage:FAIL: Expected 4 Was 0. Discovered as one of 2 failures (the other is a pre-existing, unrelated libxce/meson.build ASAN-flag gap) during worker-rejoin-resync feature-level /kiro-validate-impl full-suite run; confirmed NOT caused by worker-rejoin-resync.", "file": "src/scheduler/testsuite/test_core_events.c", @@ -22704,6 +24968,8 @@ }, { "id": "bug-1759", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T03:47:00Z", "error_message": "Live simulation run (xce_scheduler --sim, user-observed, not reproduced by me in this session) shows a JOBS_PUSH batch of 50 jobs where the first 22 (indices [0]-[21]) receive [sim] DISPATCH decisions, immediately followed by [sim] PEND decisions for ALL 50 jobs (indices [0]-[49]) -- including the 22 that had JUST been dispatched. Design expectation (confirmed correct, see root_cause): a job that has received a DISPATCH decision must be treated as RUN immediately, without waiting for any downstream job-start acknowledgment, and must never be reconsidered as a fresh scheduling candidate afterward -- in BOTH the live scheduler and simulation/replay mode.", "file": "src/scheduler/src/sim_loop.c, src/scheduler/src/plugins/builtin_fcfs.c, src/scheduler/src/scheduler.c", @@ -22735,6 +25001,9 @@ }, { "id": "bug-1760", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "88b0ac99", "timestamp": "2026-09-02T04:10:00Z", "error_message": "Live-tested against a real 3-node HA cluster + xcecd_virtual worker + xce_scheduler (worker-rejoin-resync soak-harness development): after the CURRENT primary is killed and a NEW primary is elected+reconverged, a job submitted through the new primary stays PEND for 60+ seconds with ZERO scheduler.log activity of any kind (no WARN/INFO/ERROR, no reconnect attempt) -- not a slow recovery, a genuinely stuck one. Reproduced twice in a row with the same shape.", "file": "src/scheduler/src/scheduler.c (sched_needs_rejoin:487, sched_on_reconnect:507, sched_publish_check_rejoin:1474), src/scheduler/src/main.c", @@ -22766,6 +25035,8 @@ }, { "id": "bug-1761", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T04:30:00Z", "error_message": "cli-ux audit of xce_scheduler's --interactive REPL (scheduler-sim-repl-verbosity spec discussion): repl_cmd_delete() (repl_break.c) parses its argument with atoi(), which silently returns 0 on non-numeric input instead of signalling an error -- `delete abc` reports \"[sim] delete: no breakpoint with id 0\" (a well-formed-but-missing-id message) rather than rejecting \"abc\" as not a number, masking an operator typo as a legitimate lookup miss.", "file": "src/scheduler/src/repl_break.c (repl_cmd_delete)", @@ -22786,6 +25057,8 @@ }, { "id": "bug-1762", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-02T07:50:00Z", "error_message": "scheduler-sim-repl-verbosity task 1.2 (Jobid Range Compressor, repl_fmt_jobid_range()): task 1.1's approved RED-phase meson.build target sched_repl_output_compression_test (src/scheduler/testsuite/meson.build) lists ONLY test_repl_output_compression.c as its source, with no source providing the function's implementation -- so even after adding a correct repl_fmt_jobid_range() definition to src/scheduler/src/repl.c (as design.md's File Structure Plan literally directs: 'implements the new shared jobid-range-compression helper ... in repl.c'), the target still fails to link: undefined reference to repl_fmt_jobid_range.", "file": "src/scheduler/testsuite/meson.build (sched_repl_output_compression_test), src/scheduler/src/repl.c", @@ -22807,6 +25080,9 @@ }, { "id": "bug-1763", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "test_sim_record_output.c stack-local sim read by unjoined picker thread (ASAN)", "timestamp": "2026-09-02T13:00:00Z", "error_message": "sched_sim_record_output_test (scheduler-sim-repl-verbosity task 3.2 GREEN-phase build): AddressSanitizer stack-use-after-return in cbuf_get (lib/mylock.c:151), thread T1 (sim_picker_thread_fn, sim_loop.c:456), reading the stack of test_output_file_is_decodable_with_full_stream_symmetry's stack-local `sim` (xce_sched_sim_driver_t, test_sim_record_output.c:341) after that frame returned. Fired once during a `/xce-build scheduler --incremental` run after this task's repl_break.c edits; 4/4 direct re-runs of the isolated binary afterward were clean (OK).", "file": "src/scheduler/testsuite/test_sim_record_output.c", @@ -22833,6 +25109,8 @@ }, { "id": "bug-1764", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-03T03:01:26Z", "error_message": "info worker's \"assigned jobs\" listing (repl_info_worker_one(), repl_info.c) can show a job that has already reached a terminal state (DONE/EXIT) as still assigned to a worker, contradicting info summary's running count. Live-observed in a real interactive session (session=59): info summary reported \"100 jobs (0 pending, 7 running)\" and worker pc13v's slots showed used=7.00, but the same info worker output listed assigned jobs: 0-11-1788164699-0[0-49] -- the full 50-index range of one array job, all still shown assigned.", "file": "src/scheduler/src/sim_loop.c", @@ -22860,6 +25138,9 @@ }, { "id": "bug-1765", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "frontend tests/regression/run.sh jobid regex rejects dashed jobid format", "timestamp": "2026-09-03T14:51:54Z", "error_message": "tests/regression/run.sh's 'jobs submit' test FAILs deterministically (28 passed / 1 failed / 1 skipped instead of 29/0/0), with the cascading 'jobs kill' test SKIPped as a result -- reproduced identically regardless of which build system (Makefile vs Meson) produced the xcecli binary under test, confirming it predates and is unrelated to the eliminate-frontend-makefile migration.", "root_cause": "CONFIRMED via source + git history. src/frontend/apiserver/controllers/jobs.go:1518 formats the submit-response message with a plain-digit '%d' placeholder ('Job <%d> is submitted...'), but src/frontend/xcecli/cmd/jobs/submit.go:500 post-processes that message client-side, replacing the '<%d>' substring via format.FormatJobID() (src/frontend/xcecli/common/format/jobid.go:40-57), which renders the packed 64-bit job ID as the dashed '{cluster}-{apiserver}-{timestamp}-{sequence}' form (e.g. '1-0-1788446837-0'). tests/regression/run.sh:143's assertion regex ('Job <[0-9]+> is submitted') was never updated for this dashed format. The CLI-side reformatting was introduced by commit d6008ead ('improve frontend jobid display', 2026-05-07) -- nearly four months before this discovery -- so the test has been silently broken (as a false FAIL, not a false PASS) since then, not a new regression.", @@ -22879,6 +25160,9 @@ }, { "id": "bug-1766", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "compound allocation keeps partial alloc on later primary failure, no rollback (scheduler.c)", "timestamp": "2026-09-04T08:03:28Z", "error_message": "OPEN, NOT A CRASH -- discovered while reviewing the scheduler-allocation spec requirements against live scheduler.c source: sched_make_compound_allocation() does not roll back already-succeeded primaries when a LATER primary in the same COMPOUND resreq fails allocation, and its own post-loop code can misattribute a partial allocation to a failed compound.", "root_cause": "sched_make_compound_allocation() (src/scheduler/src/scheduler.c:206-238) loops over a COMPOUND resreq's primaries, calling sched_make_primary_alloc_(sched, &ctx) for each with ONE shared ctx across the whole loop. On a mid-loop failure (rc <= 0) it just `break`s -- it never undoes the primaries that already succeeded (no zeroing of ctx.alloc, no reversal of any resource state those primaries touched). Worse: seqjob/parjob's job_alloc_fn sets ctx->alloc directly on success (see seqjob_allocator.c my_alloc()), so if primary 1 of a 2-primary compound succeeds (leaving ctx.alloc non-NULL) and primary 2 then fails, the function's unconditional post-loop `if (ctx.alloc != NULL) { MOVE_PTR(obj->alloc, ctx.alloc); }` still fires regardless of the overall `rc <= 0` -- so a COMPOUND job that is reported as allocation-FAILED can still end up with obj->alloc set to primary 1's partial, orphaned allocation. requirements.md for scheduler-allocation (Req 9 AC2) had assumed, based on the sibling scheduler-framework brief's own text, that this path already 'tracks allocations in ctx and cleans up on failure' -- that claim does not hold against the actual code; requirements.md was corrected during this review. Not yet confirmed whether the orphaned ctx.alloc is ever read for a job later found to be PEND/failed (i.e. whether this is dormant/leak-only or produces an observably wrong result) -- not investigated further since it is out of the scheduler-allocation spec's own boundary (Req 9 AC2 explicitly freezes COMPOUND/ALTER dispatch as unchanged).", @@ -22901,6 +25185,9 @@ }, { "id": "bug-1767", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "snapshot_load.c duplicate schedobj builder causes repeated field gaps", "timestamp": "2026-09-04T08:03:28Z", "error_message": "OPEN, NOT A CRASH -- recurring pattern finding, not a single defect: snapshot_load.c's local, hand-duplicated reconstruction of xce_sched_obj_t/schedobj (kept independent of core_events.c's core_build_schedobj() because that helper is `static` and snapshot.c/snapshot_load.c are compiled as separate objects) has now caused THREE independent field-completeness gaps in the same narrow reconstruction path, each discovered separately.", "root_cause": "snapshot.c's own header comment (Why not reuse core_events.c's core_build_job/core_build_queue/core_build_worker, snapshot.c:296-314) documents the deliberate choice to keep snapshot_load.c's job/queue/worker/schedobj reconstruction as a second, independent, field-for-field-identical implementation rather than exporting core_events.c's static builders -- justified there for core_build_job/queue/worker specifically (they also need an independent SAVE-direction implementation regardless, and funneling through xce_core_push_events() reintroduces a config/queue double-application bug). That reasoning does not equally apply to core_build_schedobj() alone: schedobj is never wire-captured (no build_wire_schedobj(), no SAVE-direction counterpart needed) and has no xce_core_push_events() involvement, so the only real objection to exporting it is 'larger, unrequested change to core_events.c's surface' -- a scope call, not an architectural wall. This duplication has now independently produced: (1) scheduler-snapshot-job-allocation spec's own motivating bug -- job->allocs/job->accounts never restored from snapshot (documented, deliberate scope decision at the time, later fixed); (2) the same spec's own further finding -- every snapshot-restored job (PENDING or RUNNING) got schedobj==NULL, an undocumented latent crash risk, fixed by adding a local field-for-field mirror of core_build_schedobj() to snapshot_load.c; (3) scheduler-allocation spec (this one) -- the new xce_resreq_fingerprint_t resreq_fp field added to xce_primary_schedobj_t (Req 7) is computed at core_build_schedobj()'s call site but has no reason to be computed by snapshot_load.c's independent copy, silently degrading Req 7's unsatisfiable-resreq cache to a permanent miss for any session started via snapshot+replay (the documented common real-investigation path) -- addressed in scheduler-allocation's own requirements.md (Req 7 AC 6) as a one-line addition, not by eliminating the duplication.", @@ -22927,6 +25214,9 @@ }, { "id": "bug-1768", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "nested COMPOUND/ALTER primaries skip slot/rusage checks (seqjob_allocator.c, parjob_allocator.c)", "timestamp": "2026-09-04T00:00:00Z", "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). seqjob_prepare_allocation() (src/scheduler/src/plugins/seqjob_allocator.c) and parjob_prepare_allocation() (src/scheduler/src/plugins/parjob_allocator.c) both silently `return 0` whenever `ctx->schedobj->tag != XCE_RESREQ_PRIMARY`. For a primary resource request nested inside a COMPOUND (or, transitively, an ALTER) job, `ctx->schedobj` remains the enclosing COMPOUND object for the entire per-primary allocation loop -- it is never swapped to a per-primary object -- so this guard fires for every nested primary and both prepare-allocation hooks no-op unconditionally.", "root_cause": "CONFIRMED via direct code read and `git log --follow -p`: the `ctx->schedobj->tag != XCE_RESREQ_PRIMARY` guard predates the entire scheduler-allocation feature (introduced in commit 12dcc2a6, months before this branch's base commit 5a670676); task 3.1's seqjob rewrap (b18c5f78) did not touch or introduce it. Effect: `ctx->alloc_req.host`/`ctx->alloc_req.shared` are never populated for a nested primary's resource want, so `sched_check_host_satisfy()` trivially passes against an empty want vector regardless of a candidate host's real free capacity. Only the SELECT expression (if the resreq specifies one) still gates candidate hosts for a nested primary -- slot count and rusage (memory, etc.) are unconditionally treated as satisfied. This means every COMPOUND job (and transitively every ALTER job, since ALTER delegates to COMPOUND dispatch) with a nested primary resource request has never had its real slots/rusage availability checked by the scheduler, since this guard was introduced.", @@ -22950,6 +25240,9 @@ }, { "id": "bug-1769", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "u.alters stores compound schedobj but read as sched_obj (OOB read, scheduler.c)", "timestamp": "2026-09-04T00:00:00Z", "error_message": "OPEN, NOT A CRASH (but reachable) -- pre-existing, out-of-scope type confusion discovered during scheduler-allocation task 5.1's review (cache/dispatch wiring). core_build_alter_schedobj() (src/scheduler/src/core_events.c) stores raw `xce_compound_schedobj_t*` (sizeof == 24 bytes) into `obj->u.alters` (an xlist_t), but sched_make_alter_allocation() (src/scheduler/src/scheduler.c) reads each entry back cast as `xce_sched_obj_t*` (sizeof == 176 bytes, with `offsetof(xce_sched_obj_t, u) == 136`) and dereferences `obj->u.compound->resreq` through that mis-typed pointer.", "root_cause": "CONFIRMED via direct code read at both the construction site (core_events.c) and the read site (scheduler.c), plus a compiled offsetof/sizeof probe against the real headers during task 5.1's review: this is a genuine struct-layout mismatch, not a false alarm -- the two types do not share a compatible prefix layout. Any real ALTER-tagged job (the resreq grammar's `{A} || {B}` alternate syntax legitimately produces XCE_RESREQ_ALTER per xce_resreq.peg) dereferencing through this cast reads ~112 bytes past the actual 24-byte allocation, an out-of-bounds heap read. Separately confirmed during the same investigation that test_snapshot_load_allocator_regression.c's header comment claiming to already exercise this ALTER path is itself mistaken: its `\"type == any\"` fixture has no `{`/`}` braces, so per xce_resreq.peg's grammar it parses as XCE_RESREQ_PRIMARY, not ALTER -- it never actually drives sched_make_alter_allocation()'s u.alters read path. No test anywhere in the tree currently constructs a real multi-entry u.alters list and drives it through sched_make_alter_allocation(), so this defect is both real and currently completely untested.", @@ -22974,6 +25267,8 @@ }, { "id": "bug-1770", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-04T00:00:00Z", "error_message": "FIXED (2026-09-06, resreq-maxtasks-wiring feature, commits 799a71ec..5c87f1cc). Originally: pre-existing, out-of-scope gap discovered during scheduler-allocation task 5.3's end-to-end integration test. The REAL job-ingest path could NEVER produce a primary resource request with maxTasks > 1, regardless of what task count a real job submission specified -- parjob_allocator.c's my_alloc() (maxTasks > 1) was therefore UNREACHABLE from any real production job submission.", "root_cause": "CONFIRMED via direct code read and a genuine RED-phase run of test_parjob_e2e_multislot.c (src/scheduler/testsuite/): parse_resreq_primary() (src/lib/resreq.c:431-432) unconditionally hardcodes `primary->minTasks = 1; primary->maxTasks = 1;` after parsing a resreq string's select/rusage/span/same/affinity clauses, regardless of their content -- there is no resreq-string syntax for task count at all (LSF-style task count is a job-submission-level parameter, `job_submit_data_t.mintasks/maxtasks`, sibling to the resreq string, not nested inside it). Neither of this codebase's two job-ingest constructors -- core_build_job()/core_build_primary_schedobj() (src/scheduler/src/core_events.c, the real production JOBS_PUSH path) nor make_sched_job() (src/scheduler/src/schedobj.c, the sibling non-event construction path) -- ever copies the job's own submitted job_submit_data_t.mintasks/maxtasks (correctly propagated onto the TOP-LEVEL xce_sched_job_t.mintasks/maxtasks by both functions, confirmed working) onto the NESTED xce_primary_resreq_t.minTasks/maxTasks that sched_make_primary_alloc_()'s maxTasks>1 dispatch check (scheduler.c:290) and parjob_allocator.c's my_alloc() actually read. Net effect: every job's primary resreq that parses successfully has maxTasks pinned to 1, so sched_make_primary_alloc_() always routes it to seqjob_allocator instead of parjob_allocator (scheduler.c:290-294) -- parjob_allocator's real combinatorial multi-host allocation (scheduler-allocation spec, tasks 4.1-4.3) is fully implemented and unit-tested but cannot be exercised by any real job submission as the system stands today. Live-confirmed: test_parjob_e2e_multislot.c's own RED-phase run (maxTasks patch temporarily disabled) showed the job routing to seqjob and dispatching onto exactly 1 host instead of the intended 2 (`Expected 2 Was 1`).", @@ -23000,6 +25295,9 @@ }, { "id": "bug-1771", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "fcfs_del_job dereferences job->queue without NULL check (plugins/fcfs.c)", "timestamp": "2026-09-04T00:00:00Z", "error_message": "ASAN SEGV (READ of address 0x0) in fcfs_del_job() (src/scheduler/src/plugins/fcfs.c:318, `job->queue->name`), reached via fcfs_commit() <- plugin_commit_decision() <- sched_commit_decision() <- commit_decision() <- sched_allocator(), the very first time a real DISPATCH decision was committed through the real, live sched_allocator() thread in this test directory. Discovered live during scheduler-allocation task 5.3's own end-to-end integration test construction (test_parjob_e2e_multislot.c) -- full ASAN backtrace captured.", "root_cause": "CONFIRMED via direct code read and the live ASAN backtrace. plugin_commit_decision() (src/scheduler/src/plugin.c:442-475) unconditionally fans a committed decision out to EVERY registered plugin's own `.hooks.commit_hook`, regardless of which plugin is the currently-ACTIVE job picker -- this includes the legacy 'xce_sched_fcfs' plugin (src/scheduler/src/plugins/fcfs.c), still registered alongside the modern default 'builtin_fcfs' by sched_register_plugins() (scheduler.c) even when it is never the resolved picker. fcfs_commit() (fcfs.c:326-337) calls fcfs_del_job(plugin, decision->u.alloc->jobid) for every XCE_DECISION_DISPATCH, and fcfs_del_job() (fcfs.c:303-324) unconditionally dereferences `job->queue->name` with no NULL check on `job->queue` -- sched_get_queue() (core.c:257-271) is get-ONLY (never auto-creates), so any job whose xce_sched_job_t.queue is NULL (i.e. its submitted queue name was never registered via a real QUEUE_UPDATE/CONFIG_PUSH event before the job reached DISPATCH) crashes this unrelated legacy plugin's own housekeeping, even though that plugin never elected or tracked the job at all. Every prior real-scheduler_srcs test in this directory that reaches a real sched_commit_decision() (test_scheduler_cache_dispatch.c, test_scheduler_cache_dispatch_compound.c, test_snapshot_load_allocator_regression.c) only ever produces PEND decisions (fcfs_commit()'s `default: break;` branch, never touching job->queue), so this NULL-deref had never been triggered before. Not reachable in real production as the system stands today (a real job's queue is always registered via controller-pushed config before any job can target it), but IS a genuine, real, unguarded NULL dereference in a code path that runs unconditionally for every real DISPATCH decision regardless of test/production context.", @@ -23023,6 +25321,8 @@ }, { "id": "bug-1772", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-05T00:00:00Z", "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during requirements scoping for the resreq-maxtasks-wiring spec (bug-1770's fix). snapshot_build_primary_schedobj() (src/scheduler/src/snapshot_load.c:81-96) never computes primary->resreq_fp at all, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135), which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing. Every job reconstructed from a snapshot (post-restart or post-failover restore) therefore gets a zeroed resreq_fp -- the same class of gap as bug-1767's broader 'recurring schedobj-reconstruction-duplication' pattern (snapshot_load.c independently re-implements core_events.c's schedobj construction rather than sharing it, and each independent copy has repeatedly drifted field-by-field). TRIGGER CORRECTION (2026-09-05, resreq-maxtasks-wiring design review): this entry's original phrasing ('post-restart or post-failover restore') overstates the live trigger. xce_sched_core_load_snapshot()'s ONLY real (non-test) caller anywhere in src/ is sim.c's sim_locate_and_load_snapshot(), reachable only via main.c's --sim FILE offline replay flag. Normal-mode main.c has no snapshot-load call or flag at all; a live scheduler restart (including post-failover, e.g. run-ha-soak.sh's restart_scheduler_for_new_primary()) starts with empty in-memory state and rejoins via on_scheduler_join() -> the same core_build_job() path fresh ingest uses -- never touching snapshot_load.c. The accurate trigger is: a job in a .replay-recorded snapshot loaded by the --sim simulation/replay driver, not any live-cluster crash-recovery or failover scenario. This does not change the bug's validity or fix (the gap is real for --sim/replay fidelity, which matters per this project's driver-equivalence invariant since --sim drives the identical allocator dispatch) -- it only corrects when it's actually reachable, which matters for scoping test coverage (an HA soak does NOT exercise this path at all).", "root_cause": "snapshot_load.c's snapshot_build_primary_schedobj() is a hand-duplicated, field-for-field port of core_events.c's core_build_primary_schedobj() (kept independent because core_build_primary_schedobj() is static and the two files are compiled as separate objects -- see bug-1767's own root-cause text for the deliberate-duplication rationale). The port was written before task 1.2 (scheduler-allocation feature) added resreq_fp/hash_rusages() wiring to the core_events.c original, and the snapshot_load.c copy was never updated to match. Effect: a snapshot-restored job's unsatisfiable-resreq cache fingerprint (used by scheduler-allocation's alloc_cache.c) is always the zero value, which is a valid-looking but WRONG fingerprint -- it could produce a false cache hit against an unrelated job that also happens to have a zeroed fingerprint (extremely unlikely today since real jobs always populate resreq_fp correctly at first ingest, but a snapshot-restored job's own fingerprint stays wrong for its entire post-restore lifetime, and any code that later relies on comparing/keying by resreq_fp for a restored job silently gets 0/0/0/0). Discovered while scoping resreq-maxtasks-wiring (bug-1770's fix spec) because that fix will make resreq_fp carry real, meaningful, non-degenerate maxTasks values for the first time -- increasing the practical consequence of this gap versus its current all-zero-everywhere baseline.", @@ -23048,6 +25348,9 @@ }, { "id": "bug-1773", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "JOB_UPDATE replaces resreq but schedobj keeps old primary (dormant, core_events.c)", "timestamp": "2026-09-05T00:00:00Z", "error_message": "OPEN, NOT A CRASH -- pre-existing, out-of-scope gap discovered during design review for the resreq-maxtasks-wiring spec (bug-1770's fix). A job's resource-request UPDATE (JOB_UPDATE core event / sched_update_existing_job()) never takes effect for scheduling purposes, for ANY field -- not just task count. sched_update_existing_job() (src/scheduler/src/schedobj.c:597-656) and core_events.c's JOB_UPDATE handler (src/scheduler/src/core_events.c:1200-1234) both do `free(job->resreq); job->resreq = newreq;` (a shallow free via FREEUP, utils.h:114) but never touch job->schedobj. ADDENDUM (2026-09-05, independent follow-up investigation): sched_update_existing_job() itself is dead code (unreachable -- its sole caller is scheduler.c's sched_update(), which has zero live callers, same root architecture as bug-1740). core_events.c's JOB_UPDATE handler IS live (traced main.c -> driver.c's xce_sched_driver_translate_pending_events() -> xce_core_push_events() -> the JOB_UPDATE case), reached routinely by ordinary job-lifecycle notifications (state/rusage/postdone), not only by a genuine resource-request change -- but grepping the whole controller finds no code path that ever mutates an existing job's submit->resreq after initial submission, so this live handler's parse_resreq() branch always re-parses the job's original, unchanged resreq string today. No real 'update a job's resource request' feature exists yet in this codebase; this bug describes what WOULD happen if one were added without also fixing job->schedobj reconstruction.", "root_cause": "CONFIRMED via direct code read: the scheduler's actual dispatch reads job->schedobj (scheduler.c:483, sched_make_allocation(sched, job->schedobj)), not job->resreq. sched_make_primary_allocation() (scheduler.c:315) reads xce_primary_resreq_t* primary = obj->u.primary->resreq -- i.e. job->schedobj->u.primary->resreq, a pointer set exactly once, at ingest, by core_build_primary_schedobj()/make_primary_schedobj() (primary->resreq = resreq, aliasing job->resreq->u.primary at that moment). job->schedobj is built ONLY in core_build_job() (core_events.c:259-265) and make_sched_job() (schedobj.c:194-202) -- confirmed via grepping every `job->schedobj =` assignment in the scheduler; neither update path is among them. So after any resreq update, job->schedobj->u.primary->resreq keeps pointing at the ORIGINAL, ingest-time xce_primary_resreq_t forever, completely disconnected from the freshly-parsed newreq the update path installed into job->resreq. Any change an operator makes to a pending job's select/rusage/task-count via a resource-request update is silently invisible to the scheduler's own dispatch decision -- only a job's original, at-submission resreq is ever actually evaluated.", @@ -23071,6 +25374,9 @@ }, { "id": "bug-1774", + "status": "wontfix", + "status_updated": "2026-09-24", + "status_note": "accepted: dead code path (make_primary_schedobj resreq_fp), zero live impact", "timestamp": "2026-09-05T00:00:00Z", "error_message": "OPEN, NOT A CRASH, DEAD-CODE-ONLY -- pre-existing gap discovered during design review for the resreq-maxtasks-wiring spec, sibling to bug-1772, CORRECTED 2026-09-05: this entry originally claimed make_sched_job()'s ingest path was \"confirmed live, not legacy\" -- that claim was WRONG and has been retracted (see root_cause). make_primary_schedobj() (src/scheduler/src/schedobj.c:45-63, backing make_sched_job()) never computes primary->resreq_fp, unlike core_build_primary_schedobj() (src/scheduler/src/core_events.c:130-135) which sets resreq_fp.min_tasks/max_tasks/select_node_id/rusage_hash from the parsed xce_primary_resreq_t immediately after parsing -- but make_sched_job() itself is unreachable in production today (see root_cause), so this gap currently has ZERO live impact.", "root_cause": "CONFIRMED via direct code read that make_primary_schedobj() calloc's the primary schedobj and leaves resreq_fp all-zero, with no equivalent computation anywhere in the function -- identical failure shape to bug-1772 (snapshot_load.c's snapshot_build_primary_schedobj() has the same gap), a third instance of the recurring schedobj-reconstruction-duplication pattern already tracked by bug-1767. HOWEVER, independently re-verified 2026-09-05 (two separate investigations, exhaustive grep of every call site of sched_update(), sched_update_jobs(), and make_sched_job() across src/ outside testsuite/): sched_update(xce_scheduler_t*) (scheduler.c:1200) has ZERO live callers anywhere in this codebase -- only its own declaration/definition and doc-comments describing it as dead/superseded exist (matching bug-1740's already-established finding for the sibling sched_apply_job_rusage() case, same root architecture). sched_update_jobs() (scheduler.c:1109) is called only from inside sched_update()'s switch, and make_sched_job() (schedobj.c:153) is called only from sched_update_jobs() -- so the ENTIRE chain is unreachable in production. The real, live production job-ingest paths are exactly two: core_build_job() (core_events.c, fresh submission via the driver-translated XCE_CORE_EVENT_JOBS_PUSH path) and snapshot_load.c's restore path -- both confirmed live by the same grep rigor. This entry's original \"confirmed live, not legacy\" parenthetical was an unverified, incorrect aside and is hereby retracted; make_sched_job()'s resreq_fp gap is real but currently dead-code-only, with no production consequence unless/until something makes this path reachable again.", @@ -23097,6 +25403,8 @@ }, { "id": "bug-1775", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-06T00:00:00Z", "error_message": "xce-test-multihost's run-multihost-verify.sh (resreq-maxtasks-wiring task 4.3, array-job scenario): verify_task_started_on_host()'s grep -E pattern embedded a raw jobid string directly in an extended-regex alternation. A bracketed array-element jobid (e.g. \"[0]\") has its \"[0]\" misparsed by grep -E as a one-character bracket-expression class (matching a literal '0') instead of the literal 3-character substring \"[0]\" -- causing a false negative: `grep -qE 'Job base[0] ntasks is:'` does not match the literal log line `\"Job base[0] ntasks is: 5\"`.", "root_cause": "Extended regular expressions treat unescaped `[...]` as a bracket-expression (character class), not a literal substring. Array-element jobids introduced by task 4.3 are the first jobid format in this harness containing ERE metacharacters (plain jobids are digits/hyphens only); the pre-existing plain-job code path never hit this because grep -E on a digits-and-hyphens-only string has no metacharacters to misparse.", @@ -23119,6 +25427,8 @@ }, { "id": "bug-1776", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-06T21:05:00Z", "error_message": "User improvement request (simulator REPL feedback): 'info worker's assigned jobs' listing gives no indication of how many 'slots' each assigned job actually consumes on that worker -- every assigned job looks identical regardless of whether it occupies 1 slot or many (e.g. a fully-packed 22-slot worker running one 2-slot job shows the exact same line shape as a 1-slot job).", "file": "src/scheduler/src/repl_info.c", @@ -23141,6 +25451,9 @@ }, { "id": "bug-1777", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test workerlist MaxSwap ignores hosts.yaml override", "timestamp": "2026-09-06T21:10:00Z", "error_message": "xce_test workerlist's MaxSwap column can permanently disagree with the scheduler's own resmgr/replay view for the same worker: workerlist always shows the join-time hardware_monitor stub/detected value, never a hosts.yaml per-host maxswap override, while the scheduler always uses the override. User noticed this while comparing a scheduler simulator replay (maxswap=10000) against xce_test workerlist (MaxSwap=1.00 GB) for the same host and asked whether the two should be consistent.", "file": "src/controller/testsuite/test.c", @@ -23166,6 +25479,8 @@ }, { "id": "bug-1778", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T06:50:00Z", "error_message": "Live-observed (interactive sim session=29, replaying a real captured trace): three plain (non-array) jobs already at JOB_STAT_DONE (0-14-1788751409-0, 0-14-1788751435-0, 0-14-1788751504-0) still appear in `info worker pc13v`'s 'assigned jobs' listing (each annotated '*2' via bug-1776's slot-count fix), alongside the genuinely-running array job elements. User: 'The finished job(exit/done) should be de-assign from worker.'", "file": "src/scheduler/src/repl_info.c", @@ -23195,6 +25510,8 @@ }, { "id": "bug-1779", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T06:58:00Z", "error_message": "Live-observed (interactive sim session=19, replaying a real captured trace, 'step 9'): within the SAME batch of scheduler decisions, jobs 0-12-1788762745-0[0..4] receive a DISPATCH decision, then immediately ALSO receive a PEND decision in the same step's output -- and `info jobs` afterward shows all 10 array elements (0-12-1788762745-0[0..9]) as PEND, including the 5 that were just dispatched. User: 'when a job has been dispatched (in scheduler), we should treat it as RUN, so it should be excluded from PEND ... we need to remove 0-12-1788762745-0[0-4] from PEND but show in RUN.'", "file": "src/scheduler/src/sim_loop.c", @@ -23223,6 +25540,8 @@ }, { "id": "bug-1780", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T07:55:00Z", "error_message": "User-reported (xcecli): 'xcecli jobs submit -n 2 --env /run/current-system/sw/bin/sleep 100' submits successfully but the job immediately fails with Status . xcecd.log shows: '[WARN][controller_connector.c:822] No execuser provided in startjob request, defaulting to nobody', then 'Starting worker as user nobody to process command...', then 'Worker 301734 exited with code 126'.", "file": "src/frontend/apiserver/backend/convert.go", @@ -23246,6 +25565,8 @@ }, { "id": "bug-1781", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T08:35:00Z", "gitea_issue": 45, "error_message": "Gitea issue #45 REOPENED after bug-1742's fix. User-reproduced: after bug-1742 fixed the earlier panic/empty-error-body symptom, `xcecli jobs kill ` (and stop/resume/checkpoint) now fails with a real, non-empty error instead: `request to /xce/v1/jobs//control failed: [failed to write data: xcecli_jobctrl failed: -22]` (-22 = -EINVAL). Reproduced live for kill/stop/resume/checkpoint, all failing.", @@ -23275,6 +25596,8 @@ }, { "id": "bug-1782", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T08:40:00Z", "gitea_issue": 45, "error_message": "Not a defect -- API ergonomics follow-up discussed while fixing bug-1781/gitea#45. The job control REST endpoint (`/xce/v1/jobs/{jobids}/control`) only accepted the packed numeric job ID; xcecli already translates the human-readable form (e.g. `1-0-1788768547-0`, as printed by `jobs list`) to that numeric form client-side (format.ToAPIJobID) before calling the API, so a caller hitting the endpoint directly (not through xcecli) had to reimplement that bit-packing itself.", @@ -23299,6 +25622,8 @@ }, { "id": "bug-1784", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T09:00:00Z", "gitea_issue": 46, "error_message": "Discovered while live-verifying gitea#46's Part 1 (dispatch-stops-at-8, previously hypothesized in bug-1743 to be limits.yaml's ug1_normal_slots cap=8 working as designed, never live-confirmed). Live test: submitted 10 single-slot jobs as user1 (a genuine ug1 member per users.yaml) on queue normal against a 22-slot worker -- all 10 ran, the supposed cap=8 usergroup limit never blocked anything.", @@ -23328,6 +25653,8 @@ }, { "id": "bug-1785", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T09:15:00Z", "gitea_issue": 46, "error_message": "Found immediately after fixing bug-1784: with named-usergroup resolution working, a limit scoped to only SOME of the four dimensions (users.yaml's demo limits.yaml sets users+queues on ug1_normal_slots but never projects/hosts) still matched no one at all.", @@ -23355,6 +25682,8 @@ }, { "id": "bug-1786", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T09:30:00Z", "gitea_issue": 46, "error_message": "Found immediately after bug-1784/1785: with group resolution and default-scope semantics both fixed, match_limit() finally reported a genuine match (rc=1) for the FIRST time in this codebase's live history -- and the very next step (usage/threshold accounting) immediately broke: limit_job_election()'s loop aborted after the first matching limit, so no limit past it in iteration order was ever evaluated for that job.", @@ -23384,6 +25713,8 @@ }, { "id": "bug-1787", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T09:45:00Z", "gitea_issue": 46, "error_message": "Found immediately after bug-1786: match_job_global_limit_'s own key_find rc=-2 masking fixed, but the FOLLOWING step -- instantiate_limit() -- now returned rc=-22/-EINVAL outright, still aborting the loop after the first limit.", @@ -23412,6 +25743,8 @@ }, { "id": "bug-1788", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T10:00:00Z", "gitea_issue": 46, "error_message": "Found immediately after bug-1787: with the resmgr type-confusion fixed, instantiate_limit() still returned rc=-2/-ENOENT, still aborting limit_job_election()'s loop after the first limit -- reproduced identically on a scheduler's FIRST-EVER join to a freshly-started controller, ruling out any rejoin/resync-specific explanation.", @@ -23442,6 +25775,8 @@ }, { "id": "bug-1789", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T10:30:00Z", "gitea_issue": 46, "error_message": "After bug-1784 through bug-1788 all fixed and live-verified (all 7 limits in limits.yaml now correctly match and instantiate usage tracking for every job), the actual numeric cap still does not reject anyone: 10 single-slot jobs submitted as user1 on queue normal against ug1_normal_slots's configured value:8 cap all still reach RUN, none PEND.", @@ -23469,6 +25804,8 @@ }, { "id": "bug-1790", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-07T00:00:00Z", "error_message": "OPEN, NOT A CRASH -- known residual gap explicitly carved out of bug-1789's fix, filed for its own tracking per user request. A hypothetical NEW global limit defined on a per-host resource OTHER than \"slots\" (e.g. a future absolute-value global mem cap, as opposed to the existing percent-based host-scoped mem limits in limits.yaml) would silently under-enforce the same way bug-1789's slots limits did, for a different reason than bug-1789's root cause.", "file": "src/lib/resreq.c", @@ -23492,6 +25829,8 @@ }, { "id": "bug-1791", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-09T08:50:58.000Z", "error_message": "controller_unit_test: interruptcall.c:142: xcetest_worker_main: Assertion \\`unlock_rc == 0' failed (pthread_mutex_unlock returned EPERM)", "file": "src/controller/testsuite/interruptcall.c", @@ -23519,6 +25858,8 @@ }, { "id": "bug-1792", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-09T12:40:00.000Z", "error_message": "test_test_cli.c tab-completion test (xce-test-cli-ux task 2.3): Tab keystroke produced zero effect (no bell, no redraw, no completion) when driven through the existing clitest_spawn() pty harness -- 'wo' + Tab + Enter submitted literally \"wo\" (\"wo: No such command for xce test.\"), never \"worker\".", "file": "src/controller/testsuite/test_test_cli.c", @@ -23544,6 +25885,9 @@ }, { "id": "bug-1793", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "xce_test join maxswap/maxtmp defaults are stale byte values (join_worker_args.c)", "timestamp": "2026-09-09T16:24:19Z", "error_message": "xce_test's worker->join command's maxswap (1024) and maxtmp (2048) defaults have the same stale-MB-scale bug as bug-1557's maxmem -- discovered while fixing bug-1557 (xce-test-cli-ux task 2.6)", "file": "src/controller/testsuite/join_worker_args.c (join_worker_parse_attrs, hostattrs_t default initialization)", @@ -23569,6 +25913,8 @@ }, { "id": "bug-1794", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-09T19:03:51Z", "error_message": "xce_test cmd_jobrun crashes (ASAN allocation-size-too-big abort in alloc_packet, requested size ~0xffffffffc3c3c4b0) when run live against a real controller, after xce-test-cli-ux task 3.2 wrapped its xcecli_jobdispatch() call through the interruptible-call adapter (heap-allocated ctx block instead of a stack-local xce_job_alloc_t).", "file": "src/controller/testsuite/test.c (cmd_jobrun, fill_job_alloc, jobrun_call_t/jobrun_thunk)", @@ -23597,6 +25943,8 @@ }, { "id": "bug-1795", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-09T20:10:00Z", "error_message": "leave_ctx() (test.c, called by cmd_up/cmd_quit) could free a connection's api_ctx_t (xcecli_stop()+xcecli_ctx_fini(), src/lib/client.c) while an abandoned worker thread from a prior Ctrl-C-interrupted RPC on that SAME connection was still running and still reading/writing it -- a genuine use-after-free, found by kiro-review during xce-test-cli-ux task 3.2's own round-1 review (not reproduced live/under ASAN on that run; identified structurally by tracing the code paths).", "file": "src/controller/testsuite/test.c (leave_ctx), src/controller/testsuite/interruptcall.c/.h (xcetest_wait_connection_idle, new), src/lib/client.c (xcecli_wakeup, xcecli_stop, xcecli_ctx_fini, read-only for this fix)", @@ -23628,6 +25976,9 @@ }, { "id": "bug-1796", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "qctrl/hostctrl perror() prints stale errno instead of RPC rc (src/harness/test.c)", "timestamp": "2026-09-09T22:00:00Z", "error_message": "cmd_qctrl/cmd_hostctrl/cmd_jobkill's RPC-failure branches perror() a fixed label (e.g. \"xcecli_queuectrl\") without ever setting errno to reflect the RPC's actual returned status, so the printed reason (e.g. \"Interrupted system call\", \"Resource temporarily unavailable\", \"Inappropriate ioctl for device\") is whatever errno happened to be left over from an unrelated earlier libc call in the same process, not the real failure reason.", "file": "src/controller/testsuite/test.c (cmd_qctrl, cmd_hostctrl, cmd_jobkill RPC-error return paths)", @@ -23656,6 +26007,9 @@ }, { "id": "bug-1797", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "cmd_qctrl leaks strdup'd queue name on RPC error (src/harness/test.c)", "timestamp": "2026-09-09T22:00:00Z", "error_message": "cmd_qctrl leaks its strdup()'d queue name on the xcecli_queuectrl() RPC-error return path (ASAN LeakSanitizer-confirmed, 14 bytes).", "file": "src/controller/testsuite/test.c (cmd_qctrl)", @@ -23683,6 +26037,8 @@ }, { "id": "bug-1798", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-10T06:57:00Z", "error_message": "Fresh (from-scratch) build fails intermittently with `fatal error: controller.capnp.h: No such file or directory`, reported live by the user running `./tools/xce-build.sh all debug` -- first for a libxce-internal consumer (utils.c.o), then AGAIN after the first fix landed, this time for scheduler/src/sim_record.c.o and scheduler/src/core_events.c.o (files entirely outside libxce). This is the root-cause fix for a class of failures already recorded across many prior recurrences in this buglog (see related_bugs) as an unexplained/unfixed 'ninja-race' -- this entry identifies and fixes the actual missing-dependency-edge cause at the meson.build level, in three parts (see fix), and additionally audited the whole project for other instances of this same bug class.", "file": "src/lib/meson.build", @@ -23709,6 +26065,9 @@ }, { "id": "bug-1799", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "25fb5c11", "timestamp": "2026-09-11T07:32:00Z", "error_message": "User-reported (xce_scheduler -i -s , live cluster-captured replay /tmp/scheduler-replay-20260911-064117.replay, worker pc13v, 22 real slots): `info worker pc13v` at session=98 shows `resource slots: value=22.00 used=22.00` while the printed `assigned job:` list for that same worker sums to 26 slots (jobs [5]-[9] each *4 = 20, plus six single-slot jobs from a second batch = 6); the very next session (99, no new DISPATCH/PEND events, same assigned-jobs list) shows `used=26.00` -- exceeding the worker's own `value=22.00` capacity, an admission-invariant violation (max slots exceeded). Session 98's own event stream shows two back-to-back `WORKERS_PUSH`+`JOB_PUSH` (njobs=20) events for the same worker/jobids, consistent with a live scheduler-connection resync/reconnect being faithfully replayed (matching bug-1734/bug-1739's documented resync scenario), plus `sched_add_job failed ... rc=-17, merging state into existing tracked job instead of discarding` warnings for nearly every job in the resync batch, and one `resync claims RUN for jobid ...[7]/[8] with no allocation data -- refusing to merge a phantom RUN` (bug-1739's guard firing as designed).", "file": "src/scheduler/src/scheduler.c, src/scheduler/src/core_session.c", @@ -23747,6 +26106,8 @@ }, { "id": "bug-1800", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-11T07:55:00Z", "error_message": "User request (live replay session, worker pc13v): `info worker`'s CONSUMABLE value=/used= line for a bytes-valued resource (mem/swp/tmp) printed raw byte counts with no unit scaling -- e.g. `resource mem: value=20684476416.00 used=87960930222080.00` -- unlike every OTHER bytes-valued field in the same listing (maxmem/maxswap/maxtmp already render `value=33182072832.000000 (30.90 GB)`, and mem/tmp/swp themselves render human-scaled via the Load Metrics path when NOT flagged consumable). bug-1727's own header comment had explicitly scoped this out: 'no bytes-valued CONSUMABLE resource is exercised anywhere in this codebase's tests today.'", "file": "src/scheduler/src/repl_info.c", @@ -23779,6 +26140,8 @@ }, { "id": "bug-1801", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-11T08:35:00Z", "error_message": "Follow-up fix for bug-1799 (user asked to 'fix bug-1799', this session): the resInUse over-commit caused by core_reset_workers()'s RECONCILING/UNAVAIL reset-skip (bug-1751) having no matching skip in sched_account_jobs()'s per-round re-derive pass.", "file": "src/scheduler/src/resource.c", @@ -23813,6 +26176,8 @@ }, { "id": "bug-1802", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-11T08:35:00Z", "error_message": "User request (same live replay session as bug-1799/1800): 'how can we step to the next session which is not dummy(something occurs)?' -- the REPL's `break ` vocabulary required setting all ten per-event breakpoint kinds individually to catch 'the next session with ANY event at all,' with no single convenience kind for that common workflow.", "file": "src/scheduler/src/repl_break.c, src/scheduler/src/repl_internal.h", @@ -23839,6 +26204,8 @@ }, { "id": "bug-1803", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-11T09:35:00Z", "error_message": "User feedback (live replay session, immediately after bug-1802's `break ANY` shipped): 'It seems that break any command will introduce too many break in single session' -- live transcript showed `break ANY` + repeated `step` firing 'hit breakpoint 1 @ event #N: any' on EVERY event (JOB_UPDATE, JOB_STATE_CHANGE, ...) inside an already-busy session (session 40), forcing many single-event `step`s just to get back OUT of that session -- the opposite of the intended 'skip past dummy sessions in one hop' workflow. User explicitly requested removing `break ANY` and instead having `info summary` display the next non-dummy session id so `step N` can jump there directly.", "file": "src/scheduler/src/repl_break.c, src/scheduler/src/repl_internal.h, src/scheduler/src/sim.c, src/scheduler/src/repl_info.c", @@ -23868,6 +26235,8 @@ }, { "id": "bug-1804", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-11T09:55:00Z", "error_message": "User design assertion (same live replay session as bug-1799/1800/1803): 'according to the design, when a job has been scheduled by scheduler, we can say job is running in scheduler angle' -- directly challenging this session's own earlier explanation that a job showing DISPATCH-then-still-PEND for multiple sessions was 'by design, not a bug' (deferred-until-confirmation semantics from the scheduler-sim-eager-reconciliation spec). User asked for confirmation and, once confirmed as a genuine design fork (not a bug in the then-current implementation), explicitly directed adopting the OTHER interpretation: revise the spec to match normal-mode's own synchronous dispatch-commit semantics (bug-1624).", "file": ".kiro/specs/scheduler-sim-eager-reconciliation/{requirements,design,research,tasks,spec.json}, src/scheduler/src/sim_loop.c", @@ -23904,6 +26273,8 @@ }, { "id": "bug-1805", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-14T05:07:17.752283Z", "error_message": "xce-harness-component task 1.1: controller_unit_test had 29 test failures (all in test_test_cli.c/test_xcetest_process_entry.c) after xce_test moved from src/controller/testsuite/ to src/harness/ -- Expected 0 Was 127 and missing xce> prompt", "file": "src/controller/meson.build", @@ -23925,6 +26296,9 @@ }, { "id": "bug-1806", + "status": "wontfix", + "status_updated": "2026-09-24", + "status_note": "accepted Meson limitation (user decision 2026-09-14)", "timestamp": "2026-09-14T09:20:27.103747+00:00", "error_message": "xce-harness-component task 4.3: /xce-build controller alone still rebuilds harness/xce_test on disk under the default asan profile, apparently violating requirement 2.2 (controller-only build shall not produce xce_test as a side effect)", "file": "src/harness/meson.build, src/tools/xce-build.sh", @@ -23947,6 +26321,8 @@ }, { "id": "bug-1807", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-14T15:50:00Z", "error_message": "harness_unit_test's test_test_cli.c: 4 of 132 tests deterministically FAIL every run -- test_migrated_commands_sample_reject_unrecognized_flag (jobkill unrecognized-flag rejection), test_qctrl_hostctrl_confirm_gate_destructiveness_determination (qctrl -c or hostctrl -c prompt missing -- WHICH one shifts between builds), test_destructive_commands_prompt_by_default_and_yes_bypasses (upgradeconfig prompt missing), test_confirm_gate_session_toggle_skips_and_restores ('set +y' restore prompt missing). Each failing assertion burns its full 5000ms clitest_wait_for_text() timeout (total suite runtime ~28-29s vs ~9s once fixed) -- not a race, a genuine miss every time.", "file": "src/harness/test.c (all ~20 cmd_* handlers), src/harness/join_worker_args.c", @@ -23969,6 +26345,8 @@ }, { "id": "bug-1808", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-14T15:50:00Z", "error_message": "src/harness/testsuite/test_test_cli.c: a spawned xce_test child process (clitest_spawn/clitest_spawn_ex) plus its pty master fd and stderr pipe fd leak whenever a TEST_ASSERT_* assertion fails partway through the owning test function, because every test relies on its OWN clitest_kill_and_reap()+clitest_close() calls placed inline at the bottom of the function body -- code that Unity's longjmp-based TEST_PROTECT()/TEST_ABORT() abort mechanism skips entirely on assertion failure. Directly observed: 4 live, idle `xce_test --norc` child processes still running (blocked forever on their own prompt) under one in-progress harness_unit_test invocation, discovered while investigating bug-1807's failures.", "file": "src/harness/testsuite/test_test_cli.c, src/harness/testsuite/harness_test.c", @@ -23992,6 +26370,8 @@ }, { "id": "bug-1809", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-14T16:20:00Z", "error_message": "User-reported: live scheduler logs are flooded with WARN lines for every ordinary XCE_CORE_EVENT_JOBS_PUSH resync redelivery: \"xce_core_push_events JOBS_PUSH: sched_add_job failed for jobid (rc=-17), merging state into existing tracked job instead of discarding\" (src/scheduler/src/core_events.c). rc=-17 is -EEXIST -- the expected, routine outcome of on_scheduler_join()'s full resync (bug-1734) or the array-push dedup race (bug-1714), already correctly handled by the bug-1734/bug-1738/bug-1739 merge logic in the same branch, not a functional defect.", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge branch, ~line 845)", @@ -24025,6 +26405,8 @@ }, { "id": "bug-1810", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T03:55:00Z", "error_message": "User-reported live hang: xce_test (harness REPL) never exits on Ctrl-D (EOF) after entering a live connection context (client/worker/scheduler). Live gdb attach on the user's machine showed the main thread parked forever in leave_ctx() -> xcecli_ctx_fini() -> xce_ctx_fini() -> vtaskmgr_fini() -> vcpu_fini()'s pthread_join(vcpu->vcpu, ...); a sibling vcpu thread was simultaneously deadlocked in cbuf_put() at mylock.c:101's pthread_mutex_lock(&(b->mutex)) on vtaskmgr->wbuf.mutex, whose __owner TID had already exited.", "file": "src/lib/mylock.c (cbuf_put/cbuf_get/cbuf_timedget), src/lib/vtaskmgr.c (vcpu_idle, vcpu_entry)", @@ -24058,6 +26440,8 @@ }, { "id": "bug-1811", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T07:40:00Z", "error_message": "User-reported (interactive sim session=35, `info jobs`/`info worker pc13v`): job 0-14-1789369200-0[21] shows RUN in `info jobs` but is absent from `info worker pc13v`'s assigned-jobs listing (which lists [0-20,22], skipping 21) -- user's working theory was slot overcommitment (23 running jobs vs. 22-slot worker capacity).", "file": "src/scheduler/src/core_events.c, src/scheduler/src/sim_loop.c, .kiro/specs/scheduler-sim-decision-parity/{requirements,design}.md", @@ -24083,6 +26467,8 @@ }, { "id": "bug-1812", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T08:05:00Z", "error_message": "User-reported (interactive sim session=34->35 stepped live, AND independently confirmed via a real production scheduler.log from rshen@pc13v /t/xce-ha-deploy): job 0-14-1789369200-0[21] holds allocs=1 (state=RUN) at the end of session 34; stepping into session 35 replays a JOBS_PUSH resync (njobs=60, including all of array job 0-14-1789369200-0[0-39]) followed by a JOB_UPDATE and a redundant JOB_STATE_CHANGE(RUN) for [21]; immediately after, `info job [21]` shows state=RUN but allocs=0. The real scheduler.log shows the identical mechanism live: `sched_job_state_change ... state 2` (schedStartJob confirmation) at 15:00:05.196, followed 16ms later by `JOBS_PUSH: sched_add_job failed for jobid ...[21] (rc=-17), merging state into existing tracked job instead of discarding` -- repeating in bursts on later reconnects (six WARNs in ~2s at 15:01:51-53).", "file": "src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOBS_PUSH duplicate-jobid merge, bug-1734/bug-1739's own code)", @@ -24107,6 +26493,8 @@ }, { "id": "bug-1813", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T08:35:00Z", "error_message": "Follow-up to bug-1812's investigation (user-directed architectural fix, not a live-reproduced crash): while investigating why a scheduler reconnect/resync storm collides with in-flight array-job dispatches (bug-1812's own root cause), user asked to investigate the connection-loss trigger itself. Traced the scheduler's array-job receive path (handler.c's sched_on_job_array(), reached via scheduler_req_handler() on the shared vtaskmgr thread pool, default 8 threads) and found it does the FULL per-element expansion (dedup check via sched_get_job() + N independent deep clones via clone_job_submit_data()/clone_job_startinfo()) inline, synchronously, on whatever vtask thread handles the incoming push -- the exact 'long/unbounded work on a pool that must stay free for liveness-critical messages' anti-pattern .kiro/steering/ha-async-io.md's Thread-role contract warns about (the same pool is shared with the scheduler's own controller connection traffic). A prior, independent finding (2026-09-04 discovery, this project) already showed a burst of 6x50-element array pushes (300 job events, landing within ~2ms of each other) caused the scheduler to lose its connection to the primary controller within 245ms and enter a failover/reconnect loop -- directly consistent with this mechanism. User proposed the fix directly: the request-handling thread should just requeue the raw request, with the actual (expensive) work deferred to run alongside 'begin session' -- i.e. the round-processing thread's own xce_sched_driver_translate_pending_events()/xce_core_begin_session() sequence (main.c's xce_sched_driver_run_normal_mode_round(), driver.c), which carries no connection-liveness responsibility and already establishes this exact deferred-dispatch pattern for XCE_SCHED_OP_ADMIN_RECORD.", "file": "src/scheduler/include/scheduler.h, src/scheduler/src/handler.c, src/scheduler/src/driver.c, src/scheduler/testsuite/test_job_array_expand.c, src/scheduler/testsuite/test_job_array_wire_e2e.c, src/scheduler/testsuite/test_driver_translate.c", @@ -24133,6 +26521,8 @@ }, { "id": "bug-1814", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T09:31:00Z", "error_message": "User-reported, live replay session (/tmp/scheduler-replay-20260915-091847.replay, worker pc13v): `info worker pc13v` at session=2 (right after a burst of 19 JOB_UPDATE/JOB_RUSAGE event pairs for jobid[0]-[18]) showed `resource mem: value=22346629120.00 (20.81 GB) used=334251534843904.000000 (304.00 TB)` -- a used figure ~15000x larger than the worker's own total capacity, for a 22-slot/20.81GB host. Session 1 (before any JOB_RUSAGE events) showed a sane used=225280.000000 (220.00 KB) baseline.", "file": "src/worker/src/xcecd_cgroups.c (collect_job_rusages() -- the fix), src/scheduler/src/resource.c (sched_reconcile_job_resource_usage()/sched_reconcile_host_resource_usage() -- the consumer that sums the corrupted per-job value into w->resInUse, unmodified, already correct on its own terms), src/scheduler/src/core_events.c (XCE_CORE_EVENT_JOB_RUSAGE handler -- confirmed the single, shared call site both normal-mode (driver.c) and sim/replay-mode (sim_loop.c) events funnel through, so this bug is not replay-specific)", @@ -24179,6 +26569,8 @@ }, { "id": "bug-1815", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T09:40:00Z", "error_message": "Follow-up to bug-1814: user reported that tracing a job's 'mem used' figure across multiple live replay sessions showed it never self-corrects, staying elevated until the job finishes -- not a one-time blip. User also live-verified on the real pc13v host (`systemctl show -p MemoryCurrent -p MemoryAccounting`) that systemd genuinely reports correct, sane MemoryCurrent values for a healthy running job (MemoryCurrent=5525504, MemoryAccounting=yes) -- disproving the hypothesis that bug-1814's UINT64_MAX sentinel is a persistent/structural condition on this host. This left the 'why does it never self-correct' question open.", "file": "src/scheduler/src/resource.c (sched_reconcile_host_resource_usage() -- the fix), src/scheduler/src/core_events.c (JOBS_PUSH resync-merge RUN branch, lines ~913-924 -- confirmed the account-object-replacement mechanism that interacts with this gap, not itself modified), src/scheduler/testsuite/test_resource_accounting.c (new regression test)", @@ -24217,6 +26609,8 @@ }, { "id": "bug-1816", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T13:05:00Z", "error_message": "User request, following the bug-1814/1815 mem-accounting investigation: a live 3-node HA cluster on pc13v went through a repeated PreVote-failover storm (term climbing 2->6, standbys losing primary_ctrl_fd RDHUP repeatedly, xceadmin scheduler record status failing with 'scheduler not connected'/'too many redirects') while load was confirmed low (loadavg 0.5-1.6 on 22 cores). Root cause found live by the user: `ps axwww | grep control` on pc13v showed SIX xce_controller processes -- two full sets of ctrl-1/ctrl-2/ctrl-3 (PIDs 294848/294869/294905 and 610530/610546/610561) all bound to the identical ports (9901-9903) and identical --workdir/--certdir/--logfile paths under /tmp/xce-deploy-ha. The user's own hand-typed cleanup command (`ps axwww | grep xce-ha-deploy | ... | kill -9`) had a typo -- the actual workdir is /tmp/xce-deploy-ha/... ('xce-deploy-ha'), not 'xce-ha-deploy' (words transposed) -- so it matched nothing and silently failed to kill the first deploy's processes before a second deploy started on the same ports. User asked to enhance the xce-deploy-ha skill's shutdown-all mechanism so this class of mistake becomes structurally hard to repeat.", "file": "src/tools/deploy-ha.sh (port pre-flight check, cleanup.sh generation fix, new stop-all.sh generation, start-all.sh wait_for_pidfile race fix), .claude/skills/xce-deploy-ha/SKILL.md (Step 2, Clean restart, Troubleshooting documentation updates)", @@ -24251,6 +26645,8 @@ }, { "id": "bug-1817", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-15T19:20:00Z", "error_message": "User-reported live core dump: xce_scheduler (build.dbg, HA-connected normal mode, --peers ctrl-1/2/3) crashed with SIGSEGV in its dedicated allocator thread (sched_allocator, scheduler.c:484). gdb backtrace: zarray_get_ex (lib/zarray.c:332, `data[i] = array->data[idx]`) <- zarray_get <- match_job_host_limit_ (plugins/limit.c:370) <- limit_prepare_allocation <- plugin_prepare_allocation <- sched_try_alloc_on_host_ (resource.c:528) <- parjob_try_plan_ <- my_alloc <- sched_make_parjob_allocation <- sched_make_primary_alloc_ <- sched_make_primary_allocation <- sched_make_allocation <- sched_allocator <- start_thread. gdb locals at the crash frame: i=0, array->data=0x0 (NULL), while the bounds check `idx (0) >= array->caps` had evidently passed (i.e. array->caps was nonzero) -- a zarray_t with data==NULL but caps>0, an internally inconsistent/torn state that should be structurally impossible for a correctly-initialized array.", "file": "src/lib/zarray.c (zarray_fini() -- primary fix), src/lib/xlist.c (xlist_fini() -- identical fix, same bug class), src/scheduler/src/schedobj.c (reset_sched_obj() -- the reachable production caller that exposed the bug, NOT itself modified: correct by construction once the library primitives are fixed), src/scheduler/src/plugins/limit.c (limit_job_election()/match_job_host_limit_() -- the call chain that reaches the torn array, not modified), src/lib/testsuite/test_zarray.c, src/lib/testsuite/test_xlist.c (new regression tests)", @@ -24294,6 +26690,8 @@ }, { "id": "bug-1818", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-16T08:20:00Z", "error_message": "Live 3-node HA E2E validation (controller-status spec, Requirement 12 delta, task 7.6): cmd_ctrlstatus (xce_test client) printed electedTerm: unknown / electedAt: unknown for the new primary-election-term/time fields on EVERY standby query, both before and after a real primary kill/failover (confirmed after ~4 heartbeat intervals, ruling out a convergence-timing gap) -- even though the same query correctly showed non-stale, converged term/committedLsn/appliedLsn values for the same standby at the same moment.", "file": "src/harness/test.c (cmd_ctrlstatus -- fix), .kiro/specs/controller-status/design.md (Design Delta -- Requirement 12, Locked Decision 3 -- corrected)", @@ -24318,6 +26716,8 @@ }, { "id": "bug-1819", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-16T17:15:00Z", "error_message": "controller-status spec Requirement 13 delta, task 8.6 (bug found live during task 8.5's optional E2E attempt): ctrlstatus_fill's primary-side peer-status loop (src/controller/ctrlstatus.c) scanned replicator_t.standbys[] for a host/port match and stopped at the FIRST match by slot index. When a standby was killed and later reconnected into a DIFFERENT replicator slot than the one it originally occupied (its old slot keeps host/port populated with ep==NULL after the kill), the lower-indexed stale slot won the scan and the peer was reported dead PERMANENTLY -- even minutes after it had genuinely reconnected and fully caught up. Task 8.5's live run observed this directly: primary log showed acked_lsn converging to the committed tip at a specific timestamp while ctrlstatus kept reporting dead from restart through +5m45s.", "file": "src/controller/ctrlstatus.c (ctrlstatus_fill, primary-side peer-status matching loop), src/controller/testsuite/test_ctrlstatus.c", @@ -24345,6 +26745,8 @@ }, { "id": "bug-1820", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-17T02:45:00Z", "error_message": "User report: after deploying a real (root, sudo-launched) xcecd worker via /xce-deploy-ha and stopping the cluster with scripts/stop-all.sh, xcecd keeps running -- ps shows a leftover xcecd process (a different PID than the one stop-all.sh reported stopping) still bound to the deploy dir's logs after stop-all.sh reported \"Stopping worker-1 (PID )\" with no warning.", "file": "src/tools/deploy-ha.sh (generated scripts/stop-all.sh and scripts/cleanup.sh kill loops; generated scripts/start-worker.sh's sudo branch)", @@ -24371,6 +26773,8 @@ }, { "id": "bug-1821", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-17T04:10:00Z", "error_message": "User-reported: xceadmin's --controller option (config push and all 3 scheduler record start/stop/status commands) accepted only a SINGLE host:port -- passing a comma-separated list of all configured controllers (the natural way to query an HA cluster without knowing in advance which node is alive/primary, e.g. `--controller \"localhost:9901,localhost:9902,localhost:9903\"` with 9901 dead) failed outright: `Error: cannot connect to localhost:9901,localhost:9902,localhost:9903: [Errno -2] Name or service not known` -- the whole comma-joined string was parsed as one bogus hostname.", "file": "src/xceadmin/src/xceadmin/discovery.py (new resolve_controllers/follow_redirects_any), src/xceadmin/src/xceadmin/push.py, src/xceadmin/src/xceadmin/scheduler.py (both migrated off their own duplicated _split_controller), src/xceadmin/src/xceadmin/cli.py (--controller help text)", @@ -24399,6 +26803,8 @@ }, { "id": "bug-1822", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-17T04:10:00Z", "error_message": "Found live while verifying bug-1821's --controller failover-list fix: querying `xceadmin scheduler record status --controller \"host1,host2,host3\"` immediately after killing the primary hard-failed with `Error: no primary available: none of the configured controllers responded (...)` instead of riding out the cluster's own re-election, even though _send_admin_request() already has a bounded retry loop (bug-1713/1715) built for exactly this 'cluster still settling' condition.", "file": "src/xceadmin/src/xceadmin/scheduler.py (_send_admin_request)", @@ -24428,6 +26834,8 @@ }, { "id": "bug-1823", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-17T04:20:00Z", "error_message": "User-reported (regression on top of bug-1820's own fix, live on the user's real host, 'pc13v'): after ./scripts/stop-all.sh reported 'Stopping worker-1 (PID N)' with no warning, the real xcecd worker was STILL running afterward as a different PID -- and a second `sudo ./scripts/stop-all.sh` run immediately reported 'Nothing was running for this deploy' (the pidfile had already been deleted by the first run despite the kill not actually succeeding).", "file": "src/tools/deploy-ha.sh (generated scripts/stop-all.sh and scripts/cleanup.sh)", @@ -24455,6 +26863,9 @@ }, { "id": "bug-1824", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "repl_info.c bounds check uses zarray count instead of caps", "timestamp": "2026-09-17T02:45:00Z", "error_message": "NOT A CRASH -- pre-existing, out-of-scope latent bug discovered while implementing scheduler-sim-diagnostics task 3.1 (new repl_info_limit() live-usage display). repl_info_resource_one() (src/scheduler/src/repl_info.c ~line 996) bounds-checks a resource's zarray index via `(r->idx < 0) || (r->idx >= sim->core.sharedInst.count)` before calling zarray_get() -- but zarray_t.count tracks the number of POPULATED slots (incremented only when a previously-NULL slot becomes non-NULL via zarray_put_ex/zarray_store), not the array's allocated capacity (.caps, grown by zarray_ensure_items). A sparsely-populated zarray (e.g. only a high index ever zarray_put() -- realistic if resources are registered out of index order, or an index is skipped) can have count=1 while a LOWER, unpopulated index is itself < count and would incorrectly pass this check and be read as though populated, or a HIGHER populated index can exceed .count and be incorrectly rejected as absent even though zarray_get()'s own .caps-based check would have safely returned it (or correctly returned NULL for a genuinely-unpopulated-but-in-capacity slot, since zarray_ensure_items() NULL-initializes newly-grown slots). Net effect: repl_info_resource_one() can report a resource as having no value ('value=(none)') when a real, populated value exists at that index but the array is sparse, OR (less likely) skip a real bounds-safety net zarray_get_ex() would otherwise provide.", "file": "src/scheduler/src/repl_info.c", @@ -24477,6 +26888,8 @@ }, { "id": "bug-1825", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-17T07:10:00Z", "error_message": "NOT A CRASH -- significant, verified architectural gap discovered while closing out scheduler-sim-diagnostics task 5 (final validation task for the new `info limit` REPL command). The limit plugin is never registered in ANY simulation-mode driver path, so `info limit ` will ALWAYS report \"no limits configured\"/\"no such limit\" for a real operator running `xce_scheduler -i`/`-s ` against a genuine recording -- even one whose original live run had limits configured and actively enforced. The entire feature built across scheduler-sim-diagnostics tasks 1-5 is correctly implemented, thoroughly tested, and read-only/deterministic as designed, but is currently INERT in real production usage: every positive-display test path (tasks 2-3, showing a limit's configured setting and live usage) only exercises the code via a test-only helper that hand-registers the plugin into core->plugins directly, which has no production equivalent in sim mode.", "file": "src/scheduler/src/sim_picker.c", @@ -24500,6 +26913,8 @@ }, { "id": "bug-1826", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-17T21:50:00Z", "error_message": "NOT A CRASH -- known, deliberately out-of-scope follow-up discovered while fixing bug-1825 (sim-mode limit plugin bootstrap, scheduler-sim-diagnostics task 6). Bug-1825's fix makes the limit plugin's CONFIGURED SETTING visible in sim mode (info limit now shows a real replay's actual limit configuration). But limit *matching* -- the mechanism that actually populates live usage instances (xce_vres_t) as jobs get evaluated -- is still never invoked during replay. So info limit's live-usage display will correctly show 'no active usage instance yet' for every limit against any real replay, even one whose original live session actively enforced and charged against that limit.", "file": "src/scheduler/src/plugin.c", @@ -24525,6 +26940,8 @@ }, { "id": "bug-1827", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-18T07:15:00Z", "error_message": "NOT A CRASH -- coverage gap found during an /xce-audit-invariants scheduler run (dimension 2: core module I/O isolation), while investigating driver state-equivalence for bug-1826. test_core_isolation.c's own header comment claims core_clock.c joins ONLY the nm(1) undefined-symbol enforcement layer, not the wall-clock text-scan check -- implying it is still covered by the OTHER two text-scan checks (forbidden #include list, forbidden symbol substrings). In reality, core_clock.c is absent from all three text-scan functions (test_core_never_includes_forbidden_headers(), test_core_never_calls_forbidden_symbols(), and the wall-clock scan), not just the wall-clock one -- no XCE_CORE_CLOCK_C_PATH macro is defined anywhere in test_core_isolation.c or testsuite/meson.build's c_args for that test.", "file": "src/scheduler/testsuite/test_core_isolation.c", @@ -24545,6 +26962,8 @@ }, { "id": "bug-1828", + "status": "fixed", + "status_updated": "2026-09-24", "error_message": "IMPACT CORRECTED 2026-09-18 (independent review of scheduler-sim-live-allocation task 2.3): this is NOT just a display/accounting gap -- it meant host-scoped resource limits were NEVER ACTUALLY ENFORCED, in ANY mode (normal or simulation), for this function's entire history. Originally discovered while writing scheduler-sim-live-allocation spec task 2.3's own required TDD test (a host-scoped limit match, via the simulation-mode-registered limit plugin, must create a real usage instance verifiable through limit_get_usage_instances()). limit_prepare_allocation()/match_job_host_limit_() consistently returned rc=-2 (-ENOENT) instead of >= 0, and no usage instance was ever created, even though match_limit() itself genuinely matched (rc=1) and is_host_based_limit() correctly identified the limit as host-based. Tracing the call chain (plugin.c's plugin_prepare_allocation() hook loop -> sched_fill_limit_requests()) shows the missing usage instance meant a host-scoped limit's vres was never added to obj->hlimits, so sched_fill_limit_requests() always iterated an empty list -- the limit's numeric cap was silently never applied to any job's resource request, in production, since this code existed. This does not corrupt the dispatch/pend decision machinery itself (plugin_prepare_allocation()'s hook-calling loop discards the hook's return value, so the bug never fed into a control-flow branch) -- it is a dormant-limit-never-binds defect, but a real enforcement gap, not merely a cosmetic one.", "root_cause": "match_job_host_limit_() (src/scheduler/src/plugins/limit.c) is the host-scoped sibling of match_job_global_limit_() and has the same defect CLASS bug-1786 fixed in that function alone (misinterpreting key_find()'s negative \"not found\" return as fatal, rather than the correct not-yet-instantiated signal) -- not byte-for-byte identical wording (bug-1786's own guard was `if (rc <= 0) { return rc; }`; this sibling's was `if (rc < 0) { return rc; }`), same underlying mistake. Right after a successful match_limit(), it does `rc = key_find(privdata->instances, vresname, &vres); if (rc < 0) { return rc; }` before the `if (vres == NULL) { instantiate_limit(...) }` block below it. On the FIRST-EVER lookup of a given vresname (always true the first time a host-scoped limit ever matches), key_find() signals \"not found\" as a NEGATIVE rc (-ENOENT), not 0 -- this codebase's established kvmgr convention (get_group(), is_host_based_limit(), and bug-1786/bug-1495's own root causes) -- so this early-return unconditionally caught the expected miss and returned before instantiate_limit() could ever run. bug-1786 (2026-09-07) fixed this exact defect class in match_job_global_limit_() but never touched its host-scoped sibling, which retained the bug because no test exercised match_job_host_limit_()/limit_prepare_allocation() with real matching until this task added one (verified via grep: zero prior test call sites for either function anywhere in the test suite).", "fix": "FIXED (2026-09-18, scheduler-sim-live-allocation task 2.3). Removed the erroneous `if (rc < 0) { return rc; }` guard in match_job_host_limit_() entirely, mirroring match_job_global_limit_()'s own bug-1786 fix exactly: `vres` (calloc'd NULL, only ever reassigned by a successful key_find) is already the correct, sufficient signal for \"needs instantiation\" via the existing `if (vres == NULL)` block immediately below. TDD: new test test_limit_prepare_allocation_matches_host_limit_via_sim_registered_plugin (test_repl_limit_table.c) drives the real limit_prepare_allocation()/match_job_host_limit_() path against a plugin registered via plugin_register_core() and a real percent-based host-scoped limit; RED (rc=-2, confirmed via temporary debug instrumentation tracing is_host_based_limit()=1, match_limit()=1, but limit_prepare_allocation()=-2) before this fix, GREEN (rc>=0, usage instance created and visible via limit_get_usage_instances()) after. Full scheduler suite: 79/79, zero regressions.", @@ -24569,6 +26988,8 @@ }, { "id": "bug-1829", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-19T04:00:00Z", "error_message": "AddressSanitizer: SEGV (NULL pointer read) in sched_make_seqjob_allocation (src/scheduler/src/scheduler.c:59, `plugin->job_alloc_fn` where `plugin = core->seqjob_allocator` is NULL), called from sched_make_primary_alloc_ -> sched_make_primary_allocation -> sched_make_allocation -> sched_allocator (sched_election.c) -> the real pthread spawned by sched_sim_driver_run() (sim_loop.c). Discovered while writing the mandated TDD test for scheduler-sim-live-allocation task 4.1 (its own Observable: 'a simulation-mode session ... produces a genuine dispatch or pend decision for a job based on its actual resource availability, instead of unconditionally pending it') -- the crash is 100% reproducible for ANY job reaching primary resource allocation in simulation mode, regardless of resreq shape or dispatch/pend outcome (sched_make_parjob_allocation has the identical NULL-core->parjob_allocator crash for maxTasks>1 jobs).", "root_cause": "core->seqjob_allocator/core->parjob_allocator (xce_sched_core_t fields, core.h) are never populated for the simulation driver, anywhere. Normal mode populates them via sched_init()'s sched_register_plugins() -> register_seqjob_allocator()/register_parjob_allocator() (plugins/seqjob_allocator.c, plugins/parjob_allocator.c), whose init_fn does `((xce_scheduler_t*)plugin->sched)->core.seqjob_allocator = plugin` -- REQUIRING a genuine, live xce_scheduler_t*. sched_sim_driver_start() (sim.c) registers no plugin at all (sim.h's own documented 'Known scope boundary'), and xce_sched_sim_driver_t deliberately has no xce_scheduler_t*/xce_scheduler_t field of its own (Req 5 AC 17, simulation isolation). This is NOT limited to the init_fn: every one of these two plugins' RUNTIME hooks (seqjob_prepare_allocation()/my_alloc(), parjob_prepare_allocation()/my_alloc(), and several internal static helpers taking xce_scheduler_t* sched as a parameter) ALSO read `plugin->sched->core.*` (resmgr for the 'slots'/rusage lookups, workers for host_iter, and sched_make_allocs()/sched_try_alloc_on_host_() in scheduler.c/resource.c which ALSO take xce_scheduler_t*/need `sched->core.resmgr`) on every single allocation attempt, not just at registration. This gap was already discovered and explicitly deferred by scheduler-sim-live-allocation task 3.1 (see test_sched_election_sim_fixture.c's own header comment: 'a pre-existing, tracked, OUT-OF-SCOPE gap ... NOT one of task 3.1's five permitted changes -- fixing it would be scope creep into a different, not-yet-assigned task') -- task 3.1's own fixture never reached this crash only because IT registers seqjob_allocator directly against a real, throwaway xce_scheduler_t wrapper whose OWN .core IS the core under test (single-core fixture), a shape simulation mode itself can never have (its core lives inside xce_sched_sim_driver_t, never xce_scheduler_t). Task 4.1 is the first task whose OWN production change (spawning the real sched_job_picker()/sched_allocator() thread pair against simulation mode's own core, replacing the old always-PEND sim_picker_thread_fn() placeholder that never called sched_make_allocation() at all) makes this code path reachable in simulation mode for the first time ever.", @@ -24610,6 +27031,10 @@ }, { "id": "bug-1830", + "status": "cannot-reproduce", + "status_updated": "2026-09-24", + "status_note": "downgraded 07b5c989; user's own retest showed mem limit usage correctly", + "repro_attempts": "user retest 2026-09-20, not reproduced", "timestamp": "2026-09-20T09:00:00Z", "error_message": "CORRECTED 2026-09-20 (user's own independent retest) -- original framing was too strong. A mem-based limit (limits2, from the real src/configs/limits.yaml) DID correctly populate a live usage instance in a user-run retest: `info limit limits2` against a real replay showed `usage limit=limits2/u=rshen/q=xce/p= host=pc13v: accounted=18022400.000000 (17.19 MB) capacity=4635344384.000000 (4.32 GB)` -- fully populated and correctly human-formatted on both accounted and capacity. This directly contradicts this entry's original claim that mem-based limits 'never' populate a usage instance via replay. ORIGINAL FINDING (retained for context, now understood to be narrower than stated): an earlier live-verification session (agent-run, single short-lived deploy+capture cycle) saw every mem-based limit show 'no active usage instance yet' while a slots-based limit in the same replay populated correctly. Given the user's retest succeeded, the real condition is likely session-shape/timing-dependent (e.g. elapsed replay rounds, WORKERS_PUSH arrival timing relative to job dispatch, or how long the recording ran) rather than a categorical mem-vs-slots failure.", "file": "src/scheduler/src/plugins/limit.c", @@ -24639,6 +27064,8 @@ }, { "id": "bug-1831", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-20T10:00:00Z", "error_message": "Live-verification finding (user's own retest of scheduler-sim-diagnostics Requirement 6): a correctly-configured 20% memory limit (limits2, resname: mem, value: 0.2, percent: true) displayed as 'mem = 0.200000%' in `info limit` -- reading as 0.2 percent to a human, not the 20 percent it actually computes and enforces.", "file": "src/lib/xcelimit.c", @@ -24664,6 +27091,8 @@ }, { "id": "bug-1832", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-20T10:00:00Z", "error_message": "src/configs/limits.yaml's night_mem_pct limit (resname: mem, value: 50, percent: true) is functionally dead -- it can never bind or deny any job, since its actual computed capacity is 50x (5000%) the resource's current value, not 50% of it. Surfaced while investigating bug-1831's percent-display discrepancy.", "file": "src/configs/limits.yaml", @@ -24689,6 +27118,8 @@ }, { "id": "bug-1833", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-20T11:00:00Z", "error_message": "NOT A CRASH -- proactive hardening, user-requested, following bug-1831/bug-1832. Nothing in xceadmin's config validation caught a percent-based limit's value being outside the valid 0.0-1.0 fraction range (e.g. 'value: 50, percent: true', meant to express 50% but computing as 5000% of the resource) -- both night_mem_pct in the real src/configs/limits.yaml (bug-1832) and, discovered while adding this validation, an identical stale value in the xceadmin test suite's own checked-in golden fixture (src/xceadmin/tests/fixtures/full/configs/limits.yaml) had this exact defect.", "file": "src/configs/schema/limits.schema.json", @@ -24715,6 +27146,8 @@ }, { "id": "bug-1834", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-20T09:35:00Z", "error_message": "User-identified design gap (not yet reproduced as a live symptom, code-traced and confirmed): for a percent-based limit whose `resname` is a dynamic/consumable resource (e.g. `mem`, `swap`), the limit's CAPACITY is computed as percent * the resource's current LIVE value, not percent * the resource's static total. This means the same limit's enforced cap drifts up and down as unrelated memory pressure on the host changes, rather than staying pinned to a stable fraction of total capacity (e.g. 'this queue may use up to 20% of the host's total memory' should not shrink because some other job is currently using more memory).", "file": "src/scheduler/src/plugins/limit.c", @@ -24746,6 +27179,8 @@ }, { "id": "bug-1835", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-21T03:47:10Z", "error_message": "Live 3-node HA cluster crash: xce_controller SIGSEGV in __memcpy_avx_unaligned_erms, called from capnp's copy_ptr/capn_setp, called from write_CtrlHelloRequest (lib/controller.capnp.c:22821) via encode_CtrlHelloRequest_ptr (lib/controller.capnp.c:22935), via xce_encode/xce_request, via ha_send_ctrlhello_to_all_standbys (controller/cluster.c:2432), via raft_cb_become_leader (controller/ha.c:2113), via raft_on_vote_reply -> vote_reply_dispatch -> peer_ctrl_msg_handler_inline -> peer_ctrl_vtask_entry, on the vcpu thread handling the winning vote reply. Core confirmed built from commit 1e43345b (xce v0.0.1-1e43345b, 2026-09-21T03:10:39Z); ctrl-1 had just won its Raft election (term 2, votes_granted=2) and crashed while broadcasting CtrlHello to its 2 standbys.", "file": "src/schema/controller.capnp", @@ -24780,6 +27215,8 @@ }, { "id": "bug-1836", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-20T22:00:00Z", "error_message": "User-reported live over-dispatch in xce_scheduler -i -s : replaying a captured session (JOB_PUSH of a 50-element array job) against a real single worker with 22 slots dispatched 44 concurrent single-slot jobs (`assigned jobs: 0-3-1789963203-0[0-43]`), exactly double the host's real capacity, while `info worker` simultaneously showed `resource slots: value=22.00 used=22.00` -- the accounting display looked self-consistent (used == value) even though double the job count the display's own 'used' figure implied was actually running. User confirmed normal mode (the live controller-connected driver) does not exhibit this under equivalent conditions, isolating the defect to simulation/replay mode specifically.", "file": "src/scheduler/src/sim_loop.c", @@ -24813,6 +27250,8 @@ }, { "id": "bug-1837", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-20T22:05:00Z", "error_message": "Found during /xce-audit-invariants scheduler (dimension 3, jbuf capacity-1 rendezvous audit), not user-reported and not a live symptom: sim_reconcile_unaccounted_jobs() (sim_loop.c) calls picker->picker_iface->on_dispatch(picker, job->jobid) directly from the sim driver thread (sched_sim_driver_run()'s own thread, between xce_core_begin_session() and xce_core_signal_go()), but plugin.h documents on_dispatch()/on_pend() as 'Post-allocation hooks (MUST) -- allocator thread, after each allocation attempt' -- a MUST-level threading contract this call site violates by construction (driver-thread code calling an allocator-thread-only hook).", "file": "src/scheduler/src/sim_loop.c", @@ -24843,6 +27282,8 @@ }, { "id": "bug-1838", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-21T00:30:00Z", "error_message": "Follow-up architectural correction after bug-1836 (found during user discussion, not a separately-reported live symptom): even with bug-1836's capacity gate in place, sim_reconcile_unaccounted_jobs() still unnecessarily PREEMPTED the live picker/allocator for any freshly-PEND job with a matching recorded DISPATCH decision, rather than letting the live path -- now genuinely capacity-checked, per scheduler-sim-live-allocation -- decide it on its own the way normal mode always has. User specifically asked why sim mode diverged from normal mode given they 'share the same core', then proposed the architecturally correct framing directly: the simulator should run the identical decision-making core (minus controller/worker I/O) and use the recording only to verify/reconstruct, not to short-circuit live decisions.", "file": "src/scheduler/src/sim_loop.c", @@ -24876,6 +27317,8 @@ }, { "id": "bug-1839", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-22T02:30:00Z", "error_message": "Live 3-node HA cluster crash: xce_controller SIGSEGV in peer_ctrl_msg_handler_inline (ha.c:7035, `ha_state_t *ha = vctx->ha;`). gdb analysis of the actual core dump (build cd9ecb41) confirmed vctx == NULL (i.e. ep->data == NULL) while the endpoint itself was fully alive and in active use: ep->refcnt=191964, ep->tearing_down=0, ep->heap_alloc=1, ep->msg_handler=peer_ctrl_msg_handler, ep->err_handler=standby_ep_err_handler_primary, ep->fsm_out.state=FSM_STATE_ERROR. The crash fired on the vtask-pool path (vcpu_entry -> run_task -> peer_ctrl_vtask_entry -> peer_ctrl_msg_handler_inline); g_drive_async_peerctrl_dispatch_enabled defaults to 1 (ON, the production path).", "file": "src/controller/ha.c", @@ -24906,6 +27349,9 @@ }, { "id": "bug-1840", + "status": "investigating", + "status_updated": "2026-09-24", + "status_note": "scheduler heap corruption (malloc_consolidate abort) during TLS handshake; awaiting ASAN repro", "timestamp": "2026-09-22T02:35:00Z", "error_message": "Live 3-node HA cluster crash (same soak run as bug-1839): xce_scheduler SIGABRT, glibc `malloc_consolidate(): unaligned fastbin chunk detected`, surfacing during an OpenSSL TLS handshake's MAC-buffer growth: abort -> malloc_printerr -> malloc_consolidate -> _int_malloc -> __libc_malloc2 -> CRYPTO_malloc -> CRYPTO_clear_realloc -> BUF_MEM_grow_clean -> mem_write -> bwrite_conv -> BIO_write -> ssl3_finish_mac -> tls_get_message_body -> state_machine -> handle_ssl_handshake (netio.c:548) -> io_handler (netio.c:588) -> io_worker_dispatch_batch (netio.c:648) -> io_worker (netio.c:689).", "file": "unknown -- heap corruption detected far from its cause", @@ -24932,6 +27378,8 @@ }, { "id": "bug-1841", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-22T09:00:00Z", "error_message": "User-reported (xce_scheduler -i -s /tmp/scheduler-replay-20260922-071101.replay, worker pc13v, 22 real slots), also reported to occur in normal (non-sim) mode: after worker pc13v rejoined mid-replay (session~162, multiple WORKERS_PUSH+JOB_PUSH bursts, several `JOBS_PUSH: refusing resync state 0x1 for jobid ... existing job is RUN with a real allocation on record` WARNs -- bug-1812's guard correctly refusing a stale PEND resync), every one of that worker's jobs eventually legitimately finished (`info jobs` at session=423 shows all of jobids [0-22] DONE, [23-59] PEND, zero RUN), yet `info worker pc13v` still showed `resource slots: value=22.00 used=22.00` for the rest of the run (session 162 through 423, ~18 simulated minutes) -- the worker was permanently starved even though nothing was actually running on it.", "file": "src/controller/resync.c (resync_resolve_round() -- the fix), src/controller/persist.c/src/include/persist.h (new persist_notify_worker_status_updated(), mirroring persist_notify_job_updated_already_persisted()/bug-1679), src/controller/testsuite/test_worker_resync.c (new regression test)", @@ -24961,6 +27409,8 @@ }, { "id": "bug-1842", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-22T19:35:00Z", "error_message": "User-reported: replaying a real captured .replay file (scheduler-replay-20260922-095207.replay) via --sim --sim-verify logged a MISMATCH for every single DISPATCH decision, even though the printed 'expected:'/'actual:' summaries (jobid, nhosts) were byte-for-byte identical on both sides -- e.g. 'expected: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1 -- actual: DISPATCH jobid=0-13-1790071174-0[44] nhosts=1' still reported as MISMATCH. User asked whether the comparison should stop using a timestamp as part of its check condition.", "file": "src/scheduler/src/sim_verify.c (sim_verify_alloc_eq, the fix), src/lib/xcealloc.c (make_alloc, the field's origin), src/scheduler/testsuite/test_sim_verify.c (new regression test), src/scheduler/testsuite/test_sim_live_allocation.c (stale-comment cleanup)", @@ -24993,6 +27443,8 @@ }, { "id": "bug-1843", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-22T19:35:00Z", "error_message": "Same user report as bug-1842: replaying scheduler-replay-20260922-095207.replay via --sim --sim-verify also logs a small number of 'expected: -- actual: PEND jobid=... main_reason=0' mismatches -- live decisions with no recorded counterpart at all (beyond decoded_session->n_decisions), distinct from bug-1842's DISPATCH-vs-DISPATCH false positives (already fixed). 61 occurrences total across the full 643-session replay file (0.02% of the 3431 live PEND decisions produced).", "file": "src/scheduler/src/sim_loop.c (sched_sim_driver_run()'s decision-drain loop, sim_reconcile_unaccounted_jobs()), src/scheduler/src/sched_election.c (sched_job_picker()/sched_allocator())", @@ -25024,6 +27476,8 @@ }, { "id": "bug-1844", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-23T09:15:00Z", "error_message": "User-reported follow-up to bug-1843: replaying a fresh capture (scheduler-replay-20260923-085356.replay, built after bug-1843's fix) with --sim --sim-verify still shows a huge volume of MISMATCH lines (4881 in one run) on DISPATCH decisions whose printed description is IDENTICAL on both sides -- e.g. 'expected: DISPATCH jobid=...[40] nhosts=1 -- actual: DISPATCH jobid=...[40] nhosts=1' still flagged as a mismatch.", "file": "src/scheduler/src/sim_verify.c (sim_verify_alloc_eq())", @@ -25052,6 +27506,8 @@ }, { "id": "bug-1845", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-23T10:00:00Z", "error_message": "Follow-up to bug-1843/bug-1844: replaying a fresh capture (scheduler-replay-20260923-085356.replay) with --sim --sim-verify still showed a large volume of MISMATCH lines on PEND decisions where recorded and replay agreed on jobid/main_reason but the deep host/resource reason-detail comparison disagreed (e.g. recorded {main_reason=0, nhost_reasons=0, nres_reasons=0} vs replay {main_reason=0, nhost_reasons=0, nres_reasons=2}), live-inspected via gdb by the user.", "file": "src/scheduler/src/sim_verify.c (sim_verify_job_preason_eq())", @@ -25080,6 +27536,9 @@ }, { "id": "bug-1846", + "status": "fixed", + "status_updated": "2026-09-24", + "status_note": "77f047f2, a9feded8, ac033f47", "timestamp": "2026-09-23T10:05:00Z", "error_message": "OPEN, NOT YET ROOT-CAUSED. Same replay session as bug-1844/1845 (scheduler-replay-20260923-085356.replay): after excluding instid (bug-1844) and reason-detail (bug-1845), --sim-verify still shows one job (jobid=0-8-1790154325-0[39] in the observed capture) genuinely DISPATCHING one round earlier in replay than in the original live recording -- not a false-positive field comparison, a real positional divergence: replay's round begin_round=...436582151823 dispatches a 13-job batch (jobs [27]..[39]) while the SAME round in the original recording only dispatched 12 (jobs [27]..[38]); job [39]'s recorded dispatch instead lands in the NEXT round, cascading into ~40+ downstream positional MISMATCH lines for that round pair alone.", "file": "src/scheduler/src/plugins/limit.c (match_job_host_limit_); src/scheduler/src/handler.c + main.c + scheduler.c (config push deferral)", @@ -25114,6 +27573,8 @@ }, { "id": "bug-1847", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-24T05:30:00Z", "error_message": "bug-1846 follow-up: a scheduler config push (sent by the controller on every scheduler join) was never written to a .replay recording, so a replay spanning a rejoin or a genuine config change never saw it.", "file": "src/scheduler/src/scheduler.c (sched_apply_pending_config)", @@ -25129,6 +27590,8 @@ }, { "id": "bug-1848", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-24T05:30:00Z", "error_message": "Leak: xce_sched_obj_t.hlimits was created with a NULL zarray item killer, so reset_sched_obj() (limit_job_election()'s config-version rebuild) and schedobj teardown leaked the per-worker xlist_t that match_job_host_limit_() callocs, one per worker a job had tried. Visible as part of the large LeakSanitizer report on every --sim replay run.", "file": "src/scheduler/src/schedobj.c, core_events.c, snapshot_load.c (hlimits zarray_init)", @@ -25143,6 +27606,8 @@ }, { "id": "bug-1849", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-24T05:30:00Z", "error_message": "Every --sim run logged 'WARN xce_sched_core_load_snapshot failed to resolve job picker plugin (rc=-2); picker rebuild will be skipped for this load' -- misleading; flagged as a suspect during the bug-1846 investigation.", "file": "src/scheduler/src/snapshot_load.c (xce_sched_core_load_snapshot)", @@ -25157,6 +27622,9 @@ }, { "id": "bug-1850", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "client failover rotates away from healthy primary on timeout/not-ready (client_failover.c); only logging added e71d91bf", "timestamp": "2026-09-24T05:30:00Z", "error_message": "OPEN, NOT FIXED -- root trigger of bug-1846 part 2's mid-round rejoins: the live scheduler reconnected/re-joined roughly every ~100s in scheduler-replay-20260923-085356.replay (each rejoin = full config/worker/job re-push from on_scheduler_join()).", "file": "src/lib/client_failover.c (xcecli_retry_next / xce_failover_classify), src/lib/xceapi.c (10s reply deadline)", @@ -25173,6 +27641,8 @@ }, { "id": "bug-1851", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-24T07:40:00Z", "error_message": "Live sequential HA soak (run-ha-soak.sh --sequential 6, round 5): newly-promoted primary ctrl-3 (term 10) died ~8s after promotion with ASAN heap-use-after-free READ in strdup <- collect_resval (dataex.c:71) <- apiserver_worker_change (apiserver.c:450) <- apiserver_worker_hook_fun <- xce_run_hooks <- persist_notify_worker_status_updated (persist.c:7997) <- resync_resolve_round (resync.c:418) <- resync_arm_round (resync.c:315) <- apply_join_state (worker.c:351) <- rejoin_worker (worker.c:444) <- handle_worker_join. Freed by the same thread in rejoin_worker (worker.c:434, xstring_dereference of the old hostmodel). Soak surfaced it as 'FAIL: round 5: no new primary elected after kill (no serving primary at term>9)' -- the new primary crashed rather than failing to elect.", "file": "src/controller/worker.c", @@ -25190,6 +27660,8 @@ }, { "id": "bug-1852", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-24T08:10:00Z", "error_message": "Scheduler replay-parity soak gate (run-ha-soak.sh --sequential 6, post bug-1851): 1 of 7 clean recordings failed --sim-verify with 3 MISMATCHes, all in its first decision round: live 'DISPATCH jobid=... nhosts=1' vs sim 'PEND ... main_reason=0' (sim: unsatisfy_ unmet resource 0 (slots) shortfall 1). Sim trace: snapshot at record start restored the worker with 0/0 resvals; the recording's only WORKERS_PUSH 'carries 10 resval(s)' but core_apply_worker_resvals 'applied 0' -- every recorded NameValue.val was NULL.", "file": "src/scheduler/src/driver.c", @@ -25207,6 +27679,8 @@ }, { "id": "bug-1853", + "status": "fixed", + "status_updated": "2026-09-24", "timestamp": "2026-09-24T08:20:00Z", "error_message": "Test-harness false positive, not a controller safety bug. 10-round chaotic soak (run-ha-soak.sh --chaotic 10): VERDICT committed_divergence=1 committed_job_loss=0 result=FAIL. Single hit, round 3, mid-loop walcompare with boundary=auto: 'WALCMP_DIVERGENCE lsn=58 class=committed compared=3 present=3'. Final settled sweep: committed prefix consistent, all 33 jobs present.", "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", @@ -25222,6 +27696,9 @@ }, { "id": "bug-1854", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "every config push re-adds resources: ERROR noise, changed definitions not applied (scheduler.c)", "timestamp": "2026-09-24T08:35:00Z", "error_message": "OPEN, NOT FIXED, low priority. Every HA soak run logs 'ERROR [sched_init_resources] sched_init_resources add resource failed' for EVERY configured resource (~33 lines), once per scheduler process, ~30s after it joined -- i.e. when a primary failover makes the scheduler re-join and the controller re-pushes its config (231/264/429 lines in 3 soak runs today). Scheduling is unaffected in those runs (all soaks PASS, replay parity 0 mismatches).", "file": "src/scheduler/src/scheduler.c", @@ -25234,6 +27711,48 @@ "resources", "log-noise" ] + }, + { + "id": "bug-1855", + "status_note": "commit 'feat(buglog): status field, buglog skill, status-driven gitea-sync' (2026-09-24)", + "status": "fixed", + "status_updated": "2026-09-24", + "timestamp": "2026-09-24T09:39:50Z", + "error_message": "gitea-sync mirrored unfixed bugs as closed + bug:fixed: an issue was closed whenever the buglog entry's fix text was non-empty, including fix texts reading 'NOT FIXED', 'PENDING' or 'PARTIALLY FIXED' (e.g. bug-1450 #25, bug-1451 #35, bug-1744 #47)", + "file": "scripts/gitea-sync.py,.claude/skills/gitea-sync/SKILL.md", + "root_cause": "buglog.json had no status field, so scripts/gitea-sync.py and the gitea-sync skill inferred open/closed from fix being non-empty. Same script also: crashed on entries whose fix is a list (.strip() on list); replaced an issue's whole label set with just bug:fixed on close (wiping bug/category/user labels); wrote gitea_issue back only after the whole loop via a full json.dumps rewrite (a mid-run failure lost issue numbers -> duplicate issues next run, and the rewrite churned ~1,200 unrelated lines); read approvals.implementation, which no spec.json has, so spec implementation/validation issues and milestones never closed; never reopened an issue whose bug came back.", + "fix": "Added a status field to every buglog entry (backfilled 2026-09-24) and made status the only open/closed source: STATUS_MAP -> state + one bug: label; entries sharing an issue take the most-open status; only status labels are swapped (other labels kept); reopen supported; gitea_synced_status tracks what was mirrored so unchanged entries make no API calls; per-entry surgical write-back via scripts/buglog.py set_fields; spec issues close on DONE_PHASES (implementation-complete/implemented/superseded). Skill doc updated to match.", + "tags": [ + "gitea-sync", + "tooling", + "buglog" + ], + "related_bugs": [ + "bug-1677" + ], + "occurrences": 1, + "last_seen": "2026-09-24T09:39:50Z" + }, + { + "id": "bug-1856", + "status": "open", + "status_updated": "2026-09-24", + "status_note": "upstream OpenWolf defect; mitigated locally by openwolf.buglog.auto_detect=false; scripts/buglog.py add uses max id + 1; report upstream", + "timestamp": "2026-09-24T09:39:50Z", + "error_message": "OpenWolf 2.5.0 post-write hook would create a duplicate bug id: autoDetectBugFix numbers new entries bug- (next: bug-1194), but ids run to bug-1854 and bug-1194 already exists", + "file": ".wolf/hooks/post-write.js,.wolf/config.json", + "root_cause": "Upstream OpenWolf hook (.wolf/hooks/post-write.js autoDetectBugFix, nextId = bugLog.bugs.length + 1) assumes ids are dense and never renumbered; this buglog's ids are sparse after renumbering/consolidation (see bug-1630). The hook also never sets the project's status field.", + "fix": "", + "tags": [ + "openwolf", + "buglog", + "upstream" + ], + "related_bugs": [ + "bug-1630" + ], + "occurrences": 1, + "last_seen": "2026-09-24T09:39:50Z" } ] } diff --git a/.wolf/config.json b/.wolf/config.json index 361d1df3..1e16f40e 100644 --- a/.wolf/config.json +++ b/.wolf/config.json @@ -2,6 +2,9 @@ "version": 1, "openwolf": { "enabled": true, + "buglog": { + "auto_detect": false + }, "anatomy": { "auto_scan_on_init": true, "rescan_interval_hours": 6, @@ -116,4 +119,4 @@ } } } -} +} \ No newline at end of file diff --git a/scripts/buglog.py b/scripts/buglog.py new file mode 100644 index 00000000..18ae04c7 --- /dev/null +++ b/scripts/buglog.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +"""Read and edit .wolf/buglog.json without re-serializing it. + +Re-serializing the whole buglog (json.dumps / jq) rewrites ~1,200 unrelated +lines because existing entries mix raw UTF-8 and \\uXXXX escapes, so every +write here edits only the lines of the entry it touches. Every write is +re-parsed before it is saved and saved atomically (tmp file + rename), the +same way OpenWolf's own hooks write it. + +The `status` values and when to use each are defined in +.claude/rules/buglog-status.md. + +Commands: + add --error MSG [--root-cause T] [--fix T] [--file F] [--tags a,b] + [--related bug-1,bug-2] [--status S] [--note T] + Append a new entry with the next free id; prints the id. + status ID STATUS [--note T] [--duplicate-of ID] [--repro T] + Set status + status_updated (today) + status_note. --note is + required except for `fixed`; duplicate needs --duplicate-of, + cannot-reproduce needs --repro. + note ID TEXT Append an "UPDATE (date): TEXT" paragraph to `fix`. + recur ID [--note T] The bug was seen again: bump occurrences/last_seen; + cannot-reproduce -> investigating, fixed/wontfix -> open. + set ID key= ... Set arbitrary single-line fields. + show ID Print one entry. + list [--status S,S | --open] [--tag T] [--grep RE] [--since DATE] + One line per matching entry. + triage List entries with no (or an unknown) status. + next-id Print the id `add` would use. + +Importable: set_fields(path, bug_id, {key: value}), BuglogEditError. +""" +import argparse +import datetime +import json +import os +import re +import sys +from pathlib import Path + +DEFAULT_PATH = Path(__file__).resolve().parent.parent / ".wolf" / "buglog.json" + +STATUSES = ["open", "partial", "investigating", "fixed", + "cannot-reproduce", "wontfix", "duplicate"] +OPEN_STATUSES = {"open", "partial", "investigating"} + + +class BuglogEditError(Exception): + pass + + +def _today(): + return datetime.date.today().isoformat() + + +def _now(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def _load(path): + return json.loads(Path(path).read_text(encoding="utf-8"))["bugs"] + + +def _save(path, text): + """Refuse to write text that no longer parses; write atomically.""" + json.loads(text) + path = Path(path) + tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp") + tmp.write_text(text, encoding="utf-8") + os.replace(tmp, path) + + +def _entry_span(lines, bug_id): + """(start, end, indent) of the entry: start = its "id" line, end = its '}'.""" + id_re = re.compile(r'^(\s+)"id": ' + re.escape(json.dumps(bug_id)) + r",?$") + starts = [i for i, ln in enumerate(lines) if id_re.match(ln)] + if len(starts) != 1: + raise BuglogEditError(f"{bug_id}: expected exactly one entry, found {len(starts)}") + start = starts[0] + indent = id_re.match(lines[start]).group(1) + close_re = re.compile(r"^" + re.escape(indent[:-2]) + r"\},?$") + end = next(i for i in range(start, len(lines)) if close_re.match(lines[i])) + return start, end, indent + + +def set_fields(path, bug_id, fields): + """Set each key in `fields` on entry `bug_id`, touching only its lines. + + An existing single-line field is replaced in place; a new one is inserted + right after the entry's "id" line. Multi-line (array/object) fields are + refused -- edit those by hand. + """ + path = Path(path) + lines = path.read_text(encoding="utf-8").split("\n") + start, end, indent = _entry_span(lines, bug_id) + + for key, value in fields.items(): + rendered = json.dumps(value, ensure_ascii=False) + key_re = re.compile(r"^" + re.escape(indent) + re.escape(json.dumps(key)) + r": (.*?)(,?)$") + hit = next((i for i in range(start, end) if key_re.match(lines[i])), None) + if hit is not None: + old, comma = key_re.match(lines[hit]).groups() + if old in ("[", "{"): + raise BuglogEditError(f"{bug_id}.{key}: multi-line value, edit by hand") + lines[hit] = f"{indent}{json.dumps(key)}: {rendered}{comma}" + else: + lines.insert(start + 1, f"{indent}{json.dumps(key)}: {rendered},") + end += 1 + + text = "\n".join(lines) + entry = next(b for b in json.loads(text)["bugs"] if b.get("id") == bug_id) + for key, value in fields.items(): + if entry[key] != value: + raise BuglogEditError(f"{bug_id}.{key}: wrote {entry[key]!r}, wanted {value!r}") + _save(path, text) + + +def next_id(bugs): + """One past the highest numeric id. (OpenWolf's own hook uses len(bugs)+1, + which collides with existing ids once any were renumbered or skipped.)""" + nums = [int(m.group(1)) for b in bugs + if (m := re.fullmatch(r"bug-(\d+)", str(b.get("id", ""))))] + return f"bug-{max(nums, default=0) + 1:03d}" + + +def append_entry(path, entry): + """Append `entry` as the last element of the `bugs` array.""" + path = Path(path) + lines = path.read_text(encoding="utf-8").split("\n") + close = max(i for i, ln in enumerate(lines) if ln == " ]") + prev = close - 1 + if lines[prev] == " }": + lines[prev] = " }," + elif lines[prev] != " \"bugs\": [": + raise BuglogEditError(f"unexpected line before bugs array end: {lines[prev]!r}") + body = json.dumps(entry, indent=2, ensure_ascii=False).split("\n") + lines[close:close] = [" " + ln for ln in body] + text = "\n".join(lines) + if json.loads(text)["bugs"][-1] != entry: + raise BuglogEditError("appended entry did not round-trip") + _save(path, text) + + +def _split(value): + return [v.strip() for v in (value or "").split(",") if v.strip()] + + +def _text(value): + if isinstance(value, list): + return " || ".join(str(v) for v in value) + return value or "" + + +def _find(bugs, bug_id): + for b in bugs: + if b.get("id") == bug_id: + return b + raise BuglogEditError(f"{bug_id}: no such entry") + + +def _line(b): + note = b.get("status_note") or _text(b.get("error_message")) + return f"{b['id']:<10} {b.get('status') or '-':<16} {note[:110]}" + + +def cmd_add(a, path): + if a.status not in STATUSES: + sys.exit(f"unknown status {a.status!r}; one of {', '.join(STATUSES)}") + bugs = _load(path) + bid = next_id(bugs) + now = _now() + entry = {"id": bid, "status": a.status, "status_updated": _today()} + if a.note: + entry["status_note"] = a.note + entry.update({ + "timestamp": now, + "error_message": a.error, + "file": a.file or "", + "root_cause": a.root_cause or "", + "fix": a.fix or "", + "tags": _split(a.tags), + "related_bugs": _split(a.related), + "occurrences": 1, + "last_seen": now, + }) + append_entry(path, entry) + print(bid) + + +def cmd_status(a, path): + if a.status not in STATUSES: + sys.exit(f"unknown status {a.status!r}; one of {', '.join(STATUSES)}") + if a.status == "duplicate" and not a.duplicate_of: + sys.exit("duplicate needs --duplicate-of") + if a.status == "cannot-reproduce" and not a.repro: + sys.exit("cannot-reproduce needs --repro (what was tried, e.g. '0/40 runs, 2026-08-01')") + if a.note is None and a.status != "fixed": + sys.exit(f"{a.status} needs --note (what remains / why / what was seen)") + # Always (re)write the note so a stale one never outlives its status. + fields = {"status": a.status, "status_updated": _today(), + "status_note": a.note or ""} + if a.duplicate_of: + fields["duplicate_of"] = a.duplicate_of + if a.repro: + fields["repro_attempts"] = a.repro + set_fields(path, a.id, fields) + print(_line(_find(_load(path), a.id))) + + +def cmd_note(a, path): + fix = _find(_load(path), a.id).get("fix") or "" + if isinstance(fix, list): + raise BuglogEditError(f"{a.id}.fix is a list, edit by hand") + para = f"UPDATE ({_today()}): {a.text}" + set_fields(path, a.id, {"fix": f"{fix}\n\n{para}" if fix else para}) + + +def cmd_recur(a, path): + b = _find(_load(path), a.id) + fields = {"occurrences": int(b.get("occurrences") or 1) + 1, "last_seen": _now()} + reopen = {"cannot-reproduce": "investigating", "fixed": "open", "wontfix": "open"} + if b.get("status") in reopen: + fields.update(status=reopen[b["status"]], status_updated=_today()) + fields["status_note"] = a.note or f"recurred {_today()} (was {b['status']})" + elif a.note: + fields["status_note"] = a.note + set_fields(path, a.id, fields) + print(_line(_find(_load(path), a.id))) + + +def cmd_set(a, path): + fields = {} + for kv in a.pairs: + key, sep, raw = kv.partition("=") + if not sep or not key: + sys.exit(f"bad argument {kv!r}: expected key=value") + try: + fields[key] = json.loads(raw) + except json.JSONDecodeError: + fields[key] = raw + set_fields(path, a.id, fields) + + +def cmd_show(a, path): + print(json.dumps(_find(_load(path), a.id), indent=2, ensure_ascii=False)) + + +def cmd_list(a, path): + want = OPEN_STATUSES if a.open else set(_split(a.status)) or None + rx = re.compile(a.grep, re.I) if a.grep else None + for b in _load(path): + if want is not None and b.get("status") not in want: + continue + if a.tag and a.tag not in (b.get("tags") or []): + continue + if a.since and max(b.get("last_seen") or "", b.get("status_updated") or "") < a.since: + continue + if rx and not rx.search(json.dumps(b, ensure_ascii=False)): + continue + print(_line(b)) + + +def cmd_triage(a, path): + for b in _load(path): + if b.get("status") not in STATUSES: + print(_line(b)) + + +def main(argv=None): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0], + epilog="Status values: " + ", ".join(STATUSES)) + p.add_argument("--path", default=str(DEFAULT_PATH), help=argparse.SUPPRESS) + sub = p.add_subparsers(dest="cmd", required=True) + + s = sub.add_parser("add") + s.add_argument("--error", required=True) + s.add_argument("--root-cause") + s.add_argument("--fix") + s.add_argument("--file") + s.add_argument("--tags") + s.add_argument("--related") + s.add_argument("--status", default="open") + s.add_argument("--note") + s.set_defaults(fn=cmd_add) + + s = sub.add_parser("status") + s.add_argument("id") + s.add_argument("status") + s.add_argument("--note") + s.add_argument("--duplicate-of") + s.add_argument("--repro") + s.set_defaults(fn=cmd_status) + + s = sub.add_parser("note") + s.add_argument("id") + s.add_argument("text") + s.set_defaults(fn=cmd_note) + + s = sub.add_parser("recur") + s.add_argument("id") + s.add_argument("--note") + s.set_defaults(fn=cmd_recur) + + s = sub.add_parser("set") + s.add_argument("id") + s.add_argument("pairs", nargs="+") + s.set_defaults(fn=cmd_set) + + s = sub.add_parser("show") + s.add_argument("id") + s.set_defaults(fn=cmd_show) + + s = sub.add_parser("list") + s.add_argument("--status") + s.add_argument("--open", action="store_true") + s.add_argument("--tag") + s.add_argument("--grep") + s.add_argument("--since") + s.set_defaults(fn=cmd_list) + + s = sub.add_parser("triage") + s.set_defaults(fn=cmd_triage) + + s = sub.add_parser("next-id") + s.set_defaults(fn=lambda a, path: print(next_id(_load(path)))) + + a = p.parse_args(argv) + try: + a.fn(a, a.path) + except BuglogEditError as e: + sys.exit(str(e)) + + +if __name__ == "__main__": + main() diff --git a/scripts/gitea-sync.py b/scripts/gitea-sync.py index afefd9ff..961d370d 100755 --- a/scripts/gitea-sync.py +++ b/scripts/gitea-sync.py @@ -34,6 +34,9 @@ import urllib.parse import urllib.request from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from buglog import BuglogEditError, set_fields # noqa: E402 + # ── Helpers ────────────────────────────────────────────────────────────────── @@ -120,6 +123,27 @@ LABELS = [ "description": "Bug found during development"}, {"name": "bug:fixed", "color": "27AE60", "description": "Bug resolved and fix committed"}, + {"name": "bug:partial", "color": "F39C12", + "description": "Partly fixed, remainder still open"}, + {"name": "bug:investigating", "color": "E67E22", + "description": "Real failure, root cause not found yet"}, + {"name": "bug:cannot-reproduce", "color": "95A5A6", + "description": "Reproduction attempted and failed"}, + {"name": "bug:wontfix", "color": "7F8C8D", + "description": "Not a bug / accepted limitation"}, + {"name": "bug:duplicate", "color": "BDC3C7", + "description": "Duplicate of another bug"}, + # Bug category labels (from buglog tags, see TAG_TO_LABEL) + {"name": "bug:asan", "color": "9B59B6", + "description": "ASAN/UBSAN caught bug"}, + {"name": "bug:crash", "color": "C0392B", + "description": "SEGV/UAF/double-free crash"}, + {"name": "bug:data-loss", "color": "C0392B", + "description": "Committed data loss bug"}, + {"name": "bug:ha", "color": "E67E22", + "description": "HA/Raft/consensus bug"}, + {"name": "bug:race", "color": "F39C12", + "description": "Data race / TOCTOU bug"}, # State labels {"name": "blocked", "color": "E74C3C", "description": "Blocked on external dependency or upstream spec"}, @@ -143,6 +167,45 @@ TAG_TO_LABEL = { "failover": "bug:ha", } +# buglog `status` -> (Gitea issue state, status label). The buglog's `status` +# field is the only source of truth for open/closed (see +# .claude/rules/buglog-status.md); never infer it from `fix`, which often says +# "NOT FIXED" / "PENDING" / "PARTIALLY FIXED". An entry with no status (e.g. +# created by an OpenWolf hook) is mirrored as open and reported for triage. +STATUS_MAP = { + "open": ("open", None), + "partial": ("open", "bug:partial"), + "investigating": ("open", "bug:investigating"), + "fixed": ("closed", "bug:fixed"), + "cannot-reproduce": ("closed", "bug:cannot-reproduce"), + "wontfix": ("closed", "bug:wontfix"), + "duplicate": ("closed", "bug:duplicate"), +} + + +# When several buglog entries share one Gitea issue (e.g. a user-reported +# issue whose cause was split into several bugs), the issue takes the status +# of its "most open" entry, first in this list. +STATUS_PRIORITY = ["open", "investigating", "partial", None, + "cannot-reproduce", "duplicate", "wontfix", "fixed"] + + +def bug_status(bug_entry): + """The entry's status, or None when it has none / an unknown one.""" + status = bug_entry.get("status") + return status if status in STATUS_MAP else None + + +def bug_state(bug_entry): + return STATUS_MAP.get(bug_status(bug_entry), ("open", None))[0] + + +def _text(value): + """buglog text fields are a string or, in some entries, a list of strings.""" + if isinstance(value, list): + return "\n\n".join(str(v).strip() for v in value if str(v).strip()) + return (value or "").strip() + def api_url(path): """Build full API URL. All paths are repo-scoped.""" @@ -245,9 +308,18 @@ class GiteaAPI: return issue def update_issue(self, issue_number, **fields): - """Update issue fields (state, title, body, milestone, labels).""" + """Update issue fields (state, title, body, milestone).""" request("PATCH", f"/issues/{issue_number}", fields) + def add_labels(self, issue_number, label_ids): + request("POST", f"/issues/{issue_number}/labels", {"labels": label_ids}) + + def remove_label(self, issue_number, label_id): + request("DELETE", f"/issues/{issue_number}/labels/{label_id}") + + def add_comment(self, issue_number, body): + request("POST", f"/issues/{issue_number}/comments", {"body": body}) + # ── Label Sync ─────────────────────────────────────────────────────────────── @@ -259,40 +331,27 @@ def ensure_all_labels(api): # ── Bug Tag → Label Mapping ────────────────────────────────────────────────── -def bug_labels(bug_entry): - """Derive Gitea labels from buglog tags.""" - labels = ["bug"] - tags = [t.lower() for t in bug_entry.get("tags", [])] +def status_label(status): + return STATUS_MAP.get(status, (None, None))[1] - # Fixed bugs get bug:fixed label - fix = bug_entry.get("fix", "") - if fix and fix.strip(): - labels.append("bug:fixed") + +STATUS_LABELS = {lbl for _, lbl in STATUS_MAP.values() if lbl} + + +def bug_labels(bug_entry): + """Derive Gitea labels from the entry's status and tags.""" + labels = ["bug"] + if status_label(bug_status(bug_entry)): + labels.append(status_label(bug_status(bug_entry))) # Map known tag patterns to labels - for tag in tags: + for tag in (t.lower() for t in bug_entry.get("tags", [])): for pattern, label in TAG_TO_LABEL.items(): - if pattern.lower() in tag: - if label not in labels: - labels.append(label) - - # Ensure any needed labels exist in Gitea - # (bug:asan, bug:crash, bug:data-loss, bug:ha, bug:race may not exist yet) - extra_labels = { - "bug:asan": ("9B59B6", "ASAN/UBSAN caught bug"), - "bug:crash": ("C0392B", "SEGV/UAF/double-free crash"), - "bug:data-loss": ("C0392B", "Committed data loss bug"), - "bug:ha": ("E67E22", "HA/Raft/consensus bug"), - "bug:race": ("F39C12", "Data race / TOCTOU bug"), - } - for lbl in labels: - if lbl in extra_labels and lbl not in LABEL_NAMES: - pass # handled below - - return labels, extra_labels + if pattern.lower() in tag and label not in labels: + labels.append(label) + return labels -LABEL_NAMES = {l["name"] for l in LABELS} # ── Body Formatting ────────────────────────────────────────────────────────── @@ -301,35 +360,44 @@ def bug_body(bug_entry): """Format a buglog entry as a Gitea issue body.""" parts = [] - error = bug_entry.get("error_message", "") + status = bug_entry.get("status") or "unknown (needs triage)" + note = _text(bug_entry.get("status_note")) + parts.append(f"**Status:** {status}" + (f" — {note}" if note else "")) + if bug_entry.get("duplicate_of"): + parts.append(f"Duplicate of {bug_entry['duplicate_of']}") + if bug_entry.get("repro_attempts"): + parts.append(f"Repro attempts: {_text(bug_entry['repro_attempts'])}") + parts.append("") + + error = _text(bug_entry.get("error_message")) if error: parts.append("## Error\n") - parts.append(error.strip()) + parts.append(error) parts.append("") - root_cause = bug_entry.get("root_cause", "") + root_cause = _text(bug_entry.get("root_cause")) if root_cause: parts.append("## Root Cause\n") - parts.append(root_cause.strip()) + parts.append(root_cause) parts.append("") - fix = bug_entry.get("fix", "") - if fix and fix.strip(): + fix = _text(bug_entry.get("fix")) + if fix: parts.append("## Fix\n") - parts.append(fix.strip()) + parts.append(fix) parts.append("") else: parts.append("## Fix\n") parts.append("*Not yet fixed*") parts.append("") - file_ = bug_entry.get("file", "") - if file_: + files = [f.strip() for f in (bug_entry.get("file") or "").split(",")] + files += [str(f).strip() for f in bug_entry.get("files") or []] + files = [f for f in dict.fromkeys(files) if f] + if files: parts.append("## Affected Files\n") - for f in file_.split(","): - f = f.strip() - if f: - parts.append(f"- `{f}`") + for f in files: + parts.append(f"- `{f}`") parts.append("") tags = bug_entry.get("tags", []) @@ -355,7 +423,13 @@ def bug_body(bug_entry): def bug_title(bug_entry): """Build a short title from a buglog entry.""" bid = bug_entry.get("id", "bug-?") - error = bug_entry.get("error_message", "") + error = _text(bug_entry.get("error_message")) + # Drop a leading status banner ("OPEN, NOT FIXED -- ...", "PRIORITY: LOW -- + # ...") -- status lives in labels/state, not the title. + error = re.sub(r"^(?:\s*(?:OPEN|NOT (?:YET )?FIXED|NOT A CRASH[^,.]*|" + r"NOT YET ROOT-CAUSED|DEAD-CODE-ONLY|PRIORITY: \w+|" + r"low priority|pre-existing)\b\s*(?:,|--|\.|;)?)+\s*", + "", error, flags=re.I) or error # Take first sentence or first 100 chars first_line = error.split("\n")[0].strip() first_sentence = re.split(r'[.。]\s+', first_line)[0].strip() @@ -533,6 +607,27 @@ PHASE_DEFS = [ ] +# spec.json `phase` values that mean the spec's work is finished. There is no +# implementation approval in spec.json (approvals cover requirements/design/ +# tasks only), and no tool distinguishes "implemented" from +# "implementation-complete", so both count as done. +DONE_PHASES = {"implementation-complete", "implemented", "superseded"} + + +def _spec_desired_states(spec_data): + """Desired Gitea state per phase-issue key, plus the milestone's.""" + approvals = spec_data.get("approvals", {}) + req_ok = approvals.get("requirements", {}).get("approved", False) + des_ok = approvals.get("design", {}).get("approved", False) + done = spec_data.get("phase") in DONE_PHASES + closed = lambda cond: "closed" if cond else "open" + return { + "requirements_design": closed((req_ok and des_ok) or done), + "implementation": closed(done), + "validation": closed(done), + }, closed(done) + + def sync_spec(api, spec_name, dry_run=False, quiet=False): """Sync one spec directory to Gitea milestone + phase issues.""" spec_dir = SPECS_DIR / spec_name @@ -595,14 +690,7 @@ def sync_spec(api, spec_name, dry_run=False, quiet=False): for phase_def in PHASE_DEFS: title = f"[{spec_name}] {phase_def['title_suffix']}" - # Determine if this phase issue should start closed - state = "open" - if phase_def["key"] == "requirements_design": - approvals = spec_data.get("approvals", {}) - req_ok = approvals.get("requirements", {}).get("approved", False) - des_ok = approvals.get("design", {}).get("approved", False) - if req_ok and des_ok: - state = "closed" + state = _spec_desired_states(spec_data)[0][phase_def["key"]] # Build body if phase_def.get("body_fn_extra"): @@ -638,25 +726,9 @@ def _sync_spec_phase_states(api, spec_name, spec_data, gitea_state, print(f" [DRY-RUN] check phase states for '{spec_name}'") return - approvals = spec_data.get("approvals", {}) - req_ok = approvals.get("requirements", {}).get("approved", False) - des_ok = approvals.get("design", {}).get("approved", False) - tasks_ok = approvals.get("tasks", {}).get("approved", False) - impl_ok = approvals.get("implementation", {}).get("approved", False) - issues = gitea_state.get("issues", {}) - - # Build desired states for each phase issue - desired = {} - rd_issue = issues.get("requirements_design") - if rd_issue: - desired[rd_issue] = "closed" if (req_ok and des_ok) else "open" - impl_issue = issues.get("implementation") - if impl_issue: - desired[impl_issue] = "closed" if impl_ok else "open" - val_issue = issues.get("validation") - if val_issue: - desired[val_issue] = "closed" if impl_ok else "open" + by_key, milestone_state = _spec_desired_states(spec_data) + desired = {issues[k]: st for k, st in by_key.items() if issues.get(k)} # Fetch existing issues once existing_issues = api.list_issues( @@ -676,15 +748,14 @@ def _sync_spec_phase_states(api, spec_name, spec_data, gitea_state, f"{current['state']} -> {target_state}") # Update milestone state - all_approved = req_ok and des_ok and tasks_ok and impl_ok milestone_id = gitea_state.get("milestone_id") - if all_approved and milestone_id: - all_milestones = api.list_milestones(state="all") - ms = all_milestones.get(spec_name) - if ms and ms.get("state") != "closed": - api.update_milestone(milestone_id, state="closed") + if milestone_id: + ms = api.list_milestones(state="all").get(spec_name) + if ms and ms.get("state") != milestone_state: + api.update_milestone(milestone_id, state=milestone_state) if not quiet: - print(f" closed milestone '{spec_name}'") + print(f" milestone '{spec_name}': " + f"{ms['state']} -> {milestone_state}") def _spec_milestone_desc(spec_name, spec_data): @@ -715,93 +786,137 @@ def _resolve_label_ids(api, label_names): def sync_bugs(api, dry_run=False, quiet=False, since=None): """Sync buglog entries to Gitea issues. + Open/closed and the status label come from each entry's `status` (see + STATUS_MAP). `gitea_synced_status` records the status last mirrored, so an + entry is only touched when its status changed -- including going back to + open (reopen). Sync state is written back per entry, surgically, so a + failure part-way through never loses an already-created issue number. + Args: - since: Only sync bugs with last_seen >= this date (ISO format: YYYY-MM-DD). - Helps avoid flooding Gitea with ancient auto-detected bugs. + since: ISO date (YYYY-MM-DD). Create issues only for bugs with + last_seen >= since; update already-synced ones whose + last_seen or status_updated >= since. """ if not BUGLOG_PATH.exists(): warn(f"buglog not found: {BUGLOG_PATH}") return - buglog_wrapper = json.loads(BUGLOG_PATH.read_text()) - buglog = buglog_wrapper["bugs"] - created = 0 - closed = 0 - skipped = 0 + buglog = json.loads(BUGLOG_PATH.read_text())["bugs"] + created = closed = reopened = updated = skipped = 0 + needs_status = [] - for i, bug in enumerate(buglog): - bid = bug.get("id", f"bug-{i:04d}") + # Aggregate status per already-synced issue (see STATUS_PRIORITY). + issue_status = {} + for bug in buglog: + num = bug.get("gitea_issue") + if num: + prev = issue_status.get(num, "fixed") + cur = bug_status(bug) + issue_status[num] = min(prev, cur, key=STATUS_PRIORITY.index) + done_issues = set() - # Apply --since filter + for bug in buglog: + bid = bug.get("id") + if not bid: + continue + + # --since: new issues only for recently seen bugs; status changes + # (status_updated) are still mirrored for already-synced ones. if since: - last_seen = bug.get("last_seen", "") - if last_seen < since: + recent = (bug.get("last_seen") or "") >= since + if bug.get("gitea_issue"): + recent = recent or (bug.get("status_updated") or "") >= since + if not recent: skipped += 1 continue - gitea_issue_num = bug.get("gitea_issue") + status = bug_status(bug) + if status is None: + needs_status.append(bid) + synced_status = status or "unknown" + state = bug_state(bug) + issue_num = bug.get("gitea_issue") - if gitea_issue_num: - # Already synced — check if state changed - fix_now = bool(bug.get("fix", "").strip()) - # We don't track previous fix state, so check Gitea + if issue_num and bug.get("gitea_synced_status") == synced_status: + skipped += 1 + continue + + if issue_num: + # Status changed since the last sync (or was never recorded: + # issues synced before the status field existed were closed + # whenever `fix` was non-empty, so re-assert state for them). + # The issue itself follows the aggregate status of every entry + # linked to it; only status labels are touched, so labels a + # person put on the issue survive. + agg = issue_status[issue_num] + agg_state = STATUS_MAP.get(agg, ("open", None))[0] + if dry_run: + if issue_num not in done_issues: + print(f" [DRY-RUN] #{issue_num} ({bid}): -> {agg_state}" + f" ({agg or 'unknown'})") + done_issues.add(issue_num) + continue try: - _, issue = request("GET", f"/issues/{gitea_issue_num}") - if issue and issue.get("state") == "open" and fix_now: - if dry_run: - print(f" [DRY-RUN] close bug #{gitea_issue_num}: {bid}") - else: - api.update_issue(gitea_issue_num, state="closed") - # Add bug:fixed label via API - request( - "PUT", - f"/issues/{gitea_issue_num}/labels", - {"labels": [api.list_labels().get("bug:fixed", {}).get("id", 0)]} - ) + if issue_num not in done_issues: + _, issue = request("GET", f"/issues/{issue_num}") + if issue is None: + warn(f" {bid}: issue #{issue_num} not found, skipping") + skipped += 1 + continue + labels = api.list_labels() + want = status_label(agg) + for lbl in issue.get("labels") or []: + if lbl["name"] in STATUS_LABELS and lbl["name"] != want: + api.remove_label(issue_num, lbl["id"]) + have = {lbl["name"] for lbl in issue.get("labels") or []} + if want and want not in have and want in labels: + api.add_labels(issue_num, [labels[want]["id"]]) + if issue.get("state") != agg_state: + api.update_issue(issue_num, state=agg_state) + note = _text(bug.get("status_note")) + api.add_comment(issue_num, f"{bid} status: **{synced_status}**" + + (f" — {note}" if note else "")) + if agg_state == "closed": + closed += 1 + else: + reopened += 1 if not quiet: - print(f" closed #{gitea_issue_num}: {bid}") - closed += 1 - else: - skipped += 1 - except Exception as e: - warn(f" {bid}: could not check issue #{gitea_issue_num}: {e}") + print(f" #{issue_num} {bid}: {issue['state']} -> " + f"{agg_state} ({agg or 'unknown'})") + else: + updated += 1 + done_issues.add(issue_num) + set_fields(BUGLOG_PATH, bid, {"gitea_synced_status": synced_status}) + except (RuntimeError, BuglogEditError) as e: + warn(f" {bid}: could not update issue #{issue_num}: {e}") skipped += 1 continue - # New bug — create issue + # New bug -- create issue title = bug_title(bug) - body = bug_body(bug) - fix = bug.get("fix", "").strip() - state = "closed" if fix else "open" - if dry_run: - print(f" [DRY-RUN] create {'CLOSED' if state == 'closed' else 'OPEN'} " - f"issue: {title}") + print(f" [DRY-RUN] create {state.upper()} issue: {title}") created += 1 continue - # Determine labels - labels, extra = bug_labels(bug) - for ename, (ecolor, edesc) in extra.items(): - if ename in labels: - api.ensure_label(ename, ecolor, edesc) - - label_ids = _resolve_label_ids(api, labels) - - issue = api.create_issue(title, body, labels=label_ids, state=state) - bug["gitea_issue"] = issue["number"] - state_mark = " (closed)" if state == "closed" else "" + label_ids = _resolve_label_ids(api, bug_labels(bug)) + issue = api.create_issue(title, bug_body(bug), labels=label_ids) + if state == "closed": + # Gitea may ignore `state` on create; close explicitly. + api.update_issue(issue["number"], state="closed") + set_fields(BUGLOG_PATH, bid, {"gitea_issue": issue["number"], + "gitea_synced_status": synced_status}) if not quiet: - print(f" created #{issue['number']}: {title}{state_mark}") + mark = " (closed)" if state == "closed" else "" + print(f" created #{issue['number']}: {title}{mark}") created += 1 - # Write back updated buglog if any new issues were created - if created > 0 and not dry_run: - buglog_wrapper["bugs"] = buglog - _write_json(BUGLOG_PATH, buglog_wrapper) - if not quiet: - print(f"Bugs: {created} created, {closed} closed, {skipped} skipped") + print(f"Bugs: {created} created, {closed} closed, {reopened} reopened, " + f"{updated} relabelled, {skipped} skipped") + if needs_status: + warn(f"{len(needs_status)} bug(s) have no status -- triage per " + f".claude/rules/buglog-status.md: {' '.join(needs_status)}") # ── JSON Writing ───────────────────────────────────────────────────────────── @@ -828,7 +943,7 @@ def main(): parser.add_argument("--bugs", action="store_true", help="Sync bugs only") parser.add_argument("--since", metavar="DATE", - help="Only sync bugs with last_seen >= DATE (YYYY-MM-DD)") + help="Create issues only for bugs last seen >= DATE; update synced ones changed since (YYYY-MM-DD)") parser.add_argument("--quiet", action="store_true", help="Suppress progress output, print only errors") args = parser.parse_args()