diff --git a/.gitignore b/.gitignore index fc13b3c..91ccefa 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,10 @@ graphify-out/ .codegraph/ .xce-durability-acked.* *.bak -.claude/ucai-formatter-cache.local.json +ucai-formatter-cache.local.json + +# --- session checkpoint 2026-07-03: scratch soak-run output at repo root --- +/codegraph.html +/final_suite_run.log +/xce-durability-*.log +/xce-ha-soak*.log diff --git a/.wolf/anatomy.md b/.wolf/anatomy.md index a598611..8d5de03 100644 --- a/.wolf/anatomy.md +++ b/.wolf/anatomy.md @@ -1,6 +1,6 @@ # anatomy.md -> Auto-maintained by OpenWolf. Last scanned: 2026-07-03T02:40:28.319Z +> Auto-maintained by OpenWolf. Last scanned: 2026-07-03T03:23:30.967Z > Files: 569 tracked | Anatomy hits: 0 | Misses: 0 ## ../../../../../../tmp/claude-1000/-home-rshen-projects-xce-worktrees-raft-tla/c9cdeb23-6930-42d0-adab-2651b711793f/scratchpad/ @@ -21,7 +21,7 @@ ## ./ - `.clang-format` (~226 tok) -- `.gitignore` — Git ignore rules (~131 tok) +- `.gitignore` — Git ignore rules (~170 tok) - `.mcp.json` (~7 tok) - `.opencode.json` (~7 tok) - `CLAUDE.md` — OpenWolf (~1400 tok) @@ -351,7 +351,7 @@ - `design.md` — Design: ha-consistency-invariants (~8316 tok) - `requirements.md` — Requirements Document (~3772 tok) - `research.md` — Gap Analysis: ha-consistency-invariants (~10252 tok) -- `spec.json` (~259 tok) +- `spec.json` (~462 tok) ## .kiro/specs/ha-convergence-recovery/ @@ -359,7 +359,7 @@ - `design.md` — Design: ha-convergence-recovery (~9928 tok) - `requirements.md` — Requirements Document (~3586 tok) - `research.md` — Research & Design Decisions: ha-convergence-recovery (~8994 tok) -- `spec.json` (~142 tok) +- `spec.json` (~190 tok) - `tasks.md` — Implementation Plan: ha-convergence-recovery (~5999 tok) ## .kiro/specs/ha-elect-ctx-lifecycle/ @@ -425,7 +425,7 @@ - `design.md` — Design: ha-single-leader-role (~8611 tok) - `requirements.md` — Requirements Document (~2177 tok) - `research.md` — Research & Discovery Log: ha-single-leader-role (~5686 tok) -- `spec.json` (~142 tok) +- `spec.json` (~177 tok) - `tasks.md` — Implementation Plan: ha-single-leader-role (~5040 tok) ## .kiro/specs/ha-term-safety/ @@ -945,7 +945,7 @@ ## src/lib/testsuite/ - `test_ctrlstatus.c` (~4821 tok) -- `test_wal_wire.c` — include (~4273 tok) +- `test_wal_wire.c` — include (~4238 tok) ## src/schema/ diff --git a/.wolf/buglog.json b/.wolf/buglog.json index a7d6b6f..a1fb0c9 100644 --- a/.wolf/buglog.json +++ b/.wolf/buglog.json @@ -1,23346 +1,12328 @@ -{ - "version": 1, - "bugs": [ - { - "id": "bug-001", - "timestamp": "2026-05-21T07:38:53.760Z", - "error_message": "Significant refactor of ", - "file": "CLAUDE.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 23\u219230 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T07:38:53.760Z" - }, - { - "id": "bug-002", - "timestamp": "2026-05-21T07:46:54.198Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/kiro-spec-init/SKILL.md", - "root_cause": "Had `.kiro/settings/templates/specs/init.json`", - "fix": "Changed to `templates/specs/init.json`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 12, - "last_seen": "2026-05-21T07:48:24.177Z" - }, - { - "id": "bug-003", - "timestamp": "2026-05-21T07:57:57.180Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 14\u219219 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T07:57:57.180Z" - }, - { - "id": "bug-004", - "timestamp": "2026-05-21T07:58:54.673Z", - "error_message": "Wrong reference: g_tmpdir should be g_instdir", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Used \"g_tmpdir\" instead of \"g_instdir\"", - "fix": "Changed g_tmpdir \u2192 g_instdir", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-21T07:59:00.347Z" - }, - { - "id": "bug-005", - "timestamp": "2026-05-21T08:24:10.738Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: rc == 0", - "fix": "Added guard clause: if (rc == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T08:24:10.738Z" - }, - { - "id": "bug-006", - "timestamp": "2026-05-21T08:30:20.719Z", - "error_message": "Significant refactor of ", - "file": "src/docs/config-replication-design.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 17\u219220 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T08:30:20.719Z" - }, - { - "id": "bug-007", - "timestamp": "2026-05-21T12:53:13.789Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: ep == NULL", - "fix": "Added guard clause: if (ep == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T12:53:13.789Z" - }, - { - "id": "bug-008", - "timestamp": "2026-05-21T13:20:28.953Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: getaddrinfo(host, port_str, &hints, &res", - "fix": "Added guard clause: if (getaddrinfo(host, port_str, &hints, &res)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T13:20:28.953Z" - }, - { - "id": "bug-009", - "timestamp": "2026-05-21T13:42:53.395Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21929 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T13:42:53.395Z" - }, - { - "id": "bug-010", - "timestamp": "2026-05-21T13:43:03.953Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 11\u21925 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-21T13:43:03.953Z" - }, - { - "id": "bug-011", - "timestamp": "2026-05-22T08:45:33.856Z", - "error_message": "Significant refactor of ", - "file": "src/docs/config-replication-impl-status.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 10\u219212 lines (4 removed) | Also: | `handle_config_request()` (primary handler) | \u2705 ; | `fetch_config_from_primary()` (standby fetch) | | Also: | `handle_upgrade_config()` | \u2705 Compiled, gated | ; | `push_config_to_standbys()` | \u26a0 Stub (returns 0)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-22T08:46:24.228Z" - }, - { - "id": "bug-012", - "timestamp": "2026-05-22T09:18:14.674Z", - "error_message": "Significant refactor of ", - "file": "src/docs/config-replication-impl-status.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 17\u219221 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-22T09:18:14.674Z" - }, - { - "id": "bug-013", - "timestamp": "2026-05-25T09:00:50.645Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21927 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T09:00:50.645Z" - }, - { - "id": "bug-014", - "timestamp": "2026-05-25T10:34:21.916Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 13\u219224 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T10:34:21.916Z" - }, - { - "id": "bug-015", - "timestamp": "2026-05-25T10:34:43.964Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u219210 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T10:34:43.964Z" - }, - { - "id": "bug-016", - "timestamp": "2026-05-25T11:03:39.152Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u21929 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T11:03:39.152Z" - }, - { - "id": "bug-017", - "timestamp": "2026-05-25T11:05:06.112Z", - "error_message": "Missing error handling in send", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T11:05:06.112Z" - }, - { - "id": "bug-018", - "timestamp": "2026-05-25T11:06:26.871Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T11:06:26.871Z" - }, - { - "id": "bug-019", - "timestamp": "2026-05-25T11:06:52.401Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 23\u219240 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T11:06:52.401Z" - }, - { - "id": "bug-020", - "timestamp": "2026-05-25T11:31:55.826Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T11:31:55.826Z" - }, - { - "id": "bug-021", - "timestamp": "2026-05-25T11:32:18.221Z", - "error_message": "Missing error handling in types", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-25T11:32:18.221Z" - }, - { - "id": "bug-022", - "timestamp": "2026-05-26T07:16:49.294Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "36 lines replaced/restructured", - "fix": "Rewrote 63\u219279 lines (36 removed) | Also: 6. **No simultaneous primaries**: Only one node ca; 7. **Majority-commit WAL replication (CP writes)** | Also: - `handle_request_vote()` handler on all peers; - Config tag + LSN comparison for vote decision", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-26T07:19:18.498Z" - }, - { - "id": "bug-023", - "timestamp": "2026-05-26T07:17:31.980Z", - "error_message": "Missing guard clause", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "No early return/throw for edge case: req->term < my_term", - "fix": "Added guard clause: if (req->term < my_term)", - "tags": [ - "auto-detected", - "guard-clause", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:17:31.980Z" - }, - { - "id": "bug-024", - "timestamp": "2026-05-26T07:18:36.066Z", - "error_message": "Missing guard clause", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "No early return/throw for edge case: n_peers == 2", - "fix": "Added guard clause: if (n_peers == 2)", - "tags": [ - "auto-detected", - "guard-clause", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:18:36.066Z" - }, - { - "id": "bug-025", - "timestamp": "2026-05-26T07:26:22.871Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 11\u219228 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:26:22.871Z" - }, - { - "id": "bug-026", - "timestamp": "2026-05-26T07:26:48.363Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Was returning: 1;", - "fix": "Now returns: 1; // single-node: always primary", - "tags": [ - "auto-detected", - "return-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:26:48.363Z" - }, - { - "id": "bug-027", - "timestamp": "2026-05-26T07:27:34.925Z", - "error_message": "Wrong condition in logic", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Condition was: if (n_peers == 2)", - "fix": "Changed to: if (n_peers == 1)", - "tags": [ - "auto-detected", - "logic-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:27:34.925Z" - }, - { - "id": "bug-028", - "timestamp": "2026-05-26T07:28:37.278Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 13\u219228 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:28:37.278Z" - }, - { - "id": "bug-029", - "timestamp": "2026-05-26T07:29:11.741Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `main.c`", - "fix": "Changed to `-EINVAL`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T07:29:11.741Z" - }, - { - "id": "bug-030", - "timestamp": "2026-05-26T08:52:54.129Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 17\u219228 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T08:52:54.129Z" - }, - { - "id": "bug-031", - "timestamp": "2026-05-26T09:07:01.238Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 28\u219245 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T09:07:01.238Z" - }, - { - "id": "bug-032", - "timestamp": "2026-05-26T09:07:23.015Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `-EINVAL`", - "fix": "Changed to `--vcpus N`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T09:07:23.015Z" - }, - { - "id": "bug-033", - "timestamp": "2026-05-26T09:07:46.892Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 22\u219235 lines (13 removed) | Also: **Files**: `src/controller/main.c`, `src/lib/netio; 1. **Peer-count validation in `main.c`** (before `", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-26T09:08:37.743Z" - }, - { - "id": "bug-034", - "timestamp": "2026-05-26T09:23:02.818Z", - "error_message": "Missing guard clause", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "No early return/throw for edge case: rc == 0", - "fix": "Added guard clause: if (rc == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T09:23:02.818Z" - }, - { - "id": "bug-035", - "timestamp": "2026-05-26T09:46:12.902Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `sysconf(_SC_NPROCESSORS_ONLN)`", - "fix": "Changed to ` when not specified. Values `", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T09:46:12.902Z" - }, - { - "id": "bug-036", - "timestamp": "2026-05-26T09:48:08.636Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T09:48:08.636Z" - }, - { - "id": "bug-037", - "timestamp": "2026-05-26T10:08:57.389Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T10:08:57.389Z" - }, - { - "id": "bug-038", - "timestamp": "2026-05-26T10:09:32.424Z", - "error_message": "Missing guard clause", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", - "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T10:09:32.424Z" - }, - { - "id": "bug-039", - "timestamp": "2026-05-26T10:09:57.711Z", - "error_message": "Wrong return value", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Was returning: 0 (standby); fresher peer runs its own election", - "fix": "Now returns: `-ENODATA`. Not a normal runtime path.", - "tags": [ - "auto-detected", - "return-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T10:09:57.711Z" - }, - { - "id": "bug-040", - "timestamp": "2026-05-26T10:10:18.874Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "18 lines replaced/restructured", - "fix": "Rewrote 29\u219234 lines (18 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T10:10:18.874Z" - }, - { - "id": "bug-041", - "timestamp": "2026-05-26T10:10:40.149Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Had `test_fetch_config_from_best_peer`", - "fix": "Changed to `test_handle_config_request_standbys_serve_config`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-26T10:10:40.149Z" - }, - { - "id": "bug-042", - "timestamp": "2026-05-26T11:35:44.097Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-26T11:39:15.673Z" - }, - { - "id": "bug-043", - "timestamp": "2026-05-26T11:40:35.979Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 13\u219229 lines (8 removed) | Also: 8. **Register OP_requestVote handler** in cluster'; switch at line ~195): since vtaskmgr and iomgr run", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-26T11:41:16.284Z" - }, - { - "id": "bug-044", - "timestamp": "2026-05-26T13:07:48.902Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 13\u219260 lines (9 removed) | Also: 3. **Replace timestamp polling with `EPOLLRDHUP`**; is replaced by an epoll event handler. On `EPOLLRD | Also: - **Split vote recovery**: After failing quorum, s; (using existing `jitter_seed` xorshift64), retry o", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-26T13:09:05.649Z" - }, - { - "id": "bug-045", - "timestamp": "2026-05-26T13:09:48.347Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 8\u219234 lines (8 removed) | Also: 2. **Replace `ha_check_heartbeat_nocluster()` with; - On `EPOLLRDHUP` from `primary_ctrl_fd`: **close ", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-26T13:10:03.239Z" - }, - { - "id": "bug-046", - "timestamp": "2026-05-26T13:51:32.274Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - [ ] `test_heartbeat_carries_committed_lsn`: heartbeat payl; `committed_lsn`; standby updates `ha.last_known_committed_ls", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-26T13:51:57.860Z" - }, - { - "id": "bug-047", - "timestamp": "2026-05-26T13:52:49.044Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: // Catch-up gate (Clarification 6): behind nodes must not in; if (c->ha.in_catch_up) {", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-26T13:53:15.121Z" - }, - { - "id": "bug-048", - "timestamp": "2026-05-27T03:02:34.912Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: name, key, applied_lsn, and config_tag immediately after TCP; - [ ] `test_ctrl_hello_registers_standby_in_repl`: handle_ct", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-27T03:04:02.739Z" - }, - { - "id": "bug-049", - "timestamp": "2026-05-27T03:03:26.016Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T03:03:26.016Z" - }, - { - "id": "bug-050", - "timestamp": "2026-05-27T03:36:56.123Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 37\u219251 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T03:36:56.123Z" - }, - { - "id": "bug-051", - "timestamp": "2026-05-27T03:52:41.110Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 8\u219223 lines (8 removed) | Also: - [ ] `test_persistent_conn_single_node_noop`: ha_; when n_peers == 1; primary_ctrl_fd remains -1 (uni", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-27T03:53:14.859Z" - }, - { - "id": "bug-052", - "timestamp": "2026-05-27T03:55:31.440Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 34\u219251 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T03:55:31.440Z" - }, - { - "id": "bug-053", - "timestamp": "2026-05-27T03:56:41.037Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T03:56:41.037Z" - }, - { - "id": "bug-054", - "timestamp": "2026-05-27T03:58:16.006Z", - "error_message": "Significant refactor of ", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 21\u219228 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T03:58:16.006Z" - }, - { - "id": "bug-055", - "timestamp": "2026-05-27T03:59:16.145Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T03:59:16.145Z" - }, - { - "id": "bug-056", - "timestamp": "2026-05-27T06:45:04.111Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219213 lines (2 removed) | Also: **Replace LSF priority election with Raft-style qu; Five focused changes: | Also: 8. **Hard SSL/cert gate**: Controller refuses to b; 9. **Observable via logs**: Election results, vote", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-27T06:46:17.221Z" - }, - { - "id": "bug-057", - "timestamp": "2026-05-27T06:45:34.951Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T06:45:34.951Z" - }, - { - "id": "bug-058", - "timestamp": "2026-05-27T07:23:02.729Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T07:23:02.729Z" - }, - { - "id": "bug-059", - "timestamp": "2026-05-27T07:24:56.724Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/brief.md", - "root_cause": "Had `-ENOSYS`", - "fix": "Changed to `bootstrap_config_if_needed()`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T07:24:56.724Z" - }, - { - "id": "bug-060", - "timestamp": "2026-05-27T09:24:44.651Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-config-bootstrap/spec.json", - "root_cause": "Had \"2026-05-27T01:24:00-07:00\"", - "fix": "Changed to \"2026-05-27T01:30:00-07:00\"", - "tags": [ - "auto-detected", - "wrong-value", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-27T09:24:44.651Z" - }, - { - "id": "bug-061", - "timestamp": "2026-05-28T02:54:36.356Z", - "error_message": "Significant refactor of ", - "file": "src/schema/controller.capnp", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T02:54:36.356Z" - }, - { - "id": "bug-062", - "timestamp": "2026-05-28T02:55:40.362Z", - "error_message": "Missing error handling in unknown", - "file": "src/schema/controller.capnp", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T02:55:40.362Z" - }, - { - "id": "bug-063", - "timestamp": "2026-05-28T03:05:42.555Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u219240 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T03:05:42.556Z" - }, - { - "id": "bug-064", - "timestamp": "2026-05-28T03:39:14.341Z", - "error_message": "Missing error handling in struct", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T03:39:14.341Z" - }, - { - "id": "bug-065", - "timestamp": "2026-05-28T03:40:17.891Z", - "error_message": "Missing error handling in struct", - "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T03:40:17.891Z" - }, - { - "id": "bug-066", - "timestamp": "2026-05-28T03:41:53.373Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21926 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T03:41:53.373Z" - }, - { - "id": "bug-067", - "timestamp": "2026-05-28T03:42:53.810Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'worker.c'", - "fix": "Changed to 'startup_validation.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T03:42:53.810Z" - }, - { - "id": "bug-068", - "timestamp": "2026-05-28T04:01:56.031Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/main.c", - "root_cause": "Condition was: if (arguments->vcpus < 1)", - "fix": "Changed to: if (validate_vcpus(arguments->vcpus)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T04:01:56.031Z" - }, - { - "id": "bug-069", - "timestamp": "2026-05-28T04:02:06.925Z", - "error_message": "Wrong reference: vcpus should be nres", - "file": "src/include/clusterconfig.h", - "root_cause": "Used \"vcpus\" instead of \"nres\"", - "fix": "Changed vcpus \u2192 nres", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T04:02:06.925Z" - }, - { - "id": "bug-070", - "timestamp": "2026-05-28T04:02:19.321Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 12\u21926 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T04:02:19.322Z" - }, - { - "id": "bug-071", - "timestamp": "2026-05-28T04:02:48.429Z", - "error_message": "Incorrect value in code", - "file": "src/schema/controller.capnp", - "root_cause": "Had \"vcpus\"", - "fix": "Changed to \"resources\"", - "tags": [ - "auto-detected", - "wrong-value", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T04:02:48.429Z" - }, - { - "id": "bug-072", - "timestamp": "2026-05-28T04:33:48.790Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 12\u219218 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T04:33:48.790Z" - }, - { - "id": "bug-073", - "timestamp": "2026-05-28T04:50:36.939Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 109\u2192144 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T04:50:36.939Z" - }, - { - "id": "bug-074", - "timestamp": "2026-05-28T05:59:28.525Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", - "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcod)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T05:59:28.525Z" - }, - { - "id": "bug-075", - "timestamp": "2026-05-28T07:04:33.895Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_election.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:04:33.895Z" - }, - { - "id": "bug-076", - "timestamp": "2026-05-28T07:13:42.089Z", - "error_message": "Missing error handling in struct", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:13:42.089Z" - }, - { - "id": "bug-077", - "timestamp": "2026-05-28T07:14:03.724Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:14:03.724Z" - }, - { - "id": "bug-078", - "timestamp": "2026-05-28T07:22:56.418Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= ctx->n_peers", - "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= ctx->n_peers)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:22:56.418Z" - }, - { - "id": "bug-079", - "timestamp": "2026-05-28T07:49:42.076Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "14 lines replaced/restructured", - "fix": "Rewrote 92\u2192344 lines (14 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:49:42.076Z" - }, - { - "id": "bug-080", - "timestamp": "2026-05-28T07:50:39.610Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 25\u219249 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:50:39.610Z" - }, - { - "id": "bug-081", - "timestamp": "2026-05-28T07:53:10.883Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: role != CTRL_STANDBY", - "fix": "Added guard clause: if (role != CTRL_STANDBY)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T07:53:10.883Z" - }, - { - "id": "bug-082", - "timestamp": "2026-05-28T08:23:40.040Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ctx->done_posted", - "fix": "Added guard clause: if (ctx->done_posted)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-28T08:25:11.221Z" - }, - { - "id": "bug-083", - "timestamp": "2026-05-28T08:45:37.921Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !host || !tag_out", - "fix": "Added guard clause: if (!host || !tag_out)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T08:45:37.921Z" - }, - { - "id": "bug-084", - "timestamp": "2026-05-28T08:48:58.609Z", - "error_message": "Wrong reference: capn_str_null should be str", - "file": "src/controller/config_repl.c", - "root_cause": "Used \"capn_str_null\" instead of \"str\"", - "fix": "Changed capn_str_null \u2192 str", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T08:48:58.609Z" - }, - { - "id": "bug-085", - "timestamp": "2026-05-28T09:12:37.377Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 17\u219284 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T09:12:37.377Z" - }, - { - "id": "bug-086", - "timestamp": "2026-05-28T09:13:17.676Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_bootstrap_support.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T09:13:17.676Z" - }, - { - "id": "bug-087", - "timestamp": "2026-05-28T10:13:52.421Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !workdir", - "fix": "Added guard clause: if (!workdir)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T10:13:52.421Z" - }, - { - "id": "bug-088", - "timestamp": "2026-05-28T13:34:49.513Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T13:34:49.513Z" - }, - { - "id": "bug-089", - "timestamp": "2026-05-28T13:51:39.687Z", - "error_message": "Significant refactor of ", - "file": "src/include/repl.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (3 removed) | Also: /* ACK handling \u2014 call from io_worker read callbac; /* Majority-quorum ACK wait \u2014 stub until task 7.3 ", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-28T13:51:55.672Z" - }, - { - "id": "bug-090", - "timestamp": "2026-05-28T13:52:52.636Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T13:52:52.636Z" - }, - { - "id": "bug-091", - "timestamp": "2026-05-28T13:53:09.924Z", - "error_message": "Wrong reference: in_catch_up should be acked_config_lsn", - "file": "src/controller/repl.c", - "root_cause": "Used \"in_catch_up\" instead of \"acked_config_lsn\"", - "fix": "Changed in_catch_up \u2192 acked_config_lsn", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T13:53:09.924Z" - }, - { - "id": "bug-092", - "timestamp": "2026-05-28T14:11:04.583Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn", - "fix": "Added guard clause: if (p->current_lsn == 0 || cutoff_lsn + 1 >=)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T14:11:04.583Z" - }, - { - "id": "bug-093", - "timestamp": "2026-05-28T14:19:54.082Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 20\u219229 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T14:19:54.082Z" - }, - { - "id": "bug-094", - "timestamp": "2026-05-28T14:47:44.445Z", - "error_message": "Wrong reference: ha_state_t should be _Atomic", - "file": "src/controller/ha.h", - "root_cause": "Used \"ha_state_t\" instead of \"_Atomic\"", - "fix": "Changed ha_state_t \u2192 _Atomic", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T14:47:44.445Z" - }, - { - "id": "bug-095", - "timestamp": "2026-05-28T15:04:44.613Z", - "error_message": "Wrong reference: socket should be epoll", - "file": "src/controller/ha.c", - "root_cause": "Used \"socket\" instead of \"epoll\"", - "fix": "Changed socket \u2192 epoll", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T15:04:44.613Z" - }, - { - "id": "bug-096", - "timestamp": "2026-05-28T15:05:23.129Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook", - "fix": "Added guard clause: if (ha_tcp_connect_hook)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-28T15:05:23.129Z" - }, - { - "id": "bug-097", - "timestamp": "2026-05-29T02:48:17.371Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: (ep == NULL", - "fix": "Added guard clause: if ((ep == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T02:48:17.371Z" - }, - { - "id": "bug-098", - "timestamp": "2026-05-29T02:50:46.444Z", - "error_message": "Wrong reference: XCE_ERR_STANDBY should be E_XCE_STANDBY", - "file": "src/lib/request.c", - "root_cause": "Used \"XCE_ERR_STANDBY\" instead of \"E_XCE_STANDBY\"", - "fix": "Changed XCE_ERR_STANDBY \u2192 E_XCE_STANDBY", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T02:50:46.444Z" - }, - { - "id": "bug-099", - "timestamp": "2026-05-29T02:51:03.529Z", - "error_message": "Wrong reference: XCE_ERR_NOT_READY should be E_XCE_NOT_READY", - "file": "src/lib/testsuite/test_libxce_retry.c", - "root_cause": "Used \"XCE_ERR_NOT_READY\" instead of \"E_XCE_NOT_READY\"", - "fix": "Changed XCE_ERR_NOT_READY \u2192 E_XCE_NOT_READY", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-29T02:51:16.657Z" - }, - { - "id": "bug-100", - "timestamp": "2026-05-29T03:27:20.556Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T03:27:20.556Z" - }, - { - "id": "bug-101", - "timestamp": "2026-05-29T03:29:44.753Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T03:29:44.753Z" - }, - { - "id": "bug-102", - "timestamp": "2026-05-29T03:37:52.117Z", - "error_message": "Wrong reference: data should be ctrl_hello_reply_t", - "file": "src/include/typemap.h", - "root_cause": "Used \"data\" instead of \"ctrl_hello_reply_t\"", - "fix": "Changed data \u2192 ctrl_hello_reply_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T03:37:52.117Z" - }, - { - "id": "bug-103", - "timestamp": "2026-05-29T03:53:48.559Z", - "error_message": "Missing error handling in unknown", - "file": "src/schema/controller.capnp", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T03:53:48.559Z" - }, - { - "id": "bug-104", - "timestamp": "2026-05-29T03:54:23.205Z", - "error_message": "Missing error handling in struct", - "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: wal_catchup_entry_t* walcatchupentrymsg;; snapshot_transfer_t* snapshottransfermsg; | Also: /* Forward declarations for types defined later in this file; typedef struct snapshot_transfer_t snapshot_transfer_fw", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-29T03:56:34.078Z" - }, - { - "id": "bug-105", - "timestamp": "2026-05-29T03:57:22.758Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21925 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T03:57:22.758Z" - }, - { - "id": "bug-106", - "timestamp": "2026-05-29T03:58:07.711Z", - "error_message": "Significant refactor of ", - "file": "src/schema/controller.capnp", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21923 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T03:58:07.711Z" - }, - { - "id": "bug-107", - "timestamp": "2026-05-29T04:04:12.693Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T04:04:12.693Z" - }, - { - "id": "bug-108", - "timestamp": "2026-05-29T04:06:35.126Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219212 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T04:06:35.126Z" - }, - { - "id": "bug-109", - "timestamp": "2026-05-29T06:59:14.045Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T06:59:14.045Z" - }, - { - "id": "bug-110", - "timestamp": "2026-05-29T07:01:11.916Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 10\u219220 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T07:01:11.916Z" - }, - { - "id": "bug-111", - "timestamp": "2026-05-29T07:02:31.287Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (c->myconf && c->myconf->cluster_config)", - "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T07:02:31.287Z" - }, - { - "id": "bug-112", - "timestamp": "2026-05-29T10:40:53.626Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/manual-tests.md", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 18\u219220 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T10:40:53.626Z" - }, - { - "id": "bug-113", - "timestamp": "2026-05-29T10:59:37.144Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-config-bootstrap/manual-tests.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T10:59:37.144Z" - }, - { - "id": "bug-114", - "timestamp": "2026-05-29T11:01:49.200Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/manual-tests.md", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 20\u219236 lines (10 removed) | Also: ### 11.1 Concurrent Startup (3 nodes); - Result: PASS / FAIL", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-29T11:02:31.657Z" - }, - { - "id": "bug-115", - "timestamp": "2026-05-29T12:29:37.335Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 29\u219245 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T12:29:37.335Z" - }, - { - "id": "bug-116", - "timestamp": "2026-05-29T12:43:56.243Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_bootstrap.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T12:43:56.243Z" - }, - { - "id": "bug-117", - "timestamp": "2026-05-29T13:41:46.841Z", - "error_message": "Incorrect value in code", - "file": "src/controller/myapp.c", - "root_cause": "Had \"Pre-election config loaded: tag=%016\"", - "fix": "Changed to \"Pre-election config loaded: tag=%016llx\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T13:41:46.841Z" - }, - { - "id": "bug-118", - "timestamp": "2026-05-29T13:51:42.299Z", - "error_message": "Wrong reference: nres should be vcpus", - "file": "src/include/clusterconfig.h", - "root_cause": "Used \"nres\" instead of \"vcpus\"", - "fix": "Changed nres \u2192 vcpus", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T13:51:42.299Z" - }, - { - "id": "bug-119", - "timestamp": "2026-05-29T16:32:10.285Z", - "error_message": "Wrong reference: data should be request_vote_reply_t", - "file": "src/include/typemap.h", - "root_cause": "Used \"data\" instead of \"request_vote_reply_t\"", - "fix": "Changed data \u2192 request_vote_reply_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T16:32:10.285Z" - }, - { - "id": "bug-120", - "timestamp": "2026-05-29T16:35:39.908Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 19\u219225 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T16:35:39.908Z" - }, - { - "id": "bug-121", - "timestamp": "2026-05-29T16:36:41.731Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-29T16:36:41.731Z" - }, - { - "id": "bug-122", - "timestamp": "2026-05-30T00:05:02.052Z", - "error_message": "Wrong reference: cluster_init_phase2 should be cluster_register_listener", - "file": "src/controller/include/cluster.h", - "root_cause": "Used \"cluster_init_phase2\" instead of \"cluster_register_listener\"", - "fix": "Changed cluster_init_phase2 \u2192 cluster_register_listener", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T00:05:02.052Z" - }, - { - "id": "bug-123", - "timestamp": "2026-05-30T00:20:27.244Z", - "error_message": "Missing guard clause", - "file": "src/controller/cluster.c", - "root_cause": "No early return/throw for edge case: c->ep.flags == 0", - "fix": "Added guard clause: if (c->ep.flags == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T00:20:27.244Z" - }, - { - "id": "bug-124", - "timestamp": "2026-05-30T00:24:49.302Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "30 lines replaced/restructured", - "fix": "Rewrote 79\u219261 lines (30 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T00:24:49.302Z" - }, - { - "id": "bug-125", - "timestamp": "2026-05-30T01:12:05.791Z", - "error_message": "Significant refactor of ", - "file": "src/include/xce/cluster_def.h", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 8\u219215 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T01:12:05.791Z" - }, - { - "id": "bug-126", - "timestamp": "2026-05-30T01:12:16.318Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219216 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T01:12:16.318Z" - }, - { - "id": "bug-127", - "timestamp": "2026-05-30T01:12:38.572Z", - "error_message": "Wrong reference: CTRL_PERF_NHASH should be c", - "file": "src/controller/cluster.c", - "root_cause": "Used \"CTRL_PERF_NHASH\" instead of \"c\"", - "fix": "Changed CTRL_PERF_NHASH \u2192 c", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T01:12:38.572Z" - }, - { - "id": "bug-128", - "timestamp": "2026-05-30T01:18:19.850Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "36 lines replaced/restructured", - "fix": "Rewrote 77\u219241 lines (36 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T01:18:19.850Z" - }, - { - "id": "bug-129", - "timestamp": "2026-05-30T01:21:20.140Z", - "error_message": "Wrong reference: true should be false", - "file": ".claude/skills/xce-build/SKILL.md", - "root_cause": "Used \"true\" instead of \"false\"", - "fix": "Changed true \u2192 false", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T01:21:20.140Z" - }, - { - "id": "bug-130", - "timestamp": "2026-05-30T01:28:22.079Z", - "error_message": "Wrong reference: 120 should be 300", - "file": "src/controller/meson.build", - "root_cause": "Used \"120\" instead of \"300\"", - "fix": "Changed 120 \u2192 300", - "tags": [ - "auto-detected", - "wrong-reference", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T01:28:22.079Z" - }, - { - "id": "bug-131", - "timestamp": "2026-05-30T02:37:40.267Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/myapp.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T02:37:40.267Z" - }, - { - "id": "bug-132", - "timestamp": "2026-05-30T03:33:24.652Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 15\u219221 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T03:33:24.652Z" - }, - { - "id": "bug-133", - "timestamp": "2026-05-30T03:55:45.693Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219213 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T03:55:45.693Z" - }, - { - "id": "bug-134", - "timestamp": "2026-05-30T06:19:16.564Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 10\u219227 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T06:19:16.564Z" - }, - { - "id": "bug-135", - "timestamp": "2026-05-30T06:19:47.181Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Had `cluster_init_phase2`", - "fix": "Changed to `cluster_config == NULL`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T06:19:47.181Z" - }, - { - "id": "bug-136", - "timestamp": "2026-05-30T06:38:09.325Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* After election determines role: */; int is_first_boot = !c->running; /* running=1 only after cl | Also: #### `ha_dispatch_gate` \u2014 CTRL_PRIMARY_COMMITTING case; ```c | Also: ### 3-node startup \u2014 ctrl-a wins (with COMMITTING phase); participant PeerC as ctrl-c endpoint | Also: - **`ha_start_election_async()` is fully non-blocking**: Arm; - *Initial startup* (`ha->phase2_done == 0`): calls `cluster", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-05-30T06:43:07.267Z" - }, - { - "id": "bug-137", - "timestamp": "2026-05-30T06:44:51.916Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 19\u219226 lines (4 removed) | Also: \u251c\u2500\u2500 ha.h # Add to ha_state_t: clie; \u2502 # ha_repl_ack_re | Also: \u2514\u2500\u2500 myapp.c # Restructured: phase1 + ; # for(;;) retry loop, cluster_init_phase2,", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-30T06:45:22.626Z" - }, - { - "id": "bug-138", - "timestamp": "2026-05-30T06:56:32.965Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "12 lines replaced/restructured", - "fix": "Rewrote 15\u219220 lines (12 removed) | Also: - `src/controller/ha.h` \u2014 Add `client_ssl_ctx`, `r; - `src/controller/ha.c` \u2014 Main changes above; `ha_ | Also: \u251c\u2500\u2500 delay.c # Add delay_task_ms() (ms; \u2502 # Add delay_task_cancel()", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-30T06:57:53.882Z" - }, - { - "id": "bug-139", - "timestamp": "2026-05-30T06:58:05.115Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/election-iomgr-refactor/design.md", - "root_cause": "Had `delay_task_cancel()`", - "fix": "Changed to `clock_gettime(CLOCK_MONOTONIC)`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T06:58:05.115Z" - }, - { - "id": "bug-140", - "timestamp": "2026-05-30T07:06:20.968Z", - "error_message": "Significant refactor of ", - "file": "src/lib/delay.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 53\u219285 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T07:06:20.968Z" - }, - { - "id": "bug-141", - "timestamp": "2026-05-30T07:36:03.937Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u21928 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T07:36:03.937Z" - }, - { - "id": "bug-142", - "timestamp": "2026-05-30T07:57:14.052Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: xce_vote_request_hook != NULL", - "fix": "Added guard clause: if (xce_vote_request_hook != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T07:57:14.052Z" - }, - { - "id": "bug-143", - "timestamp": "2026-05-30T08:16:03.222Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", - "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T08:16:03.222Z" - }, - { - "id": "bug-144", - "timestamp": "2026-05-30T08:49:04.058Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T08:49:04.058Z" - }, - { - "id": "bug-145", - "timestamp": "2026-05-30T08:49:59.817Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: atomic_load(&t->cancelled", - "fix": "Added guard clause: if (atomic_load(&t->cancelled)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T08:49:59.817Z" - }, - { - "id": "bug-146", - "timestamp": "2026-05-30T09:12:38.055Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 14\u219217 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T09:12:38.055Z" - }, - { - "id": "bug-147", - "timestamp": "2026-05-30T09:13:19.377Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T09:13:19.377Z" - }, - { - "id": "bug-148", - "timestamp": "2026-05-30T09:13:54.779Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_start_election_async.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 24\u219242 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T09:13:54.779Z" - }, - { - "id": "bug-149", - "timestamp": "2026-05-30T10:31:53.836Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_vote_gate.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T10:31:53.836Z" - }, - { - "id": "bug-150", - "timestamp": "2026-05-30T11:08:59.808Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T11:08:59.808Z" - }, - { - "id": "bug-151", - "timestamp": "2026-05-30T11:09:47.905Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T11:09:47.905Z" - }, - { - "id": "bug-152", - "timestamp": "2026-05-30T11:10:30.958Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 62\u2192164 lines (8 removed) | Also: /* Store candidate own tag/lsn in votes[0] for eva; * n_peers==1 \u2192 n_other==0, so votes[] has 0 entrie | Also: * election_timeout_ctx_t: argument for the delay_t; * ha_elect_evaluate_and_signal when the election d", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-30T11:13:27.304Z" - }, - { - "id": "bug-153", - "timestamp": "2026-05-30T11:43:01.989Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_repl_push.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T11:43:01.989Z" - }, - { - "id": "bug-154", - "timestamp": "2026-05-30T11:43:44.829Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Push config to standbys with lower config_tag (Req 7.1) *; #ifdef XCE_TESTING", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-30T11:43:58.208Z" - }, - { - "id": "bug-155", - "timestamp": "2026-05-30T11:44:15.059Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T11:44:15.059Z" - }, - { - "id": "bug-156", - "timestamp": "2026-05-30T11:45:24.946Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T11:45:24.946Z" - }, - { - "id": "bug-157", - "timestamp": "2026-05-30T12:02:39.009Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-30T12:03:10.183Z" - }, - { - "id": "bug-158", - "timestamp": "2026-05-30T12:09:20.842Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 22\u219232 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:09:20.842Z" - }, - { - "id": "bug-159", - "timestamp": "2026-05-30T12:22:57.013Z", - "error_message": "Wrong return value", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Was returning: in < 10ms.", - "fix": "Now returns: in < 5ms (design spec).", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:22:57.013Z" - }, - { - "id": "bug-160", - "timestamp": "2026-05-30T12:23:18.560Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Had \"myapp_run path: ha_start_election_async must retu", - "fix": "Changed to \"myapp_run path: ha_start_election_async must retu", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:23:18.560Z" - }, - { - "id": "bug-161", - "timestamp": "2026-05-30T12:29:45.867Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:29:45.867Z" - }, - { - "id": "bug-162", - "timestamp": "2026-05-30T12:30:10.570Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-30T12:30:26.987Z" - }, - { - "id": "bug-163", - "timestamp": "2026-05-30T12:31:02.203Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_failover.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:31:02.203Z" - }, - { - "id": "bug-164", - "timestamp": "2026-05-30T12:45:22.182Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:45:22.182Z" - }, - { - "id": "bug-165", - "timestamp": "2026-05-30T12:46:02.210Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 15\u21925 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:46:02.210Z" - }, - { - "id": "bug-166", - "timestamp": "2026-05-30T12:50:35.870Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_bootstrap_support.c", - "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", - "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T12:50:35.870Z" - }, - { - "id": "bug-167", - "timestamp": "2026-05-30T13:06:57.152Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: cancelled_val", - "fix": "Added guard clause: if (cancelled_val)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T13:06:57.152Z" - }, - { - "id": "bug-168", - "timestamp": "2026-05-30T13:24:06.024Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_repl_ack.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T13:24:06.024Z" - }, - { - "id": "bug-169", - "timestamp": "2026-05-30T13:42:28.077Z", - "error_message": "Wrong reference: third should be second", - "file": "src/controller/testsuite/test_ha_repl_ack.c", - "root_cause": "Used \"third\" instead of \"second\"", - "fix": "Changed third \u2192 second", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T13:42:28.077Z" - }, - { - "id": "bug-170", - "timestamp": "2026-05-30T14:35:07.679Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T14:35:07.679Z" - }, - { - "id": "bug-171", - "timestamp": "2026-05-30T15:26:07.538Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 10\u219215 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T15:26:07.538Z" - }, - { - "id": "bug-172", - "timestamp": "2026-05-30T23:42:12.644Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T23:42:12.644Z" - }, - { - "id": "bug-173", - "timestamp": "2026-05-30T23:42:34.993Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "15 lines replaced/restructured", - "fix": "Rewrote 23\u219210 lines (15 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T23:42:34.993Z" - }, - { - "id": "bug-174", - "timestamp": "2026-05-30T23:52:01.776Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21929 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-30T23:52:01.776Z" - }, - { - "id": "bug-175", - "timestamp": "2026-05-31T00:11:29.118Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 14\u219216 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T00:11:29.118Z" - }, - { - "id": "bug-176", - "timestamp": "2026-05-31T02:09:39.630Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (c->myconf && c->myconf->peers_cfg.candidates_count)", - "fix": "Changed to: if (ha->my_priority > 0)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T02:09:39.630Z" - }, - { - "id": "bug-177", - "timestamp": "2026-05-31T02:54:09.113Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "12 lines replaced/restructured", - "fix": "Rewrote 15\u219220 lines (12 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T02:54:09.113Z" - }, - { - "id": "bug-178", - "timestamp": "2026-05-31T02:54:42.189Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T02:54:42.189Z" - }, - { - "id": "bug-179", - "timestamp": "2026-05-31T02:55:12.814Z", - "error_message": "Significant refactor of ", - "file": "src/xce.build/lib/controller.capnp.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11\u219219 lines (4 removed) | Also: encode_Data_ptr(cs, &(d->sha256), s->sha256);; encode_Data_ptr(cs, &(d->bytes), s->bytes);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-31T03:00:08.081Z" - }, - { - "id": "bug-180", - "timestamp": "2026-05-31T02:57:22.273Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "28 lines replaced/restructured", - "fix": "Rewrote 31\u21924 lines (28 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T02:57:22.273Z" - }, - { - "id": "bug-181", - "timestamp": "2026-05-31T03:01:08.147Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u219218 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T03:01:08.147Z" - }, - { - "id": "bug-182", - "timestamp": "2026-05-31T03:02:13.050Z", - "error_message": "Significant refactor of ", - "file": "src/include/request.h", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 10\u219219 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T03:02:13.050Z" - }, - { - "id": "bug-183", - "timestamp": "2026-05-31T03:02:33.311Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: req == NULL", - "fix": "Added guard clause: if (req == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T03:02:33.311Z" - }, - { - "id": "bug-184", - "timestamp": "2026-05-31T03:14:15.595Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T03:14:15.595Z" - }, - { - "id": "bug-185", - "timestamp": "2026-05-31T03:27:59.922Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T03:27:59.922Z" - }, - { - "id": "bug-186", - "timestamp": "2026-05-31T04:11:45.534Z", - "error_message": "Missing error handling in struct", - "file": ".kiro/specs/raft-library/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: Caller->>RaftSM: raft_on_vote_request(state, from, term); alt term < current_term", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-31T04:13:04.500Z" - }, - { - "id": "bug-187", - "timestamp": "2026-05-31T04:13:43.960Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-library/design.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 8\u219226 lines (5 removed) | Also: * raft_init: initialize state for a node.; * All required callbacks (except on_trace and on_v | Also: - C standard library only (`stdint.h`, `pthread.h`; - `raft.c` may include `` for lock-fr | Also: | Threading | `pthread_mutex_t` (caller-owned) | P; | Atomics | None in library | Caller uses atomics ", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-05-31T04:19:37.296Z" - }, - { - "id": "bug-188", - "timestamp": "2026-05-31T04:19:54.524Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-library/design.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to `raft.h`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T04:19:54.524Z" - }, - { - "id": "bug-189", - "timestamp": "2026-05-31T04:58:21.609Z", - "error_message": "Missing error handling in RAFT_H", - "file": "src/lib/raft/raft.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T04:58:21.609Z" - }, - { - "id": "bug-190", - "timestamp": "2026-05-31T05:07:37.759Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: cfg->n_peers < 1", - "fix": "Added guard clause: if (cfg->n_peers < 1)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T05:07:37.759Z" - }, - { - "id": "bug-191", - "timestamp": "2026-05-31T05:23:19.394Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->role == RAFT_CANDIDATE || s->role == RAFT_LEADER", - "fix": "Added guard clause: if (s->role == RAFT_CANDIDATE || s->role == )", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T05:23:19.394Z" - }, - { - "id": "bug-192", - "timestamp": "2026-05-31T05:32:36.388Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 77\u2192107 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T05:32:36.388Z" - }, - { - "id": "bug-193", - "timestamp": "2026-05-31T05:35:03.808Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 9\u219262 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T05:35:03.808Z" - }, - { - "id": "bug-194", - "timestamp": "2026-05-31T05:47:51.091Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u219257 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T05:47:51.091Z" - }, - { - "id": "bug-195", - "timestamp": "2026-05-31T05:58:45.896Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 15\u219238 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T05:58:45.896Z" - }, - { - "id": "bug-196", - "timestamp": "2026-05-31T06:10:56.560Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21923 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T06:10:56.560Z" - }, - { - "id": "bug-197", - "timestamp": "2026-05-31T07:08:23.387Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T07:08:23.387Z" - }, - { - "id": "bug-198", - "timestamp": "2026-05-31T07:10:04.095Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "32 lines replaced/restructured", - "fix": "Rewrote 49\u219216 lines (32 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T07:10:04.095Z" - }, - { - "id": "bug-199", - "timestamp": "2026-05-31T07:26:39.758Z", - "error_message": "Missing error handling in before", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T07:26:39.758Z" - }, - { - "id": "bug-200", - "timestamp": "2026-05-31T07:28:29.756Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u219219 lines (3 removed) | Also: typedef struct {; ha_state_t *ha; | Also: /* \u2500\u2500 Type definitions needed by ha_elect_evaluate; *", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-31T07:30:49.949Z" - }, - { - "id": "bug-201", - "timestamp": "2026-05-31T07:57:18.504Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "167 lines replaced/restructured", - "fix": "Rewrote 291\u2192137 lines (167 removed) | Also: /* Clear voted_for: the old primary is dead; any v; * is stale and must not block the new election. */", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-31T07:58:02.135Z" - }, - { - "id": "bug-202", - "timestamp": "2026-05-31T07:58:38.513Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "12 lines replaced/restructured", - "fix": "Rewrote 28\u219268 lines (12 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T07:58:38.513Z" - }, - { - "id": "bug-203", - "timestamp": "2026-05-31T08:16:20.727Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 16\u219221 lines (2 removed) | Also: /* Task 6.3: delegate vote reply accumulation to r; * raft_on_vote_reply owns quorum tracking and fire", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-31T08:16:28.806Z" - }, - { - "id": "bug-204", - "timestamp": "2026-05-31T08:30:55.738Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 10\u21924 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T08:30:55.738Z" - }, - { - "id": "bug-205", - "timestamp": "2026-05-31T08:38:33.200Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u219214 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T08:38:33.200Z" - }, - { - "id": "bug-206", - "timestamp": "2026-05-31T08:40:00.000Z", - "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 \u2014 ha->term must be atomically updated to peer term when peer term > ours", - "file": "src/controller/ha.c", - "root_cause": "vote_reply_msg_handler delegated term fencing entirely to raft_cb_become_follower (via raft_on_vote_reply). In XCE_TESTING builds, vote_reply_dispatch calls xce_vote_reply_hook and returns early, bypassing raft_on_vote_reply and thus never updating ha->term.", - "fix": "Added explicit CAS loop in vote_reply_msg_handler before vote_reply_dispatch to advance ha->term when vr_term > ha->term. This applies on both the production path and the XCE_TESTING hook path.", - "tags": [ - "term-fencing", - "raft", - "ha", - "xce-testing", - "test-seam", - "vote-reply" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T08:40:00.000Z" - }, - { - "id": "bug-207", - "timestamp": "2026-05-31T08:47:08.484Z", - "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_even_peers_returns_einval", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_even_peers_returns_einval\"", - "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_even_peers_returns_einval", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T08:47:08.484Z" - }, - { - "id": "bug-208", - "timestamp": "2026-05-31", - "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite \u2014 build exits with error code", - "file": "src/scheduler/testsuite/test.c", - "root_cause": "sched_meet_host_selection() lazily allocates w->host_eval_ctx via build_host_eval_ctx(). Four test functions (test_meet_host_selection_always_true_passes, test_meet_host_selection_always_false_fails, test_meet_host_selection_numeric_match, test_meet_host_selection_string_match) called zarray_fini() without first calling sched_worker_release_host_ctx() to free the lazily-allocated host_eval_ctx.", - "fix": "Added sched_worker_release_host_ctx(&w) calls before zarray_fini() in all four affected test functions.", - "tags": [ - "asan", - "leak", - "scheduler", - "test", - "host_eval_ctx" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31" - }, - { - "id": "bug-209", - "timestamp": "2026-05-31T10:19:47.014Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (ha_rdhup_election_hook != NULL)", - "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T10:19:47.014Z" - }, - { - "id": "bug-210", - "timestamp": "2026-05-31T10:21:40.444Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "24 lines replaced/restructured", - "fix": "Rewrote 28\u21923 lines (24 removed) | Also: if (my_config_tag == 0) {; INFO(\"ha_start_election_async: no local config (ta", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-05-31T10:23:32.151Z" - }, - { - "id": "bug-211", - "timestamp": "2026-05-31T11:29:03.301Z", - "error_message": "Wrong reference: 2000 should be 5000", - "file": "src/controller/ha.h", - "root_cause": "Used \"2000\" instead of \"5000\"", - "fix": "Changed 2000 \u2192 5000", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T11:29:03.301Z" - }, - { - "id": "bug-212", - "timestamp": "2026-05-31T11:33:55.599Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 15\u219225 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T11:33:55.599Z" - }, - { - "id": "bug-213", - "timestamp": "2026-05-31T12:32:33.273Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 10\u219214 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T12:32:33.273Z" - }, - { - "id": "bug-214", - "timestamp": "2026-05-31T12:42:09.400Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219217 lines (4 removed) | Also: /* Trigger async config fetch from primary so stan; #ifndef XCE_TESTING | Also: handle_request_vote_reply(vctx->ha, vctx->ctx, vct; /* Fix 5 (HIGH): do NOT free vctx here.", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-05-31T12:45:37.690Z" - }, - { - "id": "bug-215", - "timestamp": "2026-05-31T12:43:45.333Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", - "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T12:43:45.333Z" - }, - { - "id": "bug-216", - "timestamp": "2026-05-31T12:49:40.554Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-peer-connection-reuse/tasks.md", - "root_cause": "Had 's vote endpoints to standby control channels\n\n- [", - "fix": "Changed to 's vote endpoints to standby control channels\n\n- [", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T12:49:40.554Z" - }, - { - "id": "bug-217", - "timestamp": "2026-05-31T12:54:26.372Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T12:54:26.372Z" - }, - { - "id": "bug-218", - "timestamp": "2026-05-31T12:58:44.435Z", - "error_message": "Wrong reference: Request_body_nodata should be Request_body_nomsg", - "file": "src/controller/ha.c", - "root_cause": "Used \"Request_body_nodata\" instead of \"Request_body_nomsg\"", - "fix": "Changed Request_body_nodata \u2192 Request_body_nomsg", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T12:58:44.435Z" - }, - { - "id": "bug-219", - "timestamp": "2026-05-31T14:19:13.820Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 10\u219218 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T14:19:13.820Z" - }, - { - "id": "bug-220", - "timestamp": "2026-05-31T14:20:41.161Z", - "error_message": "Wrong reference: vote_reply_msg_handler should be vote_reply_msg_handler_for_test", - "file": "src/controller/ha.c", - "root_cause": "Used \"vote_reply_msg_handler\" instead of \"vote_reply_msg_handler_for_test\"", - "fix": "Changed vote_reply_msg_handler \u2192 vote_reply_msg_handler_for_test", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-05-31T14:20:41.161Z" - }, - { - "id": "bug-221", - "timestamp": "2026-06-01T01:38:35.036Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 13\u219236 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T01:38:35.036Z" - }, - { - "id": "bug-222", - "timestamp": "2026-06-01T01:47:28.189Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "20 lines replaced/restructured", - "fix": "Rewrote 36\u219213 lines (20 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T01:47:28.189Z" - }, - { - "id": "bug-223", - "timestamp": "2026-06-01T01:49:30.537Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T01:49:30.537Z" - }, - { - "id": "bug-224", - "timestamp": "2026-06-01T02:07:55.176Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c->delaymgr.running", - "fix": "Added guard clause: if (!c->delaymgr.running)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:07:55.176Z" - }, - { - "id": "bug-225", - "timestamp": "2026-06-01T02:26:54.291Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "42 lines replaced/restructured", - "fix": "Rewrote 64\u219219 lines (42 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:26:54.291Z" - }, - { - "id": "bug-226", - "timestamp": "2026-06-01T02:27:55.942Z", - "error_message": "Missing error handling in struct", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:27:55.942Z" - }, - { - "id": "bug-227", - "timestamp": "2026-06-01T02:29:00.462Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 10\u219214 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:29:00.462Z" - }, - { - "id": "bug-228", - "timestamp": "2026-06-01T02:37:56.961Z", - "error_message": "Significant refactor of ", - "file": "src/controller/CLAUDE.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 5\u219215 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:37:56.961Z" - }, - { - "id": "bug-229", - "timestamp": "2026-06-01T02:49:09.141Z", - "error_message": "Wrong operator: && should be !=", - "file": "src/controller/ha.c", - "root_cause": "Used \"&&\" instead of \"!=\"", - "fix": "Changed operator && \u2192 !=", - "tags": [ - "auto-detected", - "operator-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:49:09.141Z" - }, - { - "id": "bug-230", - "timestamp": "2026-06-01T02:56:53.219Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 16\u21927 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T02:56:53.219Z" - }, - { - "id": "bug-231", - "timestamp": "2026-06-01T03:07:11.679Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "140 lines replaced/restructured", - "fix": "Rewrote 173\u219211 lines (140 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:07:11.679Z" - }, - { - "id": "bug-232", - "timestamp": "2026-06-01T03:07:49.710Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "16 lines replaced/restructured", - "fix": "Rewrote 25\u219212 lines (16 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:07:49.710Z" - }, - { - "id": "bug-233", - "timestamp": "2026-06-01T03:07:51.264Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connect.c'", - "fix": "Changed to 'testsuite/test_ha_ctrlhello.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:07:51.264Z" - }, - { - "id": "bug-234", - "timestamp": "2026-06-01T03:12:12.378Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 31\u219237 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:12:12.378Z" - }, - { - "id": "bug-235", - "timestamp": "2026-06-01T03:13:14.521Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connection_reuse.c'", - "fix": "Changed to 'testsuite/test_ha_connect.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:13:14.521Z" - }, - { - "id": "bug-236", - "timestamp": "2026-06-01T03:15:29.129Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: rc < 0", - "fix": "Added guard clause: if (rc < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:15:29.129Z" - }, - { - "id": "bug-237", - "timestamp": "2026-06-01T03:23:30.915Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "21 lines replaced/restructured", - "fix": "Rewrote 28\u21925 lines (21 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:23:30.915Z" - }, - { - "id": "bug-238", - "timestamp": "2026-06-01T03:23:32.483Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connect.c'", - "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-01T03:24:43.046Z" - }, - { - "id": "bug-239", - "timestamp": "2026-06-01T03:28:46.956Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook != NULL", - "fix": "Added guard clause: if (ha_tcp_connect_hook != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T03:28:46.956Z" - }, - { - "id": "bug-240", - "timestamp": "2026-06-01T04:08:07.950Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_connect.c'", - "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-01T04:12:55.229Z" - }, - { - "id": "bug-241", - "timestamp": "2026-06-01T04:13:03.001Z", - "error_message": "Wrong reference: test_ha_connect should be test_ha_ctrlhello", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_ha_connect\" instead of \"test_ha_ctrlhello\"", - "fix": "Changed test_ha_connect \u2192 test_ha_ctrlhello", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-01T04:13:08.650Z" - }, - { - "id": "bug-242", - "timestamp": "2026-06-01T05:19:12.427Z", - "error_message": "Wrong reference: vcpu should be vcpus", - "file": "src/tools/deploy-ha.sh", - "root_cause": "Used \"vcpu\" instead of \"vcpus\"", - "fix": "Changed vcpu \u2192 vcpus", - "tags": [ - "auto-detected", - "wrong-reference", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T05:19:12.427Z" - }, - { - "id": "bug-243", - "timestamp": "2026-06-01T05:30:33.624Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 10\u21926 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T05:30:33.624Z" - }, - { - "id": "bug-244", - "timestamp": "2026-06-01T05:36:27.988Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u219222 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T05:36:27.988Z" - }, - { - "id": "bug-245", - "timestamp": "2026-06-01T05:48:05.878Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u219210 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T05:48:05.878Z" - }, - { - "id": "bug-246", - "timestamp": "2026-06-01T05:48:40.684Z", - "error_message": "Significant refactor of ", - "file": "src/lib/request.c", - "root_cause": "32 lines replaced/restructured", - "fix": "Rewrote 49\u219260 lines (32 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T05:48:40.684Z" - }, - { - "id": "bug-247", - "timestamp": "2026-06-01T06:02:47.254Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-deploy-ha/SKILL.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 17\u219232 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T06:02:47.254Z" - }, - { - "id": "bug-248", - "timestamp": "2026-06-01T06:03:18.973Z", - "error_message": "Significant refactor of ", - "file": "src/controller/CLAUDE.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21929 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T06:03:18.973Z" - }, - { - "id": "bug-249", - "timestamp": "2026-06-01T07:28:15.703Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T07:28:15.703Z" - }, - { - "id": "bug-250", - "timestamp": "2026-06-01T07:46:05.128Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ep || !ep->data", - "fix": "Added guard clause: if (!ep || !ep->data)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T07:46:05.128Z" - }, - { - "id": "bug-251", - "timestamp": "2026-06-01T07:46:59.891Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "19 lines replaced/restructured", - "fix": "Rewrote 67\u219286 lines (19 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T07:46:59.891Z" - }, - { - "id": "bug-252", - "timestamp": "2026-06-01T08:35:29.969Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'startup_validation.c'", - "fix": "Changed to 'testsuite/test_ha_rdhup_integration.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T08:35:29.969Z" - }, - { - "id": "bug-253", - "timestamp": "2026-06-01T10:42:39.262Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-heartbeat-liveness/requirements.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 10\u219217 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T10:42:39.262Z" - }, - { - "id": "bug-254", - "timestamp": "2026-06-01T10:54:01.806Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T10:54:01.806Z" - }, - { - "id": "bug-255", - "timestamp": "2026-06-01T12:02:55.635Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (rc != 0)", - "fix": "Changed to: if (validate_heartbeat_config(ha_cfg.heartbeat_timeout)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T12:02:55.635Z" - }, - { - "id": "bug-256", - "timestamp": "2026-06-01T12:36:28.381Z", - "error_message": "Wrong reference: startup_validation should be testsuite_test_ha_liveness", - "file": "src/xce.build/build.ninja", - "root_cause": "Used \"startup_validation\" instead of \"testsuite_test_ha_liveness\"", - "fix": "Changed startup_validation \u2192 testsuite_test_ha_liveness", - "tags": [ - "auto-detected", - "wrong-reference", - "ninja" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T12:36:28.381Z" - }, - { - "id": "bug-257", - "timestamp": "2026-06-01T12:39:55.091Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "76 lines replaced/restructured", - "fix": "Rewrote 112\u2192155 lines (76 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T12:39:55.091Z" - }, - { - "id": "bug-258", - "timestamp": "2026-06-01T12:42:30.248Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 21\u219224 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T12:42:30.248Z" - }, - { - "id": "bug-259", - "timestamp": "2026-06-01T12:43:29.946Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_failover.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 29\u219247 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T12:43:29.946Z" - }, - { - "id": "bug-260", - "timestamp": "2026-06-01T14:42:41.325Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: interval_s <= 0", - "fix": "Added guard clause: if (interval_s <= 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-01T14:44:38.543Z" - }, - { - "id": "bug-261", - "timestamp": "2026-06-01T14:44:18.059Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "22 lines replaced/restructured", - "fix": "Rewrote 53\u219221 lines (22 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T14:44:18.059Z" - }, - { - "id": "bug-262", - "timestamp": "2026-06-01T15:13:36.229Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T15:13:36.229Z" - }, - { - "id": "bug-263", - "timestamp": "2026-06-01T15:32:59.304Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: !f", - "fix": "Added guard clause: if (!f)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-01T15:32:59.304Z" - }, - { - "id": "bug-264", - "timestamp": "2026-06-03T05:20:44.299Z", - "error_message": "Missing error handling in lives", - "file": ".kiro/specs/ha-config-bootstrap/research.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T05:20:44.299Z" - }, - { - "id": "bug-265", - "timestamp": "2026-06-03T08:47:25.835Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T08:47:25.835Z" - }, - { - "id": "bug-266", - "timestamp": "2026-06-03T08:47:54.376Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T08:47:54.376Z" - }, - { - "id": "bug-267", - "timestamp": "2026-06-03T08:48:09.401Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 14\u21923 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T08:48:09.401Z" - }, - { - "id": "bug-268", - "timestamp": "2026-06-03T08:48:54.248Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "79 lines replaced/restructured", - "fix": "Rewrote 87\u21923 lines (79 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T08:48:54.248Z" - }, - { - "id": "bug-269", - "timestamp": "2026-06-03T08:50:58.837Z", - "error_message": "Wrong reference: test_am_i_primary should be test_ha_cluster", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_am_i_primary\" instead of \"test_ha_cluster\"", - "fix": "Changed test_am_i_primary \u2192 test_ha_cluster", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-03T08:51:07.382Z" - }, - { - "id": "bug-270", - "timestamp": "2026-06-03T09:07:49.592Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 10\u21923 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T09:07:49.592Z" - }, - { - "id": "bug-271", - "timestamp": "2026-06-03T09:17:11.528Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "21 lines replaced/restructured", - "fix": "Rewrote 26\u21924 lines (21 removed) | Also: /* ha_check_heartbeat_nocluster() removed from the; * ha_watchdog_thread(). XCE_TESTING shim preserve", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-03T09:18:16.045Z" - }, - { - "id": "bug-272", - "timestamp": "2026-06-03T09:22:40.327Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "63 lines replaced/restructured", - "fix": "Rewrote 68\u21924 lines (63 removed) | Also: /* SIGUSR1 must set failover_requested via async-s; static void test_sigusr1_sets_failover_requested(v | Also: RUN_TEST(test_watchdog_starts_and_stops);; RUN_TEST(test_heartbeat_no_action_when_no_primary)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-03T09:25:36.265Z" - }, - { - "id": "bug-273", - "timestamp": "2026-06-03T14:53:08.631Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/manual-tests.md", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 8\u21928 lines (6 removed) | Also: **Expected within 1 second on both standbys:**; **Expected on the winner (needs 1 vote from the ot | Also: defer_config_fetch=1; Config fetch successful from primary (tag=...) | Also: Date:; Commit: (run: cd /home/rshen/projects/xce/worktree", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-03T14:56:43.241Z" - }, - { - "id": "bug-274", - "timestamp": "2026-06-03T15:05:10.632Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-config-bootstrap/manual-tests.md", - "root_cause": "95 lines replaced/restructured", - "fix": "Rewrote 158\u219265 lines (95 removed) | Also: - [ ] `ha-term.bin` file exists after election; - [ ] Magic bytes are `0xDEADBEEFCAFEBABE` (little", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-03T15:09:51.822Z" - }, - { - "id": "bug-275", - "timestamp": "2026-06-03T23:50:28.482Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 45\u219262 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-03T23:50:28.482Z" - }, - { - "id": "bug-276", - "timestamp": "2026-06-04T00:24:46.380Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/myapp.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T00:24:46.380Z" - }, - { - "id": "bug-277", - "timestamp": "2026-06-04T09:31:40.432Z", - "error_message": "Significant refactor of ", - "file": "src/controller/CLAUDE.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 5\u21927 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T09:31:40.432Z" - }, - { - "id": "bug-278", - "timestamp": "2026-06-04T09:46:14.610Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-repl-hardening/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T09:46:14.610Z" - }, - { - "id": "bug-279", - "timestamp": "2026-06-04T11:54:56.114Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-repl-hardening/design.md", - "root_cause": "Had `(checkpoint_lsn, last_received_chunk)`", - "fix": "Changed to `snap.part`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T11:54:56.114Z" - }, - { - "id": "bug-280", - "timestamp": "2026-06-04T13:13:54.348Z", - "error_message": "Significant refactor of ", - "file": "src/schema/controller.capnp", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 4\u21924 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T13:13:54.348Z" - }, - { - "id": "bug-281", - "timestamp": "2026-06-04T13:38:04.971Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 5\u21927 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T13:38:04.971Z" - }, - { - "id": "bug-282", - "timestamp": "2026-06-04T13:48:05.580Z", - "error_message": "Significant refactor of ", - "file": "src/schema/controller.capnp", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u219213 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T13:48:05.580Z" - }, - { - "id": "bug-283", - "timestamp": "2026-06-04T13:48:16.712Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T13:48:16.712Z" - }, - { - "id": "bug-284", - "timestamp": "2026-06-04T14:33:03.464Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2500\u2500 Task 2.1: OPCode_walCatchupEntry handler \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-04T14:33:51.476Z" - }, - { - "id": "bug-285", - "timestamp": "2026-06-04T14:41:35.073Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: case OPCode_walCatchupEntry:; /* Task 2.1: standby applies a single WAL catch-up entry fro", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-04T14:41:48.239Z" - }, - { - "id": "bug-286", - "timestamp": "2026-06-04T14:44:41.114Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "57 lines replaced/restructured", - "fix": "Rewrote 82\u219216 lines (57 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T14:44:41.114Z" - }, - { - "id": "bug-287", - "timestamp": "2026-06-04T14:45:42.730Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T14:45:42.730Z" - }, - { - "id": "bug-288", - "timestamp": "2026-06-04T14:46:44.652Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: #include \"persist.h\" /* persist_t, apply_wal_entry */; #include \"typemap.h\" /* wal_catchup_entry_t */", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-04T14:47:53.528Z" - }, - { - "id": "bug-289", - "timestamp": "2026-06-04T15:09:54.280Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T15:09:54.280Z" - }, - { - "id": "bug-290", - "timestamp": "2026-06-04T15:22:45.178Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "22 lines replaced/restructured", - "fix": "Rewrote 58\u2192114 lines (22 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T15:22:45.178Z" - }, - { - "id": "bug-291", - "timestamp": "2026-06-04T15:51:04.816Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 10\u219215 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T15:51:04.816Z" - }, - { - "id": "bug-292", - "timestamp": "2026-06-04T16:15:43.743Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", - "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= MAX_HA_CANDI)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T16:15:43.743Z" - }, - { - "id": "bug-293", - "timestamp": "2026-06-04T16:19:17.173Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 44\u219289 lines (4 removed) | Also: /* Decode stateReport and call convergence tracker; memset(&sr, 0, sizeof(sr));", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-04T16:21:09.851Z" - }, - { - "id": "bug-294", - "timestamp": "2026-06-04T16:45:25.724Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: standby_tag < committed_tag", - "fix": "Added guard clause: if (standby_tag < committed_tag)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T16:45:25.724Z" - }, - { - "id": "bug-295", - "timestamp": "2026-06-04T16:57:59.213Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Had \"test_evaluate_repl_action_stream_wal_from_zero: P", - "fix": "Changed to \"test_evaluate_repl_action_send_snapshot_lsn_zero:", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T16:57:59.213Z" - }, - { - "id": "bug-296", - "timestamp": "2026-06-04T16:58:16.627Z", - "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Used \"test_evaluate_repl_action_stream_wal_from_zero\" instead of \"test_evaluate_repl_action_send_snapshot_lsn_zero\"", - "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero \u2192 test_evaluate_repl_action_send_snapshot_lsn_zero", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T16:58:16.627Z" - }, - { - "id": "bug-297", - "timestamp": "2026-06-04T17:08:18.571Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", - "fix": "Added guard clause: if (!ha || peer_idx < 0 || peer_idx >= MAX_H)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T17:08:18.571Z" - }, - { - "id": "bug-298", - "timestamp": "2026-06-04T17:34:23.150Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T17:34:23.150Z" - }, - { - "id": "bug-299", - "timestamp": "2026-06-04T17:36:34.006Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 71\u219293 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T17:36:34.006Z" - }, - { - "id": "bug-300", - "timestamp": "2026-06-04T17:40:05.798Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T17:40:05.798Z" - }, - { - "id": "bug-301", - "timestamp": "2026-06-04T23:09:43.701Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", - "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T23:09:43.701Z" - }, - { - "id": "bug-302", - "timestamp": "2026-06-04T23:10:44.426Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !snap_dir", - "fix": "Added guard clause: if (!snap_dir)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T23:10:44.426Z" - }, - { - "id": "bug-303", - "timestamp": "2026-06-04T23:11:48.274Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u219215 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T23:11:48.274Z" - }, - { - "id": "bug-304", - "timestamp": "2026-06-04T23:21:45.737Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T23:21:45.737Z" - }, - { - "id": "bug-305", - "timestamp": "2026-06-04T23:22:03.572Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219212 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-04T23:22:03.572Z" - }, - { - "id": "bug-306", - "timestamp": "2026-06-05T01:16:16.713Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-repl-hardening/tasks.md", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 9\u219235 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T01:16:16.713Z" - }, - { - "id": "bug-307", - "timestamp": "2026-06-05T01:18:21.265Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219210 lines (4 removed) | Also: /* Task 3.5: snapshot transfer.; * Creates a checkpoint in the delaymgr thread (not", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-05T01:18:36.333Z" - }, - { - "id": "bug-308", - "timestamp": "2026-06-05T01:19:20.356Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !d", - "fix": "Added guard clause: if (!d)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T01:19:20.356Z" - }, - { - "id": "bug-309", - "timestamp": "2026-06-05T01:33:20.146Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !p || !snap_dir || !chunk", - "fix": "Added guard clause: if (!ha || !p || !snap_dir || !chunk)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T01:33:20.146Z" - }, - { - "id": "bug-310", - "timestamp": "2026-06-05T01:34:06.660Z", - "error_message": "Missing guard clause", - "file": "src/controller/cluster.c", - "root_cause": "No early return/throw for edge case: !chunk", - "fix": "Added guard clause: if (!chunk)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T01:34:06.660Z" - }, - { - "id": "bug-311", - "timestamp": "2026-06-05T02:21:15.033Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ua > ub", - "fix": "Added guard clause: if (ua > ub)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T02:21:15.033Z" - }, - { - "id": "bug-312", - "timestamp": "2026-06-05T02:47:32.237Z", - "error_message": "Wrong reference: uintptr_t should be void", - "file": "src/controller/testsuite/test_ha_evaluate_committing.c", - "root_cause": "Used \"uintptr_t\" instead of \"void\"", - "fix": "Changed uintptr_t \u2192 void", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T02:47:32.237Z" - }, - { - "id": "bug-313", - "timestamp": "2026-06-05T03:35:34.699Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "149 lines replaced/restructured", - "fix": "Rewrote 207\u219238 lines (149 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T03:35:34.699Z" - }, - { - "id": "bug-314", - "timestamp": "2026-06-05T04:01:58.721Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T04:01:58.721Z" - }, - { - "id": "bug-315", - "timestamp": "2026-06-05T04:52:32.206Z", - "error_message": "Wrong reference: 10000 should be 60000", - "file": "src/controller/ha.h", - "root_cause": "Used \"10000\" instead of \"60000\"", - "fix": "Changed 10000 \u2192 60000", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-05T04:52:51.260Z" - }, - { - "id": "bug-316", - "timestamp": "2026-06-05T04:53:20.573Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 24\u219242 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T04:53:20.573Z" - }, - { - "id": "bug-317", - "timestamp": "2026-06-05T04:53:55.034Z", - "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS", - "file": "src/controller/testsuite/test_ha_evaluate_committing.c", - "root_cause": "Used \"10000\" instead of \"HA_DEFAULT_COMMITTING_TIMEOUT_MS\"", - "fix": "Changed 10000 \u2192 HA_DEFAULT_COMMITTING_TIMEOUT_MS", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T04:53:55.034Z" - }, - { - "id": "bug-318", - "timestamp": "2026-06-05T05:10:13.646Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T05:10:13.646Z" - }, - { - "id": "bug-319", - "timestamp": "2026-06-05T05:39:48.707Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T05:39:48.707Z" - }, - { - "id": "bug-320", - "timestamp": "2026-06-05T06:50:00Z", - "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN", - "file": "src/controller/ha.c", - "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always \u2014 majority convergence path could never succeed.", - "fix": "Changed ha->n_peers = n_peers to ha->n_peers = n_other (= n_peers-1 = other-node count). Moved the assignment after n_other is computed. ha_repl_ack_received quorum formula (ha->n_peers+2)/2 still yields correct majority for both old and new value.", - "tags": [ - "ha", - "majority-convergence", - "n_peers", - "off-by-one", - "integration-test" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T06:50:00Z" - }, - { - "id": "bug-321", - "timestamp": "2026-06-05T07:31:54.806Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u219212 lines (2 removed) | Also: /* Disarm any existing poll ctx */; free(old);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-05T07:32:06.251Z" - }, - { - "id": "bug-006", - "timestamp": "2026-06-05T07:23:00.000Z", - "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer", - "file": "src/controller/ha.c", - "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) \u2014 double-free at lines 3594-3595 and 3600-3601", - "fix": "Set ep->data = NULL before endpoint_fini so FREEUP becomes a no-op; then free(sctx) explicitly in the error paths", - "tags": [ - "double-free", - "asan", - "ha_connect_to_peer", - "endpoint_fini", - "integration-test" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T07:23:00.000Z" - }, - { - "id": "bug-007", - "timestamp": "2026-06-05T07:23:00.000Z", - "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 \u2014 reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", - "file": "src/controller/ha.c", - "root_cause": "ha_arm_heartbeat_timeout_vtask freed the old liveness_poll_ctx_t immediately after setting cancelled=1, but the delaymgr task was already in flight reading ctx->cancelled=0 and then continued to read ctx->ha after free", - "fix": "ha_arm_heartbeat_timeout_vtask no longer frees old ctx; liveness_poll_fn now frees ctx when it fires with cancelled=1 (guarded by #ifndef XCE_TESTING to avoid bad-free on stack-allocated test ctxs)", - "tags": [ - "heap-use-after-free", - "asan", - "liveness_poll_fn", - "ha_arm_heartbeat_timeout_vtask", - "race-condition", - "integration-test" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T07:23:00.000Z" - }, - { - "id": "bug-324", - "timestamp": "2026-06-05T08:26:02.778Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 41\u219287 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-05T08:26:02.778Z" - }, - { - "id": "bug-325", - "timestamp": "2026-06-06T11:00:00.057Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 8\u21925 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-06T11:00:00.057Z" - }, - { - "id": "bug-326", - "timestamp": "2026-06-06T11:06:46.509Z", - "error_message": "Wrong reference: DEBUG2 should be INFO", - "file": "src/controller/ha.c", - "root_cause": "Used \"DEBUG2\" instead of \"INFO\"", - "fix": "Changed DEBUG2 \u2192 INFO", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-06T11:06:46.509Z" - }, - { - "id": "bug-327", - "timestamp": "2026-06-06T12:11:21.422Z", - "error_message": "Incorrect value in code", - "file": "src/controller/ha.c", - "root_cause": "Had \"HA: heartbeat tick (role=%d, n_standbys=%d)\"", - "fix": "Changed to \"HA: primary heartbeat armed\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-06T12:11:21.422Z" - }, - { - "id": "bug-328", - "timestamp": "2026-06-06T13:02:49.052Z", - "error_message": "Missing guard clause", - "file": "src/lib/delay.c", - "root_cause": "No early return/throw for edge case: t1_is_ms && t2_is_ms", - "fix": "Added guard clause: if (t1_is_ms && t2_is_ms)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-06T13:02:49.052Z" - }, - { - "id": "bug-329", - "timestamp": "2026-06-08T13:12:54.436Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-library/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-08T13:12:54.436Z" - }, - { - "id": "bug-330", - "timestamp": "2026-06-09T08:33:08.522Z", - "error_message": "Significant refactor of ", - "file": "src/docs/cluster-config-split-analysis.md", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 11\u219221 lines (8 removed) | Also: ### Phase 1: Schema split (wire-compatible); 4. **Keep the old `ClusterConfig` fields intact** ", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-09T08:35:24.717Z" - }, - { - "id": "bug-331", - "timestamp": "2026-06-09T09:28:15.795Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 27\u219236 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T09:28:15.795Z" - }, - { - "id": "bug-332", - "timestamp": "2026-06-09T09:41:02.753Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 9\u219215 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T09:41:02.753Z" - }, - { - "id": "bug-333", - "timestamp": "2026-06-09T10:23:01.661Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "37 lines replaced/restructured", - "fix": "Rewrote 41\u21924 lines (37 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T10:23:01.661Z" - }, - { - "id": "bug-334", - "timestamp": "2026-06-09T12:56:11.095Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (my_role == CTRL_STANDBY && ha_heartbeat_is_fresh(h)", - "fix": "Changed to: if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= )", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T12:56:11.095Z" - }, - { - "id": "bug-335", - "timestamp": "2026-06-09T00:00:00Z", - "error_message": "UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving resource idx by name", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "zarray_put(&w->resvals, 0, rv->val) hardcodes index 0; worker resvals are indexed by resource idx (see schedobj.c:291)", - "fix": "FIXED 2026-06-10: UPDATE_RESVALS now resolves rv->name via key_find on sched->resmgr and writes to r->idx (matching UPDATE_RESVALS_FULL pattern) instead of hardcoding index 0.", - "tags": [ - "scheduler", - "resvals", - "latent-bug", - "gap-analysis", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T17:00:00Z" - }, - { - "id": "bug-336", - "timestamp": "2026-06-09T00:00:00Z", - "error_message": "XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST and return value is silently ignored", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "makeResource_ in src/lib/resobj.c rejects duplicates with EEXIST; dequeue case ignores NULL return, so 'add or update' (EARS-UB-1.1) is only 'add'", - "fix": "FIXED 2026-06-10: UPDATE_RESOURCES now checks makeResource return; on NULL (EEXIST) looks up existing resource by name and updates valtype/ordtype/flags/interval fields in place (add-or-update semantics).", - "tags": [ - "scheduler", - "resources", - "latent-bug", - "gap-analysis", - "silent-error", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T17:00:00Z" - }, - { - "id": "bug-337", - "timestamp": "2026-06-09T00:02:00Z", - "error_message": "Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production writer after the liveness refactor", - "file": "src/controller/ha.c", - "root_cause": "ha-heartbeat-liveness refactor moved activity tracking to ha->last_activity but ha_heartbeat_is_fresh (ha.c:1762) still reads the orphaned last_heartbeat timespec; only tests write it, so freshness is always stale after ~heartbeat_timeout_ms of uptime and standbys grant votes despite a live primary (gates at ha.c:214 and ha.c:1841)", - "fix": "FIXED 2026-06-09: ha_heartbeat_is_fresh now reads ha->last_activity (returns 0 when last_activity==0); dead last_heartbeat field removed from ha_state_t; test_ha_vote_gate.c helper rewritten as set_last_activity_age and fresh/stale tests now also set primary_ctrl_fd. All tests green.", - "tags": [ - "ha", - "heartbeat", - "liveness", - "vote-gate", - "dead-field", - "gap-analysis" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T17:30:00Z" - }, - { - "id": "bug-338", - "timestamp": "2026-06-09T00:00:00Z", - "error_message": "controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0); test_standby_fresh_heartbeat_denies_vote_request FAIL (Expected 0 Was 1, Req 6.2)", - "file": "src/controller/ha.c", - "root_cause": "Uncommitted handle_request_vote changes: (1) duplicate-vote deny made candidate-agnostic (denies same-candidate idempotent re-grant, diverging from raft.c which keeps re-grant); (2) STANDBY vote deny now requires primary_ctrl_fd >= 0, tests use ha_init default fd=-1", - "fix": "FIXED 2026-06-09: handle_request_vote Req 2.6 check restored to same-candidate idempotent re-grant (deny only when voted_for_name differs from candidate_id), matching raft.c; test_standby_fresh_heartbeat_denies_vote_request updated to set primary_ctrl_fd=999. Also fixed myapp.c leader_known check to treat a vote for myconf->name as a self-vote (production records real name, not the 'self' test placeholder). All tests green.", - "tags": [ - "ha", - "election", - "vote-gate", - "raft", - "unit-test", - "uncommitted-drift", - "gap-analysis" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T17:30:00Z" - }, - { - "id": "bug-339", - "timestamp": "2026-06-10T00:23:18.666Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: last == 0", - "fix": "Added guard clause: if (last == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T00:23:18.666Z" - }, - { - "id": "bug-340", - "timestamp": "2026-06-10T00:23:21.918Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (ha->voted_for_term == term && ha->voted_for_name[0)", - "fix": "Changed to: if (ha->voted_for_term == term && ha->voted_for_name[0)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T00:23:21.918Z" - }, - { - "id": "bug-341", - "timestamp": "2026-06-10T00:23:52.563Z", - "error_message": "Wrong reference: last_heartbeat should be voted_for_term", - "file": "src/controller/ha.h", - "root_cause": "Used \"last_heartbeat\" instead of \"voted_for_term\"", - "fix": "Changed last_heartbeat \u2192 voted_for_term", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T00:23:52.563Z" - }, - { - "id": "bug-342", - "timestamp": "2026-06-10T00:24:41.873Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 13\u219217 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T00:24:41.873Z" - }, - { - "id": "bug-339", - "timestamp": "2026-06-09T17:30:00Z", - "error_message": "raft_init discards persisted vote term: voted_for_term reset to 0 even when initial_voted_for is restored, so duplicate-vote check cannot match after restart (possible double vote in same term)", - "file": "src/lib/raft/raft.c", - "root_cause": "raft_init set s->voted_for_term=0 unconditionally; persistence writes (term, voted_for) as a pair, so a restored vote belongs to initial_term", - "fix": "FIXED 2026-06-09: voted_for_term = (initial_voted_for != -1) ? initial_term : 0. Added test_proto_H6_restored_vote_blocks_second_vote and strengthened H1 assertion in src/lib/testsuite/test_raft.c. Found as candidate bug C2 in raft-library gap analysis.", - "tags": [ - "raft", - "election-safety", - "restart", - "persistence", - "gap-analysis" - ], - "related_bugs": [ - "bug-338" - ], - "occurrences": 1, - "last_seen": "2026-06-09T17:30:00Z" - }, - { - "id": "bug-344", - "timestamp": "2026-06-10T00:32:34.054Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_bootstrap_progress.md", - "root_cause": "Had `ha->last_heartbeat`", - "fix": "Changed to `ha->last_activity`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T00:32:34.054Z" - }, - { - "id": "bug-345", - "timestamp": "2026-06-10T01:36:59.112Z", - "error_message": "Missing error handling in unknown", - "file": "src/schema/controller.capnp", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:36:59.112Z" - }, - { - "id": "bug-346", - "timestamp": "2026-06-10T01:37:29.771Z", - "error_message": "Missing error handling in struct", - "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:37:29.771Z" - }, - { - "id": "bug-347", - "timestamp": "2026-06-10T01:37:41.741Z", - "error_message": "Significant refactor of ", - "file": "src/include/typemap.h", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 16\u219231 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:37:41.741Z" - }, - { - "id": "bug-348", - "timestamp": "2026-06-10T01:38:31.262Z", - "error_message": "Missing error handling in unknown", - "file": "src/include/request.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:38:31.262Z" - }, - { - "id": "bug-349", - "timestamp": "2026-06-10T01:39:19.368Z", - "error_message": "Missing guard clause", - "file": "src/lib/request.c", - "root_cause": "No early return/throw for edge case: l.p.type == CAPN_NULL", - "fix": "Added guard clause: if (l.p.type == CAPN_NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:39:19.368Z" - }, - { - "id": "bug-350", - "timestamp": "2026-06-10T01:39:31.683Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/request.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: if (req->body_which == Request_body_walcatchupentrymsg; && req->body.walcatchupentrymsg != NULL) { | Also: /* wal_entry_to_wire: fill a wire WalCatchupEntry from a C e; * fields and raw payload bytes. Sub-entry nesting is one le", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-10T01:40:14.312Z" - }, - { - "id": "bug-351", - "timestamp": "2026-06-09T00:00:00Z", - "error_message": "request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw from incompatible pointer type (xce_peerinfo_t** vs xce_peerinfo_t*)", - "file": "src/lib/request.c", - "root_cause": "ep->peer is already xce_peerinfo_t* (netio.h endpoint struct); call sites passed &ep->peer producing xce_peerinfo_t**", - "fix": "Dropped the & at both call sites (xce_wal_batch_req_raw line 547, xce_snapshot_chunk_req_raw line 633) to pass ep->peer directly", - "tags": [ - "build-failure", - "pointer-type", - "c", - "request.c", - "ha-replication" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-09T00:00:00Z" - }, - { - "id": "bug-352", - "timestamp": "2026-06-10T01:46:28.589Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 29\u219232 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:46:28.589Z" - }, - { - "id": "bug-353", - "timestamp": "2026-06-10T01:46:45.213Z", - "error_message": "Wrong reference: payload should be raw_payload", - "file": "src/controller/config_repl.c", - "root_cause": "Used \"payload\" instead of \"raw_payload\"", - "fix": "Changed payload \u2192 raw_payload", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:46:45.213Z" - }, - { - "id": "bug-354", - "timestamp": "2026-06-10T01:47:22.301Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 13\u219216 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:47:22.301Z" - }, - { - "id": "bug-355", - "timestamp": "2026-06-10T01:49:05.465Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 49\u2192137 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:49:05.465Z" - }, - { - "id": "bug-356", - "timestamp": "2026-06-10T01:49:36.997Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T01:49:36.997Z" - }, - { - "id": "bug-352", - "timestamp": "2026-06-10T02:10:00Z", - "error_message": "ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower (loser path) <- raft_on_vote_reply; election loser crashes on vote-reply processing", - "file": "src/controller/ha.c", - "root_cause": "vote_reply_decode_and_dispatch took pkt = ep->fsm_in.pkt and freed it WITHOUT clearing ep->fsm_in.pkt; vote_reply_dispatch can synchronously reach raft_cb_become_follower -> endpoint_fini(ep), which frees the stale fsm_in.pkt again. repl_ack_decode_and_dispatch already nulls the pointer (correct pattern); the vote-reply path missed it.", - "fix": "FIXED 2026-06-10: ep->fsm_in.pkt = NULL immediately after taking ownership in vote_reply_decode_and_dispatch (ha.c:2413), matching repl_ack_decode_and_dispatch. Found by task 7.7 E2E (3-node deploy, election loser crash).", - "tags": [ - "ha", - "election", - "asan", - "use-after-free", - "packet-ownership", - "netio", - "e2e" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T02:10:00Z" - }, - { - "id": "bug-358", - "timestamp": "2026-06-10T03:55:44.698Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/config_repl.c", - "root_cause": "Condition was: if (!rcpr->raw_bytes || rcpr->raw_bytes_len == 0 || !r)", - "fix": "Changed to: if ((ctrl_role_t)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T03:55:44.698Z" - }, - { - "id": "bug-359", - "timestamp": "2026-06-10T04:16:25.052Z", - "error_message": "Missing guard clause", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "No early return/throw for edge case: r == NULL", - "fix": "Added guard clause: if (r == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-10T04:16:28.320Z" - }, - { - "id": "bug-360", - "timestamp": "2026-06-10T04:17:09.223Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u21926 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T04:17:09.223Z" - }, - { - "id": "bug-361", - "timestamp": "2026-06-10T04:47:09.048Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", - "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcode)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T04:47:09.048Z" - }, - { - "id": "bug-353", - "timestamp": "2026-06-10T17:30:00Z", - "error_message": "Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows requestVote/probeConfigTag, returning -E_XCE_ERR_NOT_READY for all other opcodes. If liveness poll fires during election window, any incoming config push is discarded.", - "file": "src/controller/ha.c", - "root_cause": "CTRL_ELECTING gate in ha_dispatch_gate (ha.c:1464) allowed only vote/probe opcodes. Config push from primary arriving during the ELECTING window (after liveness-poll-triggered failover, before election result) was rejected with -E_XCE_ERR_NOT_READY (10013). Standby never saved config, leaving conf dir empty.", - "fix": "FIXED 2026-06-10: Added OPCode_hello, OPCode_nop, OPCode_replConfigPush, OPCode_walCatchupEntry, OPCode_snapshotTransfer to CTRL_ELECTING allowlist. A campaigning node must accept replication from the current primary to stay current for both win and loss paths. Controller suite green.", - "tags": [ - "ha", - "dispatch-gate", - "election", - "config-replication", - "silent-failure" - ], - "related_bugs": [ - "bug-335", - "bug-336" - ], - "occurrences": 1, - "last_seen": "2026-06-10T17:30:00Z" - }, - { - "id": "bug-363", - "timestamp": "2026-06-10T04:58:30.903Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 9\u219215 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T04:58:30.903Z" - }, - { - "id": "bug-364", - "timestamp": "2026-06-10T06:26:24.340Z", - "error_message": "Incorrect value in code", - "file": "src/controller/CLAUDE.md", - "root_cause": "Had `CTRL_ELECTING`", - "fix": "Changed to `CTRL_INIT`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T06:26:24.340Z" - }, - { - "id": "bug-365", - "timestamp": "2026-06-10T06:32:15.362Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 8\u219213 lines (3 removed) | Also: /* Req 4.1: ctrlhello counts as primary activity \u2014; if ((ctrl_role_t)atomic_load_explicit(&c->ha.role,", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-10T06:35:56.354Z" - }, - { - "id": "bug-366", - "timestamp": "2026-06-10T06:32:18.661Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u219212 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T06:32:18.661Z" - }, - { - "id": "bug-367", - "timestamp": "2026-06-10T07:56:21.347Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* =========================================================; * Task 1.1: repl_broadcast send path tests", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-10T07:57:02.036Z" - }, - { - "id": "bug-368", - "timestamp": "2026-06-10T07:57:40.792Z", - "error_message": "Wrong reference: WAL_JOB should be WAL_JOB_SUBMIT", - "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "Used \"WAL_JOB\" instead of \"WAL_JOB_SUBMIT\"", - "fix": "Changed WAL_JOB \u2192 WAL_JOB_SUBMIT", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T07:57:40.792Z" - }, - { - "id": "bug-369", - "timestamp": "2026-06-10T07:58:15.167Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T07:58:15.167Z" - }, - { - "id": "bug-370", - "timestamp": "2026-06-10T07:58:38.900Z", - "error_message": "Significant refactor of ", - "file": "src/controller/repl.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 42\u219265 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T07:58:38.900Z" - }, - { - "id": "bug-371", - "timestamp": "2026-06-10T07:59:24.268Z", - "error_message": "Wrong reference: DEBUG3 should be TRACE", - "file": "src/controller/repl.c", - "root_cause": "Used \"DEBUG3\" instead of \"TRACE\"", - "fix": "Changed DEBUG3 \u2192 TRACE", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T07:59:24.268Z" - }, - { - "id": "bug-372", - "timestamp": "2026-06-10T08:42:26.121Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 12\u219216 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T08:42:26.121Z" - }, - { - "id": "bug-373", - "timestamp": "2026-06-10T08:45:30.400Z", - "error_message": "Wrong reference: request should be ep", - "file": "src/controller/cluster.c", - "root_cause": "Used \"request\" instead of \"ep\"", - "fix": "Changed request \u2192 ep", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T08:45:30.400Z" - }, - { - "id": "bug-374", - "timestamp": "2026-06-10T08:47:48.296Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 3.3 (ha-wal-replication): dispatch gate standby-role; RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_ro", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-10T08:48:00.267Z" - }, - { - "id": "bug-375", - "timestamp": "2026-06-10T08:50:27.397Z", - "error_message": "Missing error handling in struct", - "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T08:50:27.397Z" - }, - { - "id": "bug-376", - "timestamp": "2026-06-10T09:17:35.218Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (opcode == OPCode_nop)", - "fix": "Changed to: if (rep_s.data_which == RequestReply_data_walackreplym)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T09:17:35.218Z" - }, - { - "id": "bug-377", - "timestamp": "2026-06-10T09:59:08.979Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T09:59:08.979Z" - }, - { - "id": "bug-378", - "timestamp": "2026-06-10T10:02:00.000Z", - "error_message": "AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Tests 8 and 9 declared ep_a/ep_b/g_ep9_a/g_ep9_b as 'static int' (4 bytes) and passed them to repl_add_standby without (void*) cast. repl_broadcast then called xce_wal_batch_req_raw(s->ep, ...) using the stored int* as a real network endpoint, causing an 8-byte read past the 4-byte int. Neither test installed repl_broadcast_send_hook to intercept the call.", - "fix": "Added #include typemap.h; added extern repl_broadcast_send_hook declaration guarded by XCE_TESTING; added p2_drop_send_hook no-op; installed hook around persist_wal_sync_append in tests 8 and 9; fixed missing (void*) casts on all four ep variables", - "tags": [ - "asan", - "global-buffer-overflow", - "repl_broadcast", - "unit-test", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T10:02:00.000Z" - }, - { - "id": "bug-379", - "timestamp": "2026-06-10T10:36:53.011Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 50\u219272 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T10:36:53.011Z" - }, - { - "id": "bug-380", - "timestamp": "2026-06-10T10:37:21.924Z", - "error_message": "Wrong reference: test_catchup_full_lifecycle_integration should be test_single_node_committed_lsn_unchanged", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Used \"test_catchup_full_lifecycle_integration\" instead of \"test_single_node_committed_lsn_unchanged\"", - "fix": "Changed test_catchup_full_lifecycle_integration \u2192 test_single_node_committed_lsn_unchanged", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T10:37:21.924Z" - }, - { - "id": "bug-381", - "timestamp": "2026-06-10T10:37:49.028Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T10:37:49.028Z" - }, - { - "id": "bug-382", - "timestamp": "2026-06-10T11:54:19.749Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-wal-replication/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T11:54:19.749Z" - }, - { - "id": "bug-383", - "timestamp": "2026-06-10T14:12:27.285Z", - "error_message": "Significant refactor of ", - "file": ".kiro/steering/product.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 13\u219222 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T14:12:27.286Z" - }, - { - "id": "bug-384", - "timestamp": "2026-06-10T14:13:01.581Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/tech.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T14:13:01.581Z" - }, - { - "id": "bug-385", - "timestamp": "2026-06-10T14:38:11.810Z", - "error_message": "Significant refactor of ", - "file": "CLAUDE.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 5\u21925 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T14:38:11.810Z" - }, - { - "id": "bug-386", - "timestamp": "2026-06-10T14:43:53.161Z", - "error_message": "Function not marked async", - "file": "CLAUDE.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-10T14:43:53.161Z" - }, - { - "id": "bug-387", - "timestamp": "2026-06-10T14:53:19.493Z", - "error_message": "Significant refactor of ", - "file": "src/controller/CLAUDE.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21925 lines (3 removed) | Also: ### Controllers; - Struct: `type {Resource}Controller struct{}`", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-10T14:55:18.463Z" - }, - { - "id": "bug-388", - "timestamp": "2026-06-10T14:54:18.503Z", - "error_message": "Function not marked async", - "file": "src/controller/CLAUDE.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-10T14:54:53.661Z" - }, - { - "id": "bug-389", - "timestamp": "2026-06-11T02:46:08.749Z", - "error_message": "Wrong reference: signal should be stdio", - "file": "src/lib/delay.c", - "root_cause": "Used \"signal\" instead of \"stdio\"", - "fix": "Changed signal \u2192 stdio", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T02:46:08.749Z" - }, - { - "id": "bug-390", - "timestamp": "2026-06-11T03:30:54.341Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T03:30:54.341Z" - }, - { - "id": "bug-391", - "timestamp": "2026-06-11T03:41:14.912Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T03:41:14.912Z" - }, - { - "id": "bug-392", - "timestamp": "2026-06-11T03:46:02.642Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: sync_rc_out", - "fix": "Added guard clause: if (sync_rc_out)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T03:46:02.642Z" - }, - { - "id": "bug-393", - "timestamp": "2026-06-11T03:47:03.826Z", - "error_message": "Significant refactor of ", - "file": "src/controller/job.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 15\u219228 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T03:47:03.826Z" - }, - { - "id": "bug-394", - "timestamp": "2026-06-11T05:09:37.037Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had `persist_truncate_wal_above`", - "fix": "Changed to `restore_job`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T05:09:37.037Z" - }, - { - "id": "bug-395", - "timestamp": "2026-06-11T05:10:14.835Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u21927 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T05:10:14.835Z" - }, - { - "id": "bug-396", - "timestamp": "2026-06-10T22:45:00-07:00", - "error_message": "Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch gate rejects them (quorum loss / COMMITTING window): worker never reads replies, never retries, no rejoin resync", - "file": "src/worker/src/controller_connector.c, src/lib/xceapi.c, src/controller/cluster.c", - "root_cause": "xce_notify is enqueue-and-forget (no cbuf_get); error replies dropped via cbuf_tryput -EAGAIN (xcectx.c:304); xcecd_job_finished return ignored (FIXME at xcecd_worker_utils.c:298); xce_join_req carries no running-job state so rejoin cannot resync", - "fix": "FIXED 2026-06-12 (ha-commit-barrier-rollout tasks 1.2+4.2+4.3): dispatch gate admits fact opcodes (notify/update/setpreason/clrpreason) without write quorum (ha.c CTRL_PRIMARY allowlist); facts append via RETAIN (log-is-the-queue, no timeout) and commit when quorum returns. End-to-end retention proof: test_fact_retention_across_quorum_loss. Worker-side redelivery gap remains (worker-rejoin-resync backlog covers disconnect windows).", - "tags": [ - "ha", - "notify", - "silent-data-loss", - "worker", - "quorum", - "ha-consistency-invariants" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12" - }, - { - "id": "bug-397", - "timestamp": "2026-06-11T06:34:28.705Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-consistency-invariants/requirements.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T06:34:28.705Z" - }, - { - "id": "bug-398", - "timestamp": "2026-06-11T06:35:26.288Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had `restore_job`", - "fix": "Changed to \"irrevocable\"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T06:35:26.288Z" - }, - { - "id": "bug-399", - "timestamp": "2026-06-11T06:43:14.709Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had 't span syscalls and degrade to serial mode), C ec", - "fix": "Changed to 't span syscalls), C ecosystem dead (libitm unmain", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T06:43:14.709Z" - }, - { - "id": "bug-400", - "timestamp": "2026-06-11T07:23:53.901Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had \"return quickly\"", - "fix": "Changed to `--vcpus`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T07:23:53.901Z" - }, - { - "id": "bug-401", - "timestamp": "2026-06-11T07:47:34.756Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Had `mark_job_*`", - "fix": "Changed to \"pending facts\"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T07:47:34.756Z" - }, - { - "id": "bug-402", - "timestamp": "2026-06-11T07:53:43.868Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-consistency-invariants/research.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T07:53:43.868Z" - }, - { - "id": "bug-403", - "timestamp": "2026-06-11T08:34:47.868Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-consistency-invariants/design.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T08:34:47.868Z" - }, - { - "id": "bug-404", - "timestamp": "2026-06-11T08:46:20.815Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T08:46:20.815Z" - }, - { - "id": "bug-405", - "timestamp": "2026-06-11T09:47:16.750Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-single-leader-role/design.md", - "root_cause": "Had `writable_lsn`", - "fix": "Changed to 's \"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T09:47:16.750Z" - }, - { - "id": "bug-406", - "timestamp": "2026-06-11T09:49:04.296Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-single-leader-role/tasks.md", - "root_cause": "Had \" --exclude-dir=.git --exclude-dir=.kiro --exclude", - "fix": "Changed to \" --exclude-dir=.git --exclude-dir=specs --exclude", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T09:49:04.296Z" - }, - { - "id": "bug-407", - "timestamp": "2026-06-11T10:01:15.181Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-barrier-rollout/design.md", - "root_cause": "Had `repl.h`", - "fix": "Changed to `entry_lsn`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T10:01:15.181Z" - }, - { - "id": "bug-408", - "timestamp": "2026-06-11T10:01:39.424Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-convergence-recovery/tasks.md", - "root_cause": "Had `-XCE_ERR_NOT_READY`", - "fix": "Changed to `OPCode_notify`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-11T10:02:02.349Z" - }, - { - "id": "bug-409", - "timestamp": "2026-06-11T12:57:17.283Z", - "error_message": "Significant refactor of ", - "file": ".kiro/steering/roadmap.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 6\u21928 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T12:57:17.283Z" - }, - { - "id": "bug-410", - "timestamp": "2026-06-11T13:30:00Z", - "error_message": "test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "repl_add_standby returns the slot index (>=0) on success, not 0 \u2014 second add legitimately returns 1", - "fix": "fixture asserts TEST_ASSERT_EQUAL_INT(i, repl_add_standby(...)) per standby slot", - "tags": [ - "controller", - "testsuite", - "repl", - "return-convention" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T13:30:00Z" - }, - { - "id": "bug-411", - "timestamp": "2026-06-11T13:35:00Z", - "error_message": "3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013)", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters \u2014 main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c)", - "fix": "not fixed (out of task 1.1 boundary) \u2014 documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2)", - "tags": [ - "controller", - "testsuite", - "unity", - "exit-code", - "pre-existing", - "ha" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T13:35:00Z" - }, - { - "id": "bug-412", - "timestamp": "2026-06-11T14:59:30.999Z", - "error_message": "Significant refactor of ", - "file": "src/controller/commit.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 11\u219221 lines (2 removed) | Also: (void)timeout_ms; /* RETAIN never honors the timeo; * via commit_on_ack (Req 2.1). */", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-11T15:00:10.118Z" - }, - { - "id": "bug-413", - "timestamp": "2026-06-11T15:28:35.990Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 3.1 \u2014 membership re-evaluation (commit_on_membership_ | Also: /* Task 3.1 \u2014 membership re-evaluation */; RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pend", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-11T15:29:25.430Z" - }, - { - "id": "bug-414", - "timestamp": "2026-06-11T15:55:52.451Z", - "error_message": "Missing error handling in unknown", - "file": "src/include/repl.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T15:55:52.451Z" - }, - { - "id": "bug-415", - "timestamp": "2026-06-11T15:56:20.945Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T15:56:20.945Z" - }, - { - "id": "bug-416", - "timestamp": "2026-06-11T15:56:38.428Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219215 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T15:56:38.428Z" - }, - { - "id": "bug-417", - "timestamp": "2026-06-11T15:57:43.900Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T15:57:43.900Z" - }, - { - "id": "bug-418", - "timestamp": "2026-06-11T16:15:06.660Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-engine/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T16:15:06.660Z" - }, - { - "id": "bug-419", - "timestamp": "2026-06-11T16:19:30.370Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T16:19:30.370Z" - }, - { - "id": "bug-420", - "timestamp": "2026-06-11T16:19:51.195Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 4.2 \u2014 wired gate: registry truth at ha_dispatch_gate; RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_th", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-11T16:20:05.714Z" - }, - { - "id": "bug-421", - "timestamp": "2026-06-11T16:23:46.118Z", - "error_message": "Missing guard clause", - "file": "src/controller/repl.c", - "root_cause": "No early return/throw for edge case: !r", - "fix": "Added guard clause: if (!r)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T16:23:46.118Z" - }, - { - "id": "bug-422", - "timestamp": "2026-06-11T16:23:59.453Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T16:23:59.453Z" - }, - { - "id": "bug-423", - "timestamp": "2026-06-11T22:39:24.704Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 5.1 \u2014 integration: ACK feed and the job-submit sync-a | Also: #include \"typemap.h\" /* wal_catchup_entry_t (task 5; #include \"wal_types.h\" /* WAL_JOB_SUBMIT (task 5.1 sy", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-11T22:40:45.169Z" - }, - { - "id": "bug-424", - "timestamp": "2026-06-11T23:08:49.909Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-engine/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:08:49.909Z" - }, - { - "id": "bug-425", - "timestamp": "2026-06-11T23:15:25.014Z", - "error_message": "Wrong return value", - "file": "src/controller/repl.c", - "root_cause": "Was returning: idx;", - "fix": "Now returns: * above mutates nothing, so it deliberately skips ", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:15:25.014Z" - }, - { - "id": "bug-426", - "timestamp": "2026-06-11T23:16:25.862Z", - "error_message": "Significant refactor of ", - "file": "src/controller/repl.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 13\u219227 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:16:25.862Z" - }, - { - "id": "bug-427", - "timestamp": "2026-06-11T23:32:42.365Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-engine/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:32:42.365Z" - }, - { - "id": "bug-428", - "timestamp": "2026-06-11T23:43:45.216Z", - "error_message": "Incorrect value in code", - "file": "src/include/xce/cluster_def.h", - "root_cause": "Had \"ha.h\"", - "fix": "Changed to \"commit.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:43:45.216Z" - }, - { - "id": "bug-429", - "timestamp": "2026-06-11T23:44:16.357Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (ha_is_primary(&c->ha)", - "fix": "Changed to: if (c->commit.p == NULL)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:44:16.357Z" - }, - { - "id": "bug-430", - "timestamp": "2026-06-11T23:45:06.483Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "16 lines replaced/restructured", - "fix": "Rewrote 40\u219267 lines (16 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:45:06.483Z" - }, - { - "id": "bug-431", - "timestamp": "2026-06-11T23:45:16.518Z", - "error_message": "Incorrect value in code", - "file": "src/controller/persist.c", - "root_cause": "Had \"xce/types.h\"", - "fix": "Changed to \"xce/errnox.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:45:16.518Z" - }, - { - "id": "bug-432", - "timestamp": "2026-06-11T23:45:42.031Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:45:42.031Z" - }, - { - "id": "bug-433", - "timestamp": "2026-06-11T23:45:51.649Z", - "error_message": "Wrong reference: test_single_node_committed_lsn_unchanged should be test_single_node_committed_lsn_advances", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Used \"test_single_node_committed_lsn_unchanged\" instead of \"test_single_node_committed_lsn_advances\"", - "fix": "Changed test_single_node_committed_lsn_unchanged \u2192 test_single_node_committed_lsn_advances", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-11T23:45:51.649Z" - }, - { - "id": "bug-434", - "timestamp": "2026-06-12T00:23:08.815Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T00:23:08.815Z" - }, - { - "id": "bug-435", - "timestamp": "2026-06-11T00:00:00Z", - "error_message": "test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by bug-411)", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "Stale test expectation: pinned the old accept-on-timeout contract (rc==0 + sync_rc=-ETIMEDOUT advisory). ha-commit-engine made persist_wal_sync_append reject-on-timeout (returns -ETIMEDOUT, Req 2.1/4.4). Fixture was also engine-less (deprecated ce==NULL legacy path).", - "fix": "Updated expectation to rc==-ETIMEDOUT and wired a commit engine (n_peers=3) with a priming async append so the sync entry lands at LSN 1 on the engine path; kept the committed_lsn-unmoved assertion.", - "tags": [ - "ha-commit-engine", - "task-6.1", - "stale-expectation", - "unity-misattribution" - ], - "related_bugs": [ - "bug-411" - ], - "occurrences": 1, - "last_seen": "2026-06-11T00:00:00Z" - }, - { - "id": "bug-436", - "timestamp": "2026-06-11T00:00:00Z", - "error_message": "test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected 0 Was -10013 (NOT_READY; baseline failures, masked by bug-411)", - "file": "src/controller/testsuite/test_ha_repl_ack.c, src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "CTRL_PRIMARY dispatch gate now requires write quorum (>= n_peers/2 connected standbys). Fixtures promote via a standby ACK but never register the connected standby; legacy fallback counts ha->standby_endpoints (NULL -> 0). In the rdhup test ha_on_primary_ctrl_fd_rdhup also overwrites ha->n_peers=1 with candidates_count=2 (ha.c:3484).", - "fix": "Populated standby_endpoints with one endpoint (the ACKing standby) in both fixtures so quorum legitimately holds on a healthy primary; gate unchanged. Freed the zarray before ha_fini (ha_fini does not own it).", - "tags": [ - "ha-commit-engine", - "task-6.1", - "quorum-gate", - "fixture-fix" - ], - "related_bugs": [ - "bug-411" - ], - "occurrences": 1, - "last_seen": "2026-06-11T00:00:00Z" - }, - { - "id": "bug-437", - "timestamp": "2026-06-12T00:51:52.673Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-engine/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T00:51:52.673Z" - }, - { - "id": "bug-438", - "timestamp": "2026-06-12T03:52:57.142Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_commit_barrier_rollout.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T03:52:57.142Z" - }, - { - "id": "bug-439", - "timestamp": "2026-06-12T04:13:06.326Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-barrier-rollout/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T04:13:06.326Z" - }, - { - "id": "bug-440", - "timestamp": "2026-06-12T04:18:19.158Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: detail == HOOK_DETAIL_WORKER_JOIN ||\n detail == H", - "fix": "Added guard clause: if (detail == HOOK_DETAIL_WORKER_JOIN ||\n )", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T04:18:19.158Z" - }, - { - "id": "bug-441", - "timestamp": "2026-06-12T04:18:31.067Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219212 lines (4 removed) | Also: return persist_wal_async(p, wal_type, &js, sizeof(; persist_t* p = cls; | Also: int rc = persist_wal_async(p, wal_type, pkt, (uint; persist_t* p = cls;", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-12T04:19:00.652Z" - }, - { - "id": "bug-442", - "timestamp": "2026-06-12T04:50:32.154Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_commit_barrier_rollout.c", - "root_cause": "Had \"xce/types.h\"", - "fix": "Changed to \"xce/errnox.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T04:50:32.154Z" - }, - { - "id": "bug-443", - "timestamp": "2026-06-12T05:14:36.392Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p == NULL", - "fix": "Added guard clause: if (p == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T05:14:36.392Z" - }, - { - "id": "bug-444", - "timestamp": "2026-06-12T09:09:04.495Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21925 lines (2 removed) | Also: static xce_signal_t g_sigkill = { .signo = SIGKIL; static xce_jobctrl_t g_jobctrl;", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-12T09:09:10.703Z" - }, - { - "id": "bug-445", - "timestamp": "2026-06-12T09:30:00Z", - "error_message": "AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t test_commit_barrier_reply standalone", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "Global xstring table uninitialised in standalone runs: makeString() returns NULL, resobj name becomes NULL, key_add hashes NULL. In full-suite runs test_worker_join.c:520 initialises the table unguarded, so the bug only appears with -t selection.", - "fix": "Probe makeString(\"cbrp-probe\") in the group entry; call xstring_table_init(139) only when the probe returns NULL (avoids double init in full-suite order).", - "tags": [ - "testsuite", - "xstring", - "fixture", - "segv", - "ha-commit-barrier-rollout" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T09:30:00Z" - }, - { - "id": "bug-446", - "timestamp": "2026-06-12T09:30:00Z", - "error_message": "makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "job_init (lib/jobj.c) returns -EINVAL when xce_job_t.submit == NULL, and makeJobsub requires job_obj_init() module tables which no controller test group initialises.", - "fix": "Fixture calls job_obj_init(64, 64) once per process (static guard) and provides a zeroed static job_submit_data_t as jdef.submit.", - "tags": [ - "testsuite", - "jobj", - "fixture", - "ha-commit-barrier-rollout" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T09:30:00Z" - }, - { - "id": "bug-447", - "timestamp": "2026-06-12T09:30:00Z", - "error_message": "queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1 (key_find found-rc leak)", - "file": "src/controller/queue.c", - "root_cause": "key_find (xkvmgr_find, lib/kvmgr.c:130-151) returns 1 when the key is found. queue_ctrl (queue.c:60), worker_ctrl (worker.c:643) and job_ctrl (job.c:314) store it in rc and never reset it: the 'if (rc == 0)' AFTER_EVENT hook gate is skipped (no WAL_QUEUE_CTRL / WAL_WORKER_CTRL append on the found path) and the requester reply carries status 1 instead of 0. Production gap discovered by task-3.1 RED tests; queue/host control state changes are currently never persisted or replicated.", - "fix": "FIXED (task 3.2, 2026-06-12): normalized the key_find found-rc to 0 in queue_ctrl (queue.c), worker_ctrl (worker.c) and job_ctrl (job.c) after the miss/-ENOENT checks, so the AFTER_EVENT hook gate fires (WAL append + replication) and the handlers' commit-barrier pattern sees 0. Handlers do_jobcontrol_op/do_queuecontrol_op/do_hostcontrol_op now run persist_commit_barrier(&c->persist, 0) on rc==0 before replying; handle_worker_join barriers between AFTER_EVENT and the success reply. All 12 test_commit_barrier_reply tests PASS; full controller suite 0 FAIL.", - "tags": [ - "production-gap", - "queue_ctrl", - "worker_ctrl", - "job_ctrl", - "key_find", - "wal", - "ha-commit-barrier-rollout", - "red-finding" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T11:00:00Z" - }, - { - "id": "bug-448", - "timestamp": "2026-06-12T10:05:12.464Z", - "error_message": "Incorrect value in code", - "file": "src/controller/queue.c", - "root_cause": "Had \"qobj.h\"", - "fix": "Changed to \"persist.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T10:05:12.464Z" - }, - { - "id": "bug-448", - "timestamp": "2026-06-12T10:30:00-07:00", - "error_message": "job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446) never calls dereference(j); only the workermgr-miss error path releases it", - "file": "src/controller/job.c", - "root_cause": "asymmetric reference/dereference across exit paths, predates ha-commit-barrier-rollout", - "fix": "NOT YET FIXED - discovered by task 3.2 adversarial review; add dereference(j) on the normal exit path", - "tags": [ - "controller", - "job", - "refcount", - "leak", - "pre-existing" - ], - "related_bugs": [ - "bug-447" - ], - "occurrences": 1, - "last_seen": "2026-06-12T10:30:00-07:00" - }, - { - "id": "bug-450", - "timestamp": "2026-06-12T10:48:27.053Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 12\u219256 lines (7 removed) | Also: xce_job_t jdef;; static int jobobj_ready; /* jobj module tables \u2014 o", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-12T10:48:47.855Z" - }, - { - "id": "bug-451", - "timestamp": "2026-06-12T11:05:00-07:00", - "error_message": "va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*", - "file": "src/controller/job.c (caller) / src/controller/persist.c (consumer)", - "root_cause": "update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) \u2014 an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident)", - "fix": "NOT FIXED (discovered while writing task 3.3 receipt-ACK verdict pins; production out of boundary). Test test_update_op_rusage_receipt_reply_facts_retained uses a padded union so the pin stays in-bounds under ASAN without masking the bug; fix candidates: dedicated RUSAGE arm in persist_job_hook_fun reading xce_job_rusages_t*, or fire the hook with (c, j) like the other job hooks", - "tags": [ - "pre-existing", - "type-confusion", - "va_list", - "wal", - "rusage", - "asan", - "oob-read", - "ha-commit-barrier-rollout" - ], - "related_bugs": [ - "bug-447", - "bug-448" - ], - "occurrences": 1, - "last_seen": "2026-06-12T11:05:00-07:00" - }, - { - "id": "bug-452", - "timestamp": "2026-06-12T15:03:38.962Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-12T15:03:38.962Z" - }, - { - "id": "bug-453", - "timestamp": "2026-06-12T15:45:11.299Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_repl_majority.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u219210 lines (3 removed) | Also: * walAck half is engine-wired per-LSN; the configA; * byte-identical legacy behavior (config barrier o | Also: /* upgradeConfigAck with no pending ACKs \u2192 timeout; int rc_cfg = repl_wait_majority_ack(&g_mp.repl, 0,", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-12T15:45:39.679Z" - }, - { - "id": "bug-454", - "timestamp": "2026-06-13T00:15:40.802Z", - "error_message": "Significant refactor of ", - "file": "src/controller/repl.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 6\u219211 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T00:15:40.802Z" - }, - { - "id": "bug-455", - "timestamp": "2026-06-13T00:16:42.277Z", - "error_message": "Significant refactor of ", - "file": "src/include/repl.h", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u21927 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T00:16:42.277Z" - }, - { - "id": "bug-456", - "timestamp": "2026-06-13T00:17:27.276Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "28 lines replaced/restructured", - "fix": "Rewrote 73\u219285 lines (28 removed) | Also: /* 7. Push to standbys (best-effort; see push_conf; int acked = push_config_to_standbys(c, out_bytes, | Also: * 7. Push to standbys (best-effort); * 8. ACK admin tool", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-13T00:19:11.078Z" - }, - { - "id": "bug-457", - "timestamp": "2026-06-13T00:18:17.080Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c->ha.standby_endpoints", - "fix": "Added guard clause: if (!c->ha.standby_endpoints)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T00:18:17.080Z" - }, - { - "id": "bug-458", - "timestamp": "2026-06-13T00:19:39.908Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T00:19:39.908Z" - }, - { - "id": "bug-459", - "timestamp": "2026-06-13T00:30:00Z", - "error_message": "configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack), denominator n_active with no self-count, and n_active==0 short-circuited to success; upgradeConfig also ACKed the admin tool unconditionally and requested restart even when the majority wait timed out (logged-only result)", - "file": "src/controller/repl.c, src/controller/config_repl.c", - "root_cause": "legacy connectivity-based counting on the configAck track predated the commit engine's per-push arithmetic; push_config_to_standbys returned the transport push count and only logged the barrier outcome, so do_upgrade_config_with_bytes could not observe failure", - "fix": "task 5.2: configAck branch counts acked_config_lsn >= push_tag (entry_lsn carries the tag), majority 1+count > n_peers/2 over engine n_peers, n_peers<=1 immediate 0, ce==NULL -EINVAL fail-loud; push_config_to_standbys returns the barrier result (push count via acked_out); do_upgrade_config_with_bytes sends send_upgrade_error and skips success ACK + restart on non-zero; committed_config_tag advance stays success-branch-only", - "tags": [ - "ha", - "config-repl", - "quorum", - "configAck", - "commit-barrier", - "upgradeConfig" - ], - "related_bugs": [ - "bug-396" - ], - "occurrences": 1, - "last_seen": "2026-06-13T00:30:00Z" - }, - { - "id": "bug-460", - "timestamp": "2026-06-13T01:11:40.159Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 14\u219219 lines (7 removed) | Also: cluster_t *c, const char *msg); reply.status = -EINVAL;", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T01:14:42.432Z" - }, - { - "id": "bug-461", - "timestamp": "2026-06-13T01:11:49.673Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/config_repl.c", - "root_cause": "Condition was: if (n == 0)", - "fix": "Changed to: if (n_out)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T01:11:49.673Z" - }, - { - "id": "bug-462", - "timestamp": "2026-06-13T01:12:50.297Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.h", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 22\u219239 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T01:12:50.297Z" - }, - { - "id": "bug-463", - "timestamp": "2026-06-13T01:13:19.561Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 10\u219218 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T01:13:19.561Z" - }, - { - "id": "bug-464", - "timestamp": "2026-06-13T01:14:15.668Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/join.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T01:14:15.668Z" - }, - { - "id": "bug-465", - "timestamp": "2026-06-13T01:30:00Z", - "error_message": "Config-ACK signal never produced in production: repl_on_config_ack had zero production callers; standby replconfigpushreplymsg silently dropped in repl_ack_decode_and_dispatch (ha.c) and myreq_handler default case (cluster.c) -> every multi-member config upgrade barrier timed out (upgrades bricked)", - "file": "src/controller/ha.c", - "root_cause": "ha-config-bootstrap task 8.1's production ingestion half was never landed; the 5.2 barrier relied on a signal nothing produced (tests injected repl_on_config_ack directly)", - "fix": "Added replconfigpushreplymsg decode branch in repl_ack_decode_and_dispatch (ha.c) and an OPCode_reply ingestion case in cluster.c myreq_handler, both calling repl_on_config_ack(&c->persist.repl, ep, versionTag); wire-path integration test added (test_config_ack_wire_path_completes_barrier)", - "tags": [ - "ha", - "config-repl", - "barrier", - "capnp", - "wire-path", - "task-5.2" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T01:30:00Z" - }, - { - "id": "bug-466", - "timestamp": "2026-06-13T01:30:00Z", - "error_message": "Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll thread (ha.c raft_cb_become_leader, ha_dispatch_repl_action_for_standby, legacy election path; join.c controller join) -> blocks all controller I/O; the awaited ACK arrives on the blocked thread (structural deadlock)", - "file": "src/controller/config_repl.c", - "root_cause": "task 5.2 made the barrier unconditional inside push_config_to_standbys while best-effort catch-up/election pushes on iomgr-reachable paths still called the same entry", - "fix": "Split the API: exported transport-only fan-out push_config_transport_only (no barrier, iomgr-safe) and switched all 5 ha.c sites + join.c:187 to it; push_config_to_standbys (transport+barrier) kept for the vtask-context upgrade path only; doc comments updated in config_repl.h", - "tags": [ - "ha", - "iomgr", - "deadlock", - "config-repl", - "api-split", - "task-5.2" - ], - "related_bugs": [ - "bug-465" - ], - "occurrences": 1, - "last_seen": "2026-06-13T01:30:00Z" - }, - { - "id": "bug-467", - "timestamp": "2026-06-13T01:38:08.308Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (rep_s.data_which == RequestReply_data_replconfigpu)", - "fix": "Changed to: if (0 /* REVIEWER RED CHECK */ && rep_s.data_which == )", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T01:38:43.310Z" - }, - { - "id": "bug-468", - "timestamp": "2026-06-13T03:56:09.185Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T03:56:09.185Z" - }, - { - "id": "bug-469", - "timestamp": "2026-06-13T06:33:41.449Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_ha_cluster.c'", - "fix": "Changed to 'testsuite/test_ha_role_lifecycle.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T06:33:41.449Z" - }, - { - "id": "bug-470", - "timestamp": "2026-06-13T06:34:01.929Z", - "error_message": "Wrong reference: test_ha_cluster should be test_ha_role_lifecycle", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_ha_cluster\" instead of \"test_ha_role_lifecycle\"", - "fix": "Changed test_ha_cluster \u2192 test_ha_role_lifecycle", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T06:34:05.212Z" - }, - { - "id": "bug-471", - "timestamp": "2026-06-13T06:41:01.532Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "No early return/throw for edge case: pid < 0", - "fix": "Added guard clause: if (pid < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T06:41:01.532Z" - }, - { - "id": "bug-472", - "timestamp": "2026-06-13T07:06:06.526Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "15 lines replaced/restructured", - "fix": "Rewrote 64\u2192108 lines (15 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T07:06:06.526Z" - }, - { - "id": "bug-473", - "timestamp": "2026-06-13T07:32:43.287Z", - "error_message": "Wrong return value", - "file": "src/controller/persist.c", - "root_cause": "Was returning: WAL_MODE_LOCAL;", - "fix": "Now returns: WAL_MODE_RETAIN;", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T07:32:43.287Z" - }, - { - "id": "bug-474", - "timestamp": "2026-06-13T08:17:21.222Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "21 lines replaced/restructured", - "fix": "Rewrote 35\u219257 lines (21 removed) | Also: /* \u2500\u2500 Case 3: HOOKS REGISTERED BY THE SHARED PROMO; * canonical promotion entry point ha_promote_to_pr", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T08:17:47.506Z" - }, - { - "id": "bug-475", - "timestamp": "2026-06-13T08:35:38.780Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 10\u219214 lines (5 removed) | Also: /* Start primary heartbeat immediately in COMMITTI; * NOP during the committing window and their liven", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T08:36:06.318Z" - }, - { - "id": "bug-476", - "timestamp": "2026-06-13T08:38:00.471Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: c->ha.hooks_registered", - "fix": "Added guard clause: if (c->ha.hooks_registered)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T08:38:00.471Z" - }, - { - "error_message": "AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async <- ha_arm_own_term_gate <- ha_promote_to_primary, during test_promotion_registers_persist_hooks (test_ha_role_lifecycle.c case 3)", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", - "root_cause": "task-2.2's ha_promote_to_primary now appends a WAL_NOOP via persist_wal_async, which broadcasts to registered standbys. The lifecycle fixture (lifecycle_cluster_setup) registers fake `int g_lc_eps[]` endpoints via repl_add_standby and NEVER installs repl_broadcast_send_hook, so repl_broadcast calls xce_wal_batch_req_raw(s->ep) and reads ep->peer (offset 36) past the 4-byte int. IDENTICAL to the prior test_p2_integration tests 8/9 bug already in buglog.", - "fix": "Test-side fix (parent must apply, mirrors the documented p2_integration fix): in test_ha_role_lifecycle.c add `extern int (*repl_broadcast_send_hook)(void*, wal_catchup_entry_t**, int);` and a no-op drop hook, install it in lifecycle_cluster_setup before promotion (reset to NULL in teardown), and cast endpoints to (void*). Production ha.c is correct per design (persist_wal_async appends+broadcasts by design).", - "tags": [ - "ha-single-leader-role", - "task-2.2", - "test-fixture", - "asan", - "repl_broadcast", - "global-buffer-overflow", - "out-of-boundary" - ], - "id": "bug-477", - "timestamp": "2026-06-13", - "last_seen": "2026-06-13", - "occurrences": 1, - "related_bugs": [] - }, - { - "error_message": "task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and current_lsn==noop_lsn+1, but lifecycle_cluster_setup primes current_lsn = appended_lsn + 1, so a real persist_wal_async appends the noop at appended_lsn+1 (current_lsn->appended_lsn+2), giving noop_lsn=appended_lsn+1 != asserted appended_lsn.", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", - "root_cause": "persist_wal_async assigns hdr.lsn = current_lsn++ (entry takes the CURRENT value). The fixture comment assumes the next append takes current_lsn-1, which is off by one. To make the noop land at `appended_lsn`, the prime must be `current_lsn = appended_lsn`, not `appended_lsn + 1`.", - "fix": "Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design \u00a7Promotion Rewrite / brief.", - "tags": [ - "ha-single-leader-role", - "task-2.2", - "test-fixture", - "off-by-one", - "lsn", - "out-of-boundary" - ], - "id": "bug-478", - "timestamp": "2026-06-13", - "last_seen": "2026-06-13", - "occurrences": 1, - "related_bugs": [] - }, - { - "id": "bug-479", - "timestamp": "2026-06-13T08:56:36.044Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T08:56:36.044Z" - }, - { - "id": "bug-480", - "timestamp": "2026-06-13T08:57:05.687Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T08:57:05.687Z" - }, - { - "id": "bug-481", - "timestamp": "2026-06-13T08:57:15.674Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T08:57:15.674Z" - }, - { - "id": "bug-482", - "timestamp": "2026-06-13T09:36:17.806Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_start_election_async.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T09:36:17.806Z" - }, - { - "id": "bug-483", - "timestamp": "2026-06-13T09:45:01.531Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T09:45:01.531Z" - }, - { - "id": "bug-484", - "timestamp": "2026-06-13T10:16:06.331Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-single-leader-role/tasks.md", - "root_cause": "Had `lifecycle_cluster_setup`", - "fix": "Changed to `ha_promote_to_primary`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T10:16:06.331Z" - }, - { - "id": "bug-485", - "timestamp": "2026-06-13T10:19:08.844Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 16\u21924 lines (11 removed) | Also: /* Task 5.2: After updating convergence tracking, ; * majority of standbys have converged and promote ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T10:19:43.629Z" - }, - { - "id": "bug-486", - "timestamp": "2026-06-13T10:22:20.811Z", - "error_message": "Significant refactor of ", - "file": "src/controller/meson.build", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21923 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T10:22:20.811Z" - }, - { - "id": "bug-487", - "timestamp": "2026-06-13T10:22:24.129Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21923 lines (2 removed) | Also: TEST(test_ha_evaluate_committing);; TEST(test_ha_repl_ack);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T10:22:41.219Z" - }, - { - "id": "bug-488", - "timestamp": "2026-06-13T10:25:04.643Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T10:25:04.643Z" - }, - { - "id": "bug-489", - "timestamp": "2026-06-13T11:41:32.142Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "196 lines replaced/restructured", - "fix": "Rewrote 230\u219212 lines (196 removed) | Also: /* \u2500\u2500 Task 5.3: committing timeout default value a; * Requirements: 6.4, 7.5 (ha-repl-hardening)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T11:42:05.841Z" - }, - { - "id": "bug-490", - "timestamp": "2026-06-13T11:42:51.597Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "34 lines replaced/restructured", - "fix": "Rewrote 48\u21929 lines (34 removed) | Also: * Test 4: 3-node cluster RDHUP \u2014 two acks needed, ; * | Also: /* Drive to CTRL_PRIMARY via ack */; ha_repl_ack_received(&c.ha, &c); | Also: RUN_TEST(test_rdhup_committing_to_primary_on_ack);; RUN_TEST(test_rdhup_3node_one_ack_promotes);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-13T11:43:47.811Z" - }, - { - "id": "bug-491", - "timestamp": "2026-06-13T11:44:14.184Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "26 lines replaced/restructured", - "fix": "Rewrote 35\u21928 lines (26 removed) | Also: /* \u2500\u2500 Test 10 (Task 7.3): promoted connection carr; *", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T11:44:45.523Z" - }, - { - "id": "bug-492", - "timestamp": "2026-06-13T12:04:59.483Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-single-leader-role/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T12:04:59.483Z" - }, - { - "id": "bug-493", - "timestamp": "2026-06-13T12:08:04.621Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (role == CTRL_PRIMARY || role == CTRL_PRIMARY_COMMI)", - "fix": "Changed to: if (role == CTRL_PRIMARY)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-13T12:09:17.994Z" - }, - { - "id": "bug-494", - "timestamp": "2026-06-13T12:10:08.938Z", - "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", - "file": "src/controller/ha.c", - "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", - "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T12:10:08.938Z" - }, - { - "id": "bug-495", - "timestamp": "2026-06-13T13:04:41.816Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "21 lines replaced/restructured", - "fix": "Rewrote 24\u21925 lines (21 removed) | Also: /* Task 5.3: committing timeout config-default kno; * survives until task 5.2; the degraded-mode commi", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T13:05:15.532Z" - }, - { - "id": "bug-496", - "timestamp": "2026-06-13T13:07:42.791Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Had \"winning election must land DIRECTLY in CTRL_PRIMA", - "fix": "Changed to \"winning election must land DIRECTLY in CTRL_PRIMA", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:07:42.791Z" - }, - { - "id": "bug-497", - "timestamp": "2026-06-13T13:10:05.452Z", - "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", - "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:10:05.452Z" - }, - { - "id": "bug-498", - "timestamp": "2026-06-13T13:10:16.164Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_connection_reuse.c", - "root_cause": "Had \"err_handler: ep must be removed from standby_endp", - "fix": "Changed to \"err_handler: ep must be removed from standby_endp", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:10:16.164Z" - }, - { - "id": "bug-499", - "timestamp": "2026-06-13T13:14:03.917Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:14:03.917Z" - }, - { - "id": "bug-500", - "timestamp": "2026-06-13T13:14:11.496Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Had \"test_dispatch_gate_committing_allows_repl_ops: PA", - "fix": "Changed to \"test_dispatch_gate_primary_no_quorum_allows_repl_", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:14:11.496Z" - }, - { - "id": "bug-501", - "timestamp": "2026-06-13T13:15:33.777Z", - "error_message": "Wrong reference: test_nonblocking_single_node_election_enters_committing should be test_nonblocking_single_node_election_enters_primary_directly", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Used \"test_nonblocking_single_node_election_enters_committing\" instead of \"test_nonblocking_single_node_election_enters_primary_directly\"", - "fix": "Changed test_nonblocking_single_node_election_enters_committing \u2192 test_nonblocking_single_node_election_enters_primary_directly", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:15:33.777Z" - }, - { - "id": "bug-502", - "timestamp": "2026-06-13T13:16:26.379Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:16:26.379Z" - }, - { - "id": "bug-503", - "timestamp": "2026-06-13T13:30:00.000Z", - "error_message": "test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed, expected -XCE_ERR_NOT_READY)", - "file": "src/controller/testsuite/test_ha_startup.c", - "root_cause": "Retargeting the dispatch-gate test from CTRL_PRIMARY_COMMITTING to CTRL_PRIMARY (ha-single-leader-role task 5.1) changed gate semantics: the deleted COMMITTING branch denied job ops UNCONDITIONALLY, but the CTRL_PRIMARY branch only denies job ops when ha_has_write_quorum() is false. ha_init leaves n_peers=0, so ha_has_write_quorum returns true (single-node = self-majority, ha.c:1437) and jobsub was ALLOWED (rc=0). A blind role swap does not reproduce the original deny.", - "fix": "Set ha.n_peers = 2 in both retargeted dispatch-gate tests so a multi-node primary with zero connected standbys genuinely lacks write quorum (connected 0 < n_peers/2=1), making the universal commit barrier deny job ops with -XCE_ERR_NOT_READY while still permitting replication ops.", - "tags": [ - "ha-single-leader-role", - "dispatch-gate", - "write-quorum", - "test-retarget", - "commit-barrier", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T13:30:00.000Z" - }, - { - "id": "bug-504", - "timestamp": "2026-06-13T14:01:26.000Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21923 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T14:01:26.000Z" - }, - { - "id": "bug-505", - "timestamp": "2026-06-13T14:01:32.553Z", - "error_message": "Wrong reference: uint32_t should be ha_config_t", - "file": "src/controller/ha.h", - "root_cause": "Used \"uint32_t\" instead of \"ha_config_t\"", - "fix": "Changed uint32_t \u2192 ha_config_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T14:01:32.553Z" - }, - { - "id": "bug-506", - "timestamp": "2026-06-13T14:06:15.280Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_role.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T14:06:15.280Z" - }, - { - "id": "bug-507", - "timestamp": "2026-06-13T14:43:54.194Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/ha-async-io.md", - "root_cause": "Had `CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY`", - "fix": "Changed to 's own-term commit rule (\u00a75.4.2): an election winn", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T14:43:54.194Z" - }, - { - "id": "bug-508", - "timestamp": "2026-06-13T14:54:21.781Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-deploy-ha/SKILL.md", - "root_cause": "Had `\u2192 CTRL_PRIMARY_COMMITTING`", - "fix": "Changed to `HA FSM (raft): \u2192 CTRL_PRIMARY (term=...)`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-13T14:54:21.781Z" - }, - { - "id": "bug-509", - "timestamp": "2026-06-13T15:31:21.740Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_consistency_invariants.md", - "root_cause": "Had \"XCE HA Raft+CP umbrella \u2014 sub-specs 1&2 of 4 IMPL", - "fix": "Changed to \"XCE HA Raft+CP umbrella \u2014 sub-specs 1,2,3 of 4 IM", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-13T15:31:40.089Z" - }, - { - "id": "bug-510", - "timestamp": "2026-06-14T00:30:31.077Z", - "error_message": "Wrong reference: stdint should be stdbool", - "file": "src/controller/include/cluster.h", - "root_cause": "Used \"stdint\" instead of \"stdbool\"", - "fix": "Changed stdint \u2192 stdbool", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T00:30:31.077Z" - }, - { - "id": "bug-511", - "timestamp": "2026-06-14T00:34:17.543Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219217 lines (4 removed) | Also: /* \u2500\u2500 Cross-module link stubs (dataex.c/misc.c/sch; /* clang-format off */", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T00:35:27.905Z" - }, - { - "id": "bug-512", - "timestamp": "2026-06-14T00:34:38.745Z", - "error_message": "Wrong reference: xlist_destroy should be xlist_fini", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "Used \"xlist_destroy\" instead of \"xlist_fini\"", - "fix": "Changed xlist_destroy \u2192 xlist_fini", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T00:34:38.745Z" - }, - { - "id": "bug-513", - "timestamp": "2026-06-14T00:35:13.711Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_egress_filter.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T00:35:13.711Z" - }, - { - "id": "bug-514", - "timestamp": "2026-06-14T00:37:31.926Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "No early return/throw for edge case: iter_make(&it, c->jobmgr", - "fix": "Added guard clause: if (iter_make(&it, c->jobmgr)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T00:37:31.926Z" - }, - { - "id": "bug-515", - "timestamp": "2026-06-14T00:45:06.450Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 12\u219217 lines (2 removed) | Also: jdef.jobid = 202;; jdef.state = JOB_STAT_PEND;", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T00:46:18.449Z" - }, - { - "id": "bug-516", - "timestamp": "2026-06-14T01:46:09.600Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T01:46:09.600Z" - }, - { - "id": "bug-517", - "timestamp": "2026-06-14T01:46:53.301Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21926 lines (2 removed) | Also: extern int restore_job(cluster_t* c, uint64_t jobi; extern int restore_job_old_unused(cluster_t* c, ui", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T01:47:22.530Z" - }, - { - "id": "bug-518", - "timestamp": "2026-06-14T01:50:24.429Z", - "error_message": "Missing guard clause", - "file": "src/controller/sched.c", - "root_cause": "No early return/throw for edge case: l->count > 0", - "fix": "Added guard clause: if (l->count > 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T01:50:24.429Z" - }, - { - "id": "bug-519", - "timestamp": "2026-06-14T01:52:56.833Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "Had \"\\n>> commit-gated scheduler egress tests (RED, ta", - "fix": "Changed to \"\\n>> commit-gated scheduler egress tests (GREEN, ", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T01:52:56.833Z" - }, - { - "id": "bug-520", - "timestamp": "2026-06-14T03:52:21.920Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 9\u21924 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T03:52:21.920Z" - }, - { - "id": "bug-521", - "timestamp": "2026-06-14T03:55:55.380Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "Had \"commit.h\"", - "fix": "Changed to \"cluster.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T03:58:21.222Z" - }, - { - "id": "bug-522", - "timestamp": "2026-06-14T03:57:42.389Z", - "error_message": "Significant refactor of ", - "file": "src/controller/include/cluster.h", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 12\u21923 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T03:57:42.389Z" - }, - { - "id": "bug-523", - "timestamp": "2026-06-14T04:21:18.071Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha_convergence.c", - "root_cause": "No early return/throw for edge case: ctx == NULL || ctx->c == NULL", - "fix": "Added guard clause: if (ctx == NULL || ctx->c == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T04:21:18.071Z" - }, - { - "id": "bug-524", - "timestamp": "2026-06-14T04:31:35.539Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 16\u219234 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T04:31:35.539Z" - }, - { - "id": "bug-343", - "timestamp": "2026-06-14T04:34:32+00:00", - "error_message": "AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "dc_run_pending_task peeked tasks.head and ran entry(args) WITHOUT dequeuing. The once-guard correctly makes the 2nd ha_rejoin_reconcile schedule no new task, so head still pointed at the first (already-run) task whose body had freed its heap ctx (args). The 2nd dc_run_pending_task re-ran that task on the freed ctx. The convergence task bodies free their own args (production delaymgr killer frees only the delay_task_t node, never args; production execv never re-runs a one-shot).", - "fix": "Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) \u2014 mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct).", - "tags": [ - "asan", - "use-after-free", - "ha", - "rejoin", - "delaymgr", - "test-harness", - "ha-convergence-recovery", - "task-3.2" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T04:34:32+00:00" - }, - { - "id": "bug-526", - "timestamp": "2026-06-14T04:52:58.813Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-convergence-recovery/tasks.md", - "root_cause": "Had \"No space left on device\"", - "fix": "Changed to `/tmp/xce_*`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T04:52:58.813Z" - }, - { - "id": "bug-527", - "timestamp": "2026-06-14T04:57:22.178Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /*; * ha_walack_continuation_for_test: test-only entry point for", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T04:57:42.146Z" - }, - { - "id": "bug-528", - "timestamp": "2026-06-14T04:57:59.221Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha_convergence.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T04:57:59.221Z" - }, - { - "id": "bug-529", - "timestamp": "2026-06-14T05:23:00.737Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T05:23:00.737Z" - }, - { - "id": "bug-530", - "timestamp": "2026-06-14T05:24:33.033Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T05:24:33.033Z" - }, - { - "id": "bug-531", - "timestamp": "2026-06-14T05:25:04.524Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T05:25:04.524Z" - }, - { - "id": "bug-532", - "timestamp": "2026-06-14T06:12:36.513Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha_convergence.c", - "root_cause": "No early return/throw for edge case: ha == NULL", - "fix": "Added guard clause: if (ha == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T06:12:36.513Z" - }, - { - "id": "bug-533", - "timestamp": "2026-06-14T06:37:35.904Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: printf(\"\\n>> Regression pins: fan-out / snapshot fallback / ; \"re-inclusion (task 6.1 \u2014 PASS against current code)\\n\\n\"); | Also: /* \u2500\u2500 Fan-out counting send-hook (task 6.1 (a)/(d), umbrella; * repl_broadcast routes each enqueue through repl_broadcast_", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-14T06:40:03.882Z" - }, - { - "id": "bug-534", - "timestamp": "2026-06-14T06:38:38.792Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Had \"delay.h\"", - "fix": "Changed to \"controller.capnp.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T06:38:38.792Z" - }, - { - "id": "bug-535", - "timestamp": "2026-06-14T06:39:49.073Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "22 lines replaced/restructured", - "fix": "Rewrote 33\u21923 lines (22 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T06:39:49.073Z" - }, - { - "id": "bug-536", - "timestamp": "2026-06-14T07:56:01.819Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T07:56:01.819Z" - }, - { - "id": "bug-537", - "timestamp": "2026-06-14T08:02:40.832Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T08:02:40.832Z" - }, - { - "id": "bug-538", - "timestamp": "2026-06-14", - "error_message": "config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config, breaking replication on a live 3-node cluster", - "file": "src/controller/ha.c", - "root_cause": "ha_on_primary_ctrl_fd_rdhup did not apply the no-config voter-only guard that ha_start_election_async has; after primary RDHUP it set CTRL_ELECTING, allocated elect_ctx and called raft_on_leader_lost even for tag=0 nodes. Two tag=0 nodes granted each other (freshness gate only denies LOWER tag; tag0==tag0 granted), electing a config-less primary.", - "fix": "Added an election-safety guard in ha_on_primary_ctrl_fd_rdhup after the WAL-truncate block and before elect_ctx alloc/hook/raft_on_leader_lost: if my_config_tag==0 && candidates_count>1, log + set role=CTRL_STANDBY and return (stays passive voter). Mirrors ha_start_election_async. Single-node (n_peers==1) unaffected.", - "tags": [ - "ha", - "election-safety", - "rdhup", - "config-bootstrap", - "Req2.8", - "Req10.5", - "voter-only-guard" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14" - }, - { - "id": "bug-539", - "timestamp": "2026-06-14T08:35:02.456Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T08:35:02.456Z" - }, - { - "id": "bug-540", - "timestamp": "2026-06-14T08:42:03.810Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 12\u219217 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T08:42:03.810Z" - }, - { - "id": "bug-541", - "timestamp": "2026-06-14T08:42:17.355Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 8\u219216 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T08:42:17.355Z" - }, - { - "id": "bug-542", - "timestamp": "2026-06-14T09:29:31.490Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_coldstart_lsn0.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T09:29:31.490Z" - }, - { - "id": "bug-543", - "timestamp": "2026-06-14T09:36:36.967406Z", - "error_message": "WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0", - "file": "src/controller/config_repl.c", - "root_cause": "ha_stream_wal_to_standby filter `if (hdr.lsn > standby_lsn)` excludes lsn 0 when standby_lsn==0 (0>0 is false). standby_lsn=0 is ambiguous: it means BOTH 'empty standby (applied nothing)' AND 'standby has applied lsn 0'. The streamer cannot distinguish, computes start_lsn=standby_lsn+1=1, and drops the genuine lsn-0 entry. Only non-cosmetic because lsn-0 entries can be REAL data (first-ever WAL entry of a fresh cluster is lsn 0 from a real write OR the promotion NOOP).", - "fix": "FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry \u2014 no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine).", - "tags": [ - "ha", - "replication", - "wal", - "cold-start", - "lsn-0", - "catch-up", - "correctness", - "off-by-one" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T10:00:00Z" - }, - { - "id": "bug-544", - "timestamp": "2026-06-14T09:51:22.861Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_egress_filter.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u21929 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T09:51:22.861Z" - }, - { - "id": "bug-545", - "timestamp": "2026-06-14T09:51:50.354Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_p3_hardening.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u21926 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T09:51:50.354Z" - }, - { - "id": "bug-546", - "timestamp": "2026-06-14T10:03:46.645Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_barrier_reply.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u21929 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T10:03:46.645Z" - }, - { - "id": "bug-547", - "timestamp": "2026-06-14T10:56:37.412Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 32\u2192111 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T10:56:37.412Z" - }, - { - "id": "bug-548", - "timestamp": "2026-06-14T10:58:04.523Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219215 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T10:58:04.523Z" - }, - { - "id": "bug-549", - "timestamp": "2026-06-14T10:58:59.601Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 30\u219238 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T10:58:59.601Z" - }, - { - "id": "bug-550", - "timestamp": "2026-06-14T11:38:09.614Z", - "error_message": "Wrong reference: test_pin_dispatch_gate_degraded_allowlist should be test_empty_standby_fresh_cluster_streams_not_snapshot", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "Used \"test_pin_dispatch_gate_degraded_allowlist\" instead of \"test_empty_standby_fresh_cluster_streams_not_snapshot\"", - "fix": "Changed test_pin_dispatch_gate_degraded_allowlist \u2192 test_empty_standby_fresh_cluster_streams_not_snapshot", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T11:38:09.614Z" - }, - { - "id": "bug-551", - "timestamp": "2026-06-14T11:45:02.411Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (standby_lsn < oldest_wal_lsn)", - "fix": "Changed to: if (standby_lsn + 1 < oldest_wal_lsn)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T11:45:02.411Z" - }, - { - "id": "bug-552", - "timestamp": "2026-06-14T11:45:34.585Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_convergence_verification.c", - "root_cause": "18 lines replaced/restructured", - "fix": "Rewrote 35\u219254 lines (18 removed) | Also: /* HA_REPL_SEND_SNAPSHOT: standby_lsn predates the; \"heartbeat decision: standby_lsn below oldest_wal_", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T11:45:52.313Z" - }, - { - "id": "bug-553", - "timestamp": "2026-06-14T11:47:05.280Z", - "error_message": "Significant refactor of ", - "file": "src/controller/repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T11:47:05.280Z" - }, - { - "id": "bug-554", - "timestamp": "2026-06-14T11:47:21.564Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_p2_integration.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u219220 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T11:47:21.564Z" - }, - { - "id": "bug-555", - "timestamp": "2026-06-14T12:23:52.351Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T12:23:52.351Z" - }, - { - "id": "bug-556", - "timestamp": "2026-06-14T12:25:37.723Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T12:25:37.723Z" - }, - { - "id": "bug-557", - "timestamp": "2026-06-14T12:26:04.137Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T12:26:04.137Z" - }, - { - "id": "bug-558", - "timestamp": "2026-06-14T14:03:21.674Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 7\u21929 lines (5 removed) | Also: config.nmasters = 1;; config.masters[0] = get_setting(\"host\");", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-14T14:03:42.165Z" - }, - { - "id": "bug-559", - "timestamp": "2026-06-14T14:30:38.316Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u219214 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T14:30:38.316Z" - }, - { - "id": "bug-560", - "timestamp": "2026-06-14T19:54:55.428Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T19:54:55.428Z" - }, - { - "id": "bug-561", - "timestamp": "2026-06-14T20:05:48.755409", - "error_message": "jobsub intermittent -ETIMEDOUT \"Connection timed out\": commit: WAIT deadline expired without commit (timeout_ms=50)", - "file": "src/include/config.h", - "root_cause": "DEFAULT_SYNC_TIMEOUT_MS=50 too tight for durable sync majority replication (each standby fdatasyncs before ACK); not configurable (controller always used persist_config_defaults).", - "fix": "Raised DEFAULT_SYNC_TIMEOUT_MS 50->500; added --sync-timeout-ms CLI flag plumbed arguments->myconf.sync_timeout_ms->pcfg.sync_timeout_ms (0=use default) in main.c/myconf.h/cluster.c.", - "tags": [ - "ha", - "commit-barrier", - "timeout", - "replication", - "jobsub", - "config" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T20:05:48.755409" - }, - { - "id": "bug-562", - "timestamp": "2026-06-14T23:55:00Z", - "error_message": "AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen (lib/controller.capnp.c:13168) on ctrl-2 startup, via xce_reply->xce_encode from OPCode_nop handler (cluster.c:307). Also: job submission succeeds but joblist shows no jobs.", - "file": "src/lib/request.c", - "root_cause": "init_reply_by_req() only set seq+opcode, leaving the rest of the stack-declared xce_reply_t uninitialized. The Cap'n-C generated encode_RequestReply reads the common trailer field redirect_host UNCONDITIONALLY (gated only on != NULL), so a garbage stack pointer in reply.redirect_host was strlen'd. The garbage aliased a 4KB capnp segment freed moments earlier by the WAL-ack reply encode on the same thread -> UAF. Handlers that build a reply via init_reply_by_req and only set status/data_which/data (e.g. the NOP/state-report path) never cleared redirect_host. Empty joblist is downstream: ctrl-2 crash-looping on this UAF destabilizes replication/quorum, so a get request that fails over to the not-caught-up standby reads an empty jobmgr (standby serves get from its own committed state).", - "fix": "memset(reply,0,sizeof(*reply)) at the top of init_reply_by_req() (src/lib/request.c) before setting seq/opcode. All 30+ reply sites follow declare->init->populate, so zeroing in the shared initializer fixes the whole class (redirect_host/redirect_port/status/data garbage) at the source. Added regression test test_init_reply_by_req_zeroes_reply in src/lib/testsuite/test_request.c (poisons reply with 0xFF, asserts redirect_host==NULL etc).", - "tags": [ - "use-after-free", - "asan", - "capnp", - "encode_RequestReply", - "redirect_host", - "init_reply_by_req", - "ha", - "joblist", - "uninitialized-stack" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-14T23:55:00Z" - }, - { - "id": "bug-563", - "timestamp": "2026-06-15T09:16:53.679025Z", - "error_message": "jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works.", - "file": "src/lib/request.c", - "root_cause": "SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() \u2014 the ONLY place that sets c->running=1 \u2014 is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked.", - "fix": "Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown \u2014 the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run \u2014 moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal).", - "tags": [ - "joblist", - "workerlist", - "jobsub", - "ha", - "check_controller", - "c-running", - "cluster_start", - "orphaned-function", - "xce_reply_get", - "SOLVED" - ], - "related_bugs": [ - "bug-562" - ], - "occurrences": 1, - "last_seen": "2026-06-15T09:16:53.679025Z" - }, - { - "id": "bug-564", - "timestamp": "2026-06-15T09:50:02.164Z", - "error_message": "Significant refactor of ", - "file": "src/controller/opget.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21923 lines (2 removed) | Also: {; extern uint64_t g_diag_last_jobid;", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-15T09:50:10.236Z" - }, - { - "id": "bug-565", - "timestamp": "2026-06-15T10:46:08.441Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T10:46:08.441Z" - }, - { - "id": "bug-566", - "timestamp": "2026-06-15T10:46:31.815Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T10:46:31.815Z" - }, - { - "id": "bug-567", - "timestamp": "2026-06-15T10:46:42.282Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u21924 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T10:46:42.282Z" - }, - { - "id": "bug-339", - "timestamp": "2026-06-15T11:00:00.000Z", - "error_message": "After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter an infinite election loop: both only ever raft_cb_become_follower (never become_leader), terms climb in lockstep (4..23), RDHUP -> CTRL_ELECTING -> CTRL_STANDBY repeats forever.", - "file": "src/controller/ha.c, src/controller/myapp.c, src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "LSN-source asymmetry between candidate-advertised LSN and voter freshness floor, specific to the RDHUP failover election path. ha_on_primary_ctrl_fd_rdhup advertised c->persist.committed_lsn as the RequestVote last_log_lsn, but a STANDBY never advances persist.committed_lsn (only the primary's commit engine writes it at persist.c:1192). Standbys advance applied_lsn (ha_apply_wal_catchup_entry) and last_known_committed_lsn (heartbeat replies). The voter-side freshness floor (cluster.c OPCode_requestVote) is max(applied_lsn, last_known_committed_lsn) via ha_vote_self_freshness. So once a standby applied any WAL beyond its last snapshot, every surviving standby advertised a stale-low LSN (e.g. 0) while each peer computed a HIGH floor (e.g. 10); the freshness gate cand_lsn < my_lsn tripped on BOTH sides, each denied the other's vote, neither reached quorum (2 of 3), each election timed out -> become_follower -> 10s liveness timeout -> RDHUP -> re-elect at term+1. The boot path (myapp.c:217) already advertised applied_lsn, so cold-boot election worked; only the RDHUP failover path was inconsistent.", - "fix": "In ha_on_primary_ctrl_fd_rdhup, compute candidate freshness ONCE via ha_vote_self_freshness(ha, config_version_tag, applied_lsn, &cand_tag, &cand_lsn) right after ha->raft_c=c, and use cand_tag/cand_lsn for BOTH the elect_ctx (votes[0].my_tag/my_lsn + self_tag/self_lsn) and the XCE_TESTING ha_rdhup_election_hook args. This makes candidate-advertised LSN identical to the voter-side floor, so for any two nodes A,B: A grants B iff floor_B >= floor_A -> the freshest node is always granted by every peer; equal-floor ties are broken by raft priority delay. Regression test added: test_rdhup_candidate_advertises_freshness_floor_not_stale_committed (applied/learned=10, persist.committed=0 -> asserts advertised my_lsn==10). RED (Was 0) -> GREEN verified. Follow-up: boot election path (myapp.c) also routed through ha_vote_self_freshness (was raw applied_lsn) so both election paths are symmetric with the voter floor.", - "tags": [ - "ha", - "election", - "raft", - "rdhup", - "failover", - "freshness-gate", - "committed_lsn", - "applied_lsn", - "split-vote", - "infinite-loop" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T11:00:00.000Z" - }, - { - "id": "bug-569", - "timestamp": "2026-06-15T10:58:35.588Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T10:58:35.588Z" - }, - { - "id": "bug-570", - "timestamp": "2026-06-15T11:10:10.553Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/product.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to `src/lib/raft/`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T11:10:10.553Z" - }, - { - "id": "bug-571", - "timestamp": "2026-06-15T11:10:19.238Z", - "error_message": "Significant refactor of ", - "file": ".kiro/steering/product.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T11:10:19.238Z" - }, - { - "id": "bug-572", - "timestamp": "2026-06-15T11:10:26.312Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/tech.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to `src/lib/raft/`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T11:10:26.312Z" - }, - { - "id": "bug-573", - "timestamp": "2026-06-15T14:45:22.804Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 13\u21925 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T14:45:22.804Z" - }, - { - "id": "bug-340", - "timestamp": "2026-06-15T15:00:00.000Z", - "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup \u2192 CTRL_ELECTING \u2192 raft_cb_become_follower \u2192 CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) \u2014 a wedged outbound standby endpoint.", - "file": "src/controller/ha.c, src/controller/testsuite/test_ha_liveness.c", - "root_cause": "primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints \u2014 so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout \u2014 confirming the watchdog is correct and the sole cause is the broken channel.)", - "fix": "In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action \u2014 the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) \u2192 become_follower \u2192 STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED\u2192GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c.", - "tags": [ - "ha", - "heartbeat", - "rejoin", - "restart", - "standby-endpoints", - "reconnect", - "epoll", - "liveness", - "election-loop" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-15T15:00:00.000Z" - }, - { - "id": "bug-575", - "timestamp": "2026-06-16T00:10:49.295Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-prevote/research.md", - "root_cause": "Had 't understand PreVote (older binary) \u2014 does the ca", - "fix": "Changed to 's \"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T00:10:49.295Z" - }, - { - "id": "bug-576", - "timestamp": "2026-06-16T00:18:28.793Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-prevote/research.md", - "root_cause": "Had 's \"", - "fix": "Changed to `ha_heartbeat_is_fresh()`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T00:18:28.793Z" - }, - { - "id": "bug-577", - "timestamp": "2026-06-16T00:26:16.193Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-prevote/research.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T00:26:16.193Z" - }, - { - "id": "bug-578", - "timestamp": "2026-06-16T03:00:48.818Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-prevote/design.md", - "root_cause": "Had `ha_elect_ctx_t`", - "fix": "Changed to `raft_on_vote_reply`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 9, - "last_seen": "2026-06-16T03:02:59.202Z" - }, - { - "id": "bug-579", - "timestamp": "2026-06-16T03:01:07.659Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/ha-prevote/design.md", - "root_cause": "Was returning: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", - "fix": "Now returns: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", - "tags": [ - "auto-detected", - "return-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T03:01:07.660Z" - }, - { - "id": "bug-580", - "timestamp": "2026-06-16T03:06:57.732Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-prevote/tasks.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u21927 lines (3 removed) | Also: - [ ] 5.1 Wire the PreVote reply opcode to the can; - Dispatch the incoming PreVote-reply opcode to th", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T03:07:10.762Z" - }, - { - "id": "bug-581", - "timestamp": "2026-06-16T04:02:13.927Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21925 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:02:13.927Z" - }, - { - "id": "bug-582", - "timestamp": "2026-06-16T04:18:41.611Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:18:41.611Z" - }, - { - "id": "bug-583", - "timestamp": "2026-06-16T04:19:25.513Z", - "error_message": "Missing error handling in never", - "file": "src/lib/raft/raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:19:25.513Z" - }, - { - "id": "bug-584", - "timestamp": "2026-06-16T04:27:50.039Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:27:50.039Z" - }, - { - "id": "bug-585", - "timestamp": "2026-06-16T04:36:28.796Z", - "error_message": "Missing guard clause", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "No early return/throw for edge case: rc != 0", - "fix": "Added guard clause: if (rc != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:36:28.796Z" - }, - { - "id": "bug-586", - "timestamp": "2026-06-16T04:50:38.855Z", - "error_message": "Missing error handling in struct", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 3.1 \u2014 voter-side PreVote dispatch gate (stickiness +; * no mutation, reply term unchanged) */", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T04:50:49.601Z" - }, - { - "id": "bug-587", - "timestamp": "2026-06-16T04:51:17.702Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:51:17.703Z" - }, - { - "id": "bug-588", - "timestamp": "2026-06-16T04:57:16.230Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:57:16.230Z" - }, - { - "id": "bug-589", - "timestamp": "2026-06-16T04:57:49.245Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 20\u219225 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T04:57:49.245Z" - }, - { - "id": "bug-590", - "timestamp": "2026-06-16T05:21:02.997Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-prevote/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T05:21:02.997Z" - }, - { - "id": "bug-591", - "timestamp": "2026-06-16T05:27:09.380Z", - "error_message": "Wrong reference: test_commit_engine should be test_ha_prevote", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_prevote\"", - "fix": "Changed test_commit_engine \u2192 test_ha_prevote", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T05:27:18.877Z" - }, - { - "id": "bug-592", - "timestamp": "2026-06-16T05:28:01.236Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c || !c->myconf", - "fix": "Added guard clause: if (!c || !c->myconf)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T05:30:07.209Z" - }, - { - "id": "bug-593", - "timestamp": "2026-06-16T05:44:57.430Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-prevote/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T05:44:57.430Z" - }, - { - "id": "bug-594", - "timestamp": "2026-06-16T05:50:25.729Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_commit_engine.c'", - "fix": "Changed to 'testsuite/test_ha_failover_prevote.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T05:50:25.730Z" - }, - { - "id": "bug-595", - "timestamp": "2026-06-16T05:50:35.123Z", - "error_message": "Wrong reference: test_commit_engine should be test_ha_failover_prevote", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_failover_prevote\"", - "fix": "Changed test_commit_engine \u2192 test_ha_failover_prevote", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T05:50:39.775Z" - }, - { - "id": "bug-596", - "timestamp": "2026-06-16T05:52:32.982Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "104 lines replaced/restructured", - "fix": "Rewrote 157\u219290 lines (104 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T05:52:32.982Z" - }, - { - "id": "bug-597", - "timestamp": "2026-06-16T05:59:43.939Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T05:59:43.939Z" - }, - { - "id": "bug-598", - "timestamp": "2026-06-16T06:33:05.644Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: role == CTRL_STANDBY && ha->raft.pre_vote_in_progress", - "fix": "Added guard clause: if (role == CTRL_STANDBY && ha->raft.pre_vot)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T06:33:05.644Z" - }, - { - "id": "bug-599", - "timestamp": "2026-06-16T06:58:28.916Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_prevote.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T06:58:28.916Z" - }, - { - "id": "bug-600", - "timestamp": "2026-06-16T07:54:55.333Z", - "error_message": "Significant refactor of ", - "file": ".claude/settings.json", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 13\u21923 lines (9 removed) | Also: },; {", - "tags": [ - "auto-detected", - "refactor", - "json" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-16T07:57:04.100Z" - }, - { - "id": "bug-601", - "timestamp": "2026-06-16T07:55:35.829Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-prevote/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T07:55:35.829Z" - }, - { - "id": "bug-602", - "timestamp": "2026-06-16T08:07:10.459Z", - "error_message": "Significant refactor of ", - "file": ".claude/settings.json", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 13\u21923 lines (9 removed) | Also: },; {", - "tags": [ - "auto-detected", - "refactor", - "json" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T08:07:18.239Z" - }, - { - "id": "bug-603", - "timestamp": "2026-06-16T08:18:48.567Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T08:18:48.567Z" - }, - { - "id": "bug-604", - "timestamp": "2026-06-16T15:55:00Z", - "error_message": "controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap", - "file": "src/controller/persist.c", - "root_cause": "fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned \u2014 up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms).", - "fix": "Replaced the nanosleep with pthread_cond_timedwait on a new fsync_mutex/fsync_cond (added to persist_t, init in persist_init, destroyed in persist_fini). persist_stop now sets fsync_stop and pthread_cond_signal(fsync_cond) under fsync_mutex before join, so the worker wakes immediately. fdatasync still runs under p->lock, released before re-acquiring fsync_mutex (no lock nesting). Full controller suite now completes in ~10s (was >300s); 0 failures.", - "tags": [ - "controller", - "persist", - "fsync", - "thread-shutdown", - "test-timeout", - "pthread_join", - "nanosleep", - "condvar", - "ha" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T15:55:00Z" - }, - { - "id": "bug-605", - "timestamp": "2026-06-16T16:30:00Z", - "error_message": "3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1) loops PreVote LOST every ~11s forever; config-less peers idle as voter-only. REPRODUCED via /xce-deploy-ha + clean restart 2026-06-16.", - "file": "src/controller/ha.c (ha_dispatch_gate); test: src/controller/testsuite/test_ha_role.c", - "root_cause": "ha_dispatch_gate() admitted OPCode_requestVote in CTRL_INIT/CTRL_ELECTING/CTRL_STANDBY but NEVER OPCode_preVoteRequest (opcode 38). ha-prevote added the new opcode + handler (cluster.c:440) but missed wiring it into the role-admission gate. So a STANDBY voter rejected an incoming PreVote with -XCE_ERR_STANDBY (-10014) BEFORE the handler ran -> no preVoteReply sent -> candidate's failed/empty exchange tallied as a denial (pre_vote_reply_decode_and_dispatch counts ok=0 as non-grant; vote_ep_err_handler synthesizes a denied vote). In a cold cluster (config-holder + config-less voter-only peers) every PreVote 'lost' with NO voter ever making a liveness decision (zero 'PreVote DENIED' logs), so no primary was ever elected and the config-holder looped PreVote forever. Confirmed via gdb: myreq_handler ha_dispatch_gate rc=-10014 on opcode 38. Design invariant: a PreVote (strictly non-mutating) must be at least as admissible as a real requestVote in every role.", - "fix": "Added OPCode_preVoteRequest alongside OPCode_requestVote in ha_dispatch_gate() for CTRL_INIT, CTRL_ELECTING, and the CTRL_STANDBY switch (src/controller/ha.c). PreVote now reaches the cluster.c handler and a proper preVoteReply is sent. Added TDD test test_dispatch_gate_allows_prevote_in_voter_roles (test_ha_role.c) asserting gate(preVoteRequest)==gate(requestVote)==0 across voter roles. VERIFIED: controller unit suite OK (new test PASS); fresh /xce-deploy-ha 3-node cold start now elects ctrl-1 PRIMARY (term=2), standbys establish primary channel + config committed, prevote-LOST=0, connections 96->8.", - "tags": [ - "ha", - "prevote", - "ha-dispatch-gate", - "opcode-38", - "cold-start", - "bootstrap-deadlock", - "FIXED", - "XCE_ERR_STANDBY" - ], - "related_bugs": [ - "bug-339" - ], - "occurrences": 1, - "last_seen": "2026-06-16T09:38:00Z" - }, - { - "id": "bug-606", - "timestamp": "2026-06-16T17:46:00Z", - "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win \u2192 CTRL_ELECTING \u2192 become_follower (term bumped) \u2192 ~10s \u2192 repeat, term climbing unbounded (3\u21929\u2192\u221e), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", - "file": "src/controller/ha.c (ha_begin_failover, ha_election_stagger_ms, ha_failover_stagger_fn); src/controller/ha.h; test: src/controller/testsuite/test_ha_role.c", - "root_cause": "Dueling-candidates split-vote livelock \u2014 Raft \u00a75.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other \u2014 PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) \u2192 split vote \u2192 election times out (fixed election_timeout_ms, raft.c:105, NO jitter) \u2192 become_follower, term+1 \u2192 repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn \u2192 ha_on_primary_ctrl_fd_rdhup \u2192 PreVote \u2192 ha_start_election_async \u2192 raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it.", - "fix": "Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh \u2192 skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won \u2192 PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first \u2192 STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive \u2014 skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed \u2014 the failover-start stagger alone resolves it; left as future defense-in-depth.", - "tags": [ - "ha", - "raft", - "election", - "split-vote", - "dueling-candidates", - "livelock", - "randomized-timeout", - "failover", - "stagger", - "jitter", - "FIXED" - ], - "related_bugs": [ - "bug-605" - ], - "occurrences": 1, - "last_seen": "2026-06-16T10:04:00Z" - }, - { - "id": "bug-607", - "timestamp": "2026-06-16T10:37:37.021Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/requirements.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 5\u21928 lines (5 removed) | Also: 1. If a controller returns a not-ready (no current; shall retry the request with backoff.", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T10:38:03.618Z" - }, - { - "id": "bug-608", - "timestamp": "2026-06-16T10:36:00Z", - "error_message": "libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument' (EINVAL) indefinitely", - "file": "src/lib/request.c", - "root_cause": "Two gaps: (1) xce_request_with_retry() (request.c:1956) is implemented+unit-tested but NO production path calls it - every xce_*_req wrapper calls plain xce_request() (request.c:1385 etc). (2) The retry primitive keys on E_XCE_STANDBY + reply.redirect_host/port (request.c:2010), but the real controller emits a dedicated E_XCE_REDIRECT status with data_which=nodata (join.c:264-266) and never populates those trailer fields. Controller non-primary taxonomy is E_XCE_REDIRECT / E_XCE_NOMASTER / E_XCE_NOT_READY / E_XCE_STANDBY (errnox.h:22-25). So the client both bypasses the wrapper AND, even if wired, would not detect the controller's actual redirect signal.", - "fix": "Spec libxce-failover: wire all production request paths through transparent failover; classify on actual controller statuses (REDIRECT=fast-rotate to find existing primary, address-if-present else rotate master list; NOMASTER/NOT_READY=backoff then cluster-not-ready; STANDBY=rotate); realign the existing retry unit tests to the emitted statuses. Implementation pending (requirements phase).", - "tags": [ - "libxce", - "failover", - "ha", - "E_XCE_REDIRECT", - "retry", - "client", - "protocol-mismatch" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T10:36:00Z" - }, - { - "id": "bug-609", - "timestamp": "2026-06-16T10:52:01.800Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/requirements.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21929 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T10:52:01.800Z" - }, - { - "id": "bug-610", - "timestamp": "2026-06-16T10:52:36.535Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T10:52:36.535Z" - }, - { - "id": "bug-611", - "timestamp": "2026-06-16T13:02:01.004Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 20\u219227 lines (3 removed) | Also: - On every reply, copy `reply.status` and **deep-c; with the reply \u2014 see request.c:243-250 hazard) + ` | Also: - Preconditions: `ctx` initialized; `conf->nmaster; exhaustion returns `-E_XCE_NOT_READY`; ctx left us", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-16T13:02:46.348Z" - }, - { - "id": "bug-612", - "timestamp": "2026-06-16T13:09:39.339Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 6\u219210 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T13:09:39.339Z" - }, - { - "id": "bug-613", - "timestamp": "2026-06-16T13:15:11.048Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T13:15:11.048Z" - }, - { - "id": "bug-614", - "timestamp": "2026-06-16T13:18:29.113Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Had `xce_ctx_call`", - "fix": "Changed to `xcecli_retry_next`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T13:18:35.874Z" - }, - { - "id": "bug-615", - "timestamp": "2026-06-16T13:18:59.337Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/research.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u219210 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T13:18:59.337Z" - }, - { - "id": "bug-616", - "timestamp": "2026-06-16T13:36:08.340Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 12\u219220 lines (2 removed) | Also: F->>E: reconnect to address; F->>E: reconnect to next master | Also: | 5.1, 5.3 | Reads+writes; no silent drop | Xcecli; | XcecliRetryLoop | client (`client.c`) | `do/whil | Also: - `src/lib/client.c` \u2014 wrap each `xcecli_*` op bod; (xcecli_retry_next(ac,&rc,is_write,&st));` loop; a", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-16T13:39:20.214Z" - }, - { - "id": "bug-617", - "timestamp": "2026-06-16T23:38:24.000Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Had `redirect_host/port`", - "fix": "Changed to `host/port`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-16T23:38:53.585Z" - }, - { - "id": "bug-618", - "timestamp": "2026-06-16T23:40:05.565Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/libxce-failover/tasks.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21923 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T23:40:05.565Z" - }, - { - "id": "bug-619", - "timestamp": "2026-06-16T23:40:22.003Z", - "error_message": "Wrong reference: address should be hint", - "file": ".kiro/specs/libxce-failover/tasks.md", - "root_cause": "Used \"address\" instead of \"hint\"", - "fix": "Changed address \u2192 hint", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T23:40:22.003Z" - }, - { - "id": "bug-620", - "timestamp": "2026-06-16T23:55:21.886Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 22\u219210 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T23:55:21.886Z" - }, - { - "id": "bug-621", - "timestamp": "2026-06-16T23:56:22.835Z", - "error_message": "Significant refactor of ", - "file": "src/lib/request.c", - "root_cause": "81 lines replaced/restructured", - "fix": "Rewrote 106\u219214 lines (81 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T23:56:22.835Z" - }, - { - "id": "bug-622", - "timestamp": "2026-06-16T23:56:36.669Z", - "error_message": "Significant refactor of ", - "file": "src/include/request.h", - "root_cause": "22 lines replaced/restructured", - "fix": "Rewrote 40\u219217 lines (22 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-16T23:56:36.669Z" - }, - { - "id": "bug-623", - "timestamp": "2026-06-17T00:04:51.775Z", - "error_message": "Wrong reference: XCE_MAXHOST_LEN should be XCE_MAXHOSTNAME_LEN", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Used \"XCE_MAXHOST_LEN\" instead of \"XCE_MAXHOSTNAME_LEN\"", - "fix": "Changed XCE_MAXHOST_LEN \u2192 XCE_MAXHOSTNAME_LEN", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T00:04:51.775Z" - }, - { - "id": "bug-624", - "timestamp": "2026-06-17T02:35:29.695Z", - "error_message": "Incorrect value in code", - "file": "src/lib/meson.build", - "root_cause": "Had 'dataq.c'", - "fix": "Changed to 'client_failover.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T02:35:29.695Z" - }, - { - "id": "bug-625", - "timestamp": "2026-06-17T02:47:55.586Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client_failover.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 13\u219223 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T02:47:55.586Z" - }, - { - "id": "bug-626", - "timestamp": "2026-06-17T02:48:27.547Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL || host_out == NULL || host_sz == 0 || port_out =", - "fix": "Added guard clause: if (ac == NULL || host_out == NULL || host_s)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T02:48:27.547Z" - }, - { - "id": "bug-627", - "timestamp": "2026-06-17T02:54:16.472Z", - "error_message": "Significant refactor of ", - "file": "src/include/xce/client_failover.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u21923 lines (3 removed) | Also: * EndpointRotator \u2014 choose the next connect target; *", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-17T02:54:47.480Z" - }, - { - "id": "bug-628", - "timestamp": "2026-06-17T02:55:18.468Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client_failover.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T02:55:18.468Z" - }, - { - "id": "bug-629", - "timestamp": "2026-06-17T03:18:22.365Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: base >= (double", - "fix": "Added guard clause: if (base >= (double)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:18:22.365Z" - }, - { - "id": "bug-630", - "timestamp": "2026-06-17T03:18:49.743Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client_failover.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219221 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:18:49.743Z" - }, - { - "id": "bug-631", - "timestamp": "2026-06-17T03:31:29.120Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL", - "fix": "Added guard clause: if (ac == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:31:29.120Z" - }, - { - "id": "bug-632", - "timestamp": "2026-06-17T03:39:44.187Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client_internal.h", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 10\u21927 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:39:44.187Z" - }, - { - "id": "bug-633", - "timestamp": "2026-06-17T03:40:10.853Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL", - "fix": "Added guard clause: if (ac == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:40:10.853Z" - }, - { - "id": "bug-634", - "timestamp": "2026-06-17T03:40:45.973Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client_failover.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 35\u219253 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:40:45.973Z" - }, - { - "id": "bug-635", - "timestamp": "2026-06-17T03:51:08.767Z", - "error_message": "Wrong return value", - "file": "src/lib/client.c", - "root_cause": "Was returning: xce_reslist(&(ctx->ctx), res);", - "fix": "Now returns: XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 8, - "last_seen": "2026-06-17T03:51:29.554Z" - }, - { - "id": "bug-636", - "timestamp": "2026-06-17T03:51:27.648Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21927 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T03:51:27.648Z" - }, - { - "id": "bug-637", - "timestamp": "2026-06-17T03:57:33.232Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/libxce-failover/design.md", - "root_cause": "Had `nop`", - "fix": "Changed to `jobinfo`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-17T03:57:40.348Z" - }, - { - "id": "bug-638", - "timestamp": "2026-06-17T04:02:05.572Z", - "error_message": "Missing guard clause", - "file": "src/lib/client_failover.c", - "root_cause": "No early return/throw for edge case: ac == NULL", - "fix": "Added guard clause: if (ac == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T04:02:05.572Z" - }, - { - "id": "bug-639", - "timestamp": "2026-06-17T04:11:59.759Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_p1_integration.c'", - "fix": "Changed to 'testsuite/test_join_redirect.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T04:11:59.759Z" - }, - { - "id": "bug-640", - "timestamp": "2026-06-17T04:12:01.203Z", - "error_message": "Wrong reference: test_p1_integration should be test_join_redirect", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_p1_integration\" instead of \"test_join_redirect\"", - "fix": "Changed test_p1_integration \u2192 test_join_redirect", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-17T04:12:02.464Z" - }, - { - "id": "bug-641", - "timestamp": "2026-06-17T04:19:58.697Z", - "error_message": "Wrong reference: test_recover_resets_err_state_to_run should be test_decide_jitter_staggers_many_clients", - "file": "src/lib/testsuite/test_libxce_retry.c", - "root_cause": "Used \"test_recover_resets_err_state_to_run\" instead of \"test_decide_jitter_staggers_many_clients\"", - "fix": "Changed test_recover_resets_err_state_to_run \u2192 test_decide_jitter_staggers_many_clients", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T04:19:58.697Z" - }, - { - "id": "bug-609", - "timestamp": "2026-06-17T04:45:00Z", - "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call \u2014 transparent failover did not engage", - "file": "src/lib/client_failover.c", - "root_cause": "handle_disconnect leaves xce_ctx_t.state=XCE_CTX_ERR after the primary's connection drops; the per-op xceapi guard (xceapi.c:14) returns -EINVAL when state!=XCE_CTX_RUN, BEFORE the op reaches a primary. xcecli_retry_decide classified -EINVAL as TERMINAL (non-retriable), so the failover loop bailed without rotating/recovering. The ctx->state ERR->RUN recovery only ran AFTER a rotate, which a terminal classification never triggered. Surfaced by the live 3-node E2E (task 6.2); the unit tests fed status codes directly to decide() and bypassed the real guard, so they missed it.", - "fix": "In xcecli_retry_decide, before classifying *rc, detect ac->ctx.state==XCE_CTX_ERR (known-dead connection) and treat it as a prompt transport-style failover cause (XCE_FO_RETRY_NOW, no backoff) regardless of *rc, so the loop rotates+recovers. On budget exhaustion surface -E_XCE_NOT_READY (cluster-not-ready), never -EINVAL. Regression: test_decide_known_dead_ctx_retries_not_terminal.", - "tags": [ - "libxce", - "failover", - "EINVAL", - "XCE_CTX_ERR", - "reconnect", - "E2E" - ], - "related_bugs": [ - "bug-608" - ], - "occurrences": 1, - "last_seen": "2026-06-17T04:45:00Z" - }, - { - "id": "bug-610", - "timestamp": "2026-06-17T15:00:00Z", - "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate \u2192 xcecli_retry_next \u2192 xcecli_joblist during live master failover (NULL ep->iomgr deref)", - "file": "src/lib/client_failover.c", - "root_cause": "xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) \u2192 NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it.", - "fix": "Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini \u2192 endpoint_init \u2192 endpoint_set_peer \u2192 re-bind ep->iomgr=&ctx->iomgr \u2192 endpoint_connect \u2192 re-set owner+msg/err handlers \u2192 iomgr_monitor. xce_failover_rotate now delegates the reconnect to it.", - "tags": [ - "libxce", - "failover", - "crash", - "SEGV", - "endpoint_connect", - "iomgr", - "reconnect", - "E2E" - ], - "related_bugs": [ - "bug-609", - "bug-608" - ], - "occurrences": 1, - "last_seen": "2026-06-17T15:00:00Z" - }, - { - "id": "bug-644", - "timestamp": "2026-06-17T08:15:04.978Z", - "error_message": "Incorrect value in code", - "file": "src/lib/meson.build", - "root_cause": "Had 'testsuite/test_raft.c'", - "fix": "Changed to 'testsuite/test_netio_arm_failure.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T08:15:04.978Z" - }, - { - "id": "bug-645", - "timestamp": "2026-06-17T08:15:18.557Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 25\u219239 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T08:15:18.557Z" - }, - { - "id": "bug-646", - "timestamp": "2026-06-17T08:21:32.099Z", - "error_message": "Wrong reference: 611 should be 646", - "file": "src/lib/testsuite/test_netio_arm_failure.c", - "root_cause": "Used \"611\" instead of \"646\"", - "fix": "Changed 611 \u2192 646", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T08:21:32.099Z" - }, - { - "id": "bug-647", - "timestamp": "2026-06-17T08:21:56.735Z", - "error_message": "Wrong reference: 611 should be 646", - "file": "src/lib/netio.c", - "root_cause": "Used \"611\" instead of \"646\"", - "fix": "Changed 611 \u2192 646", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T08:21:56.735Z" - }, - { - "id": "bug-646", - "timestamp": "2026-06-17T16:30:00Z", - "error_message": "xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'.", - "file": "src/lib/netio.c", - "root_cause": "endpoint_update_epoll() swallowed iomgr_update_poll() failures: it WARN'd 'rc1=-1' but returned 0 and left ep->flags unchanged, so the poll interest was never armed. iomgr_update_poll uses EPOLL_CTL_MOD, which fails with ENOENT when the fd is no longer registered (e.g. after a failed reconnect: xce_ctx_reconnect endpoint_fini's the old fd but only iomgr_monitor/EPOLL_CTL_ADDs on a SUCCESSFUL connect). The next endpoint_enq() send then MOD-failed silently, request bytes were never flushed, no reply arrived, and the synchronous client blocked forever in cbuf_get (no timeout) with no socket-drop event to wake it (handle_disconnect only fires on a broken socket). Server side was a red herring: DEFAULT_SYNC_TIMEOUT_MS=500 bounds the commit barrier, and ha_dispatch_gate already replies STANDBY/NOT_READY for non-serving roles.", - "fix": "endpoint_update_epoll() captures errno and RETURNS -errno (not 0) on iomgr_update_poll failure. endpoint_enq(): on negative arm result, run handle_error(ep) (iomgr_unmonitor + handle_disconnect -> state=ERR + inject -ECONNRESET into wbuf, waking the blocked cbuf_get) and return the error so the failover loop rotates. handle_output(): propagates the arm failure so io_handler runs the error path. TDD: test_netio_arm_failure.c drives EPOLL_CTL_MOD ENOENT via endpoint_enq on an unregistered fd; asserts loud rc<0 + err_handler fired (verified red without fix, green with).", - "tags": [ - "libxce", - "failover", - "netio", - "epoll", - "EPOLL_CTL_MOD", - "ENOENT", - "silent-failure", - "hang", - "cbuf_get", - "reconnect" - ], - "related_bugs": [ - "bug-610", - "bug-609" - ], - "occurrences": 1, - "last_seen": "2026-06-17T16:30:00Z" - }, - { - "id": "bug-649", - "timestamp": "2026-06-17T09:26:05.914Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 48\u219266 lines (13 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T09:26:05.914Z" - }, - { - "id": "bug-648", - "timestamp": "2026-06-17T17:30:00Z", - "error_message": "Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event.", - "file": "src/lib/netio.c", - "root_cause": "io_fsm_do() computed rc = -errno when a closed/reset connection was detected. By OpenSSL 3.x contract (verified against docs + a live probe on OpenSSL 3.6.0), a peer close makes SSL_read() return 0 and is reported via SSL_get_error(), NOT via the return value or errno: clean close_notify -> SSL_ERROR_ZERO_RETURN (errno 0); unexpected EOF (FIN, no close_notify) -> SSL_ERROR_SSL since OpenSSL 3.0 (pre-3.0 it was SSL_ERROR_SYSCALL with errno 0 -- documented); RST -> SSL_ERROR_SYSCALL errno=ECONNRESET. errno is only meaningful for SSL_ERROR_SYSCALL; for SSL_ERROR_SSL the error queue is authoritative and any errno (we observed EPIPE=32) is a leaked side effect of OpenSSL's internal alert-write (write() on a broken connection returns EPIPE; read() never does). So -errno is 0 for the documented EOF cases -> the close looked like success. NOTE: on this 3.6 build a process-death FIN surfaced as SSL_ERROR_SSL/errno=32, which the OLD code propagated -- so this swallow is NOT necessarily the cause of the observed controller-kill permanent hang (that still implicates the reconnect/SSL-handshake path); it is a real, separate, errno-dependent non-determinism.", - "fix": "Extracted pure io_classify_io_error(ssl_err, ecode, io_rc): returns 0 only for WANT_READ/WANT_WRITE and EAGAIN/EWOULDBLOCK syscall; otherwise returns a NEGATIVE error ((ecode!=0)?-ecode:-ECONNRESET) so a detected close NEVER returns 0 regardless of the unreliable errno. io_fsm_do() now sets FSM_STATE_ERROR iff the classifier returns <0. TDD: test_netio_close_propagation.c asserts ZERO_RETURN/errno0 and SSL_ERROR_SSL/EOF map to <0 (verified red with old -ecode, green with fix). libxce suite 168/0; controller_unit_test links clean.", - "tags": [ - "libxce", - "netio", - "ssl", - "SSL_read", - "EOF", - "close_notify", - "openssl-3.x", - "SSL_ERROR_ZERO_RETURN", - "SSL_ERROR_SSL", - "silent-failure", - "hang" - ], - "related_bugs": [ - "bug-646", - "bug-610" - ], - "occurrences": 1, - "last_seen": "2026-06-17T17:30:00Z" - }, - { - "id": "bug-651", - "timestamp": "2026-06-17T10:44:37.350Z", - "error_message": "Wrong return value", - "file": "src/lib/request.c", - "root_cause": "Was returning: 0;", - "fix": "Now returns: (enq_rc < 0) ? enq_rc : 0;", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T10:44:37.350Z" - }, - { - "id": "bug-650", - "timestamp": "2026-06-17T18:30:00Z", - "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket \u2014 the request was never transmitted yet the client waits for a reply forever.", - "file": "src/lib/request.c", - "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) \u2014 but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", - "fix": "All six request.c senders propagate the enq result: `int enq_rc = endpoint_enq(ep,&pkt->link); return (enq_rc < 0) ? enq_rc : 0;`. A failed send now returns <0 through xce_jobsubmit_req -> xce_jobsubmit, which returns before cbuf_get; the failover loop classifies the transport error and rotates instead of blocking on a reply that never arrives. TDD: test_request_send_propagation.c builds an OPCode_nop request (encodes OK) on a dead endpoint (EPOLL_CTL_MOD ENOENT) and asserts xce_request returns <0 (verified red with old `return 0`, green with fix). libxce 169/0; controller_unit_test links clean.", - "tags": [ - "libxce", - "request", - "xce_request", - "failover", - "silent-failure", - "hang", - "cbuf_get", - "endpoint_enq", - "send-propagation" - ], - "related_bugs": [ - "bug-646", - "bug-648" - ], - "occurrences": 1, - "last_seen": "2026-06-17T18:30:00Z" - }, - { - "id": "bug-653", - "timestamp": "2026-06-17T11:21:17.173Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 14\u219219 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T11:21:17.173Z" - }, - { - "id": "bug-654", - "timestamp": "2026-06-17T11:21:51.571Z", - "error_message": "Wrong return value", - "file": "src/lib/testsuite/test_netio_arm_failure.c", - "root_cause": "Was returning: a negative error and (2) invoke the endpoint's err", - "fix": "Now returns: a negative, RETRIABLE error so the caller propagat", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T11:21:51.571Z" - }, - { - "id": "bug-655", - "timestamp": "2026-06-17T11:22:11.296Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_netio_arm_failure.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 11\u219213 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T11:22:11.296Z" - }, - { - "id": "bug-656", - "timestamp": "2026-06-17T11:22:17.916Z", - "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", - "file": "src/lib/testsuite/test_netio_arm_failure.c", - "root_cause": "Used \"test_enq_arm_failure_is_loud_and_fires_errhandler\" instead of \"test_enq_arm_failure_is_loud_no_errhandler\"", - "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler \u2192 test_enq_arm_failure_is_loud_no_errhandler", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T11:22:17.916Z" - }, - { - "id": "bug-652", - "timestamp": "2026-06-17T19:10:00Z", - "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", - "file": "src/lib/netio.c", - "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() \u2014 it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", - "fix": "endpoint_enq() no longer calls handle_error(); on arm failure it just returns a RETRIABLE -ECONNRESET. The caller propagates it (request.c senders return it -> client failover classifies ECONNRESET as RETRY_NOW and rotates; controller heartbeat path simply gets the error, no lock re-entry). The real teardown stays on the io_worker thread (handle_input/handle_output -> io_handler -> handle_error), which holds no caller locks. Updated test_netio_arm_failure.c: endpoint_enq must return <0 AND must NOT invoke the err_handler (g_err_count==0). libxce 169/0; controller_unit_test links clean.", - "tags": [ - "libxce", - "netio", - "endpoint_enq", - "deadlock", - "self-deadlock", - "ha-lock", - "reentrancy", - "regression", - "controller", - "vote_ep_err_handler" - ], - "related_bugs": [ - "bug-646", - "bug-650" - ], - "occurrences": 1, - "last_seen": "2026-06-17T19:10:00Z" - }, - { - "id": "bug-658", - "timestamp": "2026-06-17T14:46:54.135Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)", - "fix": "Changed to: if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T14:46:54.135Z" - }, - { - "id": "bug-659", - "timestamp": "2026-06-17T14:47:25.446Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (role == CTRL_STANDBY &&\n )", - "fix": "Changed to: if (role == CTRL_STANDBY &&\n )", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T14:47:25.446Z" - }, - { - "id": "bug-660", - "timestamp": "2026-06-17T22:00:00Z", - "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", - "file": "src/controller/ha.c", - "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` \u2014 checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", - "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) \u2014 so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", - "tags": [ - "controller", - "ha", - "election", - "prevote", - "convergence", - "fresh_heartbeat", - "raft_cb_vote_eligible", - "primary_ctrl_fd", - "slow-election" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T22:00:00Z" - }, - { - "id": "bug-661", - "timestamp": "2026-06-17T23:30:00Z", - "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", - "file": "src/controller/persist.c, src/controller/commit.c, src/controller/repl.c", - "root_cause": "'committed' does not imply 'durable'. WAL fsync is asynchronous (persist.c:173 fsync_worker, timer wal_sync_ms); persist_append and the standby WalAck path do NOT fdatasync before ACK. commit_on_ack (commit.c:303) advances committed_lsn on majority ACK = OS page cache only, durable on zero disks. On kill+restart, killed nodes lose the not-yet-fsynced tail; persist_recover hits CRC break at zeroed region (prev_crc=00000000, offset 40) and truncates replay to ~0 entries, yet the hollow node stays eligible, wins election under degraded required=1 quorum (PreVote+freshness gate cannot fence it because the committed entries were durable on nobody), self-promotes advertising committed_lsn=21 over an empty job table, and the node holding the real data converges DOWN. Violates Leader-Completeness assumption (majority-ACK != durable-on-majority) in src/controller/CLAUDE.md.", - "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 \u2014 matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro \u2014 this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", - "tags": [ - "controller", - "ha", - "durability", - "fsync", - "wal", - "commit-engine", - "data-loss", - "failover", - "leader-completeness", - "promotion-fence", - "async-fsync", - "deterministic-repro", - "wal-sync-ms" - ], - "related_bugs": [ - "bug-339", - "bug-447", - "bug-396" - ], - "occurrences": 2, - "last_seen": "2026-06-18T00:00:00Z" - }, - { - "id": "bug-662", - "timestamp": "2026-06-17T23:34:32.115Z", - "error_message": "Function not marked async", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T23:34:32.115Z" - }, - { - "id": "bug-663", - "timestamp": "2026-06-17T23:59:39.221Z", - "error_message": "Significant refactor of ", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 19\u219235 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-17T23:59:39.221Z" - }, - { - "id": "bug-664", - "timestamp": "2026-06-17T23:59:44.577Z", - "error_message": "Incorrect value in code", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"${2:-6}\"", - "fix": "Changed to \"${2:-8}\"", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-17T23:59:50.630Z" - }, - { - "id": "bug-665", - "timestamp": "2026-06-18T00:04:40.582Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/brief.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T00:04:40.582Z" - }, - { - "id": "bug-666", - "timestamp": "2026-06-18T02:32:07.670Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-durability/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T02:32:07.670Z" - }, - { - "id": "bug-667", - "timestamp": "2026-06-18T02:35:08.157Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T02:35:08.157Z" - }, - { - "id": "bug-668", - "timestamp": "2026-06-18T03:12:26.259Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `persist.c:515`", - "fix": "Changed to `fsync_worker`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-18T03:15:00.270Z" - }, - { - "id": "bug-669", - "timestamp": "2026-06-18T03:13:32.314Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 6\u21926 lines (5 removed) | Also: - **Persistence & consistency:** `durable_lsn` adv; - **Concurrency strategy:** advanced under `p->loc", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T03:13:49.756Z" - }, - { - "id": "bug-670", - "timestamp": "2026-06-18T03:14:30.410Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T03:14:30.410Z" - }, - { - "id": "bug-671", - "timestamp": "2026-06-18T03:15:13.287Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `fsync_worker`", - "fix": "Changed to `/kiro-validate-design`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T03:15:13.287Z" - }, - { - "id": "bug-672", - "timestamp": "2026-06-18T03:35:27.530Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 18\u219225 lines (7 removed) | Also: - _Depends: 4.1, 3.3_; - Before a member becomes primary, refuse promotio", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T03:35:46.457Z" - }, - { - "id": "bug-673", - "timestamp": "2026-06-18T03:50:26.761Z", - "error_message": "Function not marked async", - "file": "src/controller/CLAUDE.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T03:50:26.761Z" - }, - { - "id": "bug-674", - "timestamp": "2026-06-18T03:50:55.752Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21926 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T03:50:55.752Z" - }, - { - "id": "bug-675", - "timestamp": "2026-06-18T03:59:55.411Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T03:59:55.411Z" - }, - { - "id": "bug-676", - "timestamp": "2026-06-18T04:00:06.872Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `src/include/persist.h`", - "fix": "Changed to `persist_wal_sync_append`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T04:00:06.872Z" - }, - { - "id": "bug-677", - "timestamp": "2026-06-18T04:39:24.460Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T04:39:24.460Z" - }, - { - "id": "bug-678", - "timestamp": "2026-06-18T04:50:36.884Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'testsuite/test_commit_durability.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T04:50:36.884Z" - }, - { - "id": "bug-679", - "timestamp": "2026-06-18T04:51:30.430Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T04:51:30.430Z" - }, - { - "id": "bug-680", - "timestamp": "2026-06-18T05:00:22.007Z", - "error_message": "Missing error handling in unknown", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:00:22.007Z" - }, - { - "id": "bug-681", - "timestamp": "2026-06-18T05:01:45.135Z", - "error_message": "Significant refactor of ", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 85\u2192202 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:01:45.135Z" - }, - { - "id": "bug-682", - "timestamp": "2026-06-18T05:02:07.556Z", - "error_message": "Incorrect value in code", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"$(mktemp /tmp/xce-durability-acked.XXXXXX)\"", - "fix": "Changed to \"$(mktemp \"", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:02:07.556Z" - }, - { - "id": "bug-683", - "timestamp": "2026-06-18T05:07:20.589Z", - "error_message": "Missing error handling in unknown", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:07:20.589Z" - }, - { - "id": "bug-684", - "timestamp": "2026-06-18T05:28:05.352Z", - "error_message": "Function not marked async", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "sh" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-18T05:29:29.811Z" - }, - { - "id": "bug-685", - "timestamp": "2026-06-18T05:42:21.676Z", - "error_message": "Missing error handling in advances", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:42:21.676Z" - }, - { - "id": "bug-686", - "timestamp": "2026-06-18T05:43:02.761Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T05:45:15.732Z" - }, - { - "id": "bug-687", - "timestamp": "2026-06-18T05:43:49.070Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 41\u2192152 lines (7 removed) | Also: /* Wake any WAIT caller blocked on the commit eval; * advanced, so a majority candidate may now cover ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T05:44:05.035Z" - }, - { - "id": "bug-688", - "timestamp": "2026-06-18T05:54:46.908Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T05:54:55.010Z" - }, - { - "id": "bug-689", - "timestamp": "2026-06-18T05:55:28.140Z", - "error_message": "Missing error handling in FAILS", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:55:28.140Z" - }, - { - "id": "bug-690", - "timestamp": "2026-06-18T05:55:33.474Z", - "error_message": "Wrong reference: else should be RUN_TEST", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Used \"else\" instead of \"RUN_TEST\"", - "fix": "Changed else \u2192 RUN_TEST", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T05:55:33.474Z" - }, - { - "id": "bug-691", - "timestamp": "2026-06-18T05:59:05.965Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 24\u219252 lines (5 removed) | Also: * already joined and will not advance it again). *; fdatasync(p->wal_fd);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T06:00:18.921Z" - }, - { - "id": "bug-692", - "timestamp": "2026-06-18T05:59:17.869Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-18T06:00:42.436Z" - }, - { - "id": "bug-693", - "timestamp": "2026-06-18T06:05:00.000Z", - "error_message": "durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority", - "file": "src/controller/persist.c", - "root_cause": "ha-commit-durability task 2.1 flush sites called fdatasync(p->wal_fd) without capturing/checking the return code, then advanced durable_lsn unconditionally. On sync error the bytes are not on stable storage, violating Req 1.1/1.2 and the fail-loud preference.", - "fix": "Capture int rc = fdatasync(...) at every durable_lsn advance site (flusher_flush_once gating + fallback paths, persist_stop) and advance durable_lsn ONLY when rc == 0; on error leave durable_lsn unchanged and log loudly via ERROR(). persist_flush already gated on rc==0 (reference). Added XCE_TESTING fault-injection seam (test_force_fsync_fail + persist_test_set_force_fsync_fail + flusher_fsync_forced_fail helper, constant-false in production) and error-path test test_durable_lsn_holds_on_fsync_failure proving durable_lsn holds at baseline under injected failure and recovers when cleared.", - "tags": [ - "ha-commit-durability", - "durability", - "fdatasync", - "fail-loud", - "remediation", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:05:00.000Z" - }, - { - "id": "bug-694", - "timestamp": "2026-06-18T06:09:08.298Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", - "root_cause": "Had `/kiro-spec-init ha-commit-durability`", - "fix": "Changed to 't fence the stale node because the entries were d", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:09:08.298Z" - }, - { - "id": "bug-695", - "timestamp": "2026-06-18T06:12:59.924Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:12:59.924Z" - }, - { - "id": "bug-696", - "timestamp": "2026-06-18T06:13:32.891Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:13:32.891Z" - }, - { - "id": "bug-697", - "timestamp": "2026-06-18T06:13:40.659Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:13:40.659Z" - }, - { - "id": "bug-698", - "timestamp": "2026-06-18T06:13:54.196Z", - "error_message": "Significant refactor of ", - "file": "src/controller/commit.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 12\u219224 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:13:54.196Z" - }, - { - "id": "bug-699", - "timestamp": "2026-06-18T06:14:16.158Z", - "error_message": "Function not marked async", - "file": "src/controller/commit.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:14:16.158Z" - }, - { - "id": "bug-700", - "timestamp": "2026-06-18T06:15:12.451Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:15:12.451Z" - }, - { - "id": "bug-701", - "timestamp": "2026-06-18T06:15:45.248Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Had \"config.h\"", - "fix": "Changed to \"commit.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:15:45.248Z" - }, - { - "id": "bug-702", - "timestamp": "2026-06-18T06:16:09.746Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:16:09.746Z" - }, - { - "id": "bug-703", - "timestamp": "2026-06-18T06:16:49.048Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_engine.c", - "root_cause": "40 lines replaced/restructured", - "fix": "Rewrote 49\u21926 lines (40 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:16:49.048Z" - }, - { - "id": "bug-704", - "timestamp": "2026-06-18", - "error_message": "controller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0)", - "file": "src/controller/testsuite/{test_repl_majority,test_commit_barrier_rollout,test_coldstart_lsn0,test_demotion_convergence}.c", - "root_cause": "ha-commit-durability 2.2 changed majority_candidate_locked self-cap from highest_appended_lsn to persist_durable_lsn. Existing commit-path fixtures prime current_lsn (appended tip) but never set durable_lsn, so under the new cap self contributed 0 and committed_lsn never advanced.", - "fix": "In every commit-path stub/in-memory-primed fixture, set durable_lsn to the appended tip (appended_lsn / current_lsn-1) right after committed_lsn=0, preserving the pre-durability 'self covers every appended entry' model. The durability precondition itself is exercised by dedicated new tests.", - "tags": [ - "durable", - "commit", - "majority", - "self-cap", - "test-fixture", - "ha-commit-durability", - "task-2.2" - ], - "related_bugs": [ - "bug-661", - "bug-693" - ], - "occurrences": 1, - "last_seen": "2026-06-18" - }, - { - "id": "bug-705", - "timestamp": "2026-06-18T06:33:12.523Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T06:33:25.815Z" - }, - { - "id": "bug-706", - "timestamp": "2026-06-18T06:33:50.093Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u219222 lines (2 removed) | Also: int fsrc = fdatasync(p->wal_fd);; if (fsrc != 0 || flusher_fsync_forced_fail(p)) {", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T06:36:02.935Z" - }, - { - "id": "bug-707", - "timestamp": "2026-06-18T06:34:06.875Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p", - "fix": "Added guard clause: if (p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:34:06.875Z" - }, - { - "id": "bug-708", - "timestamp": "2026-06-18T06:34:36.356Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:34:36.356Z" - }, - { - "id": "bug-709", - "timestamp": "2026-06-18T06:34:53.620Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219210 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:34:53.620Z" - }, - { - "id": "bug-710", - "timestamp": "2026-06-18T06:50:13.989Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:50:13.989Z" - }, - { - "id": "bug-711", - "timestamp": "2026-06-18T06:50:49.987Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: written_lsn > durable", - "fix": "Added guard clause: if (written_lsn > durable)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:50:49.987Z" - }, - { - "id": "bug-712", - "timestamp": "2026-06-18T06:50:59.047Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T06:50:59.047Z" - }, - { - "id": "bug-713", - "timestamp": "2026-06-18T07:00:10.385Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 9\u21926 lines (5 removed) | Also: /* RED-PHASE: invariant assert DISABLED to let the; * the over-advance instead of aborting the binary.", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T07:00:44.663Z" - }, - { - "id": "bug-714", - "timestamp": "2026-06-18T07:00:36.008Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: written_lsn > durable", - "fix": "Added guard clause: if (written_lsn > durable)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T07:00:36.008Z" - }, - { - "id": "bug-715", - "timestamp": "2026-06-18T07:15:03.763Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T07:15:03.763Z" - }, - { - "id": "bug-716", - "timestamp": "2026-06-18T07:15:30.633Z", - "error_message": "Missing error handling in unknown", - "file": "src/include/persist.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T07:15:30.633Z" - }, - { - "id": "bug-717", - "timestamp": "2026-06-18T07:16:20.314Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219223 lines (2 removed) | Also: break; /* clean end of segment */; break; /* truncated or pre-allocated zeros */", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T07:16:37.162Z" - }, - { - "id": "bug-718", - "timestamp": "2026-06-18T07:19:17.397Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 12\u219214 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T07:19:17.397Z" - }, - { - "id": "bug-719", - "timestamp": "2026-06-18T07:51:32.985Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T07:51:32.985Z" - }, - { - "id": "bug-720", - "timestamp": "2026-06-18T07:52:20.595Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* task 3.2 \u2014 Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T07:52:29.952Z" - }, - { - "id": "bug-721", - "timestamp": "2026-06-18T08:04:18.857Z", - "error_message": "Missing error handling in enum", - "file": "src/include/persist.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Bounded catch-up window (ha-commit-durability task 3.3, R; * The FINITE deadline a recovery-truncated / behind member i", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T08:04:38.131Z" - }, - { - "id": "bug-722", - "timestamp": "2026-06-18T08:05:05.233Z", - "error_message": "Missing error handling in of", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:05:05.233Z" - }, - { - "id": "bug-723", - "timestamp": "2026-06-18T08:06:25.738Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219251 lines (4 removed) | Also: TEST_IGNORE_MESSAGE(; \"row 4b (CLEAN + below + no peer + bounded catch-u", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T08:06:39.996Z" - }, - { - "id": "bug-724", - "timestamp": "2026-06-18T08:07:00.123Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:07:00.123Z" - }, - { - "id": "bug-725", - "timestamp": "2026-06-18T08:18:57.946Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:18:57.946Z" - }, - { - "id": "bug-726", - "timestamp": "2026-06-18T08:19:47.036Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:19:47.036Z" - }, - { - "id": "bug-727", - "timestamp": "2026-06-18T08:32:37.878Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:32:37.878Z" - }, - { - "id": "bug-728", - "timestamp": "2026-06-18T08:39:54.889Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had `durable_floor \u2264 applied_lsn`", - "fix": "Changed to `applied_lsn`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:39:54.889Z" - }, - { - "id": "bug-729", - "timestamp": "2026-06-18T08:49:08.264Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 9\u219214 lines (3 removed) | Also: /* RED-DEMO (temporary): simulate the PRE-FIX bug ; * with a stale-high learned value so the candidate", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T08:52:37.412Z" - }, - { - "id": "bug-730", - "timestamp": "2026-06-18T08:49:30.666Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 11\u219217 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:49:30.666Z" - }, - { - "id": "bug-731", - "timestamp": "2026-06-18T08:49:41.212Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 10\u219215 lines (6 removed) | Also: uint64_t local_lsn = (uint64_t)atomic_load_explici; &c->ha.applied_lsn, memory_order_acquire);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T08:49:49.852Z" - }, - { - "id": "bug-732", - "timestamp": "2026-06-18T08:51:34.635Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "No early return/throw for edge case: !mkdtemp(tmp", - "fix": "Added guard clause: if (!mkdtemp(tmp)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:51:34.635Z" - }, - { - "id": "bug-733", - "timestamp": "2026-06-18T08:52:12.840Z", - "error_message": "Wrong return value", - "file": "src/controller/ha.c", - "root_cause": "Was returning: persist_durable_floor(p);", - "fix": "Now returns: floor > 8ULL ? floor : 8ULL; /* BUG: pretends lear", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T08:52:12.840Z" - }, - { - "id": "bug-734", - "timestamp": "2026-06-18T09:07:33.096Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T09:07:33.096Z" - }, - { - "id": "bug-735", - "timestamp": "2026-06-18T09:08:09.215Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* \u2500\u2500 ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T09:08:55.973Z" - }, - { - "id": "bug-736", - "timestamp": "2026-06-18T09:09:28.348Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fr; * member is not behind anything and must be promotable (no f", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T09:10:32.435Z" - }, - { - "id": "bug-737", - "timestamp": "2026-06-18T09:09:42.763Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 7\u219236 lines (4 removed) | Also: TEST_IGNORE_MESSAGE(; \"row 3 (TRUNCATED + peer + catch-up ok -> FENCE+CA | Also: TEST_IGNORE_MESSAGE(; \"row 5 (snapshot-sentinel CLEAN + below + peer -> ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-18T09:10:12.609Z" - }, - { - "id": "bug-738", - "timestamp": "2026-06-18T09:45:40.163Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T09:50:15.505Z" - }, - { - "id": "bug-739", - "timestamp": "2026-06-18T09:46:18.019Z", - "error_message": "Function not marked async", - "file": "src/include/persist.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T09:46:36.395Z" - }, - { - "id": "bug-740", - "timestamp": "2026-06-18T09:46:46.814Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T09:47:04.913Z" - }, - { - "id": "bug-741", - "timestamp": "2026-06-18T09:49:58.840Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Test-injectable send hook (defined in repl.c under XCE_TE; * repl_broadcast from dereferencing the dummy endpoints as r", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T09:50:07.979Z" - }, - { - "id": "bug-742", - "timestamp": "2026-06-18T10:22:00.509Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:22:00.509Z" - }, - { - "id": "bug-743", - "timestamp": "2026-06-18T10:25:02.673Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ha_cancel_promotion_fence_timer: cancel a pending bounded; * (idempotent; no-op when none is armed). Called when the m", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T10:27:05.112Z" - }, - { - "id": "bug-744", - "timestamp": "2026-06-18T10:26:12.640Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:26:12.640Z" - }, - { - "id": "bug-745", - "timestamp": "2026-06-18T10:26:34.227Z", - "error_message": "Wrong operator: != should be ==", - "file": "src/controller/ha.c", - "root_cause": "Used \"!=\" instead of \"==\"", - "fix": "Changed operator != \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:26:34.227Z" - }, - { - "id": "bug-746", - "timestamp": "2026-06-18T10:30:18.073Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* task 4.4 \u2014 bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-18T10:30:51.746Z" - }, - { - "id": "bug-747", - "timestamp": "2026-06-18T10:42:48.932Z", - "error_message": "Missing error handling in unknown", - "file": "src/include/persist.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:42:48.932Z" - }, - { - "id": "bug-748", - "timestamp": "2026-06-18T10:43:04.187Z", - "error_message": "Significant refactor of ", - "file": "src/include/persist.h", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 17\u219227 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:43:04.187Z" - }, - { - "id": "bug-749", - "timestamp": "2026-06-18T10:44:38.816Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:44:38.816Z" - }, - { - "id": "bug-750", - "timestamp": "2026-06-18T10:45:30.263Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 15\u219222 lines (11 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:45:30.263Z" - }, - { - "id": "bug-751", - "timestamp": "2026-06-18T10:47:24.587Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:47:24.587Z" - }, - { - "id": "bug-752", - "timestamp": "2026-06-18T10:47:33.818Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21929 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:47:33.818Z" - }, - { - "id": "bug-753", - "timestamp": "2026-06-18T10:48:41.920Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:48:41.920Z" - }, - { - "id": "bug-754", - "timestamp": "2026-06-18T10:54:08.903Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u21929 lines (2 removed) | Also: | Intent | Distinguish clean log end from corrupti; - FATAL policy: if a recovery-truncated member can | Also: > **Status:** DECISION recorded (task 3.2). No pro; **Decision (Req 2.2, 2.4):** A member that boots w", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-18T10:55:04.759Z" - }, - { - "id": "bug-755", - "timestamp": "2026-06-18T10:55:31.898Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/design.md", - "root_cause": "Had \"bounded window\"", - "fix": "Changed to `DEFAULT_RECOVERY_CATCHUP_WINDOW_MS`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T10:55:31.898Z" - }, - { - "id": "bug-756", - "timestamp": "2026-06-18T11:07:54.918Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/CLAUDE.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T11:07:54.918Z" - }, - { - "id": "bug-757", - "timestamp": "2026-06-18T11:08:39.844Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/steering/ha-async-io.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T11:08:39.844Z" - }, - { - "id": "bug-758", - "timestamp": "2026-06-18T11:14:25.437Z", - "error_message": "Missing error handling in gap", - "file": "src/controller/CLAUDE.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T11:14:25.437Z" - }, - { - "id": "bug-759", - "timestamp": "2026-06-18T11:21:44.844Z", - "error_message": "Incorrect value in code", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", - "root_cause": "Had `--wal-sync-ms`", - "fix": "Changed to `:FAIL:`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T11:21:44.844Z" - }, - { - "id": "bug-760", - "timestamp": "2026-06-18T13:00:00Z", - "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 \u2014 a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", - "file": "src/controller/testsuite/test_commit_barrier_rollout.c", - "root_cause": "ha-commit-durability task 2.2 capped the primary commit self-contribution at durable_lsn (advanced asynchronously by the demand-driven group-commit flusher) and wired a durable-advance commit-wake callback. This adjacent-spec (ha-commit-barrier-rollout) test uses a REAL persist+flusher and checks committed_lsn SYNCHRONOUSLY the instant after repl_on_ack(fact_lsn); the flusher has not yet fdatasync'd LSN1, so durable_lsn==0, self-cap==0, majority candidate==0, committed_lsn stays 0. Pre-spec the test passed (self counted the appended tip immediately). Task 2.2 primed durable_lsn in the in-memory stub fixtures (barrier_setup_vcpus) but did NOT update this real-flusher test. The implementation-note claim that this is a pre-existing /tmp-overlay fdatasync-EIO artifact is FALSE in this env: fdatasync returns rc=0 on /tmp here, and the test passed pre-spec.", - "fix": "FIXED \u2014 test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", - "tags": [ - "ha-commit-durability", - "commit-barrier-rollout", - "durable_lsn", - "group-commit", - "test-race", - "async-durability", - "validation", - "NO-GO" - ], - "related_bugs": [ - "bug-661", - "bug-704" - ], - "occurrences": 6, - "last_seen": "2026-06-18T13:30:00Z" - }, - { - "id": "bug-761", - "timestamp": "2026-06-18T13:13:49.968Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_commit_barrier_rollout.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T13:13:49.968Z" - }, - { - "id": "bug-762", - "timestamp": "2026-06-18T13:18:43.638Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T13:18:43.638Z" - }, - { - "id": "bug-763", - "timestamp": "2026-06-18T14:54:25.478Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T14:54:25.478Z" - }, - { - "id": "bug-764", - "timestamp": "2026-06-18T14:55:00.167Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/steering/error-handling.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T14:55:00.167Z" - }, - { - "id": "bug-765", - "timestamp": "2026-06-18T14:30:00Z", - "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) \u2192 corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", - "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob \u2192 CREATE path)", - "root_cause": "Tasks were checked [x] without the dequeue/update path and rusage storage actually being wired; gap surfaced only by cross-task re-audit (per-task review missed it).", - "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation \u2014 sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 \u2014 enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", - "tags": [ - "scheduler", - "controller", - "msg-gap", - "validate-gap", - "not-implemented", - "HIGH", - "job-state-corruption" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-18T17:00:00Z" - }, - { - "id": "bug-766", - "timestamp": "2026-06-19T02:55:49.212Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", - "root_cause": "Was returning: `-ENOENT` if job not found", - "fix": "Now returns: `-ENOENT` if job not found. _Remediation (bug-765,", - "tags": [ - "auto-detected", - "return-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T02:55:49.212Z" - }, - { - "id": "bug-767", - "timestamp": "2026-06-19T03:30:07.652Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/schedobj.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 18\u219212 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T03:30:07.652Z" - }, - { - "id": "bug-768", - "timestamp": "2026-06-19T03:30:28.056Z", - "error_message": "Wrong condition in logic", - "file": "src/scheduler/src/schedobj.c", - "root_cause": "Condition was: if (job->rusages != NULL && job->rusages != rusages)", - "fix": "Changed to: if (job->rusages != rusages)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T03:30:28.056Z" - }, - { - "id": "bug-769", - "timestamp": "2026-06-19T03:31:07.604Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/src/scheduler.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 18\u219233 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T03:31:07.604Z" - }, - { - "id": "bug-770", - "timestamp": "2026-06-19T03:48:40.507Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 9\u219211 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T03:48:40.507Z" - }, - { - "id": "bug-771", - "timestamp": "2026-06-19T03:49:03.016Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21925 lines (2 removed) | Also: #### New function: `xce_sched_op_name`; Signature: char* xce_sched_op_name(xce_sched_op_t ", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T03:49:12.246Z" - }, - { - "id": "bug-772", - "timestamp": "2026-06-19T03:49:23.624Z", - "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", - "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", - "root_cause": "Used \"xce_sched_op_name\" instead of \"sched_op_name\"", - "fix": "Changed xce_sched_op_name \u2192 sched_op_name", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T03:49:23.624Z" - }, - { - "id": "bug-773", - "timestamp": "2026-06-19T03:49:39.373Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", - "root_cause": "Had `xce_sched_op_name()`", - "fix": "Changed to `sched_op_name()`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T03:49:46.161Z" - }, - { - "id": "bug-774", - "timestamp": "2026-06-19T03:50:12.460Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", - "root_cause": "Had `xce_sched_op_name()`", - "fix": "Changed to `sched_op_name()`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T03:50:20.916Z" - }, - { - "id": "bug-775", - "timestamp": "2026-06-19T04:07:03.964Z", - "error_message": "Significant refactor of ", - "file": "src/scheduler/testsuite/test_handlers.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T04:07:03.964Z" - }, - { - "id": "bug-776", - "timestamp": "2026-06-19T04:13:42.107Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 7\u21927 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T04:13:42.107Z" - }, - { - "id": "bug-777", - "timestamp": "2026-06-19T13:00:00Z", - "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", - "file": "src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table", - "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up \u2014 so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", - "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain \u2014 no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking \u2014 the systematic trigger is gone.", - "tags": [ - "ha", - "convergence", - "promotion-fence", - "recovery-truncated", - "torn-tail", - "SIGKILL", - "availability", - "election-wedge", - "ha-commit-durability", - "ha-convergence-recovery", - "live-E2E", - "false-positive-data-loss" - ], - "related_bugs": [ - "bug-661", - "bug-339" - ], - "occurrences": 2, - "last_seen": "2026-06-19T14:30:00Z" - }, - { - "id": "bug-778", - "timestamp": "2026-06-19T13:40:00Z", - "error_message": "persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss.", - "file": "src/controller/persist.c:1336 (apply_wal_entry) + src/controller/ha.c ha_apply_single_entry (~1570-1588)", - "root_cause": "On the standby replicated-apply path, ha_apply_single_entry first calls persist_wal_write_replicated() which correctly sets p->last_crc to the standby's SELF-computed chain crc (the stream carries no crc; the on-disk CRC chain is node-local, prev_crc==last_crc). It THEN builds a synthetic wal_entry_hdr with hdr.crc=0 and calls apply_wal_entry(), whose line 1336 `p->last_crc = hdr->crc` OVERWRITES the correct chain link with 0. The next replicated write computes prev_crc=(seg_offset==0?WAL_CRC_NONE:last_crc)=0 and stamps prev_crc=0 on disk. So the on-disk CRC chain breaks at the 2nd replicated entry. Invisible while live (standby serves in-memory entities; never replays its own WAL), detonates on COLD RESTART where replay_wal validates the chain -> RECOVER_TRUNCATED -> recovers only the clean prefix -> committed jobs lost + the truncated verdict drives the promotion-fence availability wedge (the other half of bug-777). NOT an fd-position/offset race (pwrite+p->lock are correct); it is shared CRC-chain state (p->last_crc) written by two paths where the logical-apply step clobbers the physical-writer's link with 0.", - "fix": "FIXED 2026-06-19 (commit 64ab210). apply_wal_entry is now LOGICAL entity-restore only (signature (p,type,lsn,payload,payload_len); no header); it advances the current_lsn high-water but NEVER touches p->last_crc. The CRC chain has a single owner: persist_wal_write_replicated (live) + replay_wal/persist_recover (recovery). ha_apply_single_entry drops the fabricated crc=0 header. Verified: RED->GREEN test test_replicated_apply_chain_survives_cold_restart (real ha_apply_wal_catchup_entry path, cold-replay -> RECOVER_CLEAN); controller suite 0 failures; 3-node rolling-SIGKILL soak on real ext2/3 fs 5/5 (was wedge+loss at round 3); full cold restart recovers all 6 jobs (was 0), CLEAN verdicts, normal election. Test callers updated: test_repl_apply, test_ha_role_lifecycle, test_egress_filter.", - "tags": [ - "ha", - "wal", - "crc-chain", - "last_crc", - "apply_wal_entry", - "persist_wal_write_replicated", - "cold-restart", - "data-loss", - "bug-661", - "standby-replication", - "recovery", - "root-caused" - ], - "related_bugs": [ - "bug-777", - "bug-661" - ], - "occurrences": 2, - "last_seen": "2026-06-19T14:30:00Z" - }, - { - "id": "bug-779", - "timestamp": "2026-06-19T13:59:32.538Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_commit_durability.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T13:59:32.538Z" - }, - { - "id": "bug-780", - "timestamp": "2026-06-19T14:01:31.365Z", - "error_message": "Significant refactor of ", - "file": "src/include/persist.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u219212 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T14:01:31.365Z" - }, - { - "id": "bug-781", - "timestamp": "2026-06-19T14:02:49.235Z", - "error_message": "Wrong reference: hdr should be payload_len", - "file": "src/controller/persist.c", - "root_cause": "Used \"hdr\" instead of \"payload_len\"", - "fix": "Changed hdr \u2192 payload_len", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T14:02:49.235Z" - }, - { - "id": "bug-782", - "timestamp": "2026-06-19T14:03:49.830Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21925 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T14:03:49.830Z" - }, - { - "id": "bug-783", - "timestamp": "2026-06-19T14:04:05.032Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 9\u21927 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T14:04:05.032Z" - }, - { - "id": "bug-784", - "timestamp": "2026-06-19T14:07:12.930Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11\u219213 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T14:07:12.930Z" - }, - { - "id": "bug-785", - "timestamp": "2026-06-19T14:08:12.866Z", - "error_message": "Wrong reference: NULL should be type", - "file": "src/controller/testsuite/test_ha_role_lifecycle.c", - "root_cause": "Used \"NULL\" instead of \"type\"", - "fix": "Changed NULL \u2192 type", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T14:09:08.241Z" - }, - { - "id": "bug-786", - "timestamp": "2026-06-19T15:00:00Z", - "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", - "file": "src/controller/testsuite/test.c (cmd_jobsub)", - "root_cause": "On the 2nd+ submit (and during failover windows) the jobsub reply/jobid is not cleanly populated, leaving an uninitialized jobid struct; cmd_jobsub still prints 'jobsubmit ok' with the garbage id instead of verifying the controller reply status and failing. Forces E2E scripts (test-commit-durability.sh) to strict-regex-filter the 4-field id to avoid chasing phantom jobs.", - "fix": "PENDING \u2014 jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient \u2014 a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", - "tags": [ - "xce_test", - "jobsub", - "garbage-jobid", - "e2e", - "client", - "test-tooling", - "xce-test-e2e-primitives", - "stale-reply-union", - "live-soak" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-19" - }, - { - "id": "bug-787", - "timestamp": "2026-06-19T15:26:46.318Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 27\u219211 lines (13 removed) | Also: *; * XCE_TEST_VERDICT_FLAG gates the new exit-code/ab", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T15:26:54.160Z" - }, - { - "id": "bug-788", - "timestamp": "2026-06-19T15:34:41.623Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/testsuite/test.c", - "root_cause": "Condition was: if (active_ctx->parent != NULL)", - "fix": "Changed to: if (i > 1)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T15:34:41.623Z" - }, - { - "id": "bug-789", - "timestamp": "2026-06-19T15:41:13.871Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 9\u21923 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T15:41:13.871Z" - }, - { - "id": "bug-790", - "timestamp": "2026-06-19T15:58:57.460Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 7\u21923 lines (5 removed) | Also: #if JOBSUB_RED_FORCE_SUCCESS; /* RED only: simulate a non-clean ack that surface", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T15:59:25.070Z" - }, - { - "id": "bug-791", - "timestamp": "2026-06-19T16:07:20.038Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 38\u219285 lines (2 removed) | Also: * g_last_jobcount most recent job count; -1 means ; * cmd_joblist (task 5.1).", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T16:07:27.736Z" - }, - { - "id": "bug-792", - "timestamp": "2026-06-19T16:13:02.715Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"shutdown\"", - "fix": "Changed to \"expect-jobs\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-19T16:13:02.715Z" - }, - { - "id": "bug-793", - "timestamp": "2026-06-19T16:32:47.571Z", - "error_message": "Significant refactor of ", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "31 lines replaced/restructured", - "fix": "Rewrote 63\u2192115 lines (31 removed) | Also: if ! wait_for_primary 40; then; fail \"cluster did not elect an initial primary wit | Also: # Wait until some live node is CTRL_PRIMARY and th; wait_for_primary() { | Also: script=$(printf 'set masters %s\\nclient\\n' \"$MASTE; script+=$(printf 'expect-job %s\\n' \"$jid\") | Also: # format-agnostically, instead of re-imposing a br; # submit did not cleanly ack, jobsub fails -> sess", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 6, - "last_seen": "2026-06-19T16:37:04.187Z" - }, - { - "id": "bug-794", - "timestamp": "2026-06-19T16:50:45.275Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 8\u219220 lines (7 removed) | Also: if (jobid == 0) {; /* Non-negative rc but no JobId was written: not a", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-19T16:50:54.261Z" - }, - { - "id": "bug-795", - "timestamp": "2026-06-19", - "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", - "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} \u2014 UPSTREAM ha-commit-durability Phase 6.2", - "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) \u2014 broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact \u2014 reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause.", - "fix": "NOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", - "tags": [ - "ha-commit-durability", - "commit-durability", - "cold-restart", - "total-outage", - "wal-truncate", - "upstream", - "xce-test-e2e-primitives", - "needs-real-fs", - "rolling-failover", - "liveness", - "non-deterministic" - ], - "related_bugs": [ - "bug-661", - "bug-778", - "bug-786" - ], - "occurrences": 3, - "last_seen": "2026-06-19" - }, - { - "id": "bug-796", - "timestamp": "2026-06-20T00:32:43.144Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/spec.json", - "root_cause": "Had \"2026-06-18T04:00:00Z\"", - "fix": "Changed to \"2026-06-19T00:00:00Z\"", - "tags": [ - "auto-detected", - "wrong-value", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T00:32:43.144Z" - }, - { - "id": "bug-797", - "timestamp": "2026-06-20T00:42:36.967Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/xce-test-e2e-primitives/spec.json", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T00:42:36.967Z" - }, - { - "id": "bug-798", - "timestamp": "2026-06-20T01:05:04.547Z", - "error_message": "Missing error handling in unknown", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T01:05:04.547Z" - }, - { - "id": "bug-799", - "timestamp": "2026-06-20T01:05:25.091Z", - "error_message": "Incorrect value in code", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Had \"$label: DATA LOSS \u2014 a committed job vanished (bug", - "fix": "Changed to \"$label: DATA LOSS \u2014 a committed job vanished from", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T01:05:25.091Z" - }, - { - "id": "bug-800", - "timestamp": "2026-06-20T01:05:51.010Z", - "error_message": "Significant refactor of ", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u219212 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T01:05:51.010Z" - }, - { - "id": "bug-801", - "timestamp": "2026-06-20T01:14:42.334Z", - "error_message": "Significant refactor of ", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u219210 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T01:14:42.334Z" - }, - { - "id": "bug-802", - "timestamp": "2026-06-20T01:21:21.789Z", - "error_message": "Missing error handling in unknown", - "file": "src/tools/test-commit-durability.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T01:21:21.789Z" - }, - { - "id": "bug-803", - "timestamp": "2026-06-20T01:27:31.000Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T01:27:31.000Z" - }, - { - "id": "bug-804", - "timestamp": "2026-06-20T03:29:22.678Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-lsid/requirements.md", - "root_cause": "Had \"none\"", - "fix": "Changed to \"unknown\"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-20T03:33:32.399Z" - }, - { - "id": "bug-805", - "timestamp": "2026-06-20T03:29:29.570Z", - "error_message": "Wrong return value", - "file": ".kiro/specs/controller-lsid/requirements.md", - "root_cause": "Was returning: a well-defined \"unknown\" or \"none\" value for that ", - "fix": "Now returns: a well-defined \"unknown\" value for that field rath", - "tags": [ - "auto-detected", - "return-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T03:29:29.570Z" - }, - { - "id": "bug-806", - "timestamp": "2026-06-20T03:32:19.414Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/controller-lsid/requirements.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21928 lines (3 removed) | Also: ### Requirement 5: Committed configuration version; **Objective:** As an operator, I want the configur", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-20T03:32:34.774Z" - }, - { - "id": "bug-807", - "timestamp": "2026-06-20T04:08:01.204Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/controller-status/design.md", - "root_cause": "12 lines replaced/restructured", - "fix": "Rewrote 16\u219213 lines (12 removed) | Also: clusterName @0: Text;; primaryName @1: Text; # empty \u21d2 \"unknown\" | Also: /* failover-aware read (is_write=false) */; - Postconditions: on success `*status` owns a deco | Also: - `configChecksum`/`configVersion` equal across tw; - `ctrlstatus` against a healthy 3-node cluster pr | Also: - Answerable on **any** node and across failover \u2014; - Report a content checksum of the committed confi", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-20T04:11:28.379Z" - }, - { - "id": "bug-808", - "timestamp": "2026-06-20T04:08:49.684Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-status/design.md", - "root_cause": "Had `committed_config_tag`", - "fix": "Changed to `committedConfigVersion`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-20T04:10:27.793Z" - }, - { - "id": "bug-809", - "timestamp": "2026-06-20T04:40:36.648Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'commit.c'", - "fix": "Changed to 'ctrlstatus.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T04:40:36.648Z" - }, - { - "id": "bug-810", - "timestamp": "2026-06-20T04:45:48.985Z", - "error_message": "Wrong reference: Data should be Text", - "file": "src/schema/controller.capnp", - "root_cause": "Used \"Data\" instead of \"Text\"", - "fix": "Changed Data \u2192 Text", - "tags": [ - "auto-detected", - "wrong-reference", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T04:45:48.985Z" - }, - { - "id": "bug-811", - "timestamp": "2026-06-20T04:45:49.790Z", - "error_message": "Wrong reference: xce_data_t should be char", - "file": "src/include/typemap.h", - "root_cause": "Used \"xce_data_t\" instead of \"char\"", - "fix": "Changed xce_data_t \u2192 char", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T04:45:49.790Z" - }, - { - "id": "bug-812", - "timestamp": "2026-06-20T05:08:41.030Z", - "error_message": "Incorrect value in code", - "file": "src/lib/meson.build", - "root_cause": "Had 'raft/raft_sim.c'", - "fix": "Changed to 'testsuite/test_ctrlstatus.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T05:08:41.030Z" - }, - { - "id": "bug-813", - "timestamp": "2026-06-20T05:42:31.707Z", - "error_message": "Wrong operator: >= should be ==", - "file": "src/controller/testsuite/test.c", - "root_cause": "Used \">=\" instead of \"==\"", - "fix": "Changed operator >= \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-20T05:42:46.894Z" - }, - { - "id": "bug-814", - "timestamp": "2026-06-20T05:51:48.030Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ctrlstatus.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 23\u219273 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T05:51:48.030Z" - }, - { - "id": "bug-815", - "timestamp": "2026-06-20T05:52:48.719Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ctrlstatus.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u219213 lines (3 removed) | Also: /* primary identity \u2014 guarded by ha->lock (Req 2.2; st->primaryName = (ha->primary_host[0] != '\\0') ? ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-20T05:53:00.264Z" - }, - { - "id": "bug-816", - "timestamp": "2026-06-20T05:57:46.206Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ctrlstatus.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 13\u219216 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T05:57:46.206Z" - }, - { - "id": "bug-817", - "timestamp": "2026-06-19", - "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", - "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) \u2014 UPSTREAM, not controller-status", - "root_cause": "libxce client failover does not cleanly advance past a dead FIRST master to the live nodes; surfaces a typed EINVAL instead of rotating. Pre-existing (controller-status consumes failover unchanged). Surfaced by controller-status task 5.1 E2E (authoritative wait-primary timed out after a primary kill while a dead node led the masters list, even though a new primary was serving).", - "fix": "NOT fixed in controller-status (out of boundary). Route to libxce-failover. Confirmed affects plain joblist identically. Workaround: ensure the masters list does not lead with a known-dead node, or restart killed nodes (as the durability soak does).", - "tags": [ - "libxce", - "failover", - "master-rotation", - "dead-first-master", - "EINVAL", - "controller-status", - "upstream" - ], - "related_bugs": [ - "bug-646" - ], - "occurrences": 1, - "last_seen": "2026-06-19" - }, - { - "id": "bug-818", - "timestamp": "2026-06-20T06:11:49.084Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/testsuite/test.c", - "root_cause": "Condition was: if (!g_continue_on_error)", - "fix": "Changed to: if (!g_continue_on_error && !g_interactive)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-20T06:14:41.073Z" - }, - { - "id": "bug-819", - "timestamp": "2026-06-20T06:14:51.644Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u21928 lines (4 removed) | Also: /* Session verdict toggle (xce-test-e2e-primitives; * `set continue-on-error` keeps the read loop runn", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-20T06:15:17.616Z" - }, - { - "id": "bug-820", - "timestamp": "2026-06-20T06:16:34.685Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", - "root_cause": "Had `g_interactive = isatty(STDIN_FILENO)`", - "fix": "Changed to `-e`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T06:16:34.685Z" - }, - { - "id": "bug-821", - "timestamp": "2026-06-20T08:14:43.577Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-status/requirements.md", - "root_cause": "Had 's identity and role, the configured controller to", - "fix": "Changed to 's identity and role, the configured controller to", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T08:14:43.577Z" - }, - { - "id": "bug-822", - "timestamp": "2026-06-20T08:15:28.319Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/controller-status/spec.json", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 24\u219230 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T08:15:28.319Z" - }, - { - "id": "bug-823", - "timestamp": "2026-06-20T09:03:03.345Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ctrlstatus.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u219212 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T09:03:03.345Z" - }, - { - "id": "bug-824", - "timestamp": "2026-06-20T09:30:23.040Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/controller-status/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T09:30:23.040Z" - }, - { - "id": "bug-825", - "timestamp": "2026-06-20T09:31:24.076Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/controller-status/spec.json", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T09:31:24.076Z" - }, - { - "id": "bug-826", - "timestamp": "2026-06-20T10:10:14.423Z", - "error_message": "Significant refactor of ", - "file": "src/lib/utils.c", - "root_cause": "13 lines replaced/restructured", - "fix": "Rewrote 26\u219250 lines (13 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T10:10:14.423Z" - }, - { - "id": "bug-827", - "timestamp": "2026-06-20T18:30:00.000Z", - "error_message": "AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from cmd_ctrlstatus (test.c:1273) when ctrlstatus failover exhausts and returns E_XCE_NOT_READY (-10013) on a primary-less cluster", - "file": "src/lib/utils.c", - "root_cause": "Enum/table drift: errnox.h E_XCE_* enum gained NOMASTER(10011)/REDIRECT(10012)/NOT_READY(10013)/STANDBY(10014), bumping E_XCE_MAX to 10015, but xce_err_str[] still had only 10 strings + a NULL (11 slots, 88 bytes). xce_strerror_r's only upper guard was errnum>=E_XCE_MAX, so 10011-10014 passed the gate and indexed past the table: NOMASTER->xce_err_str[10]==NULL (strncpy NULL deref); NOT_READY->xce_err_str[12] = 8 bytes past the 88-byte array (the ASAN OOB). Pre-existing libxce foundation bug; controller-status ctrlstatus was the first caller to format these HA codes on a failing cluster.", - "fix": "Filled xce_err_str[] with strings for all 14 codes (enum order); hardened xce_strerror_r to bound against the ACTUAL array size (idx>=nstr) + reject NULL slots (degrade to 'unknown error', never OOB) + null-terminate; added a _Static_assert tying sizeof(xce_err_str) to (E_XCE_MAX-XCE_ERRNO_BASE) so future enum additions without a string fail the build. RED: new test_strerror_all_xce_codes SEGV'd at utils.c:335 under ASAN; GREEN: lib suite Ok:1 Fail:0. Commit 1aa0761.", - "tags": [ - "global-buffer-overflow", - "asan", - "xce_strerror_r", - "errnox", - "enum-table-drift", - "libxce", - "controller-status", - "ctrlstatus", - "E_XCE_NOT_READY" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T18:30:00.000Z" - }, - { - "id": "bug-828", - "timestamp": "2026-06-20T10:19:56.678Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-convergence-recovery/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T10:19:56.678Z" - }, - { - "id": "bug-829", - "timestamp": "2026-06-20T11:44:25.496Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'ctrlstatus.c'", - "fix": "Changed to 'testsuite/test_ha_heartbeat_committed.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T11:44:25.496Z" - }, - { - "id": "bug-830", - "timestamp": "2026-06-20T11:45:26.943Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T11:45:26.943Z" - }, - { - "id": "bug-831", - "timestamp": "2026-06-20T11:54:30.060Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_ha_heartbeat_committed.c", - "root_cause": "Had \"xce/cluster_def.h\"", - "fix": "Changed to \"typemap.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T11:54:30.060Z" - }, - { - "id": "bug-832", - "timestamp": "2026-06-20T11:56:28.240Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c || !hb", - "fix": "Added guard clause: if (!c || !hb)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T11:56:28.240Z" - }, - { - "id": "bug-833", - "timestamp": "2026-06-20T12:09:25.213Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 12\u219226 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T12:09:25.213Z" - }, - { - "id": "bug-834", - "timestamp": "2026-06-20T12:22:39.011Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", - "root_cause": "Had `heartbeatmsg`", - "fix": "Changed to `xce-build.sh controller`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T12:22:39.011Z" - }, - { - "id": "bug-835", - "timestamp": "2026-06-20T12:26:15.999Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'ctrlstatus.c'", - "fix": "Changed to 'testsuite/test_ha_vote_floor_heartbeat.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T12:26:15.999Z" - }, - { - "id": "bug-836", - "timestamp": "2026-06-20T12:31:51.359Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T12:31:51.359Z" - }, - { - "id": "bug-837", - "timestamp": "2026-06-20T12:32:25.549Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-heartbeat-committed-propagation/spec.json", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T12:32:25.549Z" - }, - { - "id": "bug-828", - "timestamp": "2026-06-20T21:30:00.000Z", - "error_message": "Controller failover test loses committed jobs: ACKed jobs (jobsubmit ok, visible in joblist) disappear after kill/restart cycles. Predates libxce-failover (long-standing, unfixed).", - "file": "src/controller/persist.c / config_repl.c (CtrlHello catch-up)", - "root_cause": "REAL data loss (not commit-index). Decoded WAL bytes at /tmp/xce-ha-deploy: committed prefix entries (LSN 2-12, the lost job submits) are PHYSICALLY ABSENT from the live nodes' segments (ctrl-2 WAL starts at LSN 16, ctrl-3 at 13) and exist only on the down node (ctrl-1, LSN 1-15). snap/ dirs empty. Mechanism: a node (re)joining with an empty WAL is brought up to the cluster's CURRENT committed watermark via WAL streaming [watermark..committed], NOT from LSN 1, and no snapshot backfills [1..watermark-1] -> permanent prefix gap. When the only full-log node is down while others rejoined high, committed jobs are gone. replay_wal (persist.c:1422) is correct (read-only recovery); the gap is in catch-up start-point + missing entity-bearing snapshots.", - "fix": "NOT YET FIXED. Full findings: .kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md. Open forks: (1) test clean-vs-preserve data on restart; (2) CtrlHello advertise real applied_lsn vs learned committed (config_repl.c handle_ctrlhello); (3) is snapshot creation implemented or a stub (XCESNAP header carries only an lsn, no entities). Likely architectural: catch-up assumes a snapshot safety net that does not exist. Secondary: WAL hdr.term always 0 (p->term never set) -> no term-based divergence detection.", - "tags": [ - "data-loss", - "failover", - "committed-job-loss", - "wal", - "catch-up", - "snapshot", - "ha-commit-durability", - "bug-795", - "architecture", - "NOT-FIXED" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T21:30:00.000Z" - }, - { - "id": "bug-839", - "timestamp": "2026-06-20T13:51:29.352Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T13:51:29.352Z" - }, - { - "id": "bug-840", - "timestamp": "2026-06-20T14:05:49.912Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"xce/cluster.h\"", - "fix": "Changed to \"waldump.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:05:49.912Z" - }, - { - "id": "bug-841", - "timestamp": "2026-06-20T14:06:11.393Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/testsuite/test.c", - "root_cause": "Condition was: if (active_ctx->parent != NULL)", - "fix": "Changed to: if (argc < 2)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:06:11.393Z" - }, - { - "id": "bug-842", - "timestamp": "2026-06-20T14:09:03.676Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 6\u21925 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:09:03.676Z" - }, - { - "id": "bug-843", - "timestamp": "2026-06-20T14:15:35.599Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `!`", - "fix": "Changed to `xce_test`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:15:35.599Z" - }, - { - "id": "bug-844", - "timestamp": "2026-06-20T14:20:00.730Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 6\u219216 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:20:00.730Z" - }, - { - "id": "bug-845", - "timestamp": "2026-06-20T14:30:00.000Z", - "error_message": "Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy.", - "file": "src/lib/hook.c (xce_add_hook idempotency); root path src/controller/{persist.c persist_start, ha.c ha_register_persist_hooks}", - "root_cause": "NOT array expansion \u2014 job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) \u2014 proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary.", - "fix": "FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners \u2014 persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) \u2014 with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) \u2014 re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) \u2014 confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup.", - "tags": [ - "double-register", - "duplicate", - "fixed", - "ha", - "hook-dedup", - "investigation", - "job-submit", - "persistence", - "wal", - "waldump" - ], - "related_bugs": [ - "bug-795", - "bug-828" - ], - "occurrences": 2, - "last_seen": "2026-06-23T23:30:00.000Z" - }, - { - "id": "bug-846", - "timestamp": "2026-06-20T14:27:17.198Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:27:17.198Z" - }, - { - "id": "bug-847", - "timestamp": "2026-06-20T14:37:08.863Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/waldump.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 27\u219238 lines (6 removed) | Also: /* Per-entry line, with jobid for submits. */; char info[64] = \"\"; | Also: fprintf(out, \" %8s | %5s | %-11s | %5s | %s\\n\",; \"LSN\", \"TERM\", \"TYPE\", \"PLEN\", \"INFO\");", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-20T14:39:19.887Z" - }, - { - "id": "bug-848", - "timestamp": "2026-06-20T14:40:02.211Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_waldump.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 13\u219216 lines (5 removed) | Also: size_t plen;; uint8_t* job = encode_job(2001, &plen);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-20T14:40:43.204Z" - }, - { - "id": "bug-849", - "timestamp": "2026-06-20T14:42:09.400Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md", - "root_cause": "Had `handle_jobsubmit`", - "fix": "Changed to `6d79074e`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T14:42:09.400Z" - }, - { - "id": "bug-850", - "timestamp": "2026-06-20T15:14:36.135Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"shutdown\"", - "fix": "Changed to \"waldump\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T15:14:36.135Z" - }, - { - "id": "bug-846", - "timestamp": "2026-06-20T15:20:00.000Z", - "error_message": "After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client.", - "file": "src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)", - "root_cause": "Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) \u2014 or yields a bare EINVAL while RUN \u2014 failover gives up permanently. Exact stuck state needs a live repro to confirm.", - "fix": "FIXED (verified 2026-07-02, no direct code trail to when \u2014 landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note.", - "tags": [ - "client", - "stale-report", - "einval", - "blocker", - "failover", - "ha", - "fixed", - "libxce" - ], - "related_bugs": [ - "bug-646" - ], - "occurrences": 1, - "last_seen": "2026-07-02T17:30:00.000Z" - }, - { - "id": "bug-852", - "timestamp": "2026-06-20T15:27:35.465Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_wal_wire.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T15:27:35.465Z" - }, - { - "id": "bug-853", - "timestamp": "2026-06-20T15:27:41.698Z", - "error_message": "Wrong reference: test_wal_batch_entry_without_payload should be test_wal_batch_roundtrip_large_payloads", - "file": "src/lib/testsuite/test_wal_wire.c", - "root_cause": "Used \"test_wal_batch_entry_without_payload\" instead of \"test_wal_batch_roundtrip_large_payloads\"", - "fix": "Changed test_wal_batch_entry_without_payload \u2192 test_wal_batch_roundtrip_large_payloads", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T15:27:41.698Z" - }, - { - "id": "bug-847", - "timestamp": "2026-06-20T15:35:00.000Z", - "error_message": "Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism.", - "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) \u2014 capnp multi-segment Data pointers", - "root_cause": "Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary \u2014 on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine \u2014 ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up.", - "fix": "FIXED \u2014 CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON \u2014 clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) \u2014 the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed \u2014 whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) \u2014 buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed.", - "tags": [ - "reproducer-unignored", - "corruption", - "ha", - "waldump", - "stale-report", - "reverified-open", - "catchup", - "data-loss", - "replication", - "capnp", - "critical", - "wal", - "fixed" - ], - "related_bugs": [ - "bug-828", - "bug-795", - "bug-661" - ], - "occurrences": 1, - "last_seen": "2026-07-03T03:15:00.000Z" - }, - { - "id": "bug-855", - "timestamp": "2026-06-20T15:30:48.257Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T15:30:48.257Z" - }, - { - "id": "bug-856", - "timestamp": "2026-06-20T15:31:23.821Z", - "error_message": "Missing error handling in unknown", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_failover_job_loss.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-20T15:31:23.821Z" - }, - { - "id": "bug-848", - "timestamp": "2026-06-20T15:50:00.000Z", - "error_message": "On a PRESERVED-DATA restart (no wipe \u2014 user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", - "file": "src/controller/cluster.c (cluster_init_phase2, after persist_recover) + src/controller/ha.c (ha_init applied_lsn=0; state report sr->applied_lsn)", - "root_cause": "ha->applied_lsn is NEVER reconstructed from the recovered WAL on boot. ha_init zeros it (ha.c:1367); persist_recover (cluster.c:1352) replays the WAL and advances persist->current_lsn/durable_lsn, but nothing copies that recovered position into ha->applied_lsn (persist_t and ha_state_t are separate; the only setter besides ha_init is the SNAPSHOT receive path ha.c:1771 = checkpoint_lsn). So the standby state report advertises sr->applied_lsn = ha->applied_lsn = 0 (ha.c:1787) and the primary's ha_evaluate_repl_action streams [0+1 .. committed]. ASYMMETRY proving it's a bug: ELECTION advertise uses ha_candidate_advertise_lsn = persist_durable_floor (the RECOVERED position, correct), but CATCH-UP advertise uses the un-recovered applied_lsn=0. The system knows its durable tip for voting but not for catch-up. Defeats Raft incremental catch-up for every restarted node; also the upstream cause of the earlier missing-prefix cases (full re-stream from a trimmed primary -> prefix gap).", - "fix": "FIXED 2026-06-20 (HA-layer). Added ha_sync_applied_lsn_from_recovery(ha,p) (ha.c/ha.h) = monotonic store of persist_durable_floor(p) into ha->applied_lsn; wired into cluster_init_phase2 right after persist_recover, plus a diagnostic INFO log of recovered durable_floor/current_lsn/applied_lsn. TDD: test_recovered_member_advertises_real_applied_lsn (test_persist_recovery.c) write->recover->sync asserts applied_lsn==durable_floor>0 and monotonic. controller_unit_test 6/6 + adjacent HA 8/8, daemon builds. CAVEAT: live ctrl-1 also showed current_lsn=1 (seg O_TRUNC'd from LSN 1) implying persist_recover may not have loaded the WAL on that restart at all; if so this HA-layer fix is necessary but not sufficient. The new INFO log reveals on the next live restart whether persist_recover is effective (durable_floor>0) or not (bug-849 candidate: recovery not running/loading on the rejoin path).", - "tags": [ - "ha", - "raft", - "recovery", - "catchup", - "applied_lsn", - "data-loss", - "root-trigger", - "critical" - ], - "related_bugs": [ - "bug-847", - "bug-828", - "bug-795" - ], - "occurrences": 1, - "last_seen": "2026-06-20T15:50:00.000Z" - }, - { - "id": "bug-858", - "timestamp": "2026-06-21T00:18:54.350Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T00:18:54.350Z" - }, - { - "id": "bug-859", - "timestamp": "2026-06-21T00:19:18.673Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T00:19:18.673Z" - }, - { - "id": "bug-860", - "timestamp": "2026-06-21T00:19:50.700Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T00:19:50.700Z" - }, - { - "id": "bug-861", - "timestamp": "2026-06-21T00:20:55.974Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test_persist_recovery.c", - "root_cause": "Had \"persist.h\"", - "fix": "Changed to \"ha.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T00:20:55.974Z" - }, - { - "id": "bug-849", - "timestamp": "2026-06-21T11:15:00.000Z", - "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only \u2014 missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", - "file": "src/controller/myapp.c (standby-rejoin path ~317-320) + src/controller/cluster.c cluster_init_phase2 (holds persist_recover)", - "root_cause": "CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED \u2014 its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done \u2014 durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY \u2014 waiting for primary to connect and push config') with NO election \u2192 cluster_init_phase2 NEVER ran \u2192 NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it.", - "fix": "FIXED 2026-06-21 (design B \u2014 boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving \u2014 so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done \u2014 durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix.", - "tags": [ - "ha", - "raft", - "recovery", - "boot", - "election-gated", - "catchup", - "data-loss", - "critical" - ], - "related_bugs": [ - "bug-848", - "bug-847", - "bug-828", - "bug-795" - ], - "occurrences": 1, - "last_seen": "2026-06-21T11:15:00.000Z" - }, - { - "id": "bug-850", - "timestamp": "2026-06-21T12:30:00.000Z", - "error_message": "A rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery.", - "file": "src/controller/repl.c repl_add_standby (line ~119) + repl_broadcast in_catch_up gate", - "root_cause": "repl_add_standby initializes a freshly-registered standby with in_catch_up=0, so it is IMMEDIATELY eligible for repl_broadcast (which correctly skips in_catch_up standbys at repl.c:27). Mechanism (confirmed from ctrl-1.log 11:07:21): ctrl-1 reconnects -> registered in_catch_up=0 -> the new job [15..16] is live-broadcast to it (not gated) -> persist_wal_write_replicated writes them, current_lsn jumps to 17 -> THEN the catch-up stream of [1..14] arrives but persist_wal_write_replicated's idempotency guard (lsn+1 <= current_lsn) skips ALL of them (1+1<=17 ... 14+1<=17) -> permanent prefix gap [1..14]. Independent of bug-849: even with recovery (current_lsn=8), a live [15..16] arriving first jumps current_lsn to 17 and the catch-up of [8..14] is skipped (gap [8..14]). Root architectural cause: catch-up streaming and live broadcast are SEPARATE unordered paths (unlike Raft AppendEntries which is one ordered per-follower stream).", - "fix": "FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby \u2014 REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it \u2014 so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) \u2014 contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together.", - "tags": [ - "ha", - "raft", - "replication", - "catchup", - "ordering", - "race", - "in_catch_up", - "data-loss", - "critical" - ], - "related_bugs": [ - "bug-849", - "bug-848", - "bug-847", - "bug-828", - "bug-795" - ], - "occurrences": 1, - "last_seen": "2026-06-21T12:30:00.000Z" - }, - { - "id": "bug-864", - "timestamp": "2026-06-21T03:39:07.483Z", - "error_message": "Missing guard clause", - "file": "src/controller/repl.c", - "root_cause": "No early return/throw for edge case: r->standbys[i].ep == ep", - "fix": "Added guard clause: if (r->standbys[i].ep == ep)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T03:39:07.483Z" - }, - { - "id": "bug-865", - "timestamp": "2026-06-21T03:39:17.593Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T03:39:17.593Z" - }, - { - "id": "bug-866", - "timestamp": "2026-06-21T03:54:46.212Z", - "error_message": "Significant refactor of ", - "file": "src/controller/repl.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 15\u21925 lines (11 removed) | Also: /* Transition-aware (bug-850): only the actual 1\u21920; * (membership re-eval + waiter wakeup) work, so th", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-21T03:55:12.202Z" - }, - { - "id": "bug-867", - "timestamp": "2026-06-21T03:55:24.501Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 13\u21927 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T03:55:24.501Z" - }, - { - "id": "bug-868", - "timestamp": "2026-06-21T03:56:40.908Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T03:56:40.908Z" - }, - { - "id": "bug-869", - "timestamp": "2026-06-21T03:57:03.979Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T03:57:03.979Z" - }, - { - "id": "bug-870", - "timestamp": "2026-06-21T04:00:51.551Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T04:00:51.551Z" - }, - { - "id": "bug-871", - "timestamp": "2026-06-21T04:01:07.089Z", - "error_message": "Wrong reference: test_repl_write_rejects_checkpoint should be test_repl_write_rejects_forward_gap", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "Used \"test_repl_write_rejects_checkpoint\" instead of \"test_repl_write_rejects_forward_gap\"", - "fix": "Changed test_repl_write_rejects_checkpoint \u2192 test_repl_write_rejects_forward_gap", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T04:01:07.089Z" - }, - { - "id": "bug-851", - "timestamp": "2026-06-21T13:30:00.000Z", - "error_message": "CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn.", - "file": "src/controller/persist.c:588/726/1040 (hdr.term=p->term, p->term never set) + src/controller/ha.c (election term = local+1, PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term,lsn) log-match)", - "root_cause": "ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) \u2014 only CHAOTIC churn triggers it.", - "fix": "NOT FIXED (HA-core + persistence => full spec workflow per CLAUDE.md, not a quick patch). Keystone lever: stamp hdr.term from live ha->term on the primary append path + carry term through the repl envelope + persist it, THEN make ha_convergence reconcile truncate any local suffix whose (term,lsn) != leader's (standard Raft log-matching). MUST pair with durably-establishing the election term before a new leader acts (no two leaders per term: persist/propagate the won term to a majority before appending the own-term noop, or bump term beyond any (term,lsn) a voter advertises). Route to ha-consistency-invariants / ha-convergence-recovery as a new sub-spec.", - "tags": [ - "ha", - "failover", - "replication", - "convergence", - "raft-safety", - "term-monotonicity", - "wal-term", - "committed-divergence", - "data-loss", - "spec-needed" - ], - "related_bugs": [ - "bug-850", - "bug-849", - "bug-848", - "bug-646" - ], - "occurrences": 2, - "last_seen": "2026-06-21T09:10:00.000Z", - "root_caused": "2026-06-21T09:10:00.000Z" - }, - { - "id": "bug-873", - "timestamp": "2026-06-21T09:15:48.912Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-commit-durability/INVESTIGATION-failover-job-loss.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T09:15:48.912Z" - }, - { - "id": "bug-854", - "timestamp": "2026-06-21T09:10:00.000Z", - "error_message": "Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok.", - "file": "src/lib/client.c:403 xcecli_ctrlstatus_primary + src/controller/ctrlstatus.c:104 (standby primaryName=\"\")", - "root_cause": "Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 \u2014 the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it \u2014 it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby).", - "fix": "FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source \u2014 it is only a latency optimization; the client rotation fix alone resolves the hang.", - "tags": [ - "libxce", - "client", - "failover", - "primary-discovery", - "ctrlstatus", - "ha", - "FIXED", - "client", - "rotation", - "xcecli_ctrlstatus_primary" - ], - "related_bugs": [ - "bug-846", - "bug-851", - "bug-646" - ], - "occurrences": 2, - "last_seen": "2026-06-23" - }, - { - "id": "bug-875", - "timestamp": "2026-06-21T10:04:37.876Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/requirements.md", - "root_cause": "Had `OPCode_replConfigPush`", - "fix": "Changed to `snapshot_transfer_s`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T10:04:37.876Z" - }, - { - "id": "bug-876", - "timestamp": "2026-06-21T10:17:35.147Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-term-safety/requirements.md", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 14\u219226 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T10:17:35.147Z" - }, - { - "id": "bug-857", - "timestamp": "2026-06-21T10:30:00.000Z", - "error_message": "Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes pushes config bytes only then restart_requested; each node re-derives state by replaying its WAL under the NEW config, with no snapshot pinning an identical post-upgrade state. Replay is config-dependent (lost_and_found re-homes orphaned jobs), so nodes can diverge; amplified by bug-851 WAL tail divergence.", - "file": "src/controller/config_repl.c do_upgrade_config_with_bytes (~1001-1163)", - "root_cause": "Config is a separate consistency domain (committed_config_tag, no term, tag anti-entropy) outside both the WAL log and raft.c. Upgrade activation = config push + restart + per-node WAL replay under new config; no snapshot transfers an identical materialized state, and state is interpreted against config. Identified via TLA+ gap analysis (config should be log-entry/snapshot-anchored). NOT empirically reproduced yet.", - "fix": "NOT FIXED. Descoped 2026-06-21 from ha-term-safety R8 into the independent `config-upgrade-snapshot` spec (.kiro/specs/config-upgrade-snapshot/, requirements generated). Fix = upgrade materializes a FULL entity-bearing state snapshot at the config-change marker (term,lsn) boundary, embeds new config, installs on every member; no per-node replay-under-new-config. HARD dependency: entity-bearing-snapshots (snapshots are a 16-byte LSN sentinel today; COW xzarray_freeze/thaw built but dormant). Verify by reproducing bug-857 (upgrade during chaotic multi-kill, diff materialized state).", - "tags": [ - "config", - "snapshot", - "ha", - "needs-verification", - "spec-needed", - "descoped", - "config-upgrade-snapshot", - "entity-bearing-snapshots", - "state-divergence", - "raft-safety", - "upgrade" - ], - "related_bugs": [ - "bug-851", - "bug-828" - ], - "occurrences": 1, - "last_seen": "2026-06-21T10:30:00.000Z" - }, - { - "id": "bug-878", - "timestamp": "2026-06-21T10:34:30.593Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T10:34:30.593Z" - }, - { - "id": "bug-879", - "timestamp": "2026-06-21T10:59:29.212Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/TLA-gap-analysis.md", - "root_cause": "Had `raft.c`", - "fix": "Changed to 's split architecture, the chosen direction is to ", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T10:59:29.212Z" - }, - { - "id": "bug-880", - "timestamp": "2026-06-21T12:56:07.740Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/requirements.md", - "root_cause": "Had `snapshot_transfer_s`", - "fix": "Changed to `(term, lsn)`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T12:56:07.740Z" - }, - { - "id": "bug-881", - "timestamp": "2026-06-21T13:00:46.210Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/design.md", - "root_cause": "Had `xce_test`", - "fix": "Changed to `OPCode_replConfigPush`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-21T13:04:57.579Z" - }, - { - "id": "bug-882", - "timestamp": "2026-06-21T13:03:32.383Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-term-safety/design.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 15\u219229 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T13:03:32.383Z" - }, - { - "id": "bug-883", - "timestamp": "2026-06-21T13:05:44.284Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/research.md", - "root_cause": "Had `config_repl.c:1857-1944`", - "fix": "Changed to `xzarray_freeze`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-21T13:06:07.689Z" - }, - { - "id": "bug-884", - "timestamp": "2026-06-21T15:20:00Z", - "error_message": "test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50)", - "file": "src/lib/testsuite/test_config.c", - "root_cause": "commit ca1d9ae raised DEFAULT_SYNC_TIMEOUT_MS 50->500 in config.h but did not update the test assertion; pre-existing red on HEAD, unrelated to raft", - "fix": "synced assertion to ==500 (commit 7b6cbb8)", - "tags": [ - "test", - "config", - "stale-assertion", - "ha-commit-engine" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:20:00Z" - }, - { - "id": "bug-885", - "timestamp": "2026-06-21T15:21:59.689Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: index < 1 || index > RAFT_SIM_MAX_LOG", - "fix": "Added guard clause: if (index < 1 || index > RAFT_SIM_MAX_LOG)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:21:59.689Z" - }, - { - "id": "bug-886", - "timestamp": "2026-06-21T15:41:42.176Z", - "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_oversize_peers_returns_einval", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_oversize_peers_returns_einval\"", - "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_oversize_peers_returns_einval", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:41:42.176Z" - }, - { - "id": "bug-887", - "timestamp": "2026-06-21T15:45:29.785Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", - "fix": "Added guard clause: if (s->role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:45:29.785Z" - }, - { - "id": "bug-888", - "timestamp": "2026-06-21T15:52:45.272Z", - "error_message": "Missing guard clause", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "No early return/throw for edge case: index >= 1 && index <= RAFT_SIM_MAX_LOG", - "fix": "Added guard clause: if (index >= 1 && index <= RAFT_SIM_MAX_LOG)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:52:45.272Z" - }, - { - "id": "bug-889", - "timestamp": "2026-06-21T15:53:14.276Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:53:14.276Z" - }, - { - "id": "bug-890", - "timestamp": "2026-06-21T15:59:59.295Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: rc != 0", - "fix": "Added guard clause: if (rc != 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T15:59:59.295Z" - }, - { - "id": "bug-891", - "timestamp": "2026-06-21T16:06:37.923Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:06:37.923Z" - }, - { - "id": "bug-892", - "timestamp": "2026-06-21T16:15:51.060Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 6\u219211 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:15:51.060Z" - }, - { - "id": "bug-893", - "timestamp": "2026-06-21T16:16:47.908Z", - "error_message": "Wrong reference: g_te_spy should be g_pe_spy", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"g_te_spy\" instead of \"g_pe_spy\"", - "fix": "Changed g_te_spy \u2192 g_pe_spy", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:16:47.908Z" - }, - { - "id": "bug-894", - "timestamp": "2026-06-21T16:20:39.535Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->cbs->on_trace", - "fix": "Added guard clause: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:20:39.535Z" - }, - { - "id": "bug-895", - "timestamp": "2026-06-21T16:29:35.778Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: c->nodes[src].partition || c->nodes[dst].partition", - "fix": "Added guard clause: if (c->nodes[src].partition || c->nodes[dst])", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:29:35.778Z" - }, - { - "id": "bug-896", - "timestamp": "2026-06-21T16:29:52.264Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:29:52.264Z" - }, - { - "id": "bug-897", - "timestamp": "2026-06-21T16:36:42.796Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u21926 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:36:42.796Z" - }, - { - "id": "bug-898", - "timestamp": "2026-06-21T16:36:58.149Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21925 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:36:58.149Z" - }, - { - "id": "bug-899", - "timestamp": "2026-06-21T16:42:30.241Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", - "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:42:30.241Z" - }, - { - "id": "bug-900", - "timestamp": "2026-06-21T16:51:25.960Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: RUN_TEST(test_sim_9_1_replication_and_partition_catch_up);; RUN_TEST(test_sim_9_2_divergent_suffix_truncation);", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-21T16:51:35.044Z" - }, - { - "id": "bug-901", - "timestamp": "2026-06-21T16:54:46.287Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 13\u219216 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T16:54:46.287Z" - }, - { - "id": "bug-902", - "timestamp": "2026-06-21T17:03:59.688Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 15\u219230 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T17:03:59.688Z" - }, - { - "id": "bug-903", - "timestamp": "2026-06-21T17:04:17.830Z", - "error_message": "Wrong operator: <= should be >=", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"<=\" instead of \">=\"", - "fix": "Changed operator <= \u2192 >=", - "tags": [ - "auto-detected", - "operator-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T17:04:17.830Z" - }, - { - "id": "bug-904", - "timestamp": "2026-06-21T21:44:26.804Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: node_idx < 0 || node_idx >= c->n", - "fix": "Added guard clause: if (node_idx < 0 || node_idx >= c->n)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T21:44:26.804Z" - }, - { - "id": "bug-905", - "timestamp": "2026-06-21T21:49:10.796Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-log-replication/tasks.md", - "root_cause": "Had `initial_log_len`", - "fix": "Changed to `raft_sim_restart_node`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T21:49:10.796Z" - }, - { - "id": "bug-906", - "timestamp": "2026-06-21T23:00:19.687Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 14\u219219 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T23:00:19.687Z" - }, - { - "id": "bug-907", - "timestamp": "2026-06-21T23:14:37.837Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft.c", - "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", - "fix": "Added guard clause: if (s->role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T23:14:37.837Z" - }, - { - "id": "bug-908", - "timestamp": "2026-06-21T23:14:55.547Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", - "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T23:14:55.547Z" - }, - { - "id": "bug-909", - "timestamp": "2026-06-21T23:30:04.636Z", - "error_message": "Missing guard clause", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "No early return/throw for edge case: g_mock.get_entry_term_log_len != 0 &&\n index == g_moc", - "fix": "Added guard clause: if (g_mock.get_entry_term_log_len != 0 &&\n )", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T23:30:04.636Z" - }, - { - "id": "bug-910", - "timestamp": "2026-06-21T23:31:46.582Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219226 lines (2 removed) | Also: * Denies if cand_term < s->term; otherwise returns; * from_idx, cand_term) (granting when the gate is ", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-21T23:31:55.967Z" - }, - { - "id": "bug-911", - "timestamp": "2026-06-21T23:32:38.225Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft.c", - "root_cause": "Condition was: if (s->cbs->on_vote_eligible != NULL)", - "fix": "Changed to: if (!raft_log_ok(s, cand_last_log_term, cand_last_log_)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-21T23:32:38.225Z" - }, - { - "id": "bug-912", - "timestamp": "2026-06-22T00:13:17.311Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T00:13:17.311Z" - }, - { - "id": "bug-913", - "timestamp": "2026-06-22T00:19:56.768Z", - "error_message": "Wrong reference: TEST_ASSERT_NOT_EQUAL_INT should be TEST_ASSERT_NOT_EQUAL", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"TEST_ASSERT_NOT_EQUAL_INT\" instead of \"TEST_ASSERT_NOT_EQUAL\"", - "fix": "Changed TEST_ASSERT_NOT_EQUAL_INT \u2192 TEST_ASSERT_NOT_EQUAL", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T00:19:56.768Z" - }, - { - "id": "bug-914", - "timestamp": "2026-06-22T01:24:34.541Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/spec.json", - "root_cause": "Had \"design-generated\"", - "fix": "Changed to \"tasks-generated\"", - "tags": [ - "auto-detected", - "wrong-value", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T01:24:34.541Z" - }, - { - "id": "bug-915", - "timestamp": "2026-06-22T01:24:46.936Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-term-safety/spec.json", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "json" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T01:24:46.936Z" - }, - { - "id": "bug-916", - "timestamp": "2026-06-22T02:56:16.383Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_prevote.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219215 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T02:56:16.383Z" - }, - { - "id": "bug-917", - "timestamp": "2026-06-22T02:57:27.792Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_e2e_capnp.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T02:57:27.792Z" - }, - { - "id": "bug-855", - "timestamp": "2026-06-22T03:10:06.653962Z", - "error_message": "controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch", - "file": "src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)", - "root_cause": "PRE-EXISTING on checkpoint 20e59d0, NOT caused by ha-term-safety 1.1/1.2. The bug-850 contiguity guard (persist.c:705 rejects lsn>current_lsn with -EAGAIN) is committed at HEAD, but the test_wal_catchup_* tests feed forward-gap LSNs (30, 42) into a FRESH persist (current_lsn~=1) and still expect them to apply -> guard returns -EAGAIN -> applied_lsn does not advance. ha-term-safety changes touch only vote/schema/typemap/stubs; git diff --stat shows ZERO changes to persist.c/config_repl.c/test_config_repl.c. Also a separate run TIMED OUT 300s (the bug-850 DNR /tmp fsync-EIO + commit-WAIT signature; environment artifact on overlayfs).", - "fix": "FIXED (verified 2026-07-02): ran controller_unit_test directly \u2014 all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to).", - "tags": [ - "controller-test", - "stale-report", - "ha-term-safety", - "wal-catchup", - "EAGAIN", - "not-a-regression", - "contiguity-guard", - "fixed", - "bug-850", - "pre-existing" - ], - "related_bugs": [ - "bug-850", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-07-02T17:30:00.000Z" - }, - { - "id": "bug-918", - "timestamp": "2026-06-22T03:30:00Z", - "error_message": "nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=)", - "file": "src/flake.nix", - "root_cause": "flake pins c-capnproto to the MUTABLE github archive/main.tar.gz; upstream main branch advanced so the tarball content/hash drifted, breaking the fixed-output derivation. Only pre-cached nix stores still build.", - "fix": "RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) \u2014 the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression.", - "tags": [ - "nix", - "flake", - "c-capnproto", - "env", - "build", - "worktree-isolation" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T03:45:00Z" - }, - { - "id": "bug-920", - "timestamp": "2026-06-22T03:58:08.538Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_term_safety_adapter.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T03:58:08.538Z" - }, - { - "id": "bug-921", - "timestamp": "2026-06-22T03:58:19.392Z", - "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_waldump\" instead of \"test_term_safety_adapter\"", - "fix": "Changed test_waldump \u2192 test_term_safety_adapter", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T03:58:28.433Z" - }, - { - "id": "bug-922", - "timestamp": "2026-06-22T03:59:43.099Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || lsn == 0", - "fix": "Added guard clause: if (!p || lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T03:59:43.099Z" - }, - { - "id": "bug-923", - "timestamp": "2026-06-22T04:01:37.467Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/persist.c", - "root_cause": "Condition was: if (hdr.lsn <= from_lsn)", - "fix": "Changed to: if (hdr.type < WAL_CHECKPOINT)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T04:01:37.467Z" - }, - { - "id": "bug-924", - "timestamp": "2026-06-22", - "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", - "file": "environment (/ overlay filesystem)", - "root_cause": "Disk filled to 100% (7466 leftover /tmp/xce_* WAL dirs from repeated ASAN test runs). make_fresh_persist's persist_init mkdir_p hit ENOSPC -> returned NULL -> cascade of 'Expected Non-NULL'. Also overlayfs returned fdatasync EIO under disk pressure. NOT a code regression: baseline full suite passes; test_config_repl catchup tests are order-dependent and fail in isolation on baseline too.", - "fix": "rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles.", - "tags": [ - "disk-space", - "enospc", - "overlayfs", - "fdatasync-eio", - "controller-tests", - "flaky", - "ha-term-safety" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22" - }, - { - "id": "bug-925", - "timestamp": "2026-06-22T04:42:28.897Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T04:42:28.897Z" - }, - { - "id": "bug-926", - "timestamp": "2026-06-22T04:45:07.018Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", - "fix": "Added guard clause: if (!ha || !ha->raft_c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T04:45:07.018Z" - }, - { - "id": "bug-927", - "timestamp": "2026-06-22T04:46:36.310Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 19\u219225 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T04:46:36.310Z" - }, - { - "id": "bug-928", - "timestamp": "2026-06-22T04:55:58.789Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T04:55:58.789Z" - }, - { - "id": "bug-929", - "timestamp": "2026-06-22T04:57:03.605Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", - "fix": "Added guard clause: if (!ha || !ha->raft_c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T04:57:15.567Z" - }, - { - "id": "bug-930", - "timestamp": "2026-06-22T04:57:25.141Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T04:57:25.141Z" - }, - { - "id": "bug-931", - "timestamp": "2026-06-22T05:45:09.373Z", - "error_message": "Function not marked async", - "file": "src/include/repl.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T05:45:09.373Z" - }, - { - "id": "bug-932", - "timestamp": "2026-06-22T05:47:00.864Z", - "error_message": "Missing error handling in xce_wal_batch_req_raw", - "file": "src/controller/repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T05:47:00.864Z" - }, - { - "id": "bug-933", - "timestamp": "2026-06-22T05:47:50.037Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !d", - "fix": "Added guard clause: if (!d)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T05:47:50.037Z" - }, - { - "id": "bug-934", - "timestamp": "2026-06-22T05:49:28.039Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 15\u219220 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T05:49:28.039Z" - }, - { - "id": "bug-935", - "timestamp": "2026-06-22T06:38:20.255Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T06:38:20.255Z" - }, - { - "id": "bug-936", - "timestamp": "2026-06-22T06:39:16.197Z", - "error_message": "Significant refactor of ", - "file": "src/controller/commit.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 10\u219225 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T06:39:16.197Z" - }, - { - "id": "bug-937", - "timestamp": "2026-06-22T06:39:51.351Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", - "fix": "Added guard clause: if (!ha || !ha->raft_c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T06:39:51.351Z" - }, - { - "id": "bug-938", - "timestamp": "2026-06-22T07:19:14.726Z", - "error_message": "Significant refactor of ", - "file": "src/controller/commit.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 12\u219218 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T07:19:14.726Z" - }, - { - "id": "bug-939", - "timestamp": "2026-06-22T07:19:47.218Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "14 lines replaced/restructured", - "fix": "Rewrote 24\u219233 lines (14 removed) | Also: /* Standby ACKs LSN 2 \u2192 durable majority covers LS; /* Fire the WIRED callback with commit_index=1 \u2192 t", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T07:22:08.898Z" - }, - { - "id": "bug-940", - "timestamp": "2026-06-22T07:39:01.964Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 17\u219247 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T07:39:01.964Z" - }, - { - "id": "bug-941", - "timestamp": "2026-06-22T07:53:06.457Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T07:53:06.457Z" - }, - { - "id": "bug-942", - "timestamp": "2026-06-22T07:54:52.956Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T07:54:52.956Z" - }, - { - "id": "bug-943", - "timestamp": "2026-06-22T07:55:30.714Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T07:55:30.714Z" - }, - { - "id": "bug-944", - "timestamp": "2026-06-22T07:56:08.623Z", - "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Used \"ha_arm_own_term_gate\" instead of \"ha_arm_own_term_gate_for_test\"", - "fix": "Changed ha_arm_own_term_gate \u2192 ha_arm_own_term_gate_for_test", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T07:56:08.623Z" - }, - { - "id": "bug-945", - "timestamp": "2026-06-22T08:09:41.508Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:09:41.508Z" - }, - { - "id": "bug-946", - "timestamp": "2026-06-22T08:16:54.023Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:16:54.023Z" - }, - { - "id": "bug-947", - "timestamp": "2026-06-22T08:19:44.572Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/request.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:19:44.572Z" - }, - { - "id": "bug-948", - "timestamp": "2026-06-22T08:21:57.062Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat ; * library as a 0-entry AppendEntries so the LIBRARY performs", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T08:23:05.006Z" - }, - { - "id": "bug-949", - "timestamp": "2026-06-22T08:22:36.807Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:22:36.807Z" - }, - { - "id": "bug-950", - "timestamp": "2026-06-22T08:24:31.832Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha_convergence.c", - "root_cause": "43 lines replaced/restructured", - "fix": "Rewrote 59\u219227 lines (43 removed) | Also: * ha_rejoin_reconcile_ctx_t \u2014 heap context for the; * Carries the clamped cutoff computed at trigger t", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T08:27:13.909Z" - }, - { - "id": "bug-951", - "timestamp": "2026-06-22T08:26:04.789Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 10\u219215 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:26:04.789Z" - }, - { - "id": "bug-952", - "timestamp": "2026-06-22T08:42:30.744Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:42:30.744Z" - }, - { - "id": "bug-953", - "timestamp": "2026-06-22T08:48:45.792Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: rc == 0 && out_lsn", - "fix": "Added guard clause: if (rc == 0 && out_lsn)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T08:48:45.792Z" - }, - { - "id": "bug-954", - "timestamp": "2026-06-22T09:04:06.849Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 39\u219279 lines (9 removed) | Also: * path; neither library call performs I/O. */; ha_raft_on_repl_ack(c, vctx->peer_idx, acked_lsn, ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T09:04:19.614Z" - }, - { - "id": "bug-955", - "timestamp": "2026-06-22T09:05:02.687Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 12\u219230 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T09:05:02.687Z" - }, - { - "id": "bug-956", - "timestamp": "2026-06-22T09:05:30.540Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T09:05:30.540Z" - }, - { - "id": "bug-957", - "timestamp": "2026-06-22T09:08:41.177Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ep_is_vote", - "fix": "Added guard clause: if (ep_is_vote)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T09:11:38.490Z" - }, - { - "id": "bug-960", - "timestamp": "2026-06-22T09:20:00.000Z", - "error_message": "ha_raft_on_repl_ack unconditionally remapped the ACK endpoint peer_idx (votes_idx->absolute); for heartbeat STANDBY endpoints peer_idx is ALREADY absolute, so it was double-mapped -> wrong-peer false quorum / out-of-bounds match_index[] write (memory corruption / data loss).", - "file": "src/controller/ha.c", - "root_cause": "The walAck call site (ha.c walAck branch) fires for BOTH promoted-vote eps (vote_ep_ctx_t, peer_idx relative) AND heartbeat standby eps (standby_ep_ctx_t, ctx==NULL, peer_idx absolute). ha_raft_on_repl_ack applied the self-excluded->absolute remap unconditionally, corrupting the heartbeat path. Library raft_on_append_entries_reply writes s->match_index[peer_idx] unchecked (raft.c:802).", - "fix": "Made ha_raft_on_repl_ack take an ALREADY-ABSOLUTE peer index + added a fail-loud [0,n_peers) self-exclusion bounds check (no assert in src/lib/raft). Added pure resolver ha_repl_ack_abs_peer_idx(ha, ep_is_vote, peer_idx) mirroring ha.c:6187-6189 (vote ep -> remap, standby ep -> passthrough). Call site resolves per endpoint type via vctx->ctx!=NULL before calling the seam. Added 3 tests: endpoint-type round-trip for my_priority 0 AND 1, heartbeat-standby absolute-slot end-to-end, and out-of-range rejection. The two heartbeat tests FAIL (Expected 1 Was 2) against the old blind remap.", - "tags": [ - "ha-term-safety", - "raft", - "peer-index", - "match_index", - "oob", - "false-quorum", - "task-3.3", - "remediation", - "c" - ], - "related_bugs": [ - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-22T09:20:00.000Z" - }, - { - "id": "bug-959", - "timestamp": "2026-06-22T09:31:11.227Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !sha256 || !buf || !out_len", - "fix": "Added guard clause: if (!sha256 || !buf || !out_len)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T09:31:11.227Z" - }, - { - "id": "bug-960", - "timestamp": "2026-06-22T09:43:23.309Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T09:43:23.309Z" - }, - { - "id": "bug-961", - "timestamp": "2026-06-22T09:45:31.779Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c || !sha256", - "fix": "Added guard clause: if (!c || !sha256)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T09:45:31.779Z" - }, - { - "id": "bug-962", - "timestamp": "2026-06-22T10:04:06.960Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: committed == 0", - "fix": "Added guard clause: if (committed == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:04:06.960Z" - }, - { - "id": "bug-963", - "timestamp": "2026-06-22T10:04:17.709Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219214 lines (2 removed) | Also: /* Sentinel header: magic (8 bytes) + checkpoint_l; char header[16];", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T10:04:29.639Z" - }, - { - "id": "bug-964", - "timestamp": "2026-06-22T10:05:22.007Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: acked_out", - "fix": "Added guard clause: if (acked_out)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:05:22.007Z" - }, - { - "id": "bug-965", - "timestamp": "2026-06-22T10:05:36.940Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 22\u219226 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:05:36.940Z" - }, - { - "id": "bug-966", - "timestamp": "2026-06-22T10:10:11.668Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p", - "fix": "Added guard clause: if (!p)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:10:11.668Z" - }, - { - "id": "bug-967", - "timestamp": "2026-06-22T10:10:17.811Z", - "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Used \"persist_roll_segment\" instead of \"persist_test_force_new_segment\"", - "fix": "Changed persist_roll_segment \u2192 persist_test_force_new_segment", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:10:17.811Z" - }, - { - "id": "bug-968", - "timestamp": "2026-06-22T10:16:44.417Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_barrier_rollout.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 12\u219225 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:16:44.417Z" - }, - { - "id": "bug-969", - "timestamp": "2026-06-22T10:19:46.334Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 25\u219234 lines (4 removed) | Also: /* Commit the marker's WAL LSN through the real co; * exactly as a majority WAL ack does in production", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T10:20:17.591Z" - }, - { - "id": "bug-970", - "timestamp": "2026-06-22T10:24:35.905518Z", - "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER \u2014 config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", - "file": "src/controller/config_repl.c, src/controller/persist.c", - "root_cause": "Finding1: do_upgrade_config_with_bytes still drove the term-blind committed_config_tag push/ack/advance and never called config_append_marker, so the derive primitive was dead code. Finding2: persist_checkpoint->trim_wal_segments unlinks older segments and the snapshot sentinel carried no config reference, so a committed marker in a trimmed segment vanished and config_committed_marker regressed/returned -ENOENT.", - "fix": "Added config_commit_change() wired into do_upgrade_config_with_bytes step 7: appends WAL_CONFIG_MARKER as the consensus record (config_append_marker), then delegates byte delivery + the existing majority barrier to push_config_to_standbys (mechanism unchanged); committed_config_tag is now only a cache, config_committed_marker (committed-prefix derive) is authoritative. Compaction safety: persist_checkpoint scans the committed prefix for the latest marker {version_tag,sha256} (scan_latest_committed_config_ref) BEFORE trim and stamps it into the snapshot sentinel (41-byte optional tail); load_snapshot reads it back into persist.cfgref_*; config_committed_marker falls back to it when no marker survives the live WAL. Added integration test (drives config_commit_change) + checkpoint-trim test + persist_test_force_new_segment seam. cb5 upgrade pins made persist-backed so the marker append succeeds. RED confirmed (Expected 1 Was 0; Expected 0 Was -110); GREEN 0 failures direct-run.", - "tags": [ - "ha-term-safety", - "task-4.2", - "config-marker", - "wal", - "checkpoint", - "compaction", - "review-remediation", - "controller" - ], - "related_bugs": [ - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-22T10:24:35.905518Z" - }, - { - "id": "bug-971", - "timestamp": "2026-06-22T10:39:44.072Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: !mc", - "fix": "Added guard clause: if (!mc)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:39:44.072Z" - }, - { - "id": "bug-972", - "timestamp": "2026-06-22T10:40:31.131Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:40:31.131Z" - }, - { - "id": "bug-973", - "timestamp": "2026-06-22T10:57:25.062Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u219218 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:57:25.062Z" - }, - { - "id": "bug-974", - "timestamp": "2026-06-22T10:58:21.742Z", - "error_message": "Significant refactor of ", - "file": "src/controller/myapp.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 8\u219226 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T10:58:21.742Z" - }, - { - "id": "bug-975", - "timestamp": "2026-06-22T11:00:42.661Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: !ha", - "fix": "Added guard clause: if (!ha)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T11:00:42.661Z" - }, - { - "id": "bug-976", - "timestamp": "2026-06-22T11:15:00.000Z", - "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", - "file": "src/controller/config_repl.c, src/controller/ha.c, src/controller/myapp.c", - "root_cause": "config_activate_committed (verify-before-activate + fence) was unit-correct but not integrated: (A) handle_repl_config_push called load_primary_config UNCONDITIONALLY before config_activate_committed, so mismatched bytes went live before the gate; (B) myapp.c boot step 7 + on_commit_advanced never called the consumer, so a recovering member loaded config blind against the committed marker; (C) ha_dispatch_gate / pre-promotion gate never read config_serve_fence, so a fenced member still served + could promote", - "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY \u2014 removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", - "tags": [ - "ha-term-safety", - "config-replication", - "verify-before-activate", - "serve-fence", - "dispatch-gate", - "promotion-gate", - "safety", - "task-4.3", - "remediation" - ], - "related_bugs": [ - "bug-851", - "bug-924" - ], - "occurrences": 1, - "last_seen": "2026-06-22T11:15:00.000Z" - }, - { - "id": "bug-977", - "timestamp": "2026-06-22T11:21:09.742Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !c", - "fix": "Added guard clause: if (!c)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T11:21:09.742Z" - }, - { - "id": "bug-978", - "timestamp": "2026-06-22T11:21:58.604Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 17\u219233 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T11:21:58.604Z" - }, - { - "id": "bug-979", - "timestamp": "2026-06-22T11:30:41.086Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 26\u219221 lines (9 removed) | Also: * commit barrier, which would advance committed_ls; * live (active tag is still the OLD marker's tag).", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T11:32:15.087Z" - }, - { - "id": "bug-980", - "timestamp": "2026-06-22T11:32:35.966Z", - "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Used \"ClusterConfig\" instead of \"cluster_config_t\"", - "fix": "Changed ClusterConfig \u2192 cluster_config_t", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T11:32:35.966Z" - }, - { - "id": "bug-981", - "timestamp": "2026-06-22T11:35:00Z", - "error_message": "ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9.", - "file": "src/controller/config_repl.c, src/controller/ha.c", - "root_cause": "No post-commit retry trigger for a config serve-fence set under the push-before-commit ordering. config_committed_marker returns the older committed marker until the new marker is in the committed prefix; the two activation call sites (boot, handle_repl_config_push) both run BEFORE the new marker commits, and ha_cb_commit_advanced_stub deliberately did nothing (async-I/O invariant forbids disk reads on the iomgr ACK thread).", - "fix": "Added config_schedule_activate_retry(c) (non-blocking ordq_put via delay_task_ms, no-op unless serve-fenced AND delaymgr running) and config_activate_retry_fn(arg) (delaymgr task body that re-runs config_activate_committed off the iomgr thread). ha_cb_commit_advanced_stub now schedules the retry whenever config_serve_fenced(c) is set after commit advances. Once the NEW marker is in the committed prefix and the saved bytes hash-verify, the retry clears the fence and activates; -ENOENT still falls back without fencing; no busy-spin (no-op once unfenced). RED captured by stubbing config_activate_retry_fn to a no-op (fence stayed set, NEW config not loaded); GREEN with the real off-thread retry.", - "tags": [ - "ha-term-safety", - "config-repl", - "serve-fence", - "liveness", - "async-io", - "delaymgr", - "commit-advance", - "push-before-commit", - "R7.9" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T11:35:00Z" - }, - { - "id": "bug-982", - "timestamp": "2026-06-22T11:52:31.750Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "12 lines replaced/restructured", - "fix": "Rewrote 16\u21928 lines (12 removed) | Also: /* A standby's durable ACK covers LSN 2; route the; * The agreed entries are current-term (T), so the ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T11:54:34.028Z" - }, - { - "id": "bug-983", - "timestamp": "2026-06-22T11:53:36.582Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: commit_index", - "fix": "Added guard clause: if (commit_index)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T11:53:36.582Z" - }, - { - "id": "bug-984", - "timestamp": "2026-06-22T12:03:15.865Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T12:03:15.865Z" - }, - { - "id": "bug-985", - "timestamp": "2026-06-22T12:06:43.530Z", - "error_message": "Wrong return value", - "file": "src/controller/testsuite/waldump.c", - "root_cause": "Was returning: -EINVAL;", - "fix": "Now returns: waldump_path_visit(path, out, stats, NULL, NULL);", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T12:06:43.530Z" - }, - { - "id": "bug-986", - "timestamp": "2026-06-22T12:08:51.529Z", - "error_message": "Incorrect value in code", - "file": "src/controller/testsuite/test.c", - "root_cause": "Had \"waldump.h\"", - "fix": "Changed to \"walcompare.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T12:08:51.529Z" - }, - { - "id": "bug-987", - "timestamp": "2026-06-22T12:09:28.306Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test.c", - "root_cause": "No early return/throw for edge case: st.n_gaps > 0 || st.crc_broken", - "fix": "Added guard clause: if (st.n_gaps > 0 || st.crc_broken)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T12:09:28.306Z" - }, - { - "id": "bug-988", - "timestamp": "2026-06-22T13:05:15.064Z", - "error_message": "Missing error handling in degrades", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T13:05:15.064Z" - }, - { - "id": "bug-989", - "timestamp": "2026-06-22T13:19:16.414Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/clean_failover_preservation.sh", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u219213 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T13:19:16.414Z" - }, - { - "id": "bug-990", - "timestamp": "2026-06-22T13:24:07.476480", - "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster.", - "file": "src/controller/repl.c (live send repl_broadcast persist.c:747) + src/controller/ha.c (ha_apply_wal_catchup_entry term!=0 legacy direct-apply)", - "root_cause": "The live replication SEND path was deferred from tasks 3.2/3.3 and still uses the legacy term-0 repl_broadcast; the standby applies via the legacy direct-apply path, NOT through raft_on_append_entries_request, so it never receives/persists the leader's term. The raft library send loop (on_send_append_entries / raft_send_append_entries) has ZERO live callers. on_persist_entry stamps the real term on the PRIMARY only.", - "fix": "NOT FIXED (out of task 5.2 testsuite-only boundary). Remaining integration work: retire the term-0 repl_broadcast live SEND and drive raft_send_append_entries/on_send_append_entries on the live primary so standbys persist the leader's term via raft_on_append_entries_request. Built clean_failover_preservation.sh (XCE_FAILOVER_STRICT_TERM=1) as the ready-to-run assertion this retirement must satisfy.", - "tags": [ - "ha-term-safety", - "task-5.2", - "live-send-retirement", - "term-0", - "repl_broadcast", - "committed-prefix-divergence", - "bug-851", - "BLOCKED" - ], - "related_bugs": [ - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-22T13:24:07.476480" - }, - { - "id": "bug-991", - "timestamp": "2026-06-22T16:30:00.000Z", - "error_message": "Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic.", - "file": "src/controller/ha.c (ha_apply_term_entry_via_library ~2115; batch ~2205; single-entry ~2225)", - "root_cause": "applied_lsn carried TWO conflicting semantics. Recovery (=durable_floor), checkpoint (=checkpoint_lsn), init (=0) and the leader's catch-up streamer + ha_evaluate_repl_action all treat it as 'last contiguously-held LSN' and stream (standby_lsn, committed]. But the three live/catch-up apply setters advanced to entry->lsn + 1 ('next-to-apply'). After a standby applied catch-up entry N it advertised N+1, so the leader's next stream began at N+2 and skipped N+1 -> logOk reject -> no WalAck.", - "fix": "fix-1: the three live-apply setters now advance applied_lsn to the LAST-HELD LSN (e->lsn / last_good_lsn / entry->lsn), not +1, so applied_lsn means 'last contiguously-held LSN' everywhere. Reader audit (all consistent with last-held, no +1 reader needed): state report (advertised LSN, ha.c:2461) now matches streamer contract; convergence reported_lsn>=committed_lsn (2536) correct; snapshot/stream boundary standby_lsn+1 routed through the library, so raft.log_len already == applied_lsn after fix-1; raft.log_len legitimately differs from applied_lsn on the legacy term-0 direct-apply and recovery paths, so re-sourcing would regress bug-848 incremental catch-up -> fix-1 alone delivers the value the streamer+logOk need. Unit RED->GREEN: test_catchup_applied_lsn_is_last_held_no_skip (Expected 1 Was 2 pre-fix, PASS post-fix). 7 legacy test_wal_catchup_* expected values updated lsn+1 -> last-held. NOTE: the LIVE clean_failover_preservation STRICT_TERM commit=5 target was NOT reachable in the sandbox: committed_lsn stalls at 1 IDENTICALLY with this fix stashed (baseline experiment) -> that stall is a SEPARATE upstream blocker (library raft.log_len / commit-advance not advancing past the own-term noop; ha-term-safety task 3.1 area), NOT the bug-991 applied_lsn skip. waldump confirms all 3 members hold LSN 1..5 byte-identical (term 1) -> the replication half (which this fix corrects) is sound.", - "tags": [ - "ha", - "controller", - "wal-catchup", - "applied_lsn", - "commit-stall", - "off-by-one", - "term-safety", - "bug-991", - "streamer-skip" - ], - "related_bugs": [ - "bug-851", - "bug-850", - "bug-848", - "bug-924", - "bug-960" - ], - "occurrences": 1, - "last_seen": "2026-06-22T16:30:00.000Z" - }, - { - "id": "bug-992", - "timestamp": "2026-06-22T15:09:54.490Z", - "error_message": "Significant refactor of ", - "file": "src/include/repl.h", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u219213 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:09:54.490Z" - }, - { - "id": "bug-993", - "timestamp": "2026-06-22T15:10:39.717Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || lsn == 0", - "fix": "Added guard clause: if (!p || lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:10:39.717Z" - }, - { - "id": "bug-994", - "timestamp": "2026-06-22T15:12:30.050Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "15 lines replaced/restructured", - "fix": "Rewrote 21\u21925 lines (15 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:12:30.050Z" - }, - { - "id": "bug-995", - "timestamp": "2026-06-22T15:13:02.631Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/request.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:13:02.631Z" - }, - { - "id": "bug-996", - "timestamp": "2026-06-22T15:14:29.189Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_repl_unit.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 15\u219223 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:14:29.189Z" - }, - { - "id": "bug-997", - "timestamp": "2026-06-22T15:26:52.514Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:26:52.514Z" - }, - { - "id": "bug-998", - "timestamp": "2026-06-22T15:27:10.703Z", - "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Used \"test_repl_ack_currentterm_quorum_advances_commit\" instead of \"test_live_broadcast_receive_persists_leader_term\"", - "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit \u2192 test_live_broadcast_receive_persists_leader_term", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:27:10.703Z" - }, - { - "id": "bug-990-fix", - "timestamp": "2026-06-22T16:00:00.000000", - "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990.", - "file": "src/controller/repl.c (repl_broadcast), src/controller/persist.c (persist_wal_async call site + entry_term_locked), src/controller/config_repl.c (ha_stream_wal_to_standby catch-up), src/controller/ha.c (ha_apply_wal_catchup_entry receive route), src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)", - "root_cause": "The LIVE replication SEND paths (repl_broadcast live append + ha_stream_wal_to_standby catch-up) built wal_catchup_entry_t via memset(0) and never set entry.term/prev_log_index/prev_log_term, so standbys received term=0 entries and applied them via the LEGACY term-0 direct-apply branch (persist_wal_write_replicated stamps the STANDBY's local p->term, not the leader's). The term-carrying library receive route (raft_on_append_entries_request, task 3.2) only fired for term!=0 entries, which the live send never produced. Additionally wal_entry_to_wire only encoded per-entry term, not the per-entry prev tuple, so multi-entry catch-up batches could not be routed through the library one-at-a-time.", - "fix": "Task 3.4 minimal cutover (chosen over driving raft_send_append_entries live, which would rip out the repl model and risk 848/849/850): (1) repl_broadcast gains a prev_log_term param and sets entry.term=hdr->term, entry.prev_log_index=hdr->lsn-1, entry.prev_log_term (sourced under p->lock via new entry_term_locked accessor in persist_wal_async). (2) ha_stream_wal_to_standby sets each catch-up entry's term + prev tuple (tracking the prior streamed entry's lsn/term in ascending order). (3) wal_entry_to_wire now encodes prevLogIndex/prevLogTerm/leaderCommit PER SUB-ENTRY so a multi-entry batch is library-routable. (4) ha_apply_wal_catchup_entry routes every term-carrying entry (single + each batch sub-entry) through new ha_apply_term_entry_via_library -> ha_consume_append_entries -> raft_on_append_entries_request -> on_persist_entry, which persist_set_term(leader_term) before persist_wal_write_replicated (bug-850 contiguity guard preserved beneath). The term-0 legacy direct-apply is retired for live/catch-up entries. ACK/commit path (3.3) unchanged.", - "tags": [ - "ha-term-safety", - "task-3.4", - "live-send-retirement", - "term-0", - "repl_broadcast", - "bug-990", - "FIXED", - "catch-up-term" - ], - "related_bugs": [ - "bug-990", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-22T16:00:00.000000", - "validation": "Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) \u2014 bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline)." - }, - { - "id": "bug-1000", - "timestamp": "2026-06-22T15:44:38.466Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T15:44:38.466Z" - }, - { - "id": "bug-1001", - "timestamp": "2026-06-22T15:44:54.113Z", - "error_message": "Missing error handling in errno", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - _Blocked (live portion only \u2014 build+unit gate CONFIRMED GR", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T15:48:39.777Z" - }, - { - "id": "bug-1002", - "timestamp": "2026-06-22T21:10:31.345Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Had 's term and standbys persist it, proven at the int", - "fix": "Changed to 's REAL term (2) for the committed prefix (pre-3.4", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T21:10:31.345Z" - }, - { - "id": "bug-1003", - "timestamp": "2026-06-22T22:09:23.064Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft.c", - "root_cause": "Condition was: if (agreed == 0)", - "fix": "Changed to: if (s->cbs->on_trace)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T22:09:23.064Z" - }, - { - "id": "bug-1004", - "timestamp": "2026-06-22T22:09:40.202Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft.c", - "root_cause": "12 lines replaced/restructured", - "fix": "Rewrote 16\u21923 lines (12 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T22:09:40.202Z" - }, - { - "id": "bug-1005", - "timestamp": "2026-06-22T22:31:03.315Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T22:31:08.791Z" - }, - { - "id": "bug-1006", - "timestamp": "2026-06-22T22:32:53.722Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 \u2014 see th", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T22:33:00.066Z" - }, - { - "id": "bug-1007", - "timestamp": "2026-06-22T22:33:09.231Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u219210 lines (3 removed) | Also: * advance applied_lsn to last-successfully-app; * directly and advance applied_lsn to lsn+1 on", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-22T22:33:23.732Z" - }, - { - "id": "bug-1008", - "timestamp": "2026-06-22T22:34:27.478Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 13\u219216 lines (2 removed) | Also: /* applied_lsn must advance to 21+1=22 */; TEST_ASSERT_EQUAL_UINT64(22, | Also: /* applied_lsn must reach at least 31 (e0 applied); TEST_ASSERT_TRUE(final_lsn >= 31);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-22T22:34:47.036Z" - }, - { - "id": "bug-1009", - "timestamp": "2026-06-22T22:44:50.365Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T22:44:50.365Z" - }, - { - "id": "bug-1010", - "timestamp": "2026-06-22T22:47:11.324Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 14\u21926 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T22:47:11.324Z" - }, - { - "id": "bug-1011", - "timestamp": "2026-06-22T23:44:22.860Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/commit.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-22T23:44:22.860Z" - }, - { - "id": "bug-1012", - "timestamp": "2026-06-22T23:46:05.735Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_term_safety_adapter.c", - "root_cause": "39 lines replaced/restructured", - "fix": "Rewrote 84\u2192147 lines (39 removed) | Also: RUN_TEST(test_commit_advanced_is_commit_source);; RUN_TEST(test_commit_advanced_ceiling_caps_durable | Also: * which raises the barrier ceiling. 3-peer cluster; * self holds LSN 1,2 at the current term; one stan | Also: /* Durable majority: the standby's durable positio; * barrier ceiling is not held below the library co | Also: /* (3) on_commit_advanced fired \u2192 the barrier ceil; * advanced to the library point under the durable ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-22T23:48:10.091Z" - }, - { - "id": "bug-1013", - "timestamp": "2026-06-22T00:00:00Z", - "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes \u2014 STRICT_TERM live failover gate fails 'committed_lsn>=5'", - "file": "src/controller/commit.c", - "root_cause": "ha-term-safety task 2.4 added a min(durable-majority, raft_commit_index) ceiling in majority_candidate_locked. raft_commit_index is fed only via the library ACK path (ha_raft_on_repl_ack -> raft_advance_commit -> on_commit_advanced), which advances from match_index[] driven by the asynchronous catch-up streamer and routinely LAGS the durable majority. When delivery goes via catch-up, raft_commit_index stays low and PINS committed_lsn below the safe durable watermark (a redundant, starvation-prone gate). The durable-majority barrier was already current-term-safe under single-writer + the own-term NOOP writable gate, making the ceiling redundant.", - "fix": "Option A: removed the raft_commit_index fold from majority_candidate_locked (commit.c). The durable-majority barrier + own-term writable_lsn gate are now authoritative for commit-advance. commit_set_raft_commit_index / ce->raft_commit_index made INERT-ADVISORY (recorded for observability/conformance-oracle only; still fed by on_commit_advanced so the library half keeps a sink). Kept durable self-cap (persist_durable_lsn), persist-before-ACK, and assert(cand<=persist_durable_lsn). Verified live: commit advances 4->5 via durable barrier (commit.c:184); walcompare OK committed prefix byte-identical across 3 nodes up to LSN 5.", - "tags": [ - "ha", - "commit", - "raft", - "term-safety", - "starvation", - "commit-ceiling", - "optA", - "liveness", - "catch-up" - ], - "related_bugs": [ - "bug-991", - "bug-661", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-22T00:00:00Z" - }, - { - "id": "bug-1014", - "timestamp": "2026-06-23T00:22:40.544Z", - "error_message": "Missing error handling in can", - "file": "src/controller/testsuite/clean_failover_preservation.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T00:22:40.544Z" - }, - { - "id": "bug-1015", - "timestamp": "2026-06-23T01:30:18.896Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_election.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T01:30:18.896Z" - }, - { - "id": "bug-1016", - "timestamp": "2026-06-23T01:31:58.770Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 56\u2192117 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T01:31:58.770Z" - }, - { - "id": "bug-828", - "timestamp": "2026-06-23T01:40:00.000Z", - "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", - "file": "src/controller/ha.c (vote_reply_dispatch), src/controller/persist.c (persist_wal_write_replicated)", - "root_cause": "vote_reply_decode_and_dispatch decoded the granting voter's advertised peerLastLogLsn (vr_lsn at ha.c ~4254) but vote_reply_dispatch DISCARDED it ('(void)peer_lsn; raft_on_vote_reply(...)'). raft_on_vote_reply only bumps s->votes_granted; it never records the peer LSN. The only writer of ctx->votes[idx].vote_granted/peer_lsn (handle_request_vote_reply) was, in production, called ONLY from the connect/error failure paths with (granted=0, lsn=0); the real-grant caller was gated behind a test-only hook (xce_vote_reply_hook, never set in production). So ctx->votes[*] stayed calloc'd zero -> ha_compute_safe_committed_lsn saw n_granted=1 (self) -> safe_committed_lsn = self_lsn. A standby promoting with its own durable_floor=1 then persist_truncate_wal_above(1) destroyed committed LSN 2..5.", - "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) \u2014 it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", - "tags": [ - "safety", - "failover", - "safe-committed-lsn", - "vote-reply", - "leader-completeness", - "ha-repl-hardening", - "PARTIAL-FIX", - "bug-795", - "c" - ], - "related_bugs": [ - "bug-795", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-23T01:40:00.000Z" - }, - { - "id": "bug-1018", - "timestamp": "2026-06-23T02:58:38.185Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T02:58:38.185Z" - }, - { - "id": "bug-828-part2", - "timestamp": "2026-06-23T03:05:00.000Z", - "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", - "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", - "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held \u2014 the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", - "fix": "No production code change required for part-2 \u2014 the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 \u2014 locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", - "tags": [ - "safety", - "failover", - "safe-committed-lsn", - "durable-tip", - "advertise", - "leader-completeness", - "bug-795", - "bug-661", - "commit-durability", - "INVESTIGATION", - "c" - ], - "related_bugs": [ - "bug-828", - "bug-795", - "bug-661", - "bug-851", - "bug-924" - ], - "occurrences": 1, - "last_seen": "2026-06-23T03:05:00.000Z" - }, - { - "id": "bug-1020", - "timestamp": "2026-06-23T03:27:59.030Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_repl_apply.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T03:27:59.030Z" - }, - { - "id": "bug-1021", - "timestamp": "2026-06-23T03:28:20.600Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 8\u219225 lines (6 removed) | Also: INFO(\"DBG828 lib_ack lsn=%\" PRIu64 \" e_term=%\" PRI; \" after_term=%\" PRIu64 \" before_term=%\" PRIu64,", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-23T03:28:37.106Z" - }, - { - "id": "bug-1022", - "timestamp": "2026-06-23T03:28:31.151Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 8\u21925 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T03:28:31.151Z" - }, - { - "id": "bug-1023", - "timestamp": "2026-06-23T03:45:00Z", - "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", - "file": "src/controller/ha.c", - "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp \u2014 NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", - "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning \u2014 it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step \u2014 committed jobs are physically present on disk (walcompare proves it).", - "tags": [ - "ha", - "raft", - "failover", - "leader-completeness", - "bug-828", - "bug-795", - "durability", - "truncate", - "election", - "durable_lsn" - ], - "related_bugs": [ - "bug-661", - "bug-795", - "bug-828", - "bug-851", - "bug-854", - "bug-339" - ], - "occurrences": 1, - "last_seen": "2026-06-23T03:45:00Z" - }, - { - "id": "bug-1024", - "timestamp": "2026-06-23T04:07:37.423Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-term-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T04:07:37.423Z" - }, - { - "id": "bug-1025", - "timestamp": "2026-06-23T04:09:25.647Z", - "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", - "file": "src/controller/testsuite/clean_failover_preservation.sh", - "root_cause": "Used \"$MASTERS\" instead of \"$MASTERS_LIVE\"", - "fix": "Changed $MASTERS \u2192 $MASTERS_LIVE", - "tags": [ - "auto-detected", - "wrong-reference", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T04:09:25.647Z" - }, - { - "id": "bug-1026", - "timestamp": "2026-06-23T04:47:06.139Z", - "error_message": "Significant refactor of ", - "file": ".kiro/steering/ha-async-io.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 10\u219210 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T04:47:06.139Z" - }, - { - "id": "bug-1027", - "timestamp": "2026-06-23T04:47:28.669Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/steering/ha-async-io.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T04:47:28.669Z" - }, - { - "id": "bug-1028", - "timestamp": "2026-06-23", - "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", - "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term), config_repl.c (ha_stream_wal_to_standby), persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK).", - "root_cause": "ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 \u2192 logOk mismatch \u2192 the new-term NOOP (the post-promotion first entry) is perpetually rejected \u2192 no WalAck \u2192 applied_lsn stuck at pre-promotion value \u2192 the heartbeat state report advertises the stale lsn \u2192 the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch \u2192 CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL.", - "fix": "FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED\u2192GREEN proven (Expected 1 Was 0 \u2192 PASS). Live: standby converge + committed prefix identical.", - "tags": [ - "failover", - "availability", - "liveness", - "catch-up", - "new-primary", - "not-serving", - "ha-commit-durability", - "NOT-FIXED", - "FIXED", - "catch-up", - "ha_stream_wal_to_standby", - "persist_entry_term", - "logOk" - ], - "related_bugs": [ - "bug-828", - "bug-854", - "bug-795", - "bug-991" - ], - "occurrences": 2, - "last_seen": "2026-06-23" - }, - { - "id": "bug-1029", - "timestamp": "2026-06-23T07:35:43.178Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T07:35:43.178Z" - }, - { - "id": "bug-1030", - "timestamp": "2026-06-23T07:35:55.634Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 6\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T07:35:55.634Z" - }, - { - "id": "bug-1031", - "timestamp": "2026-06-23T07:39:57.698Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T07:39:57.698Z" - }, - { - "id": "bug-1032", - "timestamp": "2026-06-23T07:47:02.265Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 20\u219233 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T07:47:02.265Z" - }, - { - "id": "bug-1033", - "timestamp": "2026-06-23T08:05:43.916Z", - "error_message": "Incorrect value in code", - "file": "repro854.sh", - "root_cause": "Had 'set masters %s\\nwait-primary 6000\\nctrlstatus\\nqu", - "fix": "Changed to 'set masters %s\\nset myname failover-test\\nclient\\", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T08:05:43.916Z" - }, - { - "id": "bug-1034", - "timestamp": "2026-06-23T09:44:33.687Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 53\u219271 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T09:44:33.687Z" - }, - { - "id": "bug-1035", - "timestamp": "2026-06-23T09:53:37.812Z", - "error_message": "Incorrect value in code", - "file": "src/lib/xcectx.c", - "root_cause": "Had \"xcectx.h\"", - "fix": "Changed to \"controller.capnp.h\"", - "tags": [ - "auto-detected", - "wrong-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T09:53:37.812Z" - }, - { - "id": "bug-1036", - "timestamp": "2026-06-23T10:44:48.074Z", - "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", - "file": "src/lib/netio.c", - "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", - "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T10:44:48.074Z" - }, - { - "id": "bug-1037", - "timestamp": "2026-06-23T10:48:18.493Z", - "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", - "file": "src/controller/cluster.c", - "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", - "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T10:48:18.493Z" - }, - { - "id": "bug-1038", - "timestamp": "2026-06-23T10:49:09.336Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "35 lines replaced/restructured", - "fix": "Rewrote 84\u219253 lines (35 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T10:49:09.336Z" - }, - { - "id": "bug-1039", - "timestamp": "2026-06-23T10:51:44.066Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 35\u219229 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T10:51:44.066Z" - }, - { - "id": "bug-1040", - "timestamp": "2026-06-23T10:54:14.342Z", - "error_message": "Wrong reference: endpoint_ssl_handshake should be handle_ssl_handshake", - "file": "src/lib/netio.c", - "root_cause": "Used \"endpoint_ssl_handshake\" instead of \"handle_ssl_handshake\"", - "fix": "Changed endpoint_ssl_handshake \u2192 handle_ssl_handshake", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T10:54:14.342Z" - }, - { - "id": "bug-1041", - "timestamp": "2026-06-23T13:27:41.076Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 25\u219232 lines (6 removed) | Also: local out; echo \"$out\" | sed -n 's/^Primary: \\(ctrl-[0-9]\\)$/", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-23T13:30:32.319Z" - }, - { - "id": "bug-1042", - "timestamp": "2026-06-23T13:37:30.231Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/waldump.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219215 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T13:37:30.231Z" - }, - { - "id": "bug-1043", - "timestamp": "2026-06-23T13:42:03.549Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u219210 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T13:42:03.549Z" - }, - { - "id": "bug-1044", - "timestamp": "2026-06-23T13:59:31.958Z", - "error_message": "Significant refactor of ", - "file": "src/lib/client.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 34\u219253 lines (8 removed) | Also: * rotates onward to the authoritative primary as n; /* This master is unreachable \u2014 drop the half-open", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-23T14:01:19.943Z" - }, - { - "id": "bug-1045", - "timestamp": "2026-06-23T14:08:05.466Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: # \u2500\u2500 Steps 7\u201310 looped: kill primary, verify, submit, restar; #", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-23T14:08:49.394Z" - }, - { - "id": "bug-1046", - "timestamp": "2026-06-23T14:15:02.057Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 33\u219275 lines (3 removed) | Also: log \"round $round: killing primary $PRIMARY (pid=$; # Step 7: a new primary must be elected and serve ", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-23T14:15:40.333Z" - }, - { - "id": "bug-1047", - "timestamp": "2026-06-23T14:23:30.550Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T14:23:30.550Z" - }, - { - "id": "bug-1042", - "timestamp": "2026-06-23", - "error_message": "Standby-rejoin never converges: a restarted controller recovers durable_floor=7 but the primary logs 'standby[0] lsn=7 < committed_lsn=14 -> stream WAL batches' every ~3s forever; the rejoined node's ha_consume_append_entries returns success=0 ae_reject_logmismatch for prevIdx=7..13. Surfaced by /xce-test-ha soak round 2 (kill primary, restart downed node, it never catches up).", - "file": "src/controller/myapp.c (ha_init_raft before persist_recover) + src/controller/cluster.c cluster_init_phase2 (fix site)", - "root_cause": "Init-ordering: ha_init_raft (myapp.c:263, before election) sources the raft library's initial_log_len from persist_durable_floor \u2014 but on the standby-rejoin boot persist_recover runs LATER (cluster_init_phase2, myapp.c:356). So the library comes up with log_len=0 and is never told about the 7 recovered durable entries. With log_len=0 the library's logOk gate rejects every incoming AppendEntries (prevLogIndex>=1 vs empty log), so the rejoined member is permanently stuck as a non-converging standby and the primary re-streams [tip+1..committed] forever. Distinct from bug-1028 (streamer term-source): here the LIBRARY's own log position is wrong, not the streamed prev_log_term.", - "fix": "FIXED (2026-06-23). Added ha_sync_raft_log_from_recovery(ha,p) (ha.c, decl ha.h): after persist_recover, raise ha->raft.log_len to persist_durable_floor (recovered WAL tip), commit_index to the durable floor clamped to log_len, and raft.term to ha->term \u2014 all monotonic. Called in cluster_init_phase2 right after the existing ha_sync_applied_lsn_from_recovery. Mirrors bug-848's applied_lsn sync. The library holds no entry bytes (on_get_entry_term reads the recovered per-LSN term index rebuilt by replay_wal), so raising log_len is sufficient for logOk to accept the catch-up suffix.", - "tags": [ - "ha", - "failover", - "rejoin", - "standby", - "convergence", - "raft", - "log_len", - "init-ordering", - "logOk", - "FIXED" - ], - "related_bugs": [ - "bug-1028", - "bug-848", - "bug-854" - ], - "occurrences": 1, - "last_seen": "2026-06-23" - }, - { - "id": "bug-1049", - "timestamp": "2026-06-23T14:39:01.881Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (0 && recovered_tip > ha->raft.log_len)", - "fix": "Changed to: if (recovered_tip > ha->raft.log_len)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T14:39:01.881Z" - }, - { - "id": "bug-1043", - "timestamp": "2026-06-23", - "error_message": "After a failover where the new primary caught up the missing suffix via REPLICATION (not local recovery), joblist/expect-job on that primary cannot find a committed job (e.g. 0-0-1782224710-0 at LSN 9-10) \u2014 even though the entry IS on disk on all members and walcompare passes. Surfaced by /xce-test-ha soak round 2: ctrl-1 (down during round 1) restarted, caught up entries 8..14 by replication, became primary, and could not list jobs submitted while it was down.", - "file": "src/controller/ha.c ha_apply_term_entry_via_library (catch-up apply path)", - "root_cause": "The term-safe library catch-up route (ha_apply_wal_catchup_entry \u2192 ha_apply_term_entry_via_library \u2192 ha_consume_append_entries \u2192 on_persist_entry \u2192 persist_wal_write_replicated) writes the replicated entry to the WAL but NEVER applies it to the in-memory entity state (job/queue/worker tables). The LEGACY direct-apply path (ha_apply_single_entry) called apply_wal_entry (\u2192 restore_job) right after persist_wal_write_replicated; the term-safe route (ha-term-safety task 3.4) replaced that path for term-carrying entries but dropped the apply_wal_entry call. So a member that catches up via replication holds the entries on disk but not in the queryable job table; on promotion joblist/expect-job miss them \u2014 looks like committed-job loss but is an apply gap, not data loss (walcompare byte-identical).", - "fix": "FIXED (2026-06-23). In ha_apply_term_entry_via_library, after the library accepts+persists the entry (after_term == e->term), call apply_wal_entry(p, e->entry_type, e->lsn, e->raw_payload, e->raw_payload_len) \u2014 mirroring the legacy ha_apply_single_entry path \u2014 so the entity state (restore_job etc.) is rebuilt for replicated catch-up entries. Idempotent on re-stream (restore_job upserts by jobid); WAL_NOOP/WAL_CONFIG_MARKER no-op in apply_wal_entry.", - "tags": [ - "ha", - "failover", - "catch-up", - "replication", - "job-table", - "apply", - "promotion", - "not-data-loss", - "FIXED" - ], - "related_bugs": [ - "bug-1028", - "bug-1042", - "bug-828" - ], - "occurrences": 1, - "last_seen": "2026-06-23" - }, - { - "id": "bug-1051", - "timestamp": "2026-06-23T14:48:18.998Z", - "error_message": "Wrong operator: >= should be ==", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Used \">=\" instead of \"==\"", - "fix": "Changed operator >= \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T14:48:18.998Z" - }, - { - "id": "bug-1052", - "timestamp": "2026-06-23T14:53:22.593Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T14:53:22.593Z" - }, - { - "id": "bug-1053", - "timestamp": "2026-06-23T14:54:33.747Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 25\u219238 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T14:54:33.747Z" - }, - { - "id": "bug-1054", - "timestamp": "2026-06-23T16:02:42.489Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Had \"End-to-end HA failover soak test for an XCE contr", - "fix": "Changed to \"End-to-end HA failover soak test for an XCE contr", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T16:02:42.489Z" - }, - { - "id": "bug-1055", - "timestamp": "2026-06-23T16:02:53.637Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219216 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T16:02:53.637Z" - }, - { - "id": "bug-1056", - "timestamp": "2026-06-23T16:04:15.564Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u21929 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T16:04:15.564Z" - }, - { - "id": "bug-1057", - "timestamp": "2026-06-23T22:52:51.718Z", - "error_message": "Significant refactor of ", - "file": "src/lib/hook.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 16\u219231 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-23T22:52:51.718Z" - }, - { - "id": "bug-1058", - "timestamp": "2026-06-24T03:00:17.256Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T03:00:17.256Z" - }, - { - "id": "bug-1059", - "timestamp": "2026-06-24T03:01:22.373Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T03:01:22.373Z" - }, - { - "id": "bug-1060", - "timestamp": "2026-06-24T12:05:00.000Z", - "error_message": "After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win.", - "file": "src/controller/ha.c (raft_cb_become_leader / safe_committed_lsn), src/controller/cluster.c (OPCode_preVoteRequest)", - "root_cause": "ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 \u2192 majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 \u2014 divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader.", - "fix": "IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log \u2014 the new leader NEVER truncates entries it holds (Raft \u00a75.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped \u2014 clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence.", - "tags": [ - "prevote", - "deadlock", - "safe_committed_lsn", - "truncation", - "leader-completeness", - "data-loss", - "coverage-maneuver", - "FIXED" - ], - "related_bugs": [ - "bug-795", - "bug-828" - ], - "occurrences": 1, - "last_seen": "2026-06-24T18:00:00.000Z" - }, - { - "id": "bug-1061", - "timestamp": "2026-06-24T04:20:48.527Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_election.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T04:20:48.527Z" - }, - { - "id": "bug-1062", - "timestamp": "2026-06-24T04:21:33.501Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u219220 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T04:21:33.501Z" - }, - { - "id": "bug-1063", - "timestamp": "2026-06-24T04:31:30.033Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T04:31:30.033Z" - }, - { - "id": "bug-1064", - "timestamp": "2026-06-24T04:31:50.319Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 7\u219220 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T04:31:50.319Z" - }, - { - "id": "bug-1065", - "timestamp": "2026-06-24T04:33:23.913Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_election.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T04:33:23.913Z" - }, - { - "id": "bug-1066", - "timestamp": "2026-06-24T04:36:50.602Z", - "error_message": "Incorrect value in code", - "file": "src/controller/CLAUDE.md", - "root_cause": "Had `persist_truncate_wal_above(safe_committed_lsn)`", - "fix": "Changed to 's own durable log**: the cutoff is `", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T04:37:04.779Z" - }, - { - "id": "bug-1067", - "timestamp": "2026-06-24T00:00:00.000Z", - "error_message": "Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch)", - "file": "src/controller/config_repl.c, src/controller/ha.c", - "root_cause": "AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term \u2192 ae_reject_stale) correctly rejects, but it should never fire \u2014 the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever.", - "fix": "IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct \u2014 capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) \u2014 the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) \u2192 falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED \u2014 safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures.", - "tags": [ - "catch-up", - "replication", - "raft", - "ae_reject_stale", - "ae_reject_logmismatch", - "walCatchupEntry", - "cross-term", - "FIXED" - ], - "related_bugs": [ - "bug-1028", - "bug-991" - ], - "occurrences": 1, - "last_seen": "2026-06-24T20:00:00.000Z" - }, - { - "id": "bug-1068", - "timestamp": "2026-06-24T08:17:50.386Z", - "error_message": "Missing error handling in unknown", - "file": "src/schema/controller.capnp", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "capnp" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:17:50.386Z" - }, - { - "id": "bug-1069", - "timestamp": "2026-06-24T08:17:58.670Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/request.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:17:58.670Z" - }, - { - "id": "bug-1070", - "timestamp": "2026-06-24T08:18:27.510Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: if (c && c->persist.wal_path[0]) {; /* bug-1067: pass the leader's CURRENT term so historical ca", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T08:18:42.316Z" - }, - { - "id": "bug-1071", - "timestamp": "2026-06-24T08:18:57.361Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 12\u219217 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:18:57.361Z" - }, - { - "id": "bug-1072", - "timestamp": "2026-06-24T08:19:19.691Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:19:19.691Z" - }, - { - "id": "bug-1073", - "timestamp": "2026-06-24T08:20:01.379Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:20:01.379Z" - }, - { - "id": "bug-1074", - "timestamp": "2026-06-24T08:21:09.937Z", - "error_message": "Missing error handling in unknown", - "file": "src/include/typemap.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:21:09.937Z" - }, - { - "id": "bug-1075", - "timestamp": "2026-06-24T08:23:04.980Z", - "error_message": "Wrong reference: standby_lsn should be leader_term", - "file": "src/controller/testsuite/test_coldstart_lsn0.c", - "root_cause": "Used \"standby_lsn\" instead of \"leader_term\"", - "fix": "Changed standby_lsn \u2192 leader_term", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:23:04.980Z" - }, - { - "id": "bug-1076", - "timestamp": "2026-06-24T08:32:38.896Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/CLAUDE.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T08:32:38.896Z" - }, - { - "id": "bug-1077", - "timestamp": "2026-06-24T17:30:00.000Z", - "error_message": "AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c)", - "file": "src/lib/xcectx.c", - "root_cause": "ctx->reply.u is an UNTAGGED union shared across opcodes (debugmsg/get/jobid/clusterconf/ctrlstatus alias the same 8 bytes). Commit 61e1391 ('fix bug-1060 bug-1067', unrelated Raft work) added speculative free-guards `if (ctx->reply.u.ctrlstatus) free_CtrlStatus_ptr(&ctx->reply.u.ctrlstatus)` (and the clusterconf twin). A non-NULL slot does NOT imply a CtrlStatus*: a failover-aborted reslist/joblist leaves ctx->reply.u.get == &getreply (a dead STACK pointer) because xce_get_with_condition/xce_resval only NULL the slot on the success path, not the xce_get_req<0 / cbuf_get<0 early-returns. The next ctrlstatus reply (worker thread) sees the foreign non-NULL slot and free_CtrlStatus walks it -> stack-use-after-return SEGV.", - "fix": "1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) \u2014 restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0.", - "tags": [ - "asan", - "use-after-return", - "union-type-confusion", - "ha", - "failover", - "ctrlstatus", - "libxce", - "memory-safety", - "regression-61e1391", - "VERIFIED" - ], - "related_bugs": [ - "bug-646", - "bug-1060", - "bug-1067" - ], - "occurrences": 1, - "last_seen": "2026-06-24T18:30:00.000Z" - }, - { - "id": "bug-1078", - "timestamp": "2026-06-24T09:18:52.605Z", - "error_message": "Wrong reference: handle_reply should be xce_handle_reply", - "file": "src/lib/xcectx.c", - "root_cause": "Used \"handle_reply\" instead of \"xce_handle_reply\"", - "fix": "Changed handle_reply \u2192 xce_handle_reply", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T09:18:52.605Z" - }, - { - "id": "bug-1079", - "timestamp": "2026-06-24T09:27:59.772Z", - "error_message": "Wrong reference: libxce_failover_client_hang should be project_libxce_failover_hang", - "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1077_reply_union_type_confusion.md", - "root_cause": "Used \"libxce_failover_client_hang\" instead of \"project_libxce_failover_hang\"", - "fix": "Changed libxce_failover_client_hang \u2192 project_libxce_failover_hang", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T09:27:59.772Z" - }, - { - "id": "bug-1080", - "timestamp": "2026-06-24T10:24:07.439Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "Had \"on_get_entry_term -> node.log[idx].term | 0\"", - "fix": "Changed to \"on_get_entry_term -> node.log[idx].term | 0\"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T10:24:15.191Z" - }, - { - "id": "bug-1081", - "timestamp": "2026-06-24T10:30:45.585Z", - "error_message": "Wrong reference: apply should be advanced", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "Used \"apply\" instead of \"advanced\"", - "fix": "Changed apply \u2192 advanced", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T10:30:50.917Z" - }, - { - "id": "bug-1082", - "timestamp": "2026-06-24T10:30:56.237Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "Had \"vote/AE requests + responses;
Drop/Duplicate/", - "fix": "Changed to \"vote/AE requests + responses,
Drop/Duplicate/", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T10:30:56.237Z" - }, - { - "id": "bug-1083", - "timestamp": "2026-06-24T11:45:52.473Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: n > sizeof(v", - "fix": "Added guard clause: if (n > sizeof(v)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T11:46:08.548Z" - }, - { - "id": "bug-1084", - "timestamp": "2026-06-24T11:47:47.434Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 18\u219221 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T11:47:47.434Z" - }, - { - "id": "bug-1085", - "timestamp": "2026-06-24T12:00:01.924Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i != last", - "fix": "Added guard clause: if (i != last)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T12:00:01.924Z" - }, - { - "id": "bug-1086", - "timestamp": "2026-06-24T12:13:27.532Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i < 0 || i >= c->bag_count", - "fix": "Added guard clause: if (i < 0 || i >= c->bag_count)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T12:13:27.532Z" - }, - { - "id": "bug-1087", - "timestamp": "2026-06-24T12:22:22.094Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: sa.role != RAFT_LEADER", - "fix": "Added guard clause: if (sa.role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T12:22:22.094Z" - }, - { - "id": "bug-1088", - "timestamp": "2026-06-24T12:30:35.436Z", - "error_message": "Wrong reference: test_sim_check_election_safety_fail_on_two_leaders should be test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_check_election_safety_fail_on_two_leaders\" instead of \"test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix\"", - "fix": "Changed test_sim_check_election_safety_fail_on_two_leaders \u2192 test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T12:30:35.436Z" - }, - { - "id": "bug-1089", - "timestamp": "2026-06-24T12:38:40.360Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: sl.role != RAFT_LEADER", - "fix": "Added guard clause: if (sl.role != RAFT_LEADER)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T12:38:40.360Z" - }, - { - "id": "bug-1090", - "timestamp": "2026-06-24T12:47:03.843Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: shared > RAFT_SIM_MAX_LOG", - "fix": "Added guard clause: if (shared > RAFT_SIM_MAX_LOG)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T12:47:03.843Z" - }, - { - "id": "bug-1091", - "timestamp": "2026-06-24T13:04:27.057Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: c->trace_len >= RAFT_SIM_MAX_STEPS", - "fix": "Added guard clause: if (c->trace_len >= RAFT_SIM_MAX_STEPS)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T13:04:27.057Z" - }, - { - "id": "bug-1092", - "timestamp": "2026-06-24T13:15:56.934Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T13:15:56.934Z" - }, - { - "id": "bug-1093", - "timestamp": "2026-06-24T13:16:19.217Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 8\u219210 lines (3 removed) | Also: /* Drive the published election-timeout entry poin; * the term bump, self-vote, persist and vote-reque | Also: /* Bag-only fault: re-enqueue a copy of a PRNG-pic; int i = raft_sim_bag_pick(c); | Also: /* ---- 1. Compute the ENABLED action set ----; * Build the eligible-action list for the CURRENT s | Also: enabled[n_enabled++] = RAFT_SIM_ACT_DROP_MESSAGE; ; /* DUPLICATE only when a library-idempotent member", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-24T13:25:44.223Z" - }, - { - "id": "bug-1094", - "timestamp": "2026-06-24T13:22:05.858Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP", - "fix": "Added guard clause: if (c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T13:22:05.858Z" - }, - { - "id": "bug-1095", - "timestamp": "2026-06-24T13:30:00.000Z", - "error_message": "Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a duplicated VOTE_RSP grant double-counts a single peer's vote", - "file": "src/lib/raft/raft.c", - "root_cause": "raft_on_vote_reply tallies grants with a COUNTER (s->votes_granted++) and explicitly ignores from_idx ((void)from_idx;), so DuplicateMessage of one peer's grant counts that peer twice and a candidate reaches quorum with a single real vote. Diverges from raft.tla where votesGranted is a SET (idempotent). Proven: 3-node, single peer grant duplicated -> candidate becomes LEADER with votes_granted=2.", - "fix": "NOT FIXED (out of task 4.2 boundary = raft_sim.c/test_raft.c). Library fix: dedupe per-peer in raft_on_vote_reply (track a granted-by bitset/array keyed on from_idx, or guard against re-counting the same peer in the current term). The conformance harness correctly CATCHES this on adversarial seeds.", - "tags": [ - "raft", - "vote-reply", - "idempotency", - "election-safety", - "split-brain", - "conformance", - "needs-library-fix" - ], - "related_bugs": [ - "bug-1096" - ], - "occurrences": 1, - "last_seen": "2026-06-24T13:30:00.000Z" - }, - { - "id": "bug-1096", - "timestamp": "2026-06-24T13:31:00.000Z", - "error_message": "Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a STALE-term VOTE_RSP grant (reply_term < current term) is counted toward the current election", - "file": "src/lib/raft/raft.c", - "root_cause": "raft_on_vote_reply has no stale-term DROP: it handles term > s->term (step down) but for term < s->term it still falls through and counts the grant (s->votes_granted++). A grant left over from an earlier-term candidacy, delivered after the node re-campaigns at a higher term, pushes it over quorum. Diverges from raft.tla HandleVoteResponse which only acts on responses matching the current term. Proven (seed 8): node2 at term 2 counts a term-1 grant (bag VOTE_RSP src0->dst2 term=1 granted=1) and becomes a SECOND leader at term 2 alongside node0.", - "fix": "NOT FIXED (out of task 4.2 boundary). Library fix: in raft_on_vote_reply, drop responses with reply_term != s->term (stale-term drop) before tallying. R2.6 forbids the harness from re-implementing this DropStaleResponse decision, so it cannot be masked at the sim layer. Task 4.2 liveness test pins seed 3 (a clean ordering) and documents both findings; the harness correctly halts on adversarial seeds.", - "tags": [ - "raft", - "vote-reply", - "stale-term", - "election-safety", - "split-brain", - "conformance", - "needs-library-fix" - ], - "related_bugs": [ - "bug-1095", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-06-24T13:31:00.000Z" - }, - { - "id": "bug-1095", - "timestamp": "2026-06-24", - "error_message": "raft_on_vote_reply double-counts a duplicated granted vote-reply -> two leaders at same term (Election Safety violation)", - "file": "src/lib/raft/raft.c", - "root_cause": "raft_on_vote_reply did (void)from_idx and incremented votes_granted on every granted reply; a legal raft.tla DuplicateMessage re-delivered a peer's grant and double-counted toward quorum. raft.tla votesGranted is a SET (idempotent).", - "fix": "Added per-peer vote_replies[RAFT_MAX_PEERS] flag to raft_state_t (mirrors pre_vote_replies); cleared in raft_init + raft_start_election with self marked; raft_on_vote_reply now bounds-guards from_idx and dedups per peer before counting.", - "tags": [ - "raft", - "split-brain", - "bug-851-family", - "election-safety", - "vote-reply", - "dedup" - ], - "related_bugs": [ - "bug-851", - "bug-1096" - ], - "occurrences": 1, - "last_seen": "2026-06-24" - }, - { - "id": "bug-1096", - "timestamp": "2026-06-24", - "error_message": "raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it", - "file": "src/lib/raft/raft.c", - "root_cause": "No DropStaleResponse: a reply with term < s->term fell through to replies_received++/votes_granted++, corrupting quorum accounting (raft.tla DropStaleResponse discards mterm < currentTerm).", - "fix": "FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace(\"vote_reply_stale_drop\", ...) at raft.c:455 \u2014 exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks).", - "tags": [ - "stale-report", - "split-brain", - "vote-reply", - "fixed", - "raft", - "bug-851-family", - "stale-response" - ], - "related_bugs": [ - "bug-851", - "bug-1095" - ], - "occurrences": 1, - "last_seen": "2026-07-02T17:30:00.000Z" - }, - { - "id": "bug-1099", - "timestamp": "2026-06-24T14:08:44.250Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: failing != RAFT_SIM_INV_NONE", - "fix": "Added guard clause: if (failing != RAFT_SIM_INV_NONE)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:08:44.250Z" - }, - { - "id": "bug-1100", - "timestamp": "2026-06-24T14:17:17.076Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:17:17.076Z" - }, - { - "id": "bug-1101", - "timestamp": "2026-06-24T14:18:01.518Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:18:01.518Z" - }, - { - "id": "bug-1102", - "timestamp": "2026-06-24T14:18:50.393Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:18:50.393Z" - }, - { - "id": "bug-1103", - "timestamp": "2026-06-24T14:25:38.561Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:25:38.561Z" - }, - { - "id": "bug-1104", - "timestamp": "2026-06-24T14:28:33.169Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Condition was: if (c->n >= 2 && c->nodes[0].commit_index >= 1 && c->n)", - "fix": "Changed to: if (c->n >= 2)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:28:33.169Z" - }, - { - "id": "bug-1105", - "timestamp": "2026-06-24T14:30:09.265Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:30:09.265Z" - }, - { - "id": "bug-1106", - "timestamp": "2026-06-24T14:36:43.852Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:36:43.852Z" - }, - { - "id": "bug-1107", - "timestamp": "2026-06-24T14:37:07.247Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:37:07.247Z" - }, - { - "id": "bug-1108", - "timestamp": "2026-06-24T14:38:06.432Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: chk < 0", - "fix": "Added guard clause: if (chk < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:38:06.432Z" - }, - { - "id": "bug-1109", - "timestamp": "2026-06-24T14:50:13.662Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:50:13.662Z" - }, - { - "id": "bug-1110", - "timestamp": "2026-06-24T14:51:30.855Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i == leader", - "fix": "Added guard clause: if (i == leader)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T14:51:30.855Z" - }, - { - "id": "bug-1111", - "timestamp": "2026-06-24T15:13:15.333Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: p == leader", - "fix": "Added guard clause: if (p == leader)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T15:13:15.333Z" - }, - { - "id": "bug-1112", - "timestamp": "2026-06-24T15:15:03.882281Z", - "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 \u2014 lib test binary fails to link", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Commit e445404 (task 5.3 drop/dup/reorder) added the raft.h declaration + the test_sim_5_3 test but FORGOT to add the raft_sim_scenario_drop_dup_reorder implementation to raft_sim.c. Symbol is declared+referenced but never defined, breaking the link for the whole xcelib_test binary.", - "fix": "Pre-existing, outside task 5.4 boundary. Surfaced while validating task 5.4 (figure8). Needs task 5.3 implementation added to raft_sim.c. NOT fixed by 5.4.", - "tags": [ - "raft-sim-conformance", - "link-error", - "task-5.3", - "undefined-reference", - "pre-existing" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T15:15:03.882281Z" - }, - { - "id": "bug-1113", - "timestamp": "2026-06-24T15:16:52.683Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /*; * sim_scen_catch_up_commit: drive bounded AppendEntries-to-a | Also: * on a CONNECTED quorum, yet raft_advance_commit must NOT co; * count alone \u2014 the current-term Figure-8 rule (raft.c G4) f", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-24T15:20:15.882Z" - }, - { - "id": "bug-1114", - "timestamp": "2026-06-24T15:18:09.126Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 18\u219234 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T15:18:09.126Z" - }, - { - "id": "bug-1115", - "timestamp": "2026-06-24T15:18:59.248Z", - "error_message": "Wrong operator: != should be ==", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Used \"!=\" instead of \"==\"", - "fix": "Changed operator != \u2192 ==", - "tags": [ - "auto-detected", - "operator-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T15:18:59.248Z" - }, - { - "id": "bug-1116", - "timestamp": "2026-06-24T15:25:17.886Z", - "error_message": "Missing guard clause", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "No early return/throw for edge case: i == leader", - "fix": "Added guard clause: if (i == leader)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T15:25:17.886Z" - }, - { - "id": "bug-1117", - "timestamp": "2026-06-24T15:32:34.085Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft.c", - "root_cause": "Condition was: if (s->cbs->on_get_entry_term(s->ctx, agreed)", - "fix": "Changed to: if (0 && s->cbs->on_get_entry_term(s->ctx, agreed)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T15:33:53.327Z" - }, - { - "id": "bug-1118", - "timestamp": "2026-06-24T15:41:38.140Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T15:41:38.140Z" - }, - { - "id": "bug-1119", - "timestamp": "2026-06-24T15:48:09.105Z", - "error_message": "Significant refactor of ", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u21924 lines (3 removed) | Also: fprintf(stderr, \"[INSTR-5.5] seed=%llu n=%d POST-A; (unsigned long long)seed, n,", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-24T15:48:16.229Z" - }, - { - "id": "bug-1120", - "timestamp": "2026-06-24T16:09:04.749Z", - "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_1_fault_switch_mechanism", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_1_fault_switch_mechanism\"", - "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_1_fault_switch_mechanism", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:09:04.749Z" - }, - { - "id": "bug-1121", - "timestamp": "2026-06-24T16:22:03.710Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "45 lines replaced/restructured", - "fix": "Rewrote 54\u21926 lines (45 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:22:03.710Z" - }, - { - "id": "bug-1122", - "timestamp": "2026-06-24T16:22:47.678Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:22:47.678Z" - }, - { - "id": "bug-1123", - "timestamp": "2026-06-24T16:22:58.427Z", - "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_2_negative_both_directions", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_2_negative_both_directions\"", - "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_2_negative_both_directions", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:22:58.427Z" - }, - { - "id": "bug-1124", - "timestamp": "2026-06-24T16:36:45.101Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-sim-conformance/design.md", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 15\u219225 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:36:45.101Z" - }, - { - "id": "bug-1125", - "timestamp": "2026-06-24T16:42:32.261Z", - "error_message": "Wrong reference: test_sim_4_2_step_liveness_and_coverage should be test_sim_8_1_export_trace_named_and_gated", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "Used \"test_sim_4_2_step_liveness_and_coverage\" instead of \"test_sim_8_1_export_trace_named_and_gated\"", - "fix": "Changed test_sim_4_2_step_liveness_and_coverage \u2192 test_sim_8_1_export_trace_named_and_gated", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:42:32.261Z" - }, - { - "id": "bug-1126", - "timestamp": "2026-06-24T16:43:39.427Z", - "error_message": "Wrong condition in logic", - "file": "src/lib/raft/raft_sim.c", - "root_cause": "Condition was: if (node_idx < 0 || node_idx >= c->n)", - "fix": "Changed to: if (getenv(\"RAFT_SIM_EXPORT\")", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-24T16:43:39.427Z" - }, - { - "id": "bug-1127", - "timestamp": "2026-06-25T03:17:45.530Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T03:17:45.530Z" - }, - { - "id": "bug-1128", - "timestamp": "2026-06-25T03:20:00.000Z", - "error_message": "test_demotion_replay_yields_committed_only RARE flake (~1/7, load-sensitive): assertion 'fresh->current_lsn <= committed_boundary + 1' (test_demotion_convergence.c:526) \u2014 boot-recovery over the surviving WAL replayed the already-truncated uncommitted suffix", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "TEST-HARNESS race (NOT a product durability gap). The demotion convergence truncate IS durable in production (persist_truncate_wal_above ftruncates the cutoff segment + fdatasync's it under p->lock; the WARN 'truncated WAL above committed_lsn=4' fires even in the failing run, proving the truncate ran). In production ha_self_restart execv's the process \u2014 the OLD image (all threads incl. fsync_worker + every open WAL fd) is gone before the NEW process cold-recovers, so coherence is guaranteed. The test instead opens a FRESH persist_t over the SAME 64MB-fallocate'd segment files while g_dc.persist is still LIVE: its truncated wal_fd still open and its fsync_worker thread still running. On overlayfs (/tmp is overlay here) the ftruncate's size change is intermittently NOT coherent to a freshly-opened fd until the writing context closes its fd + finally syncs, so replay_wal in 'fresh' occasionally reads past the truncation point and replays the discarded suffix.", - "fix": "Test-only: call persist_stop(&g_dc.persist) AFTER dc_run_pending_task (the convergence truncate) and BEFORE opening 'fresh'. persist_stop is the faithful analogue of the production execv teardown \u2014 it joins the fsync_worker (no async writer remains), runs the final durable fdatasync, and CLOSES wal_fd \u2014 establishing the durable+stable+unheld on-disk state the new process cold-recovers. Idempotent (dc_teardown's later persist_stop is a no-op). NOT a sleep/timing band-aid: a real durability/quiescence barrier. No production code changed; the assertion is unchanged.", - "tags": [ - "test-flake", - "demotion-convergence", - "persist", - "overlayfs", - "ha-convergence-recovery", - "quiescence", - "wal-truncate" - ], - "related_bugs": [ - "bug-343", - "bug-704", - "bug-795", - "bug-828", - "bug-9799" - ], - "occurrences": 1, - "last_seen": "2026-06-25T03:20:00.000Z" - }, - { - "id": "bug-1129", - "timestamp": "2026-06-25T03:46:16.695Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 26\u219261 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T03:46:16.695Z" - }, - { - "id": "bug-1130", - "timestamp": "2026-06-25T04:14:29.141Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 12\u21926 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T04:14:29.141Z" - }, - { - "id": "bug-1128", - "timestamp": "2026-06-25T04:30:00.000Z", - "error_message": "test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY", - "file": "src/controller/testsuite/test_demotion_convergence.c", - "root_cause": "TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence \u2014 a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs.", - "fix": "Test-only, two parts. (a) Detach the durable-advance commit wake (g_dc.persist.on_durable_advance = NULL) and re-pin committed_lsn = committed_boundary right before driving the demotion task, killing the commit-engine race deterministically (helps BOTH fs paths). (b) Hybrid gate by fs type via static dc_path_is_overlayfs() (statfs f_type == OVERLAYFS_SUPER_MAGIC 0x794c7630, defaults to real-fs path on statfs error): on a real fs keep the strong cross-context fresh boot-recovery assertion (fresh->current_lsn <= committed_boundary+1); on overlayfs skip that fd-incoherent step and assert the LIVE-context truncate effect (g_dc.persist.current_lsn <= committed_boundary+1, coherent under p->lock). Both branches prove the same property. Production persist.c truncate (ftruncate+fdatasync) untouched.", - "fix_validation": "Full controller suite 20/20 runs: 0 demotion_replay failures, 0 total failures. Group-only 60/60: 0 failures. Baseline (pre-fix) was 4/20. /tmp confirmed overlayfs (statfs magic 0x794c7630), so the overlayfs branch is the one exercised here.", - "tags": [ - "test-flake", - "overlayfs", - "sandbox", - "wal-truncate", - "demotion-convergence", - "commit-engine-race", - "deterministic", - "c" - ], - "related_bugs": [ - "bug-795", - "bug-828", - "bug-1060" - ], - "occurrences": 1, - "last_seen": "2026-06-25T04:30:00.000Z" - }, - { - "id": "bug-1132", - "timestamp": "2026-06-25T04:36:05.558Z", - "error_message": "Significant refactor of ", - "file": "src/lib/testsuite/test_raft.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 19\u219224 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T04:36:05.558Z" - }, - { - "id": "bug-1133", - "timestamp": "2026-06-25T07:12:01.309Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-log-replication/spec.json", - "root_cause": "Had \"2026-06-21T11:00:00Z\"", - "fix": "Changed to \"2026-06-25T00:00:00Z\"", - "tags": [ - "auto-detected", - "wrong-value", - "json" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-25T07:12:25.418Z" - }, - { - "id": "bug-1134", - "timestamp": "2026-06-25T08:18:43.915Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T08:18:43.915Z" - }, - { - "id": "bug-1135", - "timestamp": "2026-06-25T09:26:49.441Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T09:26:49.441Z" - }, - { - "id": "bug-1136", - "timestamp": "2026-06-25T09:27:00.312Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-apply-decoupling/brief.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T09:27:00.312Z" - }, - { - "id": "bug-1137", - "timestamp": "2026-06-25T11:01:21.356Z", - "error_message": "Significant refactor of ", - "file": ".kiro/steering/ha-async-io.md", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 11\u219219 lines (7 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T11:01:21.356Z" - }, - { - "id": "bug-1138", - "timestamp": "2026-06-25T11:36:52.942Z", - "error_message": "Function not marked async", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T11:36:52.942Z" - }, - { - "id": "bug-1139", - "timestamp": "2026-06-25T14:02:51.225Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/research.md", - "root_cause": "Had `VTASK_IO`", - "fix": "Changed to `endpoint_block_tasks()`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-25T14:09:24.614Z" - }, - { - "id": "bug-1140", - "timestamp": "2026-06-25T14:03:12.607Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-drive-async/requirements.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u21926 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T14:03:12.607Z" - }, - { - "id": "bug-1141", - "timestamp": "2026-06-25T15:12:16.193Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T15:12:16.193Z" - }, - { - "id": "bug-1142", - "timestamp": "2026-06-25T15:13:00.445Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `.kiro/specs/raft-drive-async/brief.md`", - "fix": "Changed to `delay_task_ms`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T15:13:00.445Z" - }, - { - "id": "bug-1143", - "timestamp": "2026-06-25T23:37:38.579Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - Commit-advance under pool load (R4.5): with the vtask pool; - Snapshot/checkpoint concurrency (R1.1/1.3/1.5): concurrent", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-25T23:38:18.782Z" - }, - { - "id": "bug-1144", - "timestamp": "2026-06-25T23:37:49.697Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Had `persist_checkpoint`", - "fix": "Changed to `wait_all_with_timeout`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T23:37:49.697Z" - }, - { - "id": "bug-1145", - "timestamp": "2026-06-25T23:38:08.746Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T23:38:08.746Z" - }, - { - "id": "bug-1146", - "timestamp": "2026-06-25T23:43:59.922Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-25T23:43:59.922Z" - }, - { - "id": "bug-1147", - "timestamp": "2026-06-26T03:46:38.904419+00:00", - "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor \u2192 lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", - "file": "src/controller/persist.c:1374 (trim_wal_segments)", - "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live \u2014 write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", - "fix": "FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor \u2014 comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete).", - "tags": [ - "durability", - "permanent-gap", - "stale-report", - "trim", - "bug-828", - "raft-drive-async", - "latent", - "catch-up", - "wal", - "fixed", - "snapshot", - "bug-795" - ], - "related_bugs": [ - "bug-795", - "bug-828", - "bug-1060" - ], - "occurrences": 1, - "last_seen": "2026-07-02T17:30:00.000Z" - }, - { - "id": "bug-1148", - "timestamp": "2026-06-26T03:46:51.038Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Had 's boundary or any standby'", - "fix": "Changed to `persist.c:1374`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T03:46:51.038Z" - }, - { - "id": "bug-1149", - "timestamp": "2026-06-26T03:48:48.222Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `/kiro-spec-design raft-drive-async`", - "fix": "Changed to `approved: false`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T03:49:01.467Z" - }, - { - "id": "bug-1150", - "timestamp": "2026-06-26T03:49:13.292Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T03:49:13.292Z" - }, - { - "id": "bug-1151", - "timestamp": "2026-06-26T04:31:53.879Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Had `commit_submit`", - "fix": "Changed to `commit.c`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T04:36:52.562Z" - }, - { - "id": "bug-1152", - "timestamp": "2026-06-26T04:32:20.960Z", - "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", - "file": ".kiro/specs/raft-drive-async/design.md", - "root_cause": "Used \"PeerCtrlDispatch\" instead of \"LivenessAdmission\"", - "fix": "Changed PeerCtrlDispatch \u2192 LivenessAdmission", - "tags": [ - "auto-detected", - "wrong-reference", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:32:20.960Z" - }, - { - "id": "bug-1153", - "timestamp": "2026-06-26T04:33:10.286Z", - "error_message": "Incorrect value in code", - "file": ".kiro/steering/roadmap.md", - "root_cause": "Had `COMMIT_WAIT_RESERVED=2`", - "fix": "Changed to `commit.c:16`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:33:10.287Z" - }, - { - "id": "bug-1154", - "timestamp": "2026-06-26T04:37:18.052Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `VTASK_LIVENESS_RESERVED`", - "fix": "Changed to `R = clamp(n_peers \u2212 1, 1, n_vcpus \u2212 1)`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-26T04:42:01.556Z" - }, - { - "id": "bug-1155", - "timestamp": "2026-06-26T04:47:49.937Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_drive_async_config.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:47:49.937Z" - }, - { - "id": "bug-1156", - "timestamp": "2026-06-26T04:48:34.246Z", - "error_message": "Function not marked async", - "file": "src/include/config.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:48:34.246Z" - }, - { - "id": "bug-1157", - "timestamp": "2026-06-26T04:48:50.005Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:48:50.005Z" - }, - { - "id": "bug-1158", - "timestamp": "2026-06-26T04:49:04.568Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:49:04.568Z" - }, - { - "id": "bug-1159", - "timestamp": "2026-06-26T04:49:44.821Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:49:44.821Z" - }, - { - "id": "bug-1160", - "timestamp": "2026-06-26T04:57:57.877Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T04:57:57.877Z" - }, - { - "id": "bug-1161", - "timestamp": "2026-06-26T05:02:36.720Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_slowlane.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T05:02:36.720Z" - }, - { - "id": "bug-1162", - "timestamp": "2026-06-26T05:06:30.148Z", - "error_message": "Function not marked async", - "file": "src/include/xce/cluster_def.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T05:06:30.148Z" - }, - { - "id": "bug-1163", - "timestamp": "2026-06-26T05:06:42.680Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/cluster.c", - "root_cause": "Condition was: if (ha_is_primary(&c->ha)", - "fix": "Changed to: if (c->slowlane.workers == NULL)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T05:06:42.680Z" - }, - { - "id": "bug-1164", - "timestamp": "2026-06-26T05:06:53.129Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T05:06:53.129Z" - }, - { - "id": "bug-1165", - "timestamp": "2026-06-26T06:51:00.808Z", - "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", - "file": "src/lib/testsuite/test.c", - "root_cause": "Used \"test_request_send_propagation\" instead of \"test_netio_bounded_connect\"", - "fix": "Changed test_request_send_propagation \u2192 test_netio_bounded_connect", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T06:51:00.808Z" - }, - { - "id": "bug-1166", - "timestamp": "2026-06-26T06:51:57.649Z", - "error_message": "Function not marked async", - "file": "src/lib/netio.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T06:51:57.649Z" - }, - { - "id": "bug-1167", - "timestamp": "2026-06-26T06:52:05.029Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T06:52:05.029Z" - }, - { - "id": "bug-1168", - "timestamp": "2026-06-26T06:52:21.608Z", - "error_message": "Function not marked async", - "file": "src/include/netio.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T06:52:21.608Z" - }, - { - "id": "bug-1169", - "timestamp": "2026-06-26T07:16:36.377Z", - "error_message": "Function not marked async", - "file": "src/include/vtaskmgr.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-26T07:17:06.235Z" - }, - { - "id": "bug-1170", - "timestamp": "2026-06-26T07:17:42.916Z", - "error_message": "Significant refactor of ", - "file": "src/lib/vtaskmgr.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u219210 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:17:42.916Z" - }, - { - "id": "bug-1171", - "timestamp": "2026-06-26T07:18:02.307Z", - "error_message": "Function not marked async", - "file": "src/lib/vtaskmgr.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:18:02.307Z" - }, - { - "id": "bug-1172", - "timestamp": "2026-06-26T07:19:14.857Z", - "error_message": "Function not marked async", - "file": "src/controller/include/commit.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-26T07:19:30.679Z" - }, - { - "id": "bug-1173", - "timestamp": "2026-06-26T07:19:45.059Z", - "error_message": "Function not marked async", - "file": "src/controller/commit.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T07:20:29.751Z" - }, - { - "id": "bug-1174", - "timestamp": "2026-06-26T07:20:07.233Z", - "error_message": "Wrong operator: >= should be !=", - "file": "src/controller/commit.c", - "root_cause": "Used \">=\" instead of \"!=\"", - "fix": "Changed operator >= \u2192 !=", - "tags": [ - "auto-detected", - "operator-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:20:07.233Z" - }, - { - "id": "bug-1175", - "timestamp": "2026-06-26T07:20:41.448Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:20:41.448Z" - }, - { - "id": "bug-1176", - "timestamp": "2026-06-26T07:28:30.067Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_commit_admission_pool.c", - "root_cause": "16 lines replaced/restructured", - "fix": "Rewrote 55\u219266 lines (16 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:28:30.067Z" - }, - { - "id": "bug-1177", - "timestamp": "2026-06-26T07:47:19.731Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T07:47:39.153Z" - }, - { - "id": "bug-1178", - "timestamp": "2026-06-26T07:48:00.209Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T07:48:57.917Z" - }, - { - "id": "bug-1179", - "timestamp": "2026-06-26T07:51:01.491Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: deadline.tv_nsec >= 1000000000L", - "fix": "Added guard clause: if (deadline.tv_nsec >= 1000000000L)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:51:01.491Z" - }, - { - "id": "bug-1180", - "timestamp": "2026-06-26T07:51:10.764Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:51:10.764Z" - }, - { - "id": "bug-1181", - "timestamp": "2026-06-26T07:56:06.307Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 16\u21924 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:56:06.307Z" - }, - { - "id": "bug-1182", - "timestamp": "2026-06-26T07:57:21.217Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "22 lines replaced/restructured", - "fix": "Rewrote 30\u21924 lines (22 removed) | Also: /* Lane already has a reconnect for this peer queu; * raced a not-yet-cleared flag). The existing tas", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T07:58:54.413Z" - }, - { - "id": "bug-1183", - "timestamp": "2026-06-26T07:57:35.484Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T07:57:35.484Z" - }, - { - "id": "bug-1184", - "timestamp": "2026-06-26T08:12:52.624Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T08:12:59.952Z" - }, - { - "id": "bug-1185", - "timestamp": "2026-06-26T08:13:19.511Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-26T08:15:04.588Z" - }, - { - "id": "bug-1186", - "timestamp": "2026-06-26T08:15:44.315Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx)", - "fix": "Changed to: if (0 && cur_sctx && cur_sctx->peer_idx == sctx->peer_)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T08:16:44.785Z" - }, - { - "id": "bug-1187", - "timestamp": "2026-06-26T08:21:29.383Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 14\u219220 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T08:21:29.383Z" - }, - { - "id": "bug-1188", - "timestamp": "2026-06-26T10:20:33.560Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T10:20:50.537Z" - }, - { - "id": "bug-1189", - "timestamp": "2026-06-26T10:21:12.319Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-26T10:23:40.982Z" - }, - { - "id": "bug-1190", - "timestamp": "2026-06-26T10:24:21.473Z", - "error_message": "Function not marked async", - "file": "src/controller/cluster.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T10:24:21.473Z" - }, - { - "id": "bug-1191", - "timestamp": "2026-06-26T10:27:00.557Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: atomic_load(&g_pc_gate_on", - "fix": "Added guard clause: if (atomic_load(&g_pc_gate_on)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T10:27:00.557Z" - }, - { - "id": "bug-1192", - "timestamp": "2026-06-26T10:27:14.358Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T10:27:14.358Z" - }, - { - "id": "bug-1193", - "timestamp": "2026-06-26T10:56:43.461Z", - "error_message": "Function not marked async", - "file": "src/include/vtaskmgr.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T10:56:43.461Z" - }, - { - "id": "bug-1194", - "timestamp": "2026-06-26T10:57:18.280Z", - "error_message": "Function not marked async", - "file": "src/lib/vtaskmgr.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T10:57:18.280Z" - }, - { - "id": "bug-1195", - "timestamp": "2026-06-26T10:57:44.750Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-26T10:59:54.657Z" - }, - { - "id": "bug-1196", - "timestamp": "2026-06-26T11:00:15.213Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T11:00:15.213Z" - }, - { - "id": "bug-1197", - "timestamp": "2026-06-26T11:04:56.177Z", - "error_message": "Wrong reference: 1100 should be 1197", - "file": "src/controller/ha.c", - "root_cause": "Used \"1100\" instead of \"1197\"", - "fix": "FIXED 2026-06-26 (raft-drive-async 3.4a, behind flag default OFF): deferred teardown-as-vtask on the per-ep FIFO + RCU-like dispatch_inflight drain in endpoint_mark_tearing_down. become_follower marks+unmonitors+drains each vote-ep then enqueues a VTASK_IO teardown bound to that ep (runs strictly after any in-flight same-ep peer-ctrl vtask; NULLs iod->ep to disarm run_task post-body unblock). Closes the UAF for the peer-control dispatch path. NOTE: the snapshot/configpush lane borrowed-ep race is distinct and still open before the prod flag flip.", - "tags": [ - "endpoint-quiescence", - "fixed", - "c", - "dispatch_inflight", - "task-3.4a", - "tearing_down", - "auto-detected", - "wrong-reference" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26" - }, - { - "id": "bug-1199", - "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", - "file": "src/lib/vtaskmgr.c", - "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt \u2014 its pkt lifetime is the endpoint's, so an unconditional free would double-free.", - "fix": "Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan.", - "tags": [ - "raft-drive-async", - "task-3.1", - "memory-leak", - "vtaskmgr", - "packet-ownership", - "review-finding" - ], - "related_bugs": [ - "bug-1077" - ], - "occurrences": 1, - "last_seen": "2026-06-26T00:00:00.000Z" - }, - { - "id": "bug-1197", - "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF \u2014 raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", - "file": "src/controller/ha.c", - "root_cause": "Moving peer-control off the sequential io_worker onto the pool co-locates teardown (frees ep->data) and inbound delivery (reads ep->data) on different threads with no unmonitor-before-free ordering. Safe today only because g_drive_async_peerctrl_dispatch_enabled defaults OFF.", - "fix": "SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer \u2014 a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON.", - "tags": [ - "raft-drive-async", - "task-3.1", - "use-after-free", - "flag-on-hazard", - "deferred", - "review-finding" - ], - "related_bugs": [ - "bug-1198" - ], - "occurrences": 1, - "last_seen": "2026-06-30T14:30:00Z" - }, - { - "id": "bug-1198", - "timestamp": "2026-06-26T00:00:00.000Z", - "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline \u2014 a blocking op under the never-gated liveness reservation breaks deadlock-freedom", - "file": "src/controller/ha.c", - "root_cause": "repl_ack_decode_and_dispatch -> ha_dispatch_repl_action_for_standby can run the snapshot send inline. LIVENESS is only correct once that long op is offloaded off the pool.", - "fix": "FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment \u2014 the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed.", - "tags": [ - "task-3.1", - "review-finding", - "stale-report", - "raft-drive-async", - "deferred", - "flag-on-hazard", - "fixed", - "deadlock-freedom", - "liveness-admission" - ], - "related_bugs": [ - "bug-1197" - ], - "occurrences": 1, - "last_seen": "2026-07-02T17:30:00.000Z" - }, - { - "id": "bug-1201", - "timestamp": "2026-06-26T11:31:19.401Z", - "error_message": "Missing error handling in task", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T11:31:19.401Z" - }, - { - "id": "bug-1202", - "timestamp": "2026-06-26T11:33:28.328Z", - "error_message": "Missing error handling in task", - "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Helper: check if a file exists in a directory ; static int file_exists_in(const char* dir, const char* name)", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T11:34:53.637Z" - }, - { - "id": "bug-1203", - "timestamp": "2026-06-26T11:35:17.615Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 10\u219212 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T11:35:17.615Z" - }, - { - "id": "bug-1204", - "timestamp": "2026-06-26T11:35:37.597Z", - "error_message": "Wrong reference: file_exists_for_trim_test should be seg_file_exists", - "file": "src/controller/testsuite/test_e2e_checkpoint.c", - "root_cause": "Used \"file_exists_for_trim_test\" instead of \"seg_file_exists\"", - "fix": "Changed file_exists_for_trim_test \u2192 seg_file_exists", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T11:35:37.597Z" - }, - { - "id": "bug-1205", - "timestamp": "2026-06-26T11:39:53.357Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T11:41:58.418Z" - }, - { - "id": "bug-1206", - "timestamp": "2026-06-26T15:16:40.996Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T15:16:40.996Z" - }, - { - "id": "bug-1207", - "timestamp": "2026-06-26T15:28:19.885Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_repl_send_snapshot_hook != NULL", - "fix": "Added guard clause: if (ha_repl_send_snapshot_hook != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T15:28:19.885Z" - }, - { - "id": "bug-1208", - "timestamp": "2026-06-26T15:28:53.174Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "19 lines replaced/restructured", - "fix": "Rewrote 25\u219210 lines (19 removed) | Also: * DEFAULT ON (1): the snapshot transfer runs on a ; * instead of blocking the drive thread (delaymgr t | Also: * \u2502 LIVENESS is correct ONLY once task 3.2 offload; * \u2502 pool. TODAY the CTRL_PRIMARY repl-ack path ca", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-26T15:31:59.280Z" - }, - { - "id": "bug-1209", - "timestamp": "2026-06-26T15:40:13.611Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u219210 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T15:40:13.611Z" - }, - { - "id": "bug-1210", - "timestamp": "2026-06-26", - "error_message": "raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted ON (=1); header doc also said default ON", - "file": "src/controller/ha.c, src/controller/ha.h", - "root_cause": "Snapshot offload borrows a standby endpoint shared with the io_worker epoll loop (not owned, unlike reconnect_lane_fn). With the flag ON the send runs on a slow-lane thread while io_worker can RDHUP-free that endpoint => cross-thread UAF (bug-1197 family). Defaulting ON would ship that UAF; the borrowed-endpoint lifetime discipline is not established until task 3.4.", - "fix": "Default the flag OFF (dormant landing, same protocol as task 3.1 peerctrl dispatch). Inline path runs on io_worker, serialized with teardown -> no overlap. Rewrote both the ha.c definition comment and the ha.h accessor doc to state OFF is the production default and 3.4 is the gate that flips it ON. bug-1198 stays open by design until then.", - "tags": [ - "raft-drive-async", - "task-3.2", - "snapshot", - "slow-lane", - "UAF", - "feature-flag", - "bug-1197", - "bug-1198", - "liveness" - ], - "related_bugs": [ - "bug-1197", - "bug-1198", - "bug-1147" - ], - "occurrences": 1, - "last_seen": "2026-06-26" - }, - { - "id": "bug-1211", - "timestamp": "2026-06-26T15:49:46.434Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.h", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "h" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-26T15:49:59.034Z" - }, - { - "id": "bug-1212", - "timestamp": "2026-06-26T15:50:27.084Z", - "error_message": "Function not marked async", - "file": "src/controller/ha.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-26T15:52:41.021Z" - }, - { - "id": "bug-1213", - "timestamp": "2026-06-26T15:51:29.747Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ha_configpush_probe != NULL", - "fix": "Added guard clause: if (ha_configpush_probe != NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T15:51:29.747Z" - }, - { - "id": "bug-1214", - "timestamp": "2026-06-26T15:54:05.463Z", - "error_message": "Function not marked async", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T15:54:05.463Z" - }, - { - "id": "bug-1215", - "timestamp": "2026-06-26T15:57:27.700Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219210 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T15:57:27.700Z" - }, - { - "id": "bug-1216", - "timestamp": "2026-06-26T16:54:57.384Z", - "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", - "file": "src/include/netio.h", - "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", - "fix": "Changed ssl_handshake_done \u2192 tearing_down", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T16:54:57.384Z" - }, - { - "id": "bug-1217", - "timestamp": "2026-06-26T16:55:15.332Z", - "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", - "file": "src/lib/netio.c", - "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", - "fix": "Changed ssl_handshake_done \u2192 tearing_down", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T16:55:15.332Z" - }, - { - "id": "bug-1218", - "timestamp": "2026-06-26T16:55:48.690Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: ectx", - "fix": "Added guard clause: if (ectx)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T16:55:48.690Z" - }, - { - "id": "bug-1219", - "timestamp": "2026-06-26T17:07:46.423Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 10\u219227 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T17:07:46.424Z" - }, - { - "id": "bug-1220", - "timestamp": "2026-06-26T17:08:14.641Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 22\u219243 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T17:08:14.641Z" - }, - { - "id": "bug-1221", - "timestamp": "2026-06-26T17:12:06.544Z", - "error_message": "Missing error handling in struct", - "file": "src/controller/testsuite/test_ha_vep_teardown.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T17:12:06.544Z" - }, - { - "id": "bug-1222", - "timestamp": "2026-06-26T17:44:25.670Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_ha_dispatch_integration.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-26T17:44:25.670Z" - }, - { - "id": "bug-1223", - "timestamp": "2026-06-27T00:33:17.036Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_dispatch_integration.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21923 lines (2 removed) | Also: /* Release ep_A so its worker finishes before the ; TEST_ASSERT_EQUAL_INT_MESSAGE(2, poll_atomic_ge(&g", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T00:34:00.682Z" - }, - { - "id": "bug-1224", - "timestamp": "2026-06-26", - "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently \u2014 controller_unit_test never completes (timeout-killed)", - "file": "src/controller/testsuite/test_ha_dispatch_integration.c", - "root_cause": "Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) \u2192 flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard.", - "fix": "Add end-of-body completion counters (g_pc_completed on both pc_probe paths incl. the park path; g_block_done at the end of block_task_entry) and poll them to the full expected count AFTER gate_open and BEFORE vtaskmgr_fini, so every pool body has fully returned (worker idle in cbuf_get) before the cancel-based fini. Verified deterministic 5/5 isolated runs + full suite EXIT=0, ASan-clean.", - "tags": [ - "raft-drive-async", - "task-3.4b", - "test", - "flaky", - "vtaskmgr_fini", - "pthread_cancel", - "teardown-race", - "deadlock" - ], - "related_bugs": [ - "bug-924" - ], - "occurrences": 1, - "last_seen": "2026-06-26" - }, - { - "id": "bug-1225", - "timestamp": "2026-06-26", - "error_message": "/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 \"cluster did not reconverge to 3-of-3 after restarting ctrl-2\"", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh (wait_for_full_convergence)", - "root_cause": "Pre-existing HA-core instability under 6-round CHAOTIC repeated failover, NOT a raft-drive-async regression. BASELINE-CONFIRMED: the same soak against pre-Phase-3 binaries (commit 21e26a1, release/non-ASAN) ALSO fails at round 4 (baseline symptom: no-new-primary election stall; current symptom: round-4 reconverge + committed-prefix divergence at LSN 9). Both builds preserve committed JOBS (zero job loss in all completed rounds). The committed-prefix divergence is the documented open bug-851/795 family (chaotic-failover silent divergence; separate HA-core spec). raft-drive-async Phase 3 flags are default-OFF (inert); the live trim-guard only preserves more WAL.", - "fix": "Not a raft-drive-async bug (baseline-confirmed). 4.2 regression sweep PASSES for this spec: no NEW failure introduced. Pre-existing bug-851 chaotic-failover divergence + bug-1042 reconverge flakiness remain open for a separate HA-core effort.", - "tags": [ - "ASAN", - "baseline-confirmed", - "bug-851", - "false-negative", - "not-a-regression", - "raft-drive-async", - "reconverge", - "soak", - "task-4.2", - "xce-test-ha" - ], - "related_bugs": [ - "bug-1042", - "bug-924" - ], - "occurrences": 1, - "last_seen": "2026-06-26" - }, - { - "id": "bug-1226", - "timestamp": "2026-06-27T02:53:39.273Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"$REPO/src/xce.build/controller/xce_controller\"", - "fix": "Changed to \"${XCE_HA_BIN:-$REPO/src/xce.build/controller/xce_", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T02:53:39.273Z" - }, - { - "id": "bug-1227", - "timestamp": "2026-06-27T03:04:05.891Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T03:04:05.891Z" - }, - { - "id": "bug-1228", - "timestamp": "2026-06-27T04:13:35.943Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Had `; no silent retention of a stale-term entry.\n- In", - "fix": "Changed to `; no silent retention of a stale-term entry.\n- In", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T04:13:47.424Z" - }, - { - "id": "bug-1229", - "timestamp": "2026-06-27T04:14:01.248Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (2 removed) | Also: - Truncate-clamp holds: a repair that would cut be; - Election under churn: repeated rapid kill/restar", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T04:14:43.855Z" - }, - { - "id": "bug-1230", - "timestamp": "2026-06-27T04:28:29.610Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 5\u21925 lines (4 removed) | Also: - [ ] 2.3 RED: single-seam ingest tests; - Add cases asserting that every replicated/catch-", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T04:28:46.301Z" - }, - { - "id": "bug-1231", - "timestamp": "2026-06-27T04:46:25.144Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/walcompare.c", - "root_cause": "49 lines replaced/restructured", - "fix": "Rewrote 138\u2192270 lines (49 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T04:46:25.144Z" - }, - { - "id": "bug-1232", - "timestamp": "2026-06-27T04:46:44.934Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 42\u219253 lines (5 removed) | Also: /* walcompare [--committed N] ; * committed prefixes by (lsn -> term, pcrc) and FA", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T04:46:55.867Z" - }, - { - "id": "bug-1233", - "timestamp": "2026-06-27T04:59:57.805Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: # \u2500\u2500 Chaotic-mode primitives (count outcomes into the verdic; # Run `walcompare --report` across all three members and ADD | Also: # \u2500\u2500 Chaotic mode: overlapping-kill rounds \u2192 isolated verdic; # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land insid", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-27T05:00:54.066Z" - }, - { - "id": "bug-1234", - "timestamp": "2026-06-27T05:00:06.726Z", - "error_message": "Function not marked async", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T05:00:06.726Z" - }, - { - "id": "bug-1235", - "timestamp": "2026-06-27T05:02:24.332Z", - "error_message": "Missing error handling in FAILED", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T05:02:24.333Z" - }, - { - "id": "bug-1236", - "timestamp": "2026-06-27T09:27:45.080Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Had `raft_on_append_entries_request`", - "fix": "Changed to `research.md`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-27T09:31:54.096Z" - }, - { - "id": "bug-1237", - "timestamp": "2026-06-27T09:28:07.944Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - **Technical debt addressed:** the silent empty-payload shi | Also: - `.claude/skills/xce-test-ha/run-ha-soak.sh` (and `SKILL.md; - `src/controller/` log init (only if the client-probe appro | Also: | Catch-up Payload Integrity | Adapter | Stream a non-empty ; | Persist Conflict Guard | Persist | `(term,lsn,payload-crc) | Also: - The live-repro gate (task 1.3) has **already run and refut; - A green synthetic unit test over a still-red soak is not a", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-27T09:32:08.493Z" - }, - { - "id": "bug-1238", - "timestamp": "2026-06-27T09:28:50.291Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u21925 lines (3 removed) | Also: - `src/controller/ha.c` \u2014 Collapse the term-blind ; - `src/controller/persist.c` \u2014 Make the idempotenc | Also: ### Divergence repair via unified seam (the fix); participant L as Leader | Also: | Intent | Every replicated/catch-up entry passes ; | Requirements | 1.1, 1.2, 2.1, 2.3, 3.1, 3.2, 3.4 | Also: - Wrong-term entry at LSN N then correct entry \u21d2 W; - Term-blind ingest path removed: a `term==0`-shap", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 5, - "last_seen": "2026-06-27T09:32:29.183Z" - }, - { - "id": "bug-1239", - "timestamp": "2026-06-27T09:33:26.843Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "17 lines replaced/restructured", - "fix": "Rewrote 28\u219237 lines (17 removed) | Also: - [ ] 6.1 Route live leader replication and catch-; - Ensure the leader-side streamer always emits ter", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T09:34:15.963Z" - }, - { - "id": "bug-1240", - "timestamp": "2026-06-27T09:33:51.137Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T09:33:51.137Z" - }, - { - "id": "bug-1241", - "timestamp": "2026-06-27T09:37:50.913Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/research.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T09:37:50.913Z" - }, - { - "id": "bug-1242", - "timestamp": "2026-06-27T10:02:24.877Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "34 lines replaced/restructured", - "fix": "Rewrote 54\u219289 lines (34 removed) | Also: # \u2500\u2500 Mode / flag parsing (flags may precede the po; *) ARGS+=(\"$1\") ;; | Also: ROUNDS=\"${XCE_HA_ROUNDS:-6}\"; [ \"$ROUNDS\" -lt 5 ] && ROUNDS=5 | Also: # run-ha-soak.sh [--chaotic|--sequential] [--sel; # DEPLOY_DIR optional deploy directory (posi", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-27T10:04:24.730Z" - }, - { - "id": "bug-1243", - "timestamp": "2026-06-27T10:02:56.820Z", - "error_message": "Wrong operator: == should be <<", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Used \"==\" instead of \"<<\"", - "fix": "Changed operator == \u2192 <<", - "tags": [ - "auto-detected", - "operator-fix", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:02:56.820Z" - }, - { - "id": "bug-1244", - "timestamp": "2026-06-27T10:04:31.979Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"no primary elected within 45s (check $TOP/logs)\"", - "fix": "Changed to \"no primary elected within ~90s via client probe (", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:04:31.979Z" - }, - { - "id": "bug-1245", - "timestamp": "2026-06-27T10:07:32.277Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 3\u21927 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:07:32.277Z" - }, - { - "id": "bug-1246", - "timestamp": "2026-06-27T10:07:37.692Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/SKILL.md", - "root_cause": "Had `no primary elected within 45s`", - "fix": "Changed to `no primary elected within ~90s via client probe`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:07:37.692Z" - }, - { - "id": "bug-1247", - "timestamp": "2026-06-27T10:25:38.366Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || lsn == 0", - "fix": "Added guard clause: if (!p || lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:25:38.366Z" - }, - { - "id": "bug-1248", - "timestamp": "2026-06-27T10:25:56.559Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:25:56.559Z" - }, - { - "id": "bug-1249", - "timestamp": "2026-06-27T10:45:00.686Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* (a) non-empty incoming whose payload differs at an existi; * \u2014 a genuine payload conflict. Branch on tip vs below-tip.", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-27T10:45:20.669Z" - }, - { - "id": "bug-1250", - "timestamp": "2026-06-27T10:45:34.063Z", - "error_message": "Function not marked async", - "file": "src/controller/persist.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:45:34.063Z" - }, - { - "id": "bug-1251", - "timestamp": "2026-06-27T10:46:03.261Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:46:03.261Z" - }, - { - "id": "bug-1252", - "timestamp": "2026-06-27T10:47:06.628Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/persist.c", - "root_cause": "Condition was: if (lsn != tip_lsn)", - "fix": "Changed to: if (trc0 != 0)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T10:47:06.628Z" - }, - { - "id": "bug-1253", - "timestamp": "2026-06-27T11:14:02.745Z", - "error_message": "Missing error handling in XCE_TESTING", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T11:14:02.745Z" - }, - { - "id": "bug-1254", - "timestamp": "2026-06-27T11:14:27.133Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: * TASK 2.3 \u2014 cross-term catch-up streamer payload integrity ; * | Also: /* Task 2.3 cross-term catch-up streamer payload integrity:; * CORE RED \u2014 the streamer must never ship a non-empty cro", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-27T11:18:03.504Z" - }, - { - "id": "bug-1255", - "timestamp": "2026-06-27T11:28:45.351Z", - "error_message": "Missing guard clause", - "file": "src/controller/config_repl.c", - "root_cause": "No early return/throw for edge case: !pbuf", - "fix": "Added guard clause: if (!pbuf)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T11:28:45.351Z" - }, - { - "id": "bug-1256", - "timestamp": "2026-06-27T11:29:02.472Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T11:29:02.472Z" - }, - { - "id": "bug-1257", - "timestamp": "2026-06-27T11:42:14.469Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21925 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-27T11:42:14.469Z" - }, - { - "id": "bug-1258", - "timestamp": "2026-06-28T03:22:49.419Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 13\u219215 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-28T03:22:49.419Z" - }, - { - "id": "bug-1259", - "timestamp": "2026-06-28T03:23:10.335Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-28T03:23:10.335Z" - }, - { - "id": "bug-1260", - "timestamp": "2026-06-28T03:23:33.560Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 13\u219214 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-28T03:23:33.560Z" - }, - { - "id": "bug-1261", - "timestamp": "2026-06-28T03:23:53.602Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-28T03:23:53.602Z" - }, - { - "id": "bug-1262", - "timestamp": "2026-06-28T03:34:38.617Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-28T03:34:38.617Z" - }, - { - "id": "bug-1263", - "timestamp": "2026-06-28T03:41:54.136Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21926 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-28T03:41:54.136Z" - }, - { - "id": "bug-1264", - "timestamp": "2026-06-29T04:13:02.187Z", - "error_message": "Incorrect value in code", - "file": "src/controller/meson.build", - "root_cause": "Had 'testsuite/test_waldump.c'", - "fix": "Changed to 'testsuite/test_ha_partition.c'", - "tags": [ - "auto-detected", - "wrong-value", - "build" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-29T04:13:02.187Z" - }, - { - "id": "bug-1265", - "timestamp": "2026-06-29T04:19:29.991Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_partition.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 11\u21924 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-29T04:19:29.991Z" - }, - { - "id": "bug-1266", - "timestamp": "2026-06-29T04:25:39.111Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_partition.c", - "root_cause": "No early return/throw for edge case: peer_idx == g_partition_peer_idx && g_reconnect_gate", - "fix": "Added guard clause: if (peer_idx == g_partition_peer_idx && g_re)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-29T04:25:39.111Z" - }, - { - "id": "bug-1267", - "timestamp": "2026-06-29T04:29:14.151Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_partition.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 8\u21923 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-29T04:29:14.151Z" - }, - { - "id": "bug-1268", - "timestamp": "2026-06-30T00:15:49.029Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T00:15:49.029Z" - }, - { - "id": "bug-1269", - "timestamp": "2026-06-30T01:47:07.635Z", - "error_message": "Missing await", - "file": "src/controller/ha.c", - "root_cause": "Async call without await \u2014 returned Promise instead of value", - "fix": "Added await to async call", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T01:47:07.635Z" - }, - { - "id": "bug-1270", - "timestamp": "2026-06-30T02:03:36.780Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T02:03:36.780Z" - }, - { - "id": "bug-1271", - "timestamp": "2026-06-30T02:25:32.055Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: - _Depends: 1.5, 6.2, 5.2, 5.3_; - _Blocked: first soak run (HEAD 01419e1, Phase-2+5 fixes) i", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-30T02:26:08.789Z" - }, - { - "id": "bug-1272", - "timestamp": "2026-06-30T02:25:00Z", - "error_message": "raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills", - "file": "src/controller/ha.c (election/PreVote path)", - "root_cause": "After 3 overlapping kills the only fully-up-to-date node is killed; the two survivors are both restarted-and-lagging (tips 15 and 11; leader was 19). Both hold the full committed prefix [1..11] (safe candidates) but their PreVote rounds repeatedly time out with no grant quorum (prospective terms frozen, never bumped). Distinct from the 5.2 spurious-failover flap (that is fixed and confirmed). Contributors: freshness mutual-denial between lagging survivors, in_catch_up-denies-all-votes (Req 2.5) with no live primary to catch up from, and/or peer vote connection not established between two nodes that only held a control channel to the dead primary.", - "fix": "FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED\u2192GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync\u2192EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus.", - "tags": [ - "prevote", - "ha", - "liveness", - "chaotic-soak", - "environment", - "election", - "bug-924", - "corrected", - "7.2", - "fdatasync", - "raft", - "5.3", - "in_catch_up" - ], - "related_bugs": [ - "bug-924", - "bug-851", - "bug-1042", - "bug-1060" - ], - "occurrences": 2, - "last_seen": "2026-06-30T02:55:00Z" - }, - { - "id": "bug-1273", - "timestamp": "2026-06-30T02:41:41.017Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 5.3 lagging-but-safe survivor electability under ove; * CORE RED \u2014 a catching-up survivor with NO live primary m", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-30T02:41:53.665Z" - }, - { - "id": "bug-1274", - "timestamp": "2026-06-30T02:42:59.119Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u219220 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T02:42:59.119Z" - }, - { - "id": "bug-1275", - "timestamp": "2026-06-30T02:55:23.808Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T02:55:23.808Z" - }, - { - "id": "bug-1276", - "timestamp": "2026-06-30T02:55:37.693Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `research.md`", - "fix": "Changed to 's persist flusher jams on `", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T02:55:37.693Z" - }, - { - "id": "bug-1281", - "timestamp": "2026-06-30T03:25:00Z", - "error_message": "Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall.", - "file": "src/controller (restart/recovery/election path \u2014 exact site unknown, no backtrace)", - "root_cause": "ROOT-CAUSED via gdb SIGQUIT all-thread bt: an ABBA lock-order deadlock in src/lib async dispatch. io_worker (netio.c:672) holds ep->lock then blocks on q->lock (dataq_enq); vdispatcher dataq_deq (dataq.c:65) holds q->lock then blocks on ep->lock via the vtask_eligible->endpoint_is_task_blocked (netio.c:1001) match callback. io_worker dies -> node stops accepting/answering PreVotes -> round-4 stall. In release the same ep/q race corrupts -> SIGSEGV. See bug-1282 (the actual dispatch defect).", - "fix": "The crash/hang IS bug-1282 (UPSTREAM raft-drive-async dispatch deadlock). raft-ha-core-safety 7.2 stays blocked until bug-1282 is fixed upstream, then re-run the gate. No raft-ha-core-safety code change.", - "tags": [ - "7.2", - "iomgr", - "gdb", - "election", - "crash", - "5.4", - "restart", - "recovery", - "heisenbug", - "accept-hang", - "restart-stability", - "segfault", - "chaotic-soak" - ], - "related_bugs": [ - "bug-1272", - "bug-1282", - "bug-924" - ], - "occurrences": 3, - "last_seen": "2026-06-30T04:00:00Z" - }, - { - "id": "bug-1278", - "timestamp": "2026-06-30T03:32:49.143Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T03:32:49.143Z" - }, - { - "id": "bug-1279", - "timestamp": "2026-06-30T03:33:06.273Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had 's persist flusher jams on `", - "fix": "Changed to `xce_controller`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T03:33:06.273Z" - }, - { - "id": "bug-1280", - "timestamp": "2026-06-30T03:57:32.784Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had \"Task 7.2 on ~/tmp\"", - "fix": "Changed to \"Task 7.2/5.4 \u2014 UNIFIED root cause\"", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T03:57:32.785Z" - }, - { - "id": "bug-1282", - "timestamp": "2026-06-30T04:20:00Z", - "error_message": "ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV.", - "file": "src/lib/dataq.c:65 (dataq_deq holds q->lock across match cb) + src/lib/vtaskmgr.c:255 (vtask_eligible) + src/lib/netio.c:1001 (endpoint_is_task_blocked takes ep->lock) + src/lib/netio.c:672 (io_worker holds ep->lock across vtask_enq->dataq_enq)", - "root_cause": "dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock \u2014 inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes.", - "fix": "FIXED (commit 25e741d): promoted endpoint_t.busy from a 1-bit bitfield to _Atomic int; endpoint_block_tasks/endpoint_unblock_tasks/endpoint_is_task_blocked now lockless (release/acquire), removing the ep->lock acquire from the vtask-eligibility path that ran under q->lock. Verified: post-fix gdb soak has ZERO __lll_lock_wait threads + no endpoint_is_task_blocked in any lock path; controller_unit_test all green. Mirrors dispatch_inflight (bug-1197). NOTE: a SEPARATE release-only restart segfault (bug-1283) remains and still blocks 7.2.", - "tags": [ - "deadlock", - "lock-order", - "abba", - "vtaskmgr", - "dataq", - "netio", - "io_worker", - "vdispatcher", - "raft-drive-async", - "async-io", - "upstream" - ], - "related_bugs": [ - "bug-1281" - ], - "occurrences": 2, - "last_seen": "2026-06-30T04:20:00Z" - }, - { - "id": "bug-1282", - "timestamp": "2026-06-30T04:17:08.329Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "FIXED (commit 25e741d): promoted endpoint_t.busy from a 1-bit bitfield to _Atomic int; endpoint_block_tasks/endpoint_unblock_tasks/endpoint_is_task_blocked now lockless (release/acquire), removing the ep->lock acquire from the vtask-eligibility path that ran under q->lock. Verified: post-fix gdb soak has ZERO __lll_lock_wait threads + no endpoint_is_task_blocked in any lock path; controller_unit_test all green. Mirrors dispatch_inflight (bug-1197). NOTE: a SEPARATE release-only restart segfault (bug-1283) remains and still blocks 7.2.", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T04:17:08.329Z" - }, - { - "id": "bug-1283", - "timestamp": "2026-06-30T04:17:58.579Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `xce_controller`", - "fix": "Changed to `src/lib`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T04:17:58.579Z" - }, - { - "id": "bug-1284", - "timestamp": "2026-06-30T07:56:51.924Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "10 lines replaced/restructured", - "fix": "Rewrote 30\u219231 lines (10 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T07:56:51.924Z" - }, - { - "id": "bug-1285", - "timestamp": "2026-06-30T08:24:48.860Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `vtask_eligible`", - "fix": "Changed to `endpoint_is_task_blocked`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T08:24:48.860Z" - }, - { - "id": "bug-1286", - "timestamp": "2026-06-30T05:30:00Z", - "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart \u2014 Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", - "file": "src/controller/cluster.c:251 (myreq_handler) + src/controller/ha.c:3194 (ha_dispatch_gate)", - "root_cause": "NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 \u2014 a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown).", - "fix": "FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL \u2192 &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green.", - "tags": [ - "segfault", - "null-deref", - "ep-owner", - "myreq_handler", - "cluster.c", - "ha_dispatch_gate", - "restart", - "7.2", - "ROOT-CAUSED" - ], - "related_bugs": [ - "bug-1282", - "bug-1281", - "bug-1197" - ], - "occurrences": 3, - "last_seen": "2026-06-30T14:00:00Z" - }, - { - "id": "bug-1287", - "timestamp": "2026-06-30T09:00:43.427Z", - "error_message": "Significant refactor of ", - "file": "src/tools/xce-build.sh", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 19\u219253 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T09:00:43.427Z" - }, - { - "id": "bug-1288", - "timestamp": "2026-06-30T09:02:01.055Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-build/SKILL.md", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 17\u219226 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T09:02:01.055Z" - }, - { - "id": "bug-1289", - "timestamp": "2026-06-30T09:10:50.002Z", - "error_message": "Significant refactor of ", - "file": "src/tools/xce-build.sh", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 5\u219210 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T09:10:50.002Z" - }, - { - "id": "bug-1290", - "timestamp": "2026-06-30T09:15:12.861Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had 't crash) \u2014 STILL blocks the gate; likely an endpo", - "fix": "Changed to 't crash) \u2014 is now **ROOT-CAUSED** (via the new `", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T09:15:12.861Z" - }, - { - "id": "bug-1291", - "timestamp": "2026-06-30T09:32:54.664Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T09:32:54.664Z" - }, - { - "id": "bug-1292", - "timestamp": "2026-06-30T09:33:08.918Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `1+2*njobs`", - "fix": "Changed to `c = ep->owner`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T09:33:08.918Z" - }, - { - "id": "bug-1293", - "timestamp": "2026-06-30T11:15:05.373Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 11\u219219 lines (5 removed) | Also: /* ha-term-safety task 1.2: advertise our Raft (la; * \u2014 identical sourcing to the real-vote send path ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-30T11:15:18.266Z" - }, - { - "id": "bug-1294", - "timestamp": "2026-06-30T11:42:35.428Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T11:42:35.428Z" - }, - { - "id": "bug-1295", - "timestamp": "2026-06-30T13:19:42.706Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: jmax > 0", - "fix": "Added guard clause: if (jmax > 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T13:19:42.706Z" - }, - { - "id": "bug-1296", - "timestamp": "2026-06-30T13:21:00.000Z", - "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP\u2192PreVote\u2192timeout\u2192repeat livelock \u2014 zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait \u2014 chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request \u2192 mutual timeout \u2192 RDHUP fires again \u2192 repeat forever.", - "file": "src/controller/ha.c", - "root_cause": "TASK 5.6 dueling-candidate PreVote cross-denial \u2014 DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING \u2014 it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient.", - "fix": "FIXED 2026-06-30 (subsumed by tasks 5.6 commit 81cb09a + 5.7 commit 74784b2). The RDHUP->PreVote->timeout liveloc was the same fixed-watchdog resonance as bug covered by 5.7: per-campaign election watchdog re-randomized to [T,2T] (Ongaro 3.4) + don't-preempt-in-flight-election (5.6). CONFIRMED by a 10-round chaotic soak: 10/10 rounds elect a new primary in a single term increment (4->13), zero term escalation, round 7+ (the old liveloc point) all clean.", - "tags": [ - "raft-ha-core-safety", - "election-liveness", - "prevote-cross-denial", - "dueling-candidates", - "task-5.6", - "chaotic-soak", - "round-7-stall" - ], - "related_bugs": [ - "bug-1281", - "bug-1282", - "bug-1286", - "bug-1292" - ], - "occurrences": 1, - "last_seen": "2026-06-30T14:30:00Z" - }, - { - "id": "bug-1297", - "timestamp": "2026-06-30T13:28:01.170Z", - "error_message": "Missing guard clause", - "file": "src/controller/ha.c", - "root_cause": "No early return/throw for edge case: jmax > 0", - "fix": "Added guard clause: if (jmax > 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T13:28:01.170Z" - }, - { - "id": "bug-1298", - "timestamp": "2026-06-30T13:36:00-07:00", - "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) \u2014 endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3\u21924, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", - "file": "src/lib/netio.c / src/lib/vtaskmgr.c / src/controller/cluster.c", - "root_cause": "Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error \u2014 handle_endpoint_error \u2192 zarray_del \u2192 endpoint_killer free()s the endpoint_t \u2014 while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match \u2192 vtask_eligible \u2192 endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF.", - "fix": "FIXED (commit 6e05dc1): added endpoint_t reference count (refcnt _Atomic + heap_alloc flag). endpoint_get starts refcnt=1 (owner); vtask_enq takes a per-vtask ref before publishing the task to the dispatcher, run_task/taskq_item_killer release it; every heap-ep teardown site drops its owner ref via endpoint_unref (last unref fini+frees). The dispatcher can now read ep->busy on a queued vtask without the io_worker freeing it underneath. Embedded eps (heap_alloc=0) never freed by unref. Validated: 182 unit tests/0 fail; 6-round ASAN chaotic soak ZERO ASan reports, ctrl-3 survives round 4, all rounds elect.", - "tags": [ - "netio", - "vtaskmgr", - "use-after-free", - "asan", - "endpoint-teardown", - "concurrency", - "raft-ha-core-safety", - "soak", - "quorum", - "bug-1282-family", - "FIXED" - ], - "related_bugs": [ - "bug-1282", - "bug-1197", - "bug-1286" - ], - "occurrences": 1, - "last_seen": "2026-06-30T14:25:00-07:00" - }, - { - "id": "bug-1299", - "timestamp": "2026-06-30T13:39:09.194Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T13:39:09.194Z" - }, - { - "id": "bug-1300", - "timestamp": "2026-06-30T14:04:05.998Z", - "error_message": "Significant refactor of ", - "file": "src/lib/netio.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 11\u219211 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T14:04:05.998Z" - }, - { - "id": "bug-1301", - "timestamp": "2026-06-30T14:05:02.703Z", - "error_message": "Significant refactor of ", - "file": "src/controller/cluster.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 9\u219212 lines (2 removed) | Also: /* zarray_del will finalize ep */; endpoint_fini(ep); | Also: endpoint_fini(ep2);; FREEUP(ep2);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 4, - "last_seen": "2026-06-30T14:05:54.573Z" - }, - { - "id": "bug-1302", - "timestamp": "2026-06-30T14:07:16.602Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u21927 lines (2 removed) | Also: endpoint_fini(ep);; free(ep); | Also: endpoint_fini(ep);; free(ep); /* endpoint_get() calloc'd ep; ", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-06-30T14:09:43.010Z" - }, - { - "id": "bug-1303", - "timestamp": "2026-06-30T14:14:35.148Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_vep_teardown.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 8\u21929 lines (4 removed) | Also: /* ep was not torn down by a vtask here \u2014 free it ; * by the drop path). */", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-06-30T14:16:12.183Z" - }, - { - "id": "bug-1304", - "timestamp": "2026-06-30T14:24:36.874Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/research.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T14:24:36.874Z" - }, - { - "id": "bug-1305", - "timestamp": "2026-06-30T14:25:55.429Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T14:25:55.429Z" - }, - { - "id": "bug-1306", - "timestamp": "2026-06-30T14:31:00-07:00", - "error_message": "6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean.", - "file": "src/lib/hook.c (xce_run_hooks); job.c:603 gate is correct", - "root_cause": "SHARPENED 2026-06-30 (waldump+log bisect of the 10-round soak): NOT the catch-up prefix-gap/entity-snapshot mechanism. xce_run_hooks (src/lib/hook.c:134-157) returns only the LAST-registered hook's rc (rc overwritten each loop iter; break is BEFORE_EVENT-only). The durable-majority gate works: a term-9 leader's job at LSN 30 timed out commit (peers mid-restart, repl_wait_majority_ack -> commit_submit -> -ETIMEDOUT), persist_job_hook_fun returned -ETIMEDOUT, but a later-registered JOB AFTER_EVENT hook (sched/apiserver) returned 0 and CLOBBERED it -> handle_jobsubmit saw hook_rc==0 -> replied 'jobsubmit ok' for an uncommitted entry -> term-10 leader legally overwrote LSN 30 (term-10 NOOP) -> committed-job-loss. Violates the Commit Durability invariant (reply-success => majority-durable). Evidence: all 3 WALs gapless/CRC-intact/full payloads; lost job only on ctrl-1@LSN30 term9 vs term10 NOOP on majority; ctrl-1 log has the WAIT-deadline-expired WARNs and NO 'rejecting job' line.", - "fix": "FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) \u2014 neither is job loss.", - "tags": [ - "ha-commit-durability", - "pre-existing", - "not-a-regression", - "soak", - "task-5.9", - "committed-job-loss", - "commit-durability", - "term-boundary-divergence", - "catch-up-gap", - "failover", - "hook-rc-clobber", - "ack-before-majority", - "data-loss" - ], - "related_bugs": [ - "bug-795", - "bug-828", - "bug-851", - "bug-1298" - ], - "occurrences": 1, - "last_seen": "2026-06-30T17:40:00Z" - }, - { - "id": "bug-1307", - "timestamp": "2026-06-30T15:05:00.000Z", - "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN \u2014 silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix \u2192 Leader Completeness violation.", - "file": "src/controller/ha.c", - "root_cause": "Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously \u2192 cross-denied \u2192 the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) \u2192 the early node campaigns with an incomplete log \u2192 voters grant based on stale view \u2192 winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win \u2014 with the new jitter timings this window is hit frequently.", - "fix": "CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario \u2014 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) \u2014 that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run \u2014 no residual doubt.", - "tags": [ - "data-safety-regression", - "task-5.7", - "chaotic-soak", - "leader-completeness-violation", - "confirmed-via-soak", - "committed-job-loss", - "election-liveness", - "bug-1296-followup", - "committed-divergence", - "fixed" - ], - "related_bugs": [ - "bug-1296", - "bug-1281", - "bug-1282", - "bug-1286", - "bug-851" - ], - "occurrences": 1, - "last_seen": "2026-07-03T03:15:00.000Z" - }, - { - "id": "bug-1308", - "timestamp": "2026-06-30T21:22:32.991Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Had `research.md`", - "fix": "Changed to 't-preempt-in-flight-election; reduced term churn)", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-06-30T21:22:32.991Z" - }, - { - "id": "bug-1309", - "timestamp": "2026-07-01T00:23:45.689Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "39 lines replaced/restructured", - "fix": "Rewrote 46\u219214 lines (39 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T00:23:45.689Z" - }, - { - "id": "bug-1310", - "timestamp": "2026-07-01T00:24:06.562Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: p == NULL || jlist == NULL", - "fix": "Added guard clause: if (p == NULL || jlist == NULL)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T00:24:06.562Z" - }, - { - "id": "bug-1311", - "timestamp": "2026-07-01T00:26:31.002Z", - "error_message": "Significant refactor of ", - "file": "src/controller/job.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 29\u219245 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T00:26:31.002Z" - }, - { - "id": "bug-1312", - "timestamp": "2026-07-01T00:29:56.872Z", - "error_message": "Wrong reference: test_job_hook_start_writes_wal_entry should be test_job_hook_new_is_notify_only", - "file": "src/controller/testsuite/test_persist_hooks.c", - "root_cause": "Used \"test_job_hook_start_writes_wal_entry\" instead of \"test_job_hook_new_is_notify_only\"", - "fix": "Changed test_job_hook_start_writes_wal_entry \u2192 test_job_hook_new_is_notify_only", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T00:29:56.872Z" - }, - { - "id": "bug-1313", - "timestamp": "2026-07-01T00:42:13.591Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T00:42:13.591Z" - }, - { - "id": "bug-1314", - "timestamp": "2026-07-01T01:24:37.398Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T01:24:37.398Z" - }, - { - "id": "bug-1315", - "timestamp": "2026-07-01T01:26:38.341Z", - "error_message": "Wrong reference: unistd should be time", - "file": "src/controller/testsuite/test_ha_rdhup_integration.c", - "root_cause": "Used \"unistd\" instead of \"time\"", - "fix": "Changed unistd \u2192 time", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T01:26:38.341Z" - }, - { - "id": "bug-1310", - "timestamp": "2026-06-30T18:40:00Z", - "error_message": "chaotic soak round-4 election-escalation storm: terms laddered 6->24 (~18 bumps), 48 begin_failovers, round overran the harness window", - "file": "src/controller/ha.c (ha_on_primary_ctrl_fd_rdhup)", - "root_cause": "A dead primary's control fd RDHUPs faster than an election settles. The 5.6 guard only drops an RDHUP while role is INSTANTANEOUSLY CTRL_ELECTING/prevote; an RDHUP landing in the gap after a lost election (role briefly back to CTRL_STANDBY) re-triggers a fresh ha_begin_failover -> PreVote(won) -> real election(term++). One term bump per RDHUP -> unbounded escalation under a sustained cadence. NOT RNG (seed is /dev/urandom).", - "fix": "5.10b RDHUP failover DEBOUNCE: ha_state_t.last_failover_ms (atomic) stamped at ha_begin_failover; ha_on_primary_ctrl_fd_rdhup drops an RDHUP within 2*election_timeout_ms of the last trigger (closes the dead fd, returns) so the election's own jittered retry (5.7) runs without the external RDHUP cadence adding term bumps. First trigger never debounced; genuine loss still fails over, at most once per window. Unit-proven: test_rdhup_within_cooldown_is_debounced (recent last_failover -> stays CTRL_STANDBY) vs test_rdhup_single_node_reaches_primary (last_failover=0 -> CTRL_PRIMARY) = differential proof. Soak: 10/10 elect, terms 4->14 bounded, begin_failover 14/14/12 (was 48), no regression \u2014 but the stochastic storm did NOT recur this run (debounce fired 0x), so end-to-end storm suppression is unit-proven, not yet soak-reproduced.", - "tags": [ - "election-liveness", - "rdhup", - "term-escalation", - "debounce", - "5.10b", - "raft" - ], - "related_bugs": [ - "bug-1296", - "bug-1306" - ], - "occurrences": 1, - "last_seen": "2026-06-30T18:40:00Z" - }, - { - "id": "bug-1317", - "timestamp": "2026-07-01T01:32:08.631Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T01:32:08.631Z" - }, - { - "id": "bug-1318", - "timestamp": "2026-07-01T04:03:20.762Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/ha-entity-snapshots/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 4\u21929 lines (2 removed) | Also: - Any change to the `Job` or `QueueConf` capnp sch; and compatibility with `restore_job`/`restore_queu | Also: `c->qmgr` for the duration of their walk. Per the ; convention used throughout `job.c`/`queue.c`, this | Also: - Existing patterns preserved: tmp-file-then-renam; the `collect_jobs`/`collect_queues` iteration shap | Also: - Appends, in order: a format-version marker, a jo; frames, a queue count and per-queue length-prefixe | Also: - If entity frames are present, decode and call `r; in file order. On any decode or restore failure, a | Also: [queue_count * { uint32_t payload_len; uint8_t cap; - No new capnp schema struct: each entity frame is | Also: - Per-entity encode happens while the enumerating ; Architecture), giving each entity a single consist | Also: loaded back, yields N+M records field-identical to; 2.2).", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 9, - "last_seen": "2026-07-01T04:06:08.943Z" - }, - { - "id": "bug-1319", - "timestamp": "2026-07-01T04:04:13.051Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-entity-snapshots/design.md", - "root_cause": "Had \"collect+encode jobs\\n(RTTI_LOCK_READ on jobmgr)\"", - "fix": "Changed to \"collect+encode jobs\\n(table READ lock + per-job R", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-07-01T04:04:56.893Z" - }, - { - "id": "bug-1320", - "timestamp": "2026-07-01T04:47:19.662Z", - "error_message": "Wrong reference: test_ha_partition should be test_persist_snapshot_entities", - "file": "src/controller/testsuite/controller_test.c", - "root_cause": "Used \"test_ha_partition\" instead of \"test_persist_snapshot_entities\"", - "fix": "Changed test_ha_partition \u2192 test_persist_snapshot_entities", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T04:47:19.662Z" - }, - { - "id": "bug-1321", - "timestamp": "2026-07-01T05:31:23.754Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "7 lines replaced/restructured", - "fix": "Rewrote 45\u2192231 lines (7 removed) | Also: if (n < 16); if (memcmp(header, \"XCESNAP\\0\", 8) != 0) | Also: if (read(fd, &format_version, sizeof(format_versio; if (read(fd, &job_count, sizeof(job_count)) != (ss", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-07-01T05:35:29.961Z" - }, - { - "id": "bug-1322", - "timestamp": "2026-07-01T05:50:53.822Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 7\u21924 lines (3 removed) | Also: ssize_t fv_n = read(fd, &format_version, sizeof(fo; fprintf(stderr, \"DEBUG load_snapshot header_n=%zd | Also: * (Requirement 2.4, 5.1, 5.2), not an error. */; if (read(fd, &format_version, sizeof(format_versio", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-07-01T05:55:01.663Z" - }, - { - "id": "bug-1323", - "timestamp": "2026-07-01T05:51:25.776Z", - "error_message": "Significant refactor of ", - "file": "src/controller/job.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u21926 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T05:51:25.776Z" - }, - { - "id": "bug-1324", - "timestamp": "2026-07-01T05:51:32.475Z", - "error_message": "Significant refactor of ", - "file": "src/lib/jobj.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 5\u21923 lines (2 removed) | Also: fprintf(stderr, \"DEBUG makeJobsub key_add rc=%d jo; rc, (void*)jobsub, (void*)jobsub_table.sub_table);", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-01T05:51:39.311Z" - }, - { - "id": "bug-1325", - "timestamp": "2026-07-01T06:14:23.046Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "53 lines replaced/restructured", - "fix": "Rewrote 109\u219238 lines (53 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T06:14:23.046Z" - }, - { - "id": "bug-1326", - "timestamp": "2026-07-01T06:17:52.049Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_persist_snapshot_entities.c", - "root_cause": "No early return/throw for edge case: fd < 0", - "fix": "Added guard clause: if (fd < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T06:17:52.049Z" - }, - { - "id": "bug-1327", - "timestamp": "2026-07-01T06:18:39.877Z", - "error_message": "Wrong reference: pthread should be fcntl", - "file": "src/controller/testsuite/test_persist_snapshot_entities.c", - "root_cause": "Used \"pthread\" instead of \"fcntl\"", - "fix": "Changed pthread \u2192 fcntl", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T06:18:39.877Z" - }, - { - "id": "bug-1328", - "timestamp": "2026-07-01T06:33:25.117Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_persist_snapshot_entities.c", - "root_cause": "No early return/throw for edge case: fd < 0", - "fix": "Added guard clause: if (fd < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T06:33:25.117Z" - }, - { - "id": "bug-1329", - "timestamp": "2026-07-01T06:42:17.131Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 21\u219238 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T06:42:17.131Z" - }, - { - "id": "bug-1330", - "timestamp": "2026-07-01T07:24:22.541Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_ha_liveness.c", - "root_cause": "No early return/throw for edge case: cleared == 0", - "fix": "Added guard clause: if (cleared == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T07:24:22.541Z" - }, - { - "id": "bug-1331", - "timestamp": "2026-07-01T07:46:42.046Z", - "error_message": "Wrong return value", - "file": "src/controller/persist.c", - "root_cause": "Was returning: load_snapshot(p, path, lsn);", - "fix": "Now returns: rc;", - "tags": [ - "auto-detected", - "return-value", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T07:46:42.046Z" - }, - { - "id": "bug-1332", - "timestamp": "2026-07-01T07:46:49.078Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T07:46:49.078Z" - }, - { - "id": "bug-1333", - "timestamp": "2026-07-01T08:06:27.737Z", - "error_message": "Significant refactor of ", - "file": "src/controller/job.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 20\u219245 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T08:06:27.737Z" - }, - { - "id": "bug-1334", - "timestamp": "2026-07-01T08:07:17.491Z", - "error_message": "Missing error handling in returns", - "file": "src/controller/persist.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T08:07:17.491Z" - }, - { - "id": "bug-1335", - "timestamp": "2026-07-01T08:10:41.772Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 13\u219235 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T08:10:41.772Z" - }, - { - "id": "bug-1336", - "timestamp": "2026-07-01T08:33:06.618Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T08:33:06.618Z" - }, - { - "id": "bug-1337", - "timestamp": "2026-07-01T09:03:51.864Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/ha-entity-snapshots/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T09:03:51.864Z" - }, - { - "id": "bug-1338", - "timestamp": "2026-07-01T09:08:08.965Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 13\u219227 lines (4 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T09:08:08.965Z" - }, - { - "id": "bug-1339", - "timestamp": "2026-07-01T09:53:51.404Z", - "error_message": "Significant refactor of ", - "file": "src/lib/request.c", - "root_cause": "5 lines replaced/restructured", - "fix": "Rewrote 5\u21928 lines (5 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T09:53:51.404Z" - }, - { - "id": "bug-1340", - "timestamp": "2026-07-01T09:55:17.967Z", - "error_message": "Missing guard clause", - "file": "src/controller/persist.c", - "root_cause": "No early return/throw for edge case: !p || checkpoint_lsn == 0", - "fix": "Added guard clause: if (!p || checkpoint_lsn == 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T09:55:17.967Z" - }, - { - "id": "bug-1341", - "timestamp": "2026-07-01T09:56:22.420Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 15\u219221 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T09:56:22.420Z" - }, - { - "id": "bug-1342", - "timestamp": "2026-07-01T09:59:00.099Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "9 lines replaced/restructured", - "fix": "Rewrote 17\u219222 lines (9 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T09:59:00.099Z" - }, - { - "id": "bug-1343", - "timestamp": "2026-07-01T13:27:42.519Z", - "error_message": "Missing error handling in every", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block | Also: /* Task 6.1 integration: leader->standby replication carries; * payload) faithfully end-to-end across BOTH live in-term re", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-01T13:27:51.887Z" - }, - { - "id": "bug-1344", - "timestamp": "2026-07-01T13:43:55.685Z", - "error_message": "Missing error handling in that", - "file": "src/controller/testsuite/test_ha_core_safety.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T13:43:55.685Z" - }, - { - "id": "bug-1345", - "timestamp": "2026-07-01T13:45:30.986Z", - "error_message": "Significant refactor of ", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u21928 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T13:45:30.986Z" - }, - { - "id": "bug-1346", - "timestamp": "2026-07-01T13:45:46.329Z", - "error_message": "Missing error handling in that", - "file": ".kiro/specs/raft-ha-core-safety/design.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T13:45:46.329Z" - }, - { - "id": "bug-1347", - "timestamp": "2026-07-01T14:10:36.559Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 33\u219292 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T14:10:36.559Z" - }, - { - "id": "bug-1348", - "timestamp": "2026-07-01T14:10:48.987Z", - "error_message": "Wrong reference: test_pre_vote_catch_up_denies should be test_pre_vote_catch_up_grants_when_primary_unreachable", - "file": "src/controller/testsuite/test_ha_vote_gate.c", - "root_cause": "Used \"test_pre_vote_catch_up_denies\" instead of \"test_pre_vote_catch_up_grants_when_primary_unreachable\"", - "fix": "Changed test_pre_vote_catch_up_denies \u2192 test_pre_vote_catch_up_grants_when_primary_unreachable", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T14:10:48.987Z" - }, - { - "id": "bug-1349", - "timestamp": "2026-07-01T14:24:21.310Z", - "error_message": "Wrong condition in logic", - "file": "src/controller/ha.c", - "root_cause": "Condition was: if (atomic_load_explicit(&ha->in_catch_up, memory_orde)", - "fix": "Changed to: if (0 && atomic_load_explicit(&ha->in_catch_up, memory)", - "tags": [ - "auto-detected", - "logic-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T14:24:21.310Z" - }, - { - "id": "bug-1350", - "timestamp": "2026-07-01T14:44:36.740Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 13\u219222 lines (4 removed) | Also: out=\"$(printf 'walcompare --report --committed %s ; \"$boundary\" \"$d1\" \"$d2\" \"$d3\" | xtest)\"", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-01T14:44:50.618Z" - }, - { - "id": "bug-1351", - "timestamp": "2026-07-01T15:04:48.683Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T15:04:48.683Z" - }, - { - "id": "bug-1352", - "timestamp": "2026-07-01T15:12:14.946Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T15:12:14.946Z" - }, - { - "id": "bug-1353", - "timestamp": "2026-07-01T23:48:24.726Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-ha-core-safety/research.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-01T23:48:24.726Z" - }, - { - "id": "bug-1354", - "timestamp": "2026-07-02T02:45:02.051Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/walcompare.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 23\u219230 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T02:45:02.051Z" - }, - { - "id": "bug-1355", - "timestamp": "2026-07-02T03:03:21.985Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:03:21.985Z" - }, - { - "id": "bug-1356", - "timestamp": "2026-07-02T03:03:31.659Z", - "error_message": "Missing error handling in unknown", - "file": "src/include/persist.h", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:03:31.659Z" - }, - { - "id": "bug-1357", - "timestamp": "2026-07-02T03:03:51.842Z", - "error_message": "Function not marked async", - "file": "src/controller/main.c", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:03:51.842Z" - }, - { - "id": "bug-1358", - "timestamp": "2026-07-02T03:07:46.943Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 3\u21924 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:07:46.943Z" - }, - { - "id": "bug-1359", - "timestamp": "2026-07-02T03:08:59.550Z", - "error_message": "Missing error handling in checkpoint", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:08:59.550Z" - }, - { - "id": "bug-1360", - "timestamp": "2026-07-02T03:17:30.107Z", - "error_message": "Significant refactor of ", - "file": "src/controller/persist.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 6\u21924 lines (3 removed) | Also: INFO(\"persist: schedule_checkpoint firing (TEMP DI; atomic_load(&persist->wal_record_count), persist->", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T03:17:37.196Z" - }, - { - "id": "bug-1361", - "timestamp": "2026-07-02T03:31:48.236Z", - "error_message": "Significant refactor of ", - "file": "src/include/delay.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 8\u219213 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:31:48.236Z" - }, - { - "id": "bug-1362", - "timestamp": "2026-07-02T03:50:14.650Z", - "error_message": "Wrong reference: persist_config_t should be uint32_t", - "file": "src/include/persist.h", - "root_cause": "Used \"persist_config_t\" instead of \"uint32_t\"", - "fix": "Changed persist_config_t \u2192 uint32_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:50:14.650Z" - }, - { - "id": "bug-1363", - "timestamp": "2026-07-02T03:50:42.583Z", - "error_message": "Wrong reference: WAL_SEGMENT_SIZE_BYTES should be p", - "file": "src/controller/persist.c", - "root_cause": "Used \"WAL_SEGMENT_SIZE_BYTES\" instead of \"p\"", - "fix": "Changed WAL_SEGMENT_SIZE_BYTES \u2192 p", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-07-02T03:50:52.480Z" - }, - { - "id": "bug-1364", - "timestamp": "2026-07-02T03:52:09.933Z", - "error_message": "Wrong reference: myconf_t should be uint32_t", - "file": "src/include/myconf.h", - "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", - "fix": "Changed myconf_t \u2192 uint32_t", - "tags": [ - "auto-detected", - "wrong-reference", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:52:09.933Z" - }, - { - "id": "bug-1365", - "timestamp": "2026-07-02T03:52:28.903Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/myapp.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:52:28.903Z" - }, - { - "id": "bug-1366", - "timestamp": "2026-07-02T03:52:43.112Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/cluster.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:52:43.112Z" - }, - { - "id": "bug-1367", - "timestamp": "2026-07-02T03:57:11.657Z", - "error_message": "Missing error handling in unknown", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:57:11.657Z" - }, - { - "id": "bug-1368", - "timestamp": "2026-07-02T03:57:26.467Z", - "error_message": "Significant refactor of ", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 8\u219211 lines (2 removed)", - "tags": [ - "auto-detected", - "refactor", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:57:26.467Z" - }, - { - "id": "bug-1369", - "timestamp": "2026-07-02T03:57:52.302Z", - "error_message": "Incorrect value in code", - "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", - "root_cause": "Had \"MODE=force-snapshot \u2014 forcing one full-checkpoint", - "fix": "Changed to \"MODE=force-snapshot \u2014 forcing one full-checkpoint", - "tags": [ - "auto-detected", - "wrong-value", - "sh" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T03:57:52.302Z" - }, - { - "id": "bug-1370", - "timestamp": "2026-07-02T04:10:10.430Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/raft-ha-core-safety/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T04:10:10.430Z" - }, - { - "id": "bug-1371", - "timestamp": "2026-07-02T04:24:56.109Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "4 lines replaced/restructured", - "fix": "Rewrote 4\u219213 lines (4 removed) | Also: * DEFAULT OFF (0): the registered peer-control han; * INLINE on the iomgr io_worker thread \u2014 byte-for-", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T04:25:09.263Z" - }, - { - "id": "bug-1372", - "timestamp": "2026-07-02T04:39:29.527Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: fd < 0", - "fix": "Added guard clause: if (fd < 0)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T04:39:29.527Z" - }, - { - "id": "bug-1373", - "timestamp": "2026-07-02T04:40:28.552Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/config_repl.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T04:40:28.552Z" - }, - { - "id": "bug-1374", - "timestamp": "2026-07-02T04:47:10.613Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 19\u219226 lines (8 removed) | Also: * DEFAULT ON (1): the registered peer-control hand; * packet to the vtask pool and returns promptly, i", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T04:47:31.936Z" - }, - { - "id": "bug-1375", - "timestamp": "2026-07-02T04:53:43.568Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T04:53:43.569Z" - }, - { - "id": "bug-1376", - "timestamp": "2026-07-02T05:10:00.000Z", - "error_message": "raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout.", - "file": "src/controller/ha.c", - "root_cause": "peer_ctrl_msg_handler (ha.c) held ep->dispatch_inflight incremented across the ENTIRE peer_ctrl_dispatch_to_pool call, including its inline fallback (the common cold-start shape when ha->pool is not yet wired). That inline body can reach raft_cb_become_follower, which -- now that cluster_init_phase2 ran synchronously earlier in the SAME call chain and wired the pool -- tears down its own loser vote-ep (the very ep whose reply triggered this call) via the deferred path (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting for dispatch_inflight==0 on that SAME ep. That count is held only by this exact call, on this exact thread, and cannot reach 0 until the spin itself returns -- a permanent self-deadlock. Confirmed live via a gdb thread backtrace (GDB_WRAP debug seam added to run-ha-soak.sh, launching the target node directly under `gdb --batch` so it becomes ptrace-legal as gdb's own direct child, then SIGINT to gdb once the log went stale for a bounded interrupt + full backtrace): frame `peer_ctrl_dispatch_to_pool` showed pool=0x0 at dispatch time; the same ep pointer appeared in the nested vote_ep_teardown_async -> endpoint_mark_tearing_down frame, spinning on sched_yield().", - "fix": "Moved the ep->dispatch_inflight release into peer_ctrl_dispatch_to_pool itself, at the exact right point per exit path: released BEFORE calling the inline body on every fallback path (pool==NULL, OOM, enqueue failure) -- safe because io_worker is single-threaded and the inline path never enqueues anything, so there is no pool-worker race left to protect against once inline execution is committed to -- and only AFTER the (non-blocking) enqueue completes on the success path, preserving the original protection for the genuinely concurrent pool-worker case. New unit test test_vep_self_teardown_does_not_deadlock (test_ha_vep_teardown.c) uses a bounded-timeout pthread_timedjoin_np so a regression fails loud instead of hanging the suite; RED confirmed via a temporary manual revert (Expected 0 Was 110 == ETIMEDOUT) before restoring the fix. Live-verified with both g_drive_async_snapshot_offload and g_drive_async_peerctrl_dispatch_enabled flipped ON: --force-snapshot soak (the exact repro shape) completes without hanging across repeated runs, confirmed real snapshot installs, committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak with both flags ON also passed clean; full controller suite green throughout. Both flags now default ON in production (raft-drive-async tasks 3.1/3.2 closed).", - "tags": [ - "raft-drive-async", - "cold-start", - "vtask-pool", - "self-deadlock", - "fixed" - ], - "related_bugs": [ - "bug-1197", - "bug-1298", - "bug-1310" - ], - "occurrences": 1, - "last_seen": "2026-07-02T00:20:00.000Z" - }, - { - "id": "bug-1377", - "timestamp": "2026-07-02T07:03:54.861Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/testsuite/test_ha_vep_teardown.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T07:03:54.861Z" - }, - { - "id": "bug-1378", - "timestamp": "2026-07-02T07:05:07.924Z", - "error_message": "Missing error handling in unknown", - "file": "src/controller/ha.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T07:05:07.924Z" - }, - { - "id": "bug-1379", - "timestamp": "2026-07-02T07:12:59.577Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "23 lines replaced/restructured", - "fix": "Rewrote 30\u219242 lines (23 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T07:12:59.577Z" - }, - { - "id": "bug-1380", - "timestamp": "2026-07-02T07:16:35.420Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `, differential-verified: this specific soak scena", - "fix": "Changed to `, differential-verified). (4) Re-ran `", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T07:17:36.171Z" - }, - { - "id": "bug-1381", - "timestamp": "2026-07-02T00:35:00.000Z", - "error_message": "raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has a pre-existing endpoint-lifetime race, independent of the g_drive_async_configpush_offload flag -- it grabs raw endpoint_t* pointers for every registered standby under c->ha.lock, releases the lock, then spawns one pthread per standby (push_thread) that reads/writes those endpoints unprotected for up to CONF_PUSH_TIMEOUT_MS. If a standby RDHUP-disconnects during that window, the io_worker's teardown path (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) can free the endpoint while a push_thread is still using it -- a UAF. This is NOT specific to the lane offload: the transport fan-out spawns its own independent OS threads regardless of which caller (inline drive-thread or lane worker) invokes it, so the documented 'runs inline on io_worker so teardown and push are serialized on one thread' rationale for keeping the flag OFF does not actually hold -- the race exists in the inline path today too.", - "file": "src/controller/config_repl.c", - "root_cause": "push_config_transport_only (config_repl.c ~line 1366) takes c->ha.lock only long enough to copy standby ep pointers into a push_ctx_t array, then releases the lock before pthread_create-ing one push_thread per standby. Those threads read/write the raw endpoint_t* for the full duration of the send + wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS), with no ref taken on the endpoints and no coordination with concurrent teardown (handle_endpoint_error -> zarray_del -> endpoint_killer -> endpoint_unref). Unlike the snapshot-lane fix (bug-1197/bug-1376 session), no endpoint_ref/endpoint_unref bracket protects this fan-out.", - "fix": "FIXED. push_config_transport_only (config_repl.c) now takes endpoint_ref() on each standby ep while the pointer is snapshotted under c->ha.lock, mirroring the snapshot-lane fix (bug-1197/bug-1376). Every ref has exactly one matching endpoint_unref(): from push_thread after it finishes using ctx->ep (new close in push_thread itself), or from push_config_transport_only inline if pthread_create fails for that standby, or from a cleanup loop if the threads[] allocation fails entirely. A concurrent RDHUP teardown (endpoint_killer -> endpoint_unref) can now only drop the OWNER reference while a push is in flight -- it can no longer free ep out from under push_thread. New regression test test_configpush_transport_holds_endpoint_ref_across_send (test_config_repl.c) uses a new push_thread_send_hook test seam (same pattern as xce_wal_stream_request_hook) to park a push_thread mid-send, simulate the owner teardown, and assert ep survives with refcnt intact. RED->GREEN verified via temporary revert (Expected 2 Was 1 with the ref-take removed); full controller suite green (230/230) after restore.", - "tags": [ - "raft-drive-async", - "configpush", - "endpoint-lifetime", - "uaf-suspect", - "fixed" - ], - "related_bugs": [ - "bug-1197", - "bug-1298", - "bug-1376" - ], - "occurrences": 1, - "last_seen": "2026-07-02T00:35:00.000Z" - }, - { - "id": "bug-1382", - "timestamp": "2026-07-02T07:23:50.910Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `snapshot_lane_fn`", - "fix": "Changed to `configpush_offload`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T07:24:12.475Z" - }, - { - "id": "bug-1383", - "timestamp": "2026-07-02T07:31:28.737Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: waited >= timeout_ms", - "fix": "Added guard clause: if (waited >= timeout_ms)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T07:31:28.737Z" - }, - { - "id": "bug-1384", - "timestamp": "2026-07-02T07:37:33.427Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had ` (test_ha_liveness.c). Blocked on **bug-1381** (o", - "fix": "Changed to ` (test_ha_liveness.c). **bug-1381 CLOSED 2026-07-", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T07:37:48.507Z" - }, - { - "id": "bug-1385", - "timestamp": "2026-07-02T08:09:06.284Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "8 lines replaced/restructured", - "fix": "Rewrote 34\u219245 lines (8 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T08:09:06.284Z" - }, - { - "id": "bug-1386", - "timestamp": "2026-07-02T08:15:00.000Z", - "error_message": "/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install (\"HA: snapshot loaded/legacy\" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip.", - "file": "src/controller/config_repl.c", - "root_cause": "ha_send_snapshot_to_standby (config_repl.c, ~line 2596) loops over up to 12 65536-byte chunks calling xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len) WITHOUT checking or propagating its return value. xce_snapshot_chunk_req_raw -> endpoint_enq can return -ECONNRESET (e.g. if the standby endpoint is mid-teardown or reconnecting during the empty-rejoin window) but the loop keeps incrementing offset/chunk_index regardless, and the function still logs \"transfer complete\" and returns rc=0 at the end. Confirmed via ctrl-2 log evidence: primary logs start+complete for a 735546-byte (12-chunk) snapshot within ~2ms, but ctrl-2 only ever receives a handful of leading chunks (2 in one run, 5 in another) before the stream silently stops -- no ERROR on either side.", - "fix": "FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) \u2014 the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups).", - "tags": [ - "raft-drive-async", - "snapshot-transfer", - "silent-failure", - "force-snapshot-soak", - "fixed" - ], - "related_bugs": [ - "bug-1197", - "bug-1376", - "bug-1381" - ], - "occurrences": 1, - "last_seen": "2026-07-02T09:30:00.000Z" - }, - { - "id": "bug-1387", - "timestamp": "2026-07-02T08:18:34.580Z", - "error_message": "Significant refactor of ", - "file": "src/controller/config_repl.c", - "root_cause": "2 lines replaced/restructured", - "fix": "Rewrote 31\u219248 lines (2 removed) | Also: /* bug-1386: a failed chunk send (e.g. -ECONNRESET; * endpoint is mid-teardown/reconnect) MUST stop th | Also: xce_snapshot_chunk_hook(ep, &req);; xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-07-02T08:23:56.681Z" - }, - { - "id": "bug-1388", - "timestamp": "2026-07-02T08:19:04.015Z", - "error_message": "Missing guard clause", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", - "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", - "tags": [ - "auto-detected", - "guard-clause", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T08:19:04.015Z" - }, - { - "id": "bug-1389", - "timestamp": "2026-07-02T09:05:00.000Z", - "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times \u2014 the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT \u2014 the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) \u2014 nondeterministic, not a fixed structural bug.", - "file": "src/lib/request.c (suspected: xce_snapshot_chunk_encode_raw capnp Data encode) or src/controller/ha.c (ha_handle_snapshot_chunk per-chunk append)", - "root_cause": "NOT YET PINNED to an exact line \u2014 root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 \u2014 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' \u2014 a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing \u2014 vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race \u2014 xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation \u2014 alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug \u2014 it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities).", - "fix": "FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) \u2014 the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen \"ab\"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward \u2014 exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append \u2014 order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' \u2014 under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found \u2014 permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS.", - "tags": [ - "raft-drive-async", - "snapshot-transfer", - "data-corruption", - "capnp", - "force-snapshot-soak", - "fixed" - ], - "related_bugs": [ - "bug-1386", - "bug-847", - "bug-1197" - ], - "occurrences": 2, - "last_seen": "2026-07-02T12:10:00.000Z" - }, - { - "id": "bug-1390", - "timestamp": "2026-07-02T08:58:44.362Z", - "error_message": "Missing error handling in unknown", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T08:58:44.362Z" - }, - { - "id": "bug-1391", - "timestamp": "2026-07-02T11:47:56.980Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "11 lines replaced/restructured", - "fix": "Rewrote 43\u219276 lines (11 removed) | Also: /* bug-1389: chunks of one transfer are dispatched; * and are NOT guaranteed to arrive/execute in chun | Also: /* TEMP REVERT FOR RED CONFIRMATION \u2014 bug-1389 */; FILE *f = fopen(part_path, \"ab\");", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 3, - "last_seen": "2026-07-02T11:54:09.889Z" - }, - { - "id": "bug-1392", - "timestamp": "2026-07-02T11:49:56.186Z", - "error_message": "Significant refactor of ", - "file": "src/controller/testsuite/test_config_repl.c", - "root_cause": "6 lines replaced/restructured", - "fix": "Rewrote 13\u219223 lines (6 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T11:49:56.186Z" - }, - { - "id": "bug-1393", - "timestamp": "2026-07-02T12:05:51.415Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `~/tmp/xce-ha-deploy/controller/ctrl-{1,2}/data/sn", - "fix": "Changed to `ha_send_snapshot_to_standby`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T12:05:51.415Z" - }, - { - "id": "bug-1394", - "timestamp": "2026-07-02T13:09:34.387Z", - "error_message": "Significant refactor of ", - "file": "src/controller/ha.c", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 11\u219233 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T13:09:34.387Z" - }, - { - "id": "bug-1395", - "timestamp": "2026-07-02T13:25:19.081Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/raft-drive-async/tasks.md", - "root_cause": "Had `[~]`", - "fix": "Changed to `[x]`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T13:25:28.568Z" - }, - { - "id": "bug-1396", - "timestamp": "2026-07-02T14:00:00.000Z", - "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle \u2014 the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) \u2014 an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", - "file": "src/tools/test-commit-durability.sh (harness) or controller job-recovery path (unconfirmed which)", - "root_cause": "NOT INVESTIGATED \u2014 deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race \u2014 ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE \u2014 a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) \u2014 rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) \u2014 almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected.", - "fix": "NOT YET ROOT-CAUSED (still needs a live re-run to capture the actual extra job's identity). Made the harness able to diagnose it next time it fires: src/tools/test-commit-durability.sh's assert_job_count() now dumps the live `joblist` output AND the ACKED_FILE ledger (sorted, deduplicated) on any count mismatch, instead of only printing the count as before. This was the bug's own recorded 'next step'. Bash syntax-checked (bash -n); not yet re-run against a live cluster in this session (that requires reproducing the original 5-round real-fs soak, which is long-running).", - "tags": [ - "ha-commit-durability", - "low-priority", - "cold-restart-all", - "job-count-mismatch", - "harness-instrumented", - "unconfirmed" - ], - "related_bugs": [ - "bug-661" - ], - "occurrences": 1, - "last_seen": "2026-07-02T16:00:00.000Z" - }, - { - "id": "bug-1397", - "timestamp": "2026-07-02T13:57:56.610Z", - "error_message": "Function not marked async", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Function uses await but wasn't declared async", - "fix": "Added async modifier", - "tags": [ - "auto-detected", - "async-fix", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T13:57:56.610Z" - }, - { - "id": "bug-1398", - "timestamp": "2026-07-02T13:58:18.885Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/ha-commit-durability/tasks.md", - "root_cause": "Had `/kiro-validate-impl`", - "fix": "Changed to `src/tools/test-commit-durability.sh ~/tmp/xce-dur", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T13:58:18.885Z" - }, - { - "id": "bug-1399", - "timestamp": "2026-07-02T14:03:20.814Z", - "error_message": "Incorrect value in code", - "file": "src/controller/CLAUDE.md", - "root_cause": "Had `xce-test-e2e-primitives`", - "fix": "Changed to `ha-commit-durability`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T14:03:20.814Z" - }, - { - "id": "bug-1400", - "timestamp": "2026-07-02T14:20:00.000Z", - "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` \u2014 positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", - "file": "src/controller/testsuite/test.c (cmd_jobsub, ~line 1748)", - "root_cause": "CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 \u2014 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) \u2014 a ~128-byte leak per malformed jobsub invocation.", - "fix": "SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label \u2014 mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change.", - "tags": [ - "stale-report", - "low-priority", - "test-client-only", - "input-validation", - "crash", - "cmd_jobsub", - "fixed", - "xce_test" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T16:00:00.000Z" - }, - { - "id": "bug-1401", - "timestamp": "2026-07-02T14:25:00.000Z", - "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) \u2014 observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", - "file": "src/lib/controller.capnp.c (decode_CtrlStatus_ptr, generated codec) or its caller in request.c/xcecli", - "root_cause": "CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` \u2014 the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct \u2014 the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code.", - "fix": "src/include/client.h:119 changed to `#define xce_free_ctrlstatus(s) free_CtrlStatus_ptr(&(s))`. Added regression test test_xce_free_ctrlstatus_frees_the_struct_itself (src/lib/testsuite/test_ctrlstatus.c), which decodes via decode_CtrlStatus_ptr (the real allocation path, not the stack-struct helper the other roundtrip tests use) and frees via the xce_free_ctrlstatus macro. RED confirmed: reverting the fix reproduces the exact reported leak signature (LeakSanitizer: 120 bytes in 1 object, decode_CtrlStatus_ptr calloc). GREEN after fix: 285 lib tests / 0 failures / 0 leaks. Controller ASAN build + controller_unit_test also green post-fix (client.h is shared).", - "tags": [ - "asan", - "fixed", - "low-priority", - "controller-status", - "ctrlstatus", - "memory-leak" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-02T16:00:00.000Z" - }, - { - "id": "bug-1402", - "timestamp": "2026-07-02T14:21:35.080Z", - "error_message": "Incorrect value in code", - "file": ".kiro/specs/controller-status/tasks.md", - "root_cause": "Had `/xce-deploy-ha`", - "fix": "Changed to `~/tmp`", - "tags": [ - "auto-detected", - "wrong-value", - "md" - ], - "related_bugs": [], - "occurrences": 2, - "last_seen": "2026-07-02T14:22:28.117Z" - }, - { - "id": "bug-1403", - "timestamp": "2026-07-03T00:02:30.175Z", - "error_message": "Significant refactor of ", - "file": "src/include/client.h", - "root_cause": "3 lines replaced/restructured", - "fix": "Rewrote 4\u21927 lines (3 removed)", - "tags": [ - "auto-detected", - "refactor", - "h" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-03T00:02:30.175Z" - }, - { - "id": "bug-1404", - "timestamp": "2026-07-03T00:08:14.291Z", - "error_message": "Missing error handling in unknown", - "file": "src/lib/testsuite/test_ctrlstatus.c", - "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", - "fix": "Added try/catch block", - "tags": [ - "auto-detected", - "error-handling", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-03T00:08:14.291Z" - }, - { - "id": "bug-1405", - "timestamp": "2026-07-03T00:08:25.320Z", - "error_message": "Wrong reference: test_xcecli_ctrlstatus_null_ctx_typed_failure should be test_xce_free_ctrlstatus_frees_the_struct_itself", - "file": "src/lib/testsuite/test_ctrlstatus.c", - "root_cause": "Used \"test_xcecli_ctrlstatus_null_ctx_typed_failure\" instead of \"test_xce_free_ctrlstatus_frees_the_struct_itself\"", - "fix": "Changed test_xcecli_ctrlstatus_null_ctx_typed_failure \u2192 test_xce_free_ctrlstatus_frees_the_struct_itself", - "tags": [ - "auto-detected", - "wrong-reference", - "c" - ], - "related_bugs": [], - "occurrences": 1, - "last_seen": "2026-07-03T00:08:25.320Z" - } - ] -} +[ + { + "id": "bug-002", + "timestamp": "2026-05-21T07:46:54.198Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/kiro-spec-init/SKILL.md", + "root_cause": "Had `.kiro/settings/templates/specs/init.json`", + "fix": "Changed to `templates/specs/init.json`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 12, + "last_seen": "2026-05-21T07:48:24.177Z" + }, + { + "id": "bug-004", + "timestamp": "2026-05-21T07:58:54.673Z", + "error_message": "Wrong reference: g_tmpdir should be g_instdir", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"g_tmpdir\" instead of \"g_instdir\"", + "fix": "Changed g_tmpdir \u2192 g_instdir", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-21T07:59:00.347Z" + }, + { + "id": "bug-005", + "timestamp": "2026-05-21T08:24:10.738Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: rc == 0", + "fix": "Added guard clause: if (rc == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-21T08:24:10.738Z" + }, + { + "id": "bug-006", + "timestamp": "2026-05-21T08:30:20.719Z", + "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer", + "file": "src/controller/ha.c", + "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) \u2014 double-free at lines 3594-3595 and 3600-3601", + "fix": "Set ep->data = NULL before endpoint_fini so FREEUP becomes a no-op; then free(sctx) explicitly in the error paths", + "tags": [ + "md", + "double-free", + "asan", + "ha_connect_to_peer", + "endpoint_fini", + "integration-test" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-05T07:23:00.000Z" + }, + { + "id": "bug-007", + "timestamp": "2026-05-21T12:53:13.789Z", + "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 \u2014 reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx", + "file": "src/controller/ha.c", + "root_cause": "ha_arm_heartbeat_timeout_vtask freed the old liveness_poll_ctx_t immediately after setting cancelled=1, but the delaymgr task was already in flight reading ctx->cancelled=0 and then continued to read ctx->ha after free", + "fix": "ha_arm_heartbeat_timeout_vtask no longer frees old ctx; liveness_poll_fn now frees ctx when it fires with cancelled=1 (guarded by #ifndef XCE_TESTING to avoid bad-free on stack-allocated test ctxs)", + "tags": [ + "guard-clause", + "c", + "heap-use-after-free", + "asan", + "liveness_poll_fn", + "ha_arm_heartbeat_timeout_vtask", + "race-condition", + "integration-test" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-05T07:23:00.000Z" + }, + { + "id": "bug-008", + "timestamp": "2026-05-21T13:20:28.953Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: getaddrinfo(host, port_str, &hints, &res", + "fix": "Added guard clause: if (getaddrinfo(host, port_str, &hints, &res)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-21T13:20:28.953Z" + }, + { + "id": "bug-020", + "timestamp": "2026-05-25T11:31:55.826Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-25T11:31:55.826Z" + }, + { + "id": "bug-023", + "timestamp": "2026-05-26T07:17:31.980Z", + "error_message": "Missing guard clause", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "No early return/throw for edge case: req->term < my_term", + "fix": "Added guard clause: if (req->term < my_term)", + "tags": [ + "auto-detected", + "guard-clause", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T07:17:31.980Z" + }, + { + "id": "bug-024", + "timestamp": "2026-05-26T07:18:36.066Z", + "error_message": "Missing guard clause", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "No early return/throw for edge case: n_peers == 2", + "fix": "Added guard clause: if (n_peers == 2)", + "tags": [ + "auto-detected", + "guard-clause", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T07:18:36.066Z" + }, + { + "id": "bug-026", + "timestamp": "2026-05-26T07:26:48.363Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Was returning: 1;", + "fix": "Now returns: 1; // single-node: always primary", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T07:26:48.363Z" + }, + { + "id": "bug-027", + "timestamp": "2026-05-26T07:27:34.925Z", + "error_message": "Wrong condition in logic", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Condition was: if (n_peers == 2)", + "fix": "Changed to: if (n_peers == 1)", + "tags": [ + "auto-detected", + "logic-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T07:27:34.925Z" + }, + { + "id": "bug-029", + "timestamp": "2026-05-26T07:29:11.741Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `main.c`", + "fix": "Changed to `-EINVAL`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T07:29:11.741Z" + }, + { + "id": "bug-032", + "timestamp": "2026-05-26T09:07:23.015Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `-EINVAL`", + "fix": "Changed to `--vcpus N`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T09:07:23.015Z" + }, + { + "id": "bug-034", + "timestamp": "2026-05-26T09:23:02.818Z", + "error_message": "Missing guard clause", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "No early return/throw for edge case: rc == 0", + "fix": "Added guard clause: if (rc == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T09:23:02.818Z" + }, + { + "id": "bug-035", + "timestamp": "2026-05-26T09:46:12.902Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `sysconf(_SC_NPROCESSORS_ONLN)`", + "fix": "Changed to ` when not specified. Values `", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T09:46:12.902Z" + }, + { + "id": "bug-038", + "timestamp": "2026-05-26T10:09:32.424Z", + "error_message": "Missing guard clause", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", + "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T10:09:32.424Z" + }, + { + "id": "bug-039", + "timestamp": "2026-05-26T10:09:57.711Z", + "error_message": "Wrong return value", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Was returning: 0 (standby); fresher peer runs its own election", + "fix": "Now returns: `-ENODATA`. Not a normal runtime path.", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T10:09:57.711Z" + }, + { + "id": "bug-041", + "timestamp": "2026-05-26T10:10:40.149Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Had `test_fetch_config_from_best_peer`", + "fix": "Changed to `test_handle_config_request_standbys_serve_config`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-26T10:10:40.149Z" + }, + { + "id": "bug-042", + "timestamp": "2026-05-26T11:35:44.097Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-26T11:39:15.673Z" + }, + { + "id": "bug-046", + "timestamp": "2026-05-26T13:51:32.274Z", + "error_message": "Missing error handling in unknown", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - [ ] `test_heartbeat_carries_committed_lsn`: heartbeat payl; `committed_lsn`; standby updates `ha.last_known_committed_ls", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-26T13:51:57.860Z" + }, + { + "id": "bug-047", + "timestamp": "2026-05-26T13:52:49.044Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: // Catch-up gate (Clarification 6): behind nodes must not in; if (c->ha.in_catch_up) {", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-26T13:53:15.121Z" + }, + { + "id": "bug-048", + "timestamp": "2026-05-27T03:02:34.912Z", + "error_message": "Missing error handling in unknown", + "file": "../../../../../claude/.claude/plans/prancy-herding-nest.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: name, key, applied_lsn, and config_tag immediately after TCP; - [ ] `test_ctrl_hello_registers_standby_in_repl`: handle_ct", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-27T03:04:02.739Z" + }, + { + "id": "bug-059", + "timestamp": "2026-05-27T07:24:56.724Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/brief.md", + "root_cause": "Had `-ENOSYS`", + "fix": "Changed to `bootstrap_config_if_needed()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-27T07:24:56.724Z" + }, + { + "id": "bug-060", + "timestamp": "2026-05-27T09:24:44.651Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-config-bootstrap/spec.json", + "root_cause": "Had \"2026-05-27T01:24:00-07:00\"", + "fix": "Changed to \"2026-05-27T01:30:00-07:00\"", + "tags": [ + "auto-detected", + "wrong-value", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-27T09:24:44.651Z" + }, + { + "id": "bug-067", + "timestamp": "2026-05-28T03:42:53.810Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'worker.c'", + "fix": "Changed to 'startup_validation.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T03:42:53.810Z" + }, + { + "id": "bug-068", + "timestamp": "2026-05-28T04:01:56.031Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/main.c", + "root_cause": "Condition was: if (arguments->vcpus < 1)", + "fix": "Changed to: if (validate_vcpus(arguments->vcpus)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T04:01:56.031Z" + }, + { + "id": "bug-069", + "timestamp": "2026-05-28T04:02:06.925Z", + "error_message": "Wrong reference: vcpus should be nres", + "file": "src/include/clusterconfig.h", + "root_cause": "Used \"vcpus\" instead of \"nres\"", + "fix": "Changed vcpus \u2192 nres", + "tags": [ + "auto-detected", + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T04:02:06.925Z" + }, + { + "id": "bug-071", + "timestamp": "2026-05-28T04:02:48.429Z", + "error_message": "Incorrect value in code", + "file": "src/schema/controller.capnp", + "root_cause": "Had \"vcpus\"", + "fix": "Changed to \"resources\"", + "tags": [ + "auto-detected", + "wrong-value", + "capnp" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T04:02:48.429Z" + }, + { + "id": "bug-074", + "timestamp": "2026-05-28T05:59:28.525Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", + "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcod)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T05:59:28.525Z" + }, + { + "id": "bug-075", + "timestamp": "2026-05-28T07:04:33.895Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_election.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T07:04:33.895Z" + }, + { + "id": "bug-078", + "timestamp": "2026-05-28T07:22:56.418Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= ctx->n_peers", + "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= ctx->n_peers)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T07:22:56.418Z" + }, + { + "id": "bug-081", + "timestamp": "2026-05-28T07:53:10.883Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: role != CTRL_STANDBY", + "fix": "Added guard clause: if (role != CTRL_STANDBY)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T07:53:10.883Z" + }, + { + "id": "bug-082", + "timestamp": "2026-05-28T08:23:40.040Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ctx->done_posted", + "fix": "Added guard clause: if (ctx->done_posted)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-28T08:25:11.221Z" + }, + { + "id": "bug-083", + "timestamp": "2026-05-28T08:45:37.921Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !host || !tag_out", + "fix": "Added guard clause: if (!host || !tag_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T08:45:37.921Z" + }, + { + "id": "bug-084", + "timestamp": "2026-05-28T08:48:58.609Z", + "error_message": "Wrong reference: capn_str_null should be str", + "file": "src/controller/config_repl.c", + "root_cause": "Used \"capn_str_null\" instead of \"str\"", + "fix": "Changed capn_str_null \u2192 str", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T08:48:58.609Z" + }, + { + "id": "bug-086", + "timestamp": "2026-05-28T09:13:17.676Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_bootstrap_support.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T09:13:17.676Z" + }, + { + "id": "bug-087", + "timestamp": "2026-05-28T10:13:52.421Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !workdir", + "fix": "Added guard clause: if (!workdir)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T10:13:52.421Z" + }, + { + "id": "bug-091", + "timestamp": "2026-05-28T13:53:09.924Z", + "error_message": "Wrong reference: in_catch_up should be acked_config_lsn", + "file": "src/controller/repl.c", + "root_cause": "Used \"in_catch_up\" instead of \"acked_config_lsn\"", + "fix": "Changed in_catch_up \u2192 acked_config_lsn", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T13:53:09.924Z" + }, + { + "id": "bug-092", + "timestamp": "2026-05-28T14:11:04.583Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p->current_lsn == 0 || cutoff_lsn + 1 >= p->current_lsn", + "fix": "Added guard clause: if (p->current_lsn == 0 || cutoff_lsn + 1 >=)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T14:11:04.583Z" + }, + { + "id": "bug-094", + "timestamp": "2026-05-28T14:47:44.445Z", + "error_message": "Wrong reference: ha_state_t should be _Atomic", + "file": "src/controller/ha.h", + "root_cause": "Used \"ha_state_t\" instead of \"_Atomic\"", + "fix": "Changed ha_state_t \u2192 _Atomic", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T14:47:44.445Z" + }, + { + "id": "bug-095", + "timestamp": "2026-05-28T15:04:44.613Z", + "error_message": "Wrong reference: socket should be epoll", + "file": "src/controller/ha.c", + "root_cause": "Used \"socket\" instead of \"epoll\"", + "fix": "Changed socket \u2192 epoll", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T15:04:44.613Z" + }, + { + "id": "bug-096", + "timestamp": "2026-05-28T15:05:23.129Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook", + "fix": "Added guard clause: if (ha_tcp_connect_hook)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-28T15:05:23.129Z" + }, + { + "id": "bug-097", + "timestamp": "2026-05-29T02:48:17.371Z", + "error_message": "Missing guard clause", + "file": "src/lib/request.c", + "root_cause": "No early return/throw for edge case: (ep == NULL", + "fix": "Added guard clause: if ((ep == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T02:48:17.371Z" + }, + { + "id": "bug-098", + "timestamp": "2026-05-29T02:50:46.444Z", + "error_message": "Wrong reference: XCE_ERR_STANDBY should be E_XCE_STANDBY", + "file": "src/lib/request.c", + "root_cause": "Used \"XCE_ERR_STANDBY\" instead of \"E_XCE_STANDBY\"", + "fix": "Changed XCE_ERR_STANDBY \u2192 E_XCE_STANDBY", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T02:50:46.444Z" + }, + { + "id": "bug-099", + "timestamp": "2026-05-29T02:51:03.529Z", + "error_message": "Wrong reference: XCE_ERR_NOT_READY should be E_XCE_NOT_READY", + "file": "src/lib/testsuite/test_libxce_retry.c", + "root_cause": "Used \"XCE_ERR_NOT_READY\" instead of \"E_XCE_NOT_READY\"", + "fix": "Changed XCE_ERR_NOT_READY \u2192 E_XCE_NOT_READY", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-29T02:51:16.657Z" + }, + { + "id": "bug-102", + "timestamp": "2026-05-29T03:37:52.117Z", + "error_message": "Wrong reference: data should be ctrl_hello_reply_t", + "file": "src/include/typemap.h", + "root_cause": "Used \"data\" instead of \"ctrl_hello_reply_t\"", + "fix": "Changed data \u2192 ctrl_hello_reply_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T03:37:52.117Z" + }, + { + "id": "bug-104", + "timestamp": "2026-05-29T03:54:23.205Z", + "error_message": "Missing error handling in struct", + "file": "src/include/typemap.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: wal_catchup_entry_t* walcatchupentrymsg;; snapshot_transfer_t* snapshottransfermsg; | Also: /* Forward declarations for types defined later in this file; typedef struct snapshot_transfer_t snapshot_transfer_fw", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-05-29T03:56:34.078Z" + }, + { + "id": "bug-109", + "timestamp": "2026-05-29T06:59:14.045Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T06:59:14.045Z" + }, + { + "id": "bug-111", + "timestamp": "2026-05-29T07:02:31.287Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (c->myconf && c->myconf->cluster_config)", + "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T07:02:31.287Z" + }, + { + "id": "bug-117", + "timestamp": "2026-05-29T13:41:46.841Z", + "error_message": "Incorrect value in code", + "file": "src/controller/myapp.c", + "root_cause": "Had \"Pre-election config loaded: tag=%016\"", + "fix": "Changed to \"Pre-election config loaded: tag=%016llx\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T13:41:46.841Z" + }, + { + "id": "bug-118", + "timestamp": "2026-05-29T13:51:42.299Z", + "error_message": "Wrong reference: nres should be vcpus", + "file": "src/include/clusterconfig.h", + "root_cause": "Used \"nres\" instead of \"vcpus\"", + "fix": "Changed nres \u2192 vcpus", + "tags": [ + "auto-detected", + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T13:51:42.299Z" + }, + { + "id": "bug-119", + "timestamp": "2026-05-29T16:32:10.285Z", + "error_message": "Wrong reference: data should be request_vote_reply_t", + "file": "src/include/typemap.h", + "root_cause": "Used \"data\" instead of \"request_vote_reply_t\"", + "fix": "Changed data \u2192 request_vote_reply_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-29T16:32:10.285Z" + }, + { + "id": "bug-122", + "timestamp": "2026-05-30T00:05:02.052Z", + "error_message": "Wrong reference: cluster_init_phase2 should be cluster_register_listener", + "file": "src/controller/include/cluster.h", + "root_cause": "Used \"cluster_init_phase2\" instead of \"cluster_register_listener\"", + "fix": "Changed cluster_init_phase2 \u2192 cluster_register_listener", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T00:05:02.052Z" + }, + { + "id": "bug-123", + "timestamp": "2026-05-30T00:20:27.244Z", + "error_message": "Missing guard clause", + "file": "src/controller/cluster.c", + "root_cause": "No early return/throw for edge case: c->ep.flags == 0", + "fix": "Added guard clause: if (c->ep.flags == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T00:20:27.244Z" + }, + { + "id": "bug-127", + "timestamp": "2026-05-30T01:12:38.572Z", + "error_message": "Wrong reference: CTRL_PERF_NHASH should be c", + "file": "src/controller/cluster.c", + "root_cause": "Used \"CTRL_PERF_NHASH\" instead of \"c\"", + "fix": "Changed CTRL_PERF_NHASH \u2192 c", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T01:12:38.572Z" + }, + { + "id": "bug-129", + "timestamp": "2026-05-30T01:21:20.140Z", + "error_message": "Wrong reference: true should be false", + "file": ".claude/skills/xce-build/SKILL.md", + "root_cause": "Used \"true\" instead of \"false\"", + "fix": "Changed true \u2192 false", + "tags": [ + "auto-detected", + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T01:21:20.140Z" + }, + { + "id": "bug-130", + "timestamp": "2026-05-30T01:28:22.079Z", + "error_message": "Wrong reference: 120 should be 300", + "file": "src/controller/meson.build", + "root_cause": "Used \"120\" instead of \"300\"", + "fix": "Changed 120 \u2192 300", + "tags": [ + "auto-detected", + "wrong-reference", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T01:28:22.079Z" + }, + { + "id": "bug-135", + "timestamp": "2026-05-30T06:19:47.181Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/election-iomgr-refactor/design.md", + "root_cause": "Had `cluster_init_phase2`", + "fix": "Changed to `cluster_config == NULL`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T06:19:47.181Z" + }, + { + "id": "bug-136", + "timestamp": "2026-05-30T06:38:09.325Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/election-iomgr-refactor/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* After election determines role: */; int is_first_boot = !c->running; /* running=1 only after cl | Also: #### `ha_dispatch_gate` \u2014 CTRL_PRIMARY_COMMITTING case; ```c | Also: ### 3-node startup \u2014 ctrl-a wins (with COMMITTING phase); participant PeerC as ctrl-c endpoint | Also: - **`ha_start_election_async()` is fully non-blocking**: Arm; - *Initial startup* (`ha->phase2_done == 0`): calls `cluster", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-05-30T06:43:07.267Z" + }, + { + "id": "bug-139", + "timestamp": "2026-05-30T06:58:05.115Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/election-iomgr-refactor/design.md", + "root_cause": "Had `delay_task_cancel()`", + "fix": "Changed to `clock_gettime(CLOCK_MONOTONIC)`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T06:58:05.115Z" + }, + { + "id": "bug-142", + "timestamp": "2026-05-30T07:57:14.052Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: xce_vote_request_hook != NULL", + "fix": "Added guard clause: if (xce_vote_request_hook != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T07:57:14.052Z" + }, + { + "id": "bug-143", + "timestamp": "2026-05-30T08:16:03.222Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", + "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T08:16:03.222Z" + }, + { + "id": "bug-144", + "timestamp": "2026-05-30T08:49:04.058Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T08:49:04.058Z" + }, + { + "id": "bug-145", + "timestamp": "2026-05-30T08:49:59.817Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: atomic_load(&t->cancelled", + "fix": "Added guard clause: if (atomic_load(&t->cancelled)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T08:49:59.817Z" + }, + { + "id": "bug-147", + "timestamp": "2026-05-30T09:13:19.377Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T09:13:19.377Z" + }, + { + "id": "bug-149", + "timestamp": "2026-05-30T10:31:53.836Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_vote_gate.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T10:31:53.836Z" + }, + { + "id": "bug-153", + "timestamp": "2026-05-30T11:43:01.989Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_repl_push.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T11:43:01.989Z" + }, + { + "id": "bug-154", + "timestamp": "2026-05-30T11:43:44.829Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Push config to standbys with lower config_tag (Req 7.1) *; #ifdef XCE_TESTING", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-30T11:43:58.208Z" + }, + { + "id": "bug-157", + "timestamp": "2026-05-30T12:02:39.009Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-30T12:03:10.183Z" + }, + { + "id": "bug-159", + "timestamp": "2026-05-30T12:22:57.013Z", + "error_message": "Wrong return value", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Was returning: in < 10ms.", + "fix": "Now returns: in < 5ms (design spec).", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T12:22:57.013Z" + }, + { + "id": "bug-160", + "timestamp": "2026-05-30T12:23:18.560Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Had \"myapp_run path: ha_start_election_async must retu", + "fix": "Changed to \"myapp_run path: ha_start_election_async must retu", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T12:23:18.560Z" + }, + { + "id": "bug-161", + "timestamp": "2026-05-30T12:29:45.867Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T12:29:45.867Z" + }, + { + "id": "bug-162", + "timestamp": "2026-05-30T12:30:10.570Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-30T12:30:26.987Z" + }, + { + "id": "bug-163", + "timestamp": "2026-05-30T12:31:02.203Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_failover.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T12:31:02.203Z" + }, + { + "id": "bug-164", + "timestamp": "2026-05-30T12:45:22.182Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T12:45:22.182Z" + }, + { + "id": "bug-166", + "timestamp": "2026-05-30T12:50:35.870Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_bootstrap_support.c", + "root_cause": "No early return/throw for edge case: c->myconf->config_version_tag != 0", + "fix": "Added guard clause: if (c->myconf->config_version_tag != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T12:50:35.870Z" + }, + { + "id": "bug-167", + "timestamp": "2026-05-30T13:06:57.152Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: cancelled_val", + "fix": "Added guard clause: if (cancelled_val)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T13:06:57.152Z" + }, + { + "id": "bug-168", + "timestamp": "2026-05-30T13:24:06.024Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_repl_ack.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T13:24:06.024Z" + }, + { + "id": "bug-169", + "timestamp": "2026-05-30T13:42:28.077Z", + "error_message": "Wrong reference: third should be second", + "file": "src/controller/testsuite/test_ha_repl_ack.c", + "root_cause": "Used \"third\" instead of \"second\"", + "fix": "Changed third \u2192 second", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T13:42:28.077Z" + }, + { + "id": "bug-170", + "timestamp": "2026-05-30T14:35:07.679Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T14:35:07.679Z" + }, + { + "id": "bug-172", + "timestamp": "2026-05-30T23:42:12.644Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-30T23:42:12.644Z" + }, + { + "id": "bug-176", + "timestamp": "2026-05-31T02:09:39.630Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (c->myconf && c->myconf->peers_cfg.candidates_count)", + "fix": "Changed to: if (ha->my_priority > 0)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T02:09:39.630Z" + }, + { + "id": "bug-183", + "timestamp": "2026-05-31T03:02:33.311Z", + "error_message": "Missing guard clause", + "file": "src/lib/request.c", + "root_cause": "No early return/throw for edge case: req == NULL", + "fix": "Added guard clause: if (req == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T03:02:33.311Z" + }, + { + "id": "bug-186", + "timestamp": "2026-05-31T04:11:45.534Z", + "error_message": "Missing error handling in struct", + "file": ".kiro/specs/raft-library/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: Caller->>RaftSM: raft_on_vote_request(state, from, term); alt term < current_term", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-05-31T04:13:04.500Z" + }, + { + "id": "bug-188", + "timestamp": "2026-05-31T04:19:54.524Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-library/design.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to `raft.h`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T04:19:54.524Z" + }, + { + "id": "bug-190", + "timestamp": "2026-05-31T05:07:37.759Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: cfg->n_peers < 1", + "fix": "Added guard clause: if (cfg->n_peers < 1)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T05:07:37.759Z" + }, + { + "id": "bug-191", + "timestamp": "2026-05-31T05:23:19.394Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->role == RAFT_CANDIDATE || s->role == RAFT_LEADER", + "fix": "Added guard clause: if (s->role == RAFT_CANDIDATE || s->role == )", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T05:23:19.394Z" + }, + { + "id": "bug-197", + "timestamp": "2026-05-31T07:08:23.387Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T07:08:23.387Z" + }, + { + "id": "bug-206", + "timestamp": "2026-05-31T08:40:00.000Z", + "error_message": "test_vote_reply_handler_term_fencing FAIL: Expected 10 Was 3 \u2014 ha->term must be atomically updated to peer term when peer term > ours", + "file": "src/controller/ha.c", + "root_cause": "vote_reply_msg_handler delegated term fencing entirely to raft_cb_become_follower (via raft_on_vote_reply). In XCE_TESTING builds, vote_reply_dispatch calls xce_vote_reply_hook and returns early, bypassing raft_on_vote_reply and thus never updating ha->term.", + "fix": "Added explicit CAS loop in vote_reply_msg_handler before vote_reply_dispatch to advance ha->term when vr_term > ha->term. This applies on both the production path and the XCE_TESTING hook path.", + "tags": [ + "term-fencing", + "raft", + "ha", + "xce-testing", + "test-seam", + "vote-reply" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T08:40:00.000Z" + }, + { + "id": "bug-207", + "timestamp": "2026-05-31T08:47:08.484Z", + "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_even_peers_returns_einval", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_even_peers_returns_einval\"", + "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_even_peers_returns_einval", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T08:47:08.484Z" + }, + { + "id": "bug-208", + "timestamp": "2026-05-31", + "error_message": "ASAN LeakSanitizer: detected memory leaks in scheduler testsuite \u2014 build exits with error code", + "file": "src/scheduler/testsuite/test.c", + "root_cause": "sched_meet_host_selection() lazily allocates w->host_eval_ctx via build_host_eval_ctx(). Four test functions (test_meet_host_selection_always_true_passes, test_meet_host_selection_always_false_fails, test_meet_host_selection_numeric_match, test_meet_host_selection_string_match) called zarray_fini() without first calling sched_worker_release_host_ctx() to free the lazily-allocated host_eval_ctx.", + "fix": "Added sched_worker_release_host_ctx(&w) calls before zarray_fini() in all four affected test functions.", + "tags": [ + "asan", + "leak", + "scheduler", + "test", + "host_eval_ctx" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31" + }, + { + "id": "bug-209", + "timestamp": "2026-05-31T10:19:47.014Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (ha_rdhup_election_hook != NULL)", + "fix": "Changed to: if (c->myconf && c->myconf->peers_cfg.candidates_count)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T10:19:47.014Z" + }, + { + "id": "bug-211", + "timestamp": "2026-05-31T11:29:03.301Z", + "error_message": "Wrong reference: 2000 should be 5000", + "file": "src/controller/ha.h", + "root_cause": "Used \"2000\" instead of \"5000\"", + "fix": "Changed 2000 \u2192 5000", + "tags": [ + "auto-detected", + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T11:29:03.301Z" + }, + { + "id": "bug-215", + "timestamp": "2026-05-31T12:43:45.333Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !pkt || !pkt->data || pkt->sz == 0", + "fix": "Added guard clause: if (!pkt || !pkt->data || pkt->sz == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T12:43:45.333Z" + }, + { + "id": "bug-216", + "timestamp": "2026-05-31T12:49:40.554Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-peer-connection-reuse/tasks.md", + "root_cause": "Had 's vote endpoints to standby control channels\n\n- [", + "fix": "Changed to 's vote endpoints to standby control channels\n\n- [", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T12:49:40.554Z" + }, + { + "id": "bug-218", + "timestamp": "2026-05-31T12:58:44.435Z", + "error_message": "Wrong reference: Request_body_nodata should be Request_body_nomsg", + "file": "src/controller/ha.c", + "root_cause": "Used \"Request_body_nodata\" instead of \"Request_body_nomsg\"", + "fix": "Changed Request_body_nodata \u2192 Request_body_nomsg", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T12:58:44.435Z" + }, + { + "id": "bug-220", + "timestamp": "2026-05-31T14:20:41.161Z", + "error_message": "Wrong reference: vote_reply_msg_handler should be vote_reply_msg_handler_for_test", + "file": "src/controller/ha.c", + "root_cause": "Used \"vote_reply_msg_handler\" instead of \"vote_reply_msg_handler_for_test\"", + "fix": "Changed vote_reply_msg_handler \u2192 vote_reply_msg_handler_for_test", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-05-31T14:20:41.161Z" + }, + { + "id": "bug-224", + "timestamp": "2026-06-01T02:07:55.176Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c->delaymgr.running", + "fix": "Added guard clause: if (!c->delaymgr.running)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T02:07:55.176Z" + }, + { + "id": "bug-229", + "timestamp": "2026-06-01T02:49:09.141Z", + "error_message": "Wrong operator: && should be !=", + "file": "src/controller/ha.c", + "root_cause": "Used \"&&\" instead of \"!=\"", + "fix": "Changed operator && \u2192 !=", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T02:49:09.141Z" + }, + { + "id": "bug-233", + "timestamp": "2026-06-01T03:07:51.264Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connect.c'", + "fix": "Changed to 'testsuite/test_ha_ctrlhello.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T03:07:51.264Z" + }, + { + "id": "bug-235", + "timestamp": "2026-06-01T03:13:14.521Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connection_reuse.c'", + "fix": "Changed to 'testsuite/test_ha_connect.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T03:13:14.521Z" + }, + { + "id": "bug-236", + "timestamp": "2026-06-01T03:15:29.129Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: rc < 0", + "fix": "Added guard clause: if (rc < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T03:15:29.129Z" + }, + { + "id": "bug-238", + "timestamp": "2026-06-01T03:23:32.483Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connect.c'", + "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-01T03:24:43.046Z" + }, + { + "id": "bug-239", + "timestamp": "2026-06-01T03:28:46.956Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_tcp_connect_hook != NULL", + "fix": "Added guard clause: if (ha_tcp_connect_hook != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T03:28:46.956Z" + }, + { + "id": "bug-240", + "timestamp": "2026-06-01T04:08:07.950Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_connect.c'", + "fix": "Changed to 'testsuite/test_ha_connection_reuse.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-01T04:12:55.229Z" + }, + { + "id": "bug-241", + "timestamp": "2026-06-01T04:13:03.001Z", + "error_message": "Wrong reference: test_ha_connect should be test_ha_ctrlhello", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_ha_connect\" instead of \"test_ha_ctrlhello\"", + "fix": "Changed test_ha_connect \u2192 test_ha_ctrlhello", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-01T04:13:08.650Z" + }, + { + "id": "bug-242", + "timestamp": "2026-06-01T05:19:12.427Z", + "error_message": "Wrong reference: vcpu should be vcpus", + "file": "src/tools/deploy-ha.sh", + "root_cause": "Used \"vcpu\" instead of \"vcpus\"", + "fix": "Changed vcpu \u2192 vcpus", + "tags": [ + "auto-detected", + "wrong-reference", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T05:19:12.427Z" + }, + { + "id": "bug-250", + "timestamp": "2026-06-01T07:46:05.128Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ep || !ep->data", + "fix": "Added guard clause: if (!ep || !ep->data)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T07:46:05.128Z" + }, + { + "id": "bug-252", + "timestamp": "2026-06-01T08:35:29.969Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'startup_validation.c'", + "fix": "Changed to 'testsuite/test_ha_rdhup_integration.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T08:35:29.969Z" + }, + { + "id": "bug-255", + "timestamp": "2026-06-01T12:02:55.635Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (rc != 0)", + "fix": "Changed to: if (validate_heartbeat_config(ha_cfg.heartbeat_timeout)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T12:02:55.635Z" + }, + { + "id": "bug-256", + "timestamp": "2026-06-01T12:36:28.381Z", + "error_message": "Wrong reference: startup_validation should be testsuite_test_ha_liveness", + "file": "src/xce.build/build.ninja", + "root_cause": "Used \"startup_validation\" instead of \"testsuite_test_ha_liveness\"", + "fix": "Changed startup_validation \u2192 testsuite_test_ha_liveness", + "tags": [ + "wrong-reference", + "ninja" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T12:36:28.381Z" + }, + { + "id": "bug-260", + "timestamp": "2026-06-01T14:42:41.325Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: interval_s <= 0", + "fix": "Added guard clause: if (interval_s <= 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-01T14:44:38.543Z" + }, + { + "id": "bug-263", + "timestamp": "2026-06-01T15:32:59.304Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: !f", + "fix": "Added guard clause: if (!f)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-01T15:32:59.304Z" + }, + { + "id": "bug-265", + "timestamp": "2026-06-03T08:47:25.835Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-03T08:47:25.835Z" + }, + { + "id": "bug-266", + "timestamp": "2026-06-03T08:47:54.376Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-03T08:47:54.376Z" + }, + { + "id": "bug-269", + "timestamp": "2026-06-03T08:50:58.837Z", + "error_message": "Wrong reference: test_am_i_primary should be test_ha_cluster", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_am_i_primary\" instead of \"test_ha_cluster\"", + "fix": "Changed test_am_i_primary \u2192 test_ha_cluster", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-03T08:51:07.382Z" + }, + { + "id": "bug-279", + "timestamp": "2026-06-04T11:54:56.114Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-repl-hardening/design.md", + "root_cause": "Had `(checkpoint_lsn, last_received_chunk)`", + "fix": "Changed to `snap.part`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T11:54:56.114Z" + }, + { + "id": "bug-284", + "timestamp": "2026-06-04T14:33:03.464Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2500\u2500 Task 2.1: OPCode_walCatchupEntry handler \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-04T14:33:51.476Z" + }, + { + "id": "bug-285", + "timestamp": "2026-06-04T14:41:35.073Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/cluster.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: case OPCode_walCatchupEntry:; /* Task 2.1: standby applies a single WAL catch-up entry fro", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-04T14:41:48.239Z" + }, + { + "id": "bug-288", + "timestamp": "2026-06-04T14:46:44.652Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: #include \"persist.h\" /* persist_t, apply_wal_entry */; #include \"typemap.h\" /* wal_catchup_entry_t */", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-04T14:47:53.528Z" + }, + { + "id": "bug-292", + "timestamp": "2026-06-04T16:15:43.743Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", + "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= MAX_HA_CANDI)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T16:15:43.743Z" + }, + { + "id": "bug-294", + "timestamp": "2026-06-04T16:45:25.724Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: standby_tag < committed_tag", + "fix": "Added guard clause: if (standby_tag < committed_tag)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T16:45:25.724Z" + }, + { + "id": "bug-295", + "timestamp": "2026-06-04T16:57:59.213Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_role.c", + "root_cause": "Had \"test_evaluate_repl_action_stream_wal_from_zero: P", + "fix": "Changed to \"test_evaluate_repl_action_send_snapshot_lsn_zero:", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T16:57:59.213Z" + }, + { + "id": "bug-296", + "timestamp": "2026-06-04T16:58:16.627Z", + "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero", + "file": "src/controller/testsuite/test_ha_role.c", + "root_cause": "Used \"test_evaluate_repl_action_stream_wal_from_zero\" instead of \"test_evaluate_repl_action_send_snapshot_lsn_zero\"", + "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero \u2192 test_evaluate_repl_action_send_snapshot_lsn_zero", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T16:58:16.627Z" + }, + { + "id": "bug-297", + "timestamp": "2026-06-04T17:08:18.571Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES", + "fix": "Added guard clause: if (!ha || peer_idx < 0 || peer_idx >= MAX_H)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T17:08:18.571Z" + }, + { + "id": "bug-301", + "timestamp": "2026-06-04T23:09:43.701Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", + "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T23:09:43.701Z" + }, + { + "id": "bug-302", + "timestamp": "2026-06-04T23:10:44.426Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !snap_dir", + "fix": "Added guard clause: if (!snap_dir)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-04T23:10:44.426Z" + }, + { + "id": "bug-308", + "timestamp": "2026-06-05T01:19:20.356Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !d", + "fix": "Added guard clause: if (!d)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T01:19:20.356Z" + }, + { + "id": "bug-309", + "timestamp": "2026-06-05T01:33:20.146Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !p || !snap_dir || !chunk", + "fix": "Added guard clause: if (!ha || !p || !snap_dir || !chunk)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T01:33:20.146Z" + }, + { + "id": "bug-310", + "timestamp": "2026-06-05T01:34:06.660Z", + "error_message": "Missing guard clause", + "file": "src/controller/cluster.c", + "root_cause": "No early return/throw for edge case: !chunk", + "fix": "Added guard clause: if (!chunk)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T01:34:06.660Z" + }, + { + "id": "bug-311", + "timestamp": "2026-06-05T02:21:15.033Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ua > ub", + "fix": "Added guard clause: if (ua > ub)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T02:21:15.033Z" + }, + { + "id": "bug-312", + "timestamp": "2026-06-05T02:47:32.237Z", + "error_message": "Wrong reference: uintptr_t should be void", + "file": "src/controller/testsuite/test_ha_evaluate_committing.c", + "root_cause": "Used \"uintptr_t\" instead of \"void\"", + "fix": "Changed uintptr_t \u2192 void", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T02:47:32.237Z" + }, + { + "id": "bug-314", + "timestamp": "2026-06-05T04:01:58.721Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T04:01:58.721Z" + }, + { + "id": "bug-315", + "timestamp": "2026-06-05T04:52:32.206Z", + "error_message": "Wrong reference: 10000 should be 60000", + "file": "src/controller/ha.h", + "root_cause": "Used \"10000\" instead of \"60000\"", + "fix": "Changed 10000 \u2192 60000", + "tags": [ + "auto-detected", + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-05T04:52:51.260Z" + }, + { + "id": "bug-317", + "timestamp": "2026-06-05T04:53:55.034Z", + "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS", + "file": "src/controller/testsuite/test_ha_evaluate_committing.c", + "root_cause": "Used \"10000\" instead of \"HA_DEFAULT_COMMITTING_TIMEOUT_MS\"", + "fix": "Changed 10000 \u2192 HA_DEFAULT_COMMITTING_TIMEOUT_MS", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T04:53:55.034Z" + }, + { + "id": "bug-320", + "timestamp": "2026-06-05T06:50:00Z", + "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN", + "file": "src/controller/ha.c", + "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always \u2014 majority convergence path could never succeed.", + "fix": "Changed ha->n_peers = n_peers to ha->n_peers = n_other (= n_peers-1 = other-node count). Moved the assignment after n_other is computed. ha_repl_ack_received quorum formula (ha->n_peers+2)/2 still yields correct majority for both old and new value.", + "tags": [ + "ha", + "majority-convergence", + "n_peers", + "off-by-one", + "integration-test" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-05T06:50:00Z" + }, + { + "id": "bug-326", + "timestamp": "2026-06-06T11:06:46.509Z", + "error_message": "Wrong reference: DEBUG2 should be INFO", + "file": "src/controller/ha.c", + "root_cause": "Used \"DEBUG2\" instead of \"INFO\"", + "fix": "Changed DEBUG2 \u2192 INFO", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-06T11:06:46.509Z" + }, + { + "id": "bug-327", + "timestamp": "2026-06-06T12:11:21.422Z", + "error_message": "Incorrect value in code", + "file": "src/controller/ha.c", + "root_cause": "Had \"HA: heartbeat tick (role=%d, n_standbys=%d)\"", + "fix": "Changed to \"HA: primary heartbeat armed\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-06T12:11:21.422Z" + }, + { + "id": "bug-328", + "timestamp": "2026-06-06T13:02:49.052Z", + "error_message": "Missing guard clause", + "file": "src/lib/delay.c", + "root_cause": "No early return/throw for edge case: t1_is_ms && t2_is_ms", + "fix": "Added guard clause: if (t1_is_ms && t2_is_ms)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-06T13:02:49.052Z" + }, + { + "id": "bug-329", + "timestamp": "2026-06-08T13:12:54.436Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-library/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-08T13:12:54.436Z" + }, + { + "id": "bug-334", + "timestamp": "2026-06-09T12:56:11.095Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (my_role == CTRL_STANDBY && ha_heartbeat_is_fresh(h)", + "fix": "Changed to: if (my_role == CTRL_STANDBY && ha->primary_ctrl_fd >= )", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-09T12:56:11.095Z" + }, + { + "id": "bug-335", + "timestamp": "2026-06-09T00:00:00Z", + "error_message": "UPDATE_RESVALS dequeue writes worker resval at index 0 unconditionally instead of resolving resource idx by name", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "zarray_put(&w->resvals, 0, rv->val) hardcodes index 0; worker resvals are indexed by resource idx (see schedobj.c:291)", + "fix": "FIXED 2026-06-10: UPDATE_RESVALS now resolves rv->name via key_find on sched->resmgr and writes to r->idx (matching UPDATE_RESVALS_FULL pattern) instead of hardcoding index 0.", + "tags": [ + "scheduler", + "resvals", + "latent-bug", + "gap-analysis", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T17:00:00Z" + }, + { + "id": "bug-336", + "timestamp": "2026-06-09T00:00:00Z", + "error_message": "XCE_SCHED_OP_UPDATE_RESOURCES cannot update existing resources; makeResource returns NULL/EEXIST and return value is silently ignored", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "makeResource_ in src/lib/resobj.c rejects duplicates with EEXIST; dequeue case ignores NULL return, so 'add or update' (EARS-UB-1.1) is only 'add'", + "fix": "FIXED 2026-06-10: UPDATE_RESOURCES now checks makeResource return; on NULL (EEXIST) looks up existing resource by name and updates valtype/ordtype/flags/interval fields in place (add-or-update semantics).", + "tags": [ + "scheduler", + "resources", + "latent-bug", + "gap-analysis", + "silent-error", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T17:00:00Z" + }, + { + "id": "bug-337", + "timestamp": "2026-06-09T00:02:00Z", + "error_message": "Vote-deny gate inert: ha_heartbeat_is_fresh reads ha->last_heartbeat which has no production writer after the liveness refactor", + "file": "src/controller/ha.c", + "root_cause": "ha-heartbeat-liveness refactor moved activity tracking to ha->last_activity but ha_heartbeat_is_fresh (ha.c:1762) still reads the orphaned last_heartbeat timespec; only tests write it, so freshness is always stale after ~heartbeat_timeout_ms of uptime and standbys grant votes despite a live primary (gates at ha.c:214 and ha.c:1841)", + "fix": "FIXED 2026-06-09: ha_heartbeat_is_fresh now reads ha->last_activity (returns 0 when last_activity==0); dead last_heartbeat field removed from ha_state_t; test_ha_vote_gate.c helper rewritten as set_last_activity_age and fresh/stale tests now also set primary_ctrl_fd. All tests green.", + "tags": [ + "ha", + "heartbeat", + "liveness", + "vote-gate", + "dead-field", + "gap-analysis" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-09T17:30:00Z" + }, + { + "id": "bug-338", + "timestamp": "2026-06-09T00:00:00Z", + "error_message": "controller_unit_test: test_vote_granted_to_same_candidate_same_term FAIL (Expected 1 Was 0); test_standby_fresh_heartbeat_denies_vote_request FAIL (Expected 0 Was 1, Req 6.2)", + "file": "src/controller/ha.c", + "root_cause": "Uncommitted handle_request_vote changes: (1) duplicate-vote deny made candidate-agnostic (denies same-candidate idempotent re-grant, diverging from raft.c which keeps re-grant); (2) STANDBY vote deny now requires primary_ctrl_fd >= 0, tests use ha_init default fd=-1", + "fix": "FIXED 2026-06-09: handle_request_vote Req 2.6 check restored to same-candidate idempotent re-grant (deny only when voted_for_name differs from candidate_id), matching raft.c; test_standby_fresh_heartbeat_denies_vote_request updated to set primary_ctrl_fd=999. Also fixed myapp.c leader_known check to treat a vote for myconf->name as a self-vote (production records real name, not the 'self' test placeholder). All tests green.", + "tags": [ + "ha", + "election", + "vote-gate", + "raft", + "unit-test", + "uncommitted-drift", + "gap-analysis" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-09T17:30:00Z" + }, + { + "id": "bug-339", + "timestamp": "2026-06-09T17:30:00Z", + "error_message": "After killing the primary in a 3-node controller HA cluster, the two surviving controllers enter an infinite election loop: both only ever raft_cb_become_follower (never become_leader), terms climb in lockstep (4..23), RDHUP -> CTRL_ELECTING -> CTRL_STANDBY repeats forever.", + "file": "src/controller/ha.c, src/controller/myapp.c, src/controller/testsuite/test_ha_rdhup_integration.c", + "root_cause": "LSN-source asymmetry between candidate-advertised LSN and voter freshness floor, specific to the RDHUP failover election path. ha_on_primary_ctrl_fd_rdhup advertised c->persist.committed_lsn as the RequestVote last_log_lsn, but a STANDBY never advances persist.committed_lsn (only the primary's commit engine writes it at persist.c:1192). Standbys advance applied_lsn (ha_apply_wal_catchup_entry) and last_known_committed_lsn (heartbeat replies). The voter-side freshness floor (cluster.c OPCode_requestVote) is max(applied_lsn, last_known_committed_lsn) via ha_vote_self_freshness. So once a standby applied any WAL beyond its last snapshot, every surviving standby advertised a stale-low LSN (e.g. 0) while each peer computed a HIGH floor (e.g. 10); the freshness gate cand_lsn < my_lsn tripped on BOTH sides, each denied the other's vote, neither reached quorum (2 of 3), each election timed out -> become_follower -> 10s liveness timeout -> RDHUP -> re-elect at term+1. The boot path (myapp.c:217) already advertised applied_lsn, so cold-boot election worked; only the RDHUP failover path was inconsistent.", + "fix": "In ha_on_primary_ctrl_fd_rdhup, compute candidate freshness ONCE via ha_vote_self_freshness(ha, config_version_tag, applied_lsn, &cand_tag, &cand_lsn) right after ha->raft_c=c, and use cand_tag/cand_lsn for BOTH the elect_ctx (votes[0].my_tag/my_lsn + self_tag/self_lsn) and the XCE_TESTING ha_rdhup_election_hook args. This makes candidate-advertised LSN identical to the voter-side floor, so for any two nodes A,B: A grants B iff floor_B >= floor_A -> the freshest node is always granted by every peer; equal-floor ties are broken by raft priority delay. Regression test added: test_rdhup_candidate_advertises_freshness_floor_not_stale_committed (applied/learned=10, persist.committed=0 -> asserts advertised my_lsn==10). RED (Was 0) -> GREEN verified. Follow-up: boot election path (myapp.c) also routed through ha_vote_self_freshness (was raw applied_lsn) so both election paths are symmetric with the voter floor.", + "tags": [ + "guard-clause", + "c", + "raft", + "election-safety", + "restart", + "persistence", + "gap-analysis", + "ha", + "election", + "rdhup", + "failover", + "freshness-gate", + "committed_lsn", + "applied_lsn", + "split-vote", + "infinite-loop" + ], + "related_bugs": [ + "bug-338" + ], + "occurrences": 3, + "last_seen": "2026-06-15T11:00:00.000Z" + }, + { + "id": "bug-340", + "timestamp": "2026-06-10T00:23:21.918Z", + "error_message": "After kill+RESTART of a controller (ctrl-1), the restarted node loops forever: ha_on_primary_ctrl_fd_rdhup \u2192 CTRL_ELECTING \u2192 raft_cb_become_follower \u2192 CTRL_STANDBY every ~heartbeat_timeout, term climbing unbounded (473+), while ctrl-2 stays PRIMARY at term 3. On ctrl-2: every heartbeat tick logs `xce_encode op 0 (NOP)` then `WARN endpoint_update_epoll rc1=-1 < 0` (netio.c:575) \u2014 a wedged outbound standby endpoint.", + "file": "src/controller/ha.c, src/controller/testsuite/test_ha_liveness.c", + "root_cause": "primary_heartbeat_fn never evicted a dead/wedged outbound standby endpoint from ha->standby_endpoints. When a peer is killed/restarted, its outbound ep on the primary wedges: handle_error (netio.c:420) runs iomgr_unmonitor (EPOLL_CTL_DEL, does NOT close the fd) and the err handler may leave the ep in standby_endpoints; epoll_ctl(EPOLL_CTL_MOD) on the DEL'd fd then returns ENOENT (rc1=-1) on every heartbeat enqueue. xce_nop_req still returns >=0 (enqueue ok) so the heartbeat layer never detected the wedge. Critically, primary_heartbeat_fn step 2 (reconnect) treats a peer as 'found' (connected) if ANY ep for it is in standby_endpoints \u2014 so the wedged ep SUPPRESSED reconnection. ctrl-2 therefore never opened a fresh connection to the restarted ctrl-1, never sent a ctrlhello, so ctrl-1 never set primary_ctrl_fd / refreshed last_activity and its liveness watchdog re-elected every heartbeat_timeout forever. (The earlier suspected 'become_follower doesn't reset last_activity' was NOT a bug: ha_arm_heartbeat_timeout_vtask already stamps last_activity=now, so the ~11s period is exactly one full timeout \u2014 confirming the watchdog is correct and the sole cause is the broken channel.)", + "fix": "In primary_heartbeat_fn (ha.c) step 1, added liveness eviction symmetric with the standby's own liveness poll: for each standby ep, if ep->ts_last_in > 0 (was responsive at least once) AND (now_wall - ep->ts_last_in) > heartbeat_timeout_s, remove it from ha->standby_endpoints (zarray_del, under ha->lock) and collect it; after releasing ha->lock, repl_remove_standby() each evicted ep. Removal is the load-bearing action \u2014 the reconnect scan (step 2) then re-establishes a fresh ctrlhello channel to the restarted peer on this/next tick, and ctrl-1 settles via ha_discover_primary (no term gate, ha.c:3807) \u2192 become_follower \u2192 STANDBY. ts_last_in/ts_last_out are existing endpoint_t fields (wall clock, set in handle_input/handle_output), so no libnetio change. The evicted ep object itself is reclaimed by its err handler when the dead connection finally surfaces an epoll event (bounded leak, consistent with codebase). Eviction keys on ts_last_in>0 so a just-connected joiner (ts_last_in==0) gets grace. Regression tests: test_primary_hb_evicts_silent_standby (RED\u2192GREEN), test_primary_hb_keeps_live_standby, test_primary_hb_keeps_unconfirmed_standby in test_ha_liveness.c.", + "tags": [ + "logic-fix", + "c", + "ha", + "heartbeat", + "rejoin", + "restart", + "standby-endpoints", + "reconnect", + "epoll", + "liveness", + "election-loop" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-15T15:00:00.000Z" + }, + { + "id": "bug-341", + "timestamp": "2026-06-10T00:23:52.563Z", + "error_message": "Wrong reference: last_heartbeat should be voted_for_term", + "file": "src/controller/ha.h", + "root_cause": "Used \"last_heartbeat\" instead of \"voted_for_term\"", + "fix": "Changed last_heartbeat \u2192 voted_for_term", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T00:23:52.563Z" + }, + { + "id": "bug-344", + "timestamp": "2026-06-10T00:32:34.054Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_bootstrap_progress.md", + "root_cause": "Had `ha->last_heartbeat`", + "fix": "Changed to `ha->last_activity`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T00:32:34.054Z" + }, + { + "id": "bug-349", + "timestamp": "2026-06-10T01:39:19.368Z", + "error_message": "Missing guard clause", + "file": "src/lib/request.c", + "root_cause": "No early return/throw for edge case: l.p.type == CAPN_NULL", + "fix": "Added guard clause: if (l.p.type == CAPN_NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T01:39:19.368Z" + }, + { + "id": "bug-350", + "timestamp": "2026-06-10T01:39:31.683Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/request.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: if (req->body_which == Request_body_walcatchupentrymsg; && req->body.walcatchupentrymsg != NULL) { | Also: /* wal_entry_to_wire: fill a wire WalCatchupEntry from a C e; * fields and raw payload bytes. Sub-entry nesting is one le", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-10T01:40:14.312Z" + }, + { + "id": "bug-351", + "timestamp": "2026-06-09T00:00:00Z", + "error_message": "request.c: passing argument 'from' of xce_wal_batch_encode_raw / xce_snapshot_chunk_encode_raw from incompatible pointer type (xce_peerinfo_t** vs xce_peerinfo_t*)", + "file": "src/lib/request.c", + "root_cause": "ep->peer is already xce_peerinfo_t* (netio.h endpoint struct); call sites passed &ep->peer producing xce_peerinfo_t**", + "fix": "Dropped the & at both call sites (xce_wal_batch_req_raw line 547, xce_snapshot_chunk_req_raw line 633) to pass ep->peer directly", + "tags": [ + "build-failure", + "pointer-type", + "c", + "request.c", + "ha-replication" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-09T00:00:00Z" + }, + { + "id": "bug-352", + "timestamp": "2026-06-10T01:46:28.589Z", + "error_message": "ASAN heap-use-after-free in free_packet (netio.c:333) via endpoint_fini <- raft_cb_become_follower (loser path) <- raft_on_vote_reply; election loser crashes on vote-reply processing", + "file": "src/controller/ha.c", + "root_cause": "vote_reply_decode_and_dispatch took pkt = ep->fsm_in.pkt and freed it WITHOUT clearing ep->fsm_in.pkt; vote_reply_dispatch can synchronously reach raft_cb_become_follower -> endpoint_fini(ep), which frees the stale fsm_in.pkt again. repl_ack_decode_and_dispatch already nulls the pointer (correct pattern); the vote-reply path missed it.", + "fix": "FIXED 2026-06-10: ep->fsm_in.pkt = NULL immediately after taking ownership in vote_reply_decode_and_dispatch (ha.c:2413), matching repl_ack_decode_and_dispatch. Found by task 7.7 E2E (3-node deploy, election loser crash).", + "tags": [ + "c", + "ha", + "election", + "asan", + "use-after-free", + "packet-ownership", + "netio", + "e2e" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-10T02:10:00Z" + }, + { + "id": "bug-353", + "timestamp": "2026-06-10T01:46:45.213Z", + "error_message": "Config push silently rejected when standby is in CTRL_ELECTING: ha_dispatch_gate only allows requestVote/probeConfigTag, returning -E_XCE_ERR_NOT_READY for all other opcodes. If liveness poll fires during election window, any incoming config push is discarded.", + "file": "src/controller/ha.c", + "root_cause": "CTRL_ELECTING gate in ha_dispatch_gate (ha.c:1464) allowed only vote/probe opcodes. Config push from primary arriving during the ELECTING window (after liveness-poll-triggered failover, before election result) was rejected with -E_XCE_ERR_NOT_READY (10013). Standby never saved config, leaving conf dir empty.", + "fix": "FIXED 2026-06-10: Added OPCode_hello, OPCode_nop, OPCode_replConfigPush, OPCode_walCatchupEntry, OPCode_snapshotTransfer to CTRL_ELECTING allowlist. A campaigning node must accept replication from the current primary to stay current for both win and loss paths. Controller suite green.", + "tags": [ + "wrong-reference", + "c", + "ha", + "dispatch-gate", + "election", + "config-replication", + "silent-failure" + ], + "related_bugs": [ + "bug-335", + "bug-336" + ], + "occurrences": 2, + "last_seen": "2026-06-10T17:30:00Z" + }, + { + "id": "bug-358", + "timestamp": "2026-06-10T03:55:44.698Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/config_repl.c", + "root_cause": "Condition was: if (!rcpr->raw_bytes || rcpr->raw_bytes_len == 0 || !r)", + "fix": "Changed to: if ((ctrl_role_t)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T03:55:44.698Z" + }, + { + "id": "bug-359", + "timestamp": "2026-06-10T04:16:25.052Z", + "error_message": "Missing guard clause", + "file": "src/scheduler/src/scheduler.c", + "root_cause": "No early return/throw for edge case: r == NULL", + "fix": "Added guard clause: if (r == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-10T04:16:28.320Z" + }, + { + "id": "bug-361", + "timestamp": "2026-06-10T04:47:09.048Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (opcode == OPCode_requestVote || opcode == OPCode_p)", + "fix": "Changed to: if (opcode == OPCode_requestVote ||\n opcode)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T04:47:09.048Z" + }, + { + "id": "bug-364", + "timestamp": "2026-06-10T06:26:24.340Z", + "error_message": "Incorrect value in code", + "file": "src/controller/CLAUDE.md", + "root_cause": "Had `CTRL_ELECTING`", + "fix": "Changed to `CTRL_INIT`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T06:26:24.340Z" + }, + { + "id": "bug-367", + "timestamp": "2026-06-10T07:56:21.347Z", + "error_message": "Missing error handling in XCE_TESTING", + "file": "src/controller/testsuite/test_repl_unit.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* =========================================================; * Task 1.1: repl_broadcast send path tests", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-10T07:57:02.036Z" + }, + { + "id": "bug-368", + "timestamp": "2026-06-10T07:57:40.792Z", + "error_message": "Wrong reference: WAL_JOB should be WAL_JOB_SUBMIT", + "file": "src/controller/testsuite/test_repl_unit.c", + "root_cause": "Used \"WAL_JOB\" instead of \"WAL_JOB_SUBMIT\"", + "fix": "Changed WAL_JOB \u2192 WAL_JOB_SUBMIT", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T07:57:40.792Z" + }, + { + "id": "bug-371", + "timestamp": "2026-06-10T07:59:24.268Z", + "error_message": "Wrong reference: DEBUG3 should be TRACE", + "file": "src/controller/repl.c", + "root_cause": "Used \"DEBUG3\" instead of \"TRACE\"", + "fix": "Changed DEBUG3 \u2192 TRACE", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T07:59:24.268Z" + }, + { + "id": "bug-373", + "timestamp": "2026-06-10T08:45:30.400Z", + "error_message": "Wrong reference: request should be ep", + "file": "src/controller/cluster.c", + "root_cause": "Used \"request\" instead of \"ep\"", + "fix": "Changed request \u2192 ep", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T08:45:30.400Z" + }, + { + "id": "bug-374", + "timestamp": "2026-06-10T08:47:48.296Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_role.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 3.3 (ha-wal-replication): dispatch gate standby-role; RUN_TEST(test_dispatch_gate_allows_wal_catchup_in_standby_ro", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-10T08:48:00.267Z" + }, + { + "id": "bug-376", + "timestamp": "2026-06-10T09:17:35.218Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (opcode == OPCode_nop)", + "fix": "Changed to: if (rep_s.data_which == RequestReply_data_walackreplym)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T09:17:35.218Z" + }, + { + "id": "bug-378", + "timestamp": "2026-06-10T10:02:00.000Z", + "error_message": "AddressSanitizer: global-buffer-overflow in dataq_enq via xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Tests 8 and 9 declared ep_a/ep_b/g_ep9_a/g_ep9_b as 'static int' (4 bytes) and passed them to repl_add_standby without (void*) cast. repl_broadcast then called xce_wal_batch_req_raw(s->ep, ...) using the stored int* as a real network endpoint, causing an 8-byte read past the 4-byte int. Neither test installed repl_broadcast_send_hook to intercept the call.", + "fix": "Added #include typemap.h; added extern repl_broadcast_send_hook declaration guarded by XCE_TESTING; added p2_drop_send_hook no-op; installed hook around persist_wal_sync_append in tests 8 and 9; fixed missing (void*) casts on all four ep variables", + "tags": [ + "asan", + "global-buffer-overflow", + "repl_broadcast", + "unit-test", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T10:02:00.000Z" + }, + { + "id": "bug-380", + "timestamp": "2026-06-10T10:37:21.924Z", + "error_message": "Wrong reference: test_catchup_full_lifecycle_integration should be test_single_node_committed_lsn_unchanged", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Used \"test_catchup_full_lifecycle_integration\" instead of \"test_single_node_committed_lsn_unchanged\"", + "fix": "Changed test_catchup_full_lifecycle_integration \u2192 test_single_node_committed_lsn_unchanged", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T10:37:21.924Z" + }, + { + "id": "bug-382", + "timestamp": "2026-06-10T11:54:19.749Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-wal-replication/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T11:54:19.749Z" + }, + { + "id": "bug-384", + "timestamp": "2026-06-10T14:13:01.581Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/tech.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T14:13:01.581Z" + }, + { + "id": "bug-386", + "timestamp": "2026-06-10T14:43:53.161Z", + "error_message": "Function not marked async", + "file": "CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-10T14:43:53.161Z" + }, + { + "id": "bug-388", + "timestamp": "2026-06-10T14:54:18.503Z", + "error_message": "Function not marked async", + "file": "src/controller/CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-10T14:54:53.661Z" + }, + { + "id": "bug-389", + "timestamp": "2026-06-11T02:46:08.749Z", + "error_message": "Wrong reference: signal should be stdio", + "file": "src/lib/delay.c", + "root_cause": "Used \"signal\" instead of \"stdio\"", + "fix": "Changed signal \u2192 stdio", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T02:46:08.749Z" + }, + { + "id": "bug-391", + "timestamp": "2026-06-11T03:41:14.912Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T03:41:14.912Z" + }, + { + "id": "bug-392", + "timestamp": "2026-06-11T03:46:02.642Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: sync_rc_out", + "fix": "Added guard clause: if (sync_rc_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T03:46:02.642Z" + }, + { + "id": "bug-394", + "timestamp": "2026-06-11T05:09:37.037Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had `persist_truncate_wal_above`", + "fix": "Changed to `restore_job`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T05:09:37.037Z" + }, + { + "id": "bug-396", + "timestamp": "2026-06-10T22:45:00-07:00", + "error_message": "Worker job state notifications (OPCode_notify) are permanently lost when the controller dispatch gate rejects them (quorum loss / COMMITTING window): worker never reads replies, never retries, no rejoin resync", + "file": "src/worker/src/controller_connector.c, src/lib/xceapi.c, src/controller/cluster.c", + "root_cause": "xce_notify is enqueue-and-forget (no cbuf_get); error replies dropped via cbuf_tryput -EAGAIN (xcectx.c:304); xcecd_job_finished return ignored (FIXME at xcecd_worker_utils.c:298); xce_join_req carries no running-job state so rejoin cannot resync", + "fix": "FIXED 2026-06-12 (ha-commit-barrier-rollout tasks 1.2+4.2+4.3): dispatch gate admits fact opcodes (notify/update/setpreason/clrpreason) without write quorum (ha.c CTRL_PRIMARY allowlist); facts append via RETAIN (log-is-the-queue, no timeout) and commit when quorum returns. End-to-end retention proof: test_fact_retention_across_quorum_loss. Worker-side redelivery gap remains (worker-rejoin-resync backlog covers disconnect windows).", + "tags": [ + "ha", + "notify", + "silent-data-loss", + "worker", + "quorum", + "ha-consistency-invariants" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12" + }, + { + "id": "bug-398", + "timestamp": "2026-06-11T06:35:26.288Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had `restore_job`", + "fix": "Changed to \"irrevocable\"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T06:35:26.288Z" + }, + { + "id": "bug-399", + "timestamp": "2026-06-11T06:43:14.709Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had 't span syscalls and degrade to serial mode), C ec", + "fix": "Changed to 't span syscalls), C ecosystem dead (libitm unmain", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T06:43:14.709Z" + }, + { + "id": "bug-400", + "timestamp": "2026-06-11T07:23:53.901Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had \"return quickly\"", + "fix": "Changed to `--vcpus`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T07:23:53.901Z" + }, + { + "id": "bug-401", + "timestamp": "2026-06-11T07:47:34.756Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-consistency-invariants/research.md", + "root_cause": "Had `mark_job_*`", + "fix": "Changed to \"pending facts\"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T07:47:34.756Z" + }, + { + "id": "bug-403", + "timestamp": "2026-06-11T08:34:47.868Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-consistency-invariants/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T08:34:47.868Z" + }, + { + "id": "bug-405", + "timestamp": "2026-06-11T09:47:16.750Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-single-leader-role/design.md", + "root_cause": "Had `writable_lsn`", + "fix": "Changed to 's \"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T09:47:16.750Z" + }, + { + "id": "bug-406", + "timestamp": "2026-06-11T09:49:04.296Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-single-leader-role/tasks.md", + "root_cause": "Had \" --exclude-dir=.git --exclude-dir=.kiro --exclude", + "fix": "Changed to \" --exclude-dir=.git --exclude-dir=specs --exclude", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T09:49:04.296Z" + }, + { + "id": "bug-407", + "timestamp": "2026-06-11T10:01:15.181Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-barrier-rollout/design.md", + "root_cause": "Had `repl.h`", + "fix": "Changed to `entry_lsn`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T10:01:15.181Z" + }, + { + "id": "bug-408", + "timestamp": "2026-06-11T10:01:39.424Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-convergence-recovery/tasks.md", + "root_cause": "Had `-XCE_ERR_NOT_READY`", + "fix": "Changed to `OPCode_notify`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-11T10:02:02.349Z" + }, + { + "id": "bug-410", + "timestamp": "2026-06-11T13:30:00Z", + "error_message": "test_commit_engine fixture: Expected 0 Was 1 from repl_add_standby", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "repl_add_standby returns the slot index (>=0) on success, not 0 \u2014 second add legitimately returns 1", + "fix": "fixture asserts TEST_ASSERT_EQUAL_INT(i, repl_add_standby(...)) per standby slot", + "tags": [ + "controller", + "testsuite", + "repl", + "return-convention" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T13:30:00Z" + }, + { + "id": "bug-411", + "timestamp": "2026-06-11T13:35:00Z", + "error_message": "3 pre-existing controller testsuite failures masked by exit code 0: test_committed_lsn_does_not_advance_on_timeout (-110), test_repl_ack_after_promote_gate_allows_jobsub (-10013), test_rdhup_primary_dispatch_gate_opens (-10013)", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "each test group calls UNITY_BEGIN/UNITY_END internally, resetting Unity counters \u2014 main()'s final UNITY_END returns 0 unless the LAST group fails, so meson reported the suite green despite 3 real failures; Unity also misattributes failure FILE names to whichever file's UNITY_BEGIN ran (e.g. reports test_repl_unit.c for a test living in test_p2_integration.c)", + "fix": "not fixed (out of task 1.1 boundary) \u2014 documented; failures exist identically on baseline without ha-commit-engine changes (verified by stash/rebuild). persist_wal_sync_append returns -ETIMEDOUT while the test expects rc==0 with sync_rc==-ETIMEDOUT; the two gate tests fail ha_has_write_quorum (standby_endpoints empty with n_peers=2)", + "tags": [ + "controller", + "testsuite", + "unity", + "exit-code", + "pre-existing", + "ha" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T13:35:00Z" + }, + { + "id": "bug-413", + "timestamp": "2026-06-11T15:28:35.990Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 3.1 \u2014 membership re-evaluation (commit_on_membership_ | Also: /* Task 3.1 \u2014 membership re-evaluation */; RUN_TEST(test_catchup_standby_ack_does_not_count_toward_pend", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-11T15:29:25.430Z" + }, + { + "id": "bug-420", + "timestamp": "2026-06-11T16:19:51.195Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 4.2 \u2014 wired gate: registry truth at ha_dispatch_gate; RUN_TEST(test_gate_wired_registry_catchup_standby_rejects_th", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-11T16:20:05.714Z" + }, + { + "id": "bug-421", + "timestamp": "2026-06-11T16:23:46.118Z", + "error_message": "Missing guard clause", + "file": "src/controller/repl.c", + "root_cause": "No early return/throw for edge case: !r", + "fix": "Added guard clause: if (!r)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T16:23:46.118Z" + }, + { + "id": "bug-423", + "timestamp": "2026-06-11T22:39:24.704Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_engine.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550; * Task 5.1 \u2014 integration: ACK feed and the job-submit sync-a | Also: #include \"typemap.h\" /* wal_catchup_entry_t (task 5; #include \"wal_types.h\" /* WAL_JOB_SUBMIT (task 5.1 sy", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-11T22:40:45.169Z" + }, + { + "id": "bug-424", + "timestamp": "2026-06-11T23:08:49.909Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-engine/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:08:49.909Z" + }, + { + "id": "bug-425", + "timestamp": "2026-06-11T23:15:25.014Z", + "error_message": "Wrong return value", + "file": "src/controller/repl.c", + "root_cause": "Was returning: idx;", + "fix": "Now returns: * above mutates nothing, so it deliberately skips ", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:15:25.014Z" + }, + { + "id": "bug-428", + "timestamp": "2026-06-11T23:43:45.216Z", + "error_message": "Incorrect value in code", + "file": "src/include/xce/cluster_def.h", + "root_cause": "Had \"ha.h\"", + "fix": "Changed to \"commit.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:43:45.216Z" + }, + { + "id": "bug-429", + "timestamp": "2026-06-11T23:44:16.357Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (ha_is_primary(&c->ha)", + "fix": "Changed to: if (c->commit.p == NULL)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:44:16.357Z" + }, + { + "id": "bug-431", + "timestamp": "2026-06-11T23:45:16.518Z", + "error_message": "Incorrect value in code", + "file": "src/controller/persist.c", + "root_cause": "Had \"xce/types.h\"", + "fix": "Changed to \"xce/errnox.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:45:16.518Z" + }, + { + "id": "bug-432", + "timestamp": "2026-06-11T23:45:42.031Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:45:42.031Z" + }, + { + "id": "bug-433", + "timestamp": "2026-06-11T23:45:51.649Z", + "error_message": "Wrong reference: test_single_node_committed_lsn_unchanged should be test_single_node_committed_lsn_advances", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Used \"test_single_node_committed_lsn_unchanged\" instead of \"test_single_node_committed_lsn_advances\"", + "fix": "Changed test_single_node_committed_lsn_unchanged \u2192 test_single_node_committed_lsn_advances", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-11T23:45:51.649Z" + }, + { + "id": "bug-434", + "timestamp": "2026-06-12T00:23:08.815Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T00:23:08.815Z" + }, + { + "id": "bug-435", + "timestamp": "2026-06-11T00:00:00Z", + "error_message": "test_committed_lsn_does_not_advance_on_timeout: Expected 0 Was -110 (baseline failure, masked by bug-411)", + "file": "src/controller/testsuite/test_p2_integration.c", + "root_cause": "Stale test expectation: pinned the old accept-on-timeout contract (rc==0 + sync_rc=-ETIMEDOUT advisory). ha-commit-engine made persist_wal_sync_append reject-on-timeout (returns -ETIMEDOUT, Req 2.1/4.4). Fixture was also engine-less (deprecated ce==NULL legacy path).", + "fix": "Updated expectation to rc==-ETIMEDOUT and wired a commit engine (n_peers=3) with a priming async append so the sync entry lands at LSN 1 on the engine path; kept the committed_lsn-unmoved assertion.", + "tags": [ + "ha-commit-engine", + "task-6.1", + "stale-expectation", + "unity-misattribution" + ], + "related_bugs": [ + "bug-411" + ], + "occurrences": 1, + "last_seen": "2026-06-11T00:00:00Z" + }, + { + "id": "bug-436", + "timestamp": "2026-06-11T00:00:00Z", + "error_message": "test_repl_ack_after_promote_gate_allows_jobsub / test_rdhup_primary_dispatch_gate_opens: Expected 0 Was -10013 (NOT_READY; baseline failures, masked by bug-411)", + "file": "src/controller/testsuite/test_ha_repl_ack.c, src/controller/testsuite/test_ha_rdhup_integration.c", + "root_cause": "CTRL_PRIMARY dispatch gate now requires write quorum (>= n_peers/2 connected standbys). Fixtures promote via a standby ACK but never register the connected standby; legacy fallback counts ha->standby_endpoints (NULL -> 0). In the rdhup test ha_on_primary_ctrl_fd_rdhup also overwrites ha->n_peers=1 with candidates_count=2 (ha.c:3484).", + "fix": "Populated standby_endpoints with one endpoint (the ACKing standby) in both fixtures so quorum legitimately holds on a healthy primary; gate unchanged. Freed the zarray before ha_fini (ha_fini does not own it).", + "tags": [ + "ha-commit-engine", + "task-6.1", + "quorum-gate", + "fixture-fix" + ], + "related_bugs": [ + "bug-411" + ], + "occurrences": 1, + "last_seen": "2026-06-11T00:00:00Z" + }, + { + "id": "bug-437", + "timestamp": "2026-06-12T00:51:52.673Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-engine/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T00:51:52.673Z" + }, + { + "id": "bug-438", + "timestamp": "2026-06-12T03:52:57.142Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_commit_barrier_rollout.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T03:52:57.142Z" + }, + { + "id": "bug-440", + "timestamp": "2026-06-12T04:18:19.158Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: detail == HOOK_DETAIL_WORKER_JOIN ||\n detail == H", + "fix": "Added guard clause: if (detail == HOOK_DETAIL_WORKER_JOIN ||\n )", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T04:18:19.158Z" + }, + { + "id": "bug-442", + "timestamp": "2026-06-12T04:50:32.154Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "Had \"xce/types.h\"", + "fix": "Changed to \"xce/errnox.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T04:50:32.154Z" + }, + { + "id": "bug-443", + "timestamp": "2026-06-12T05:14:36.392Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p == NULL", + "fix": "Added guard clause: if (p == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T05:14:36.392Z" + }, + { + "id": "bug-445", + "timestamp": "2026-06-12T09:30:00Z", + "error_message": "AddressSanitizer SEGV in hash_string via makeResource when running controller_unit_test -t test_commit_barrier_reply standalone", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "Global xstring table uninitialised in standalone runs: makeString() returns NULL, resobj name becomes NULL, key_add hashes NULL. In full-suite runs test_worker_join.c:520 initialises the table unguarded, so the bug only appears with -t selection.", + "fix": "Probe makeString(\"cbrp-probe\") in the group entry; call xstring_table_init(139) only when the probe returns NULL (avoids double init in full-suite order).", + "tags": [ + "testsuite", + "xstring", + "fixture", + "segv", + "ha-commit-barrier-rollout" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T09:30:00Z" + }, + { + "id": "bug-446", + "timestamp": "2026-06-12T09:30:00Z", + "error_message": "makeJob() returns NULL (Expected Non-NULL) in commit-barrier reply fixture", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "job_init (lib/jobj.c) returns -EINVAL when xce_job_t.submit == NULL, and makeJobsub requires job_obj_init() module tables which no controller test group initialises.", + "fix": "Fixture calls job_obj_init(64, 64) once per process (static guard) and provides a zeroed static job_submit_data_t as jdef.submit.", + "tags": [ + "testsuite", + "jobj", + "fixture", + "ha-commit-barrier-rollout" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T09:30:00Z" + }, + { + "id": "bug-447", + "timestamp": "2026-06-12T09:30:00Z", + "error_message": "queue_ctrl/worker_ctrl append NO WAL entry and reply status 1 on success; job_ctrl also returns 1 (key_find found-rc leak)", + "file": "src/controller/queue.c", + "root_cause": "key_find (xkvmgr_find, lib/kvmgr.c:130-151) returns 1 when the key is found. queue_ctrl (queue.c:60), worker_ctrl (worker.c:643) and job_ctrl (job.c:314) store it in rc and never reset it: the 'if (rc == 0)' AFTER_EVENT hook gate is skipped (no WAL_QUEUE_CTRL / WAL_WORKER_CTRL append on the found path) and the requester reply carries status 1 instead of 0. Production gap discovered by task-3.1 RED tests; queue/host control state changes are currently never persisted or replicated.", + "fix": "FIXED (task 3.2, 2026-06-12): normalized the key_find found-rc to 0 in queue_ctrl (queue.c), worker_ctrl (worker.c) and job_ctrl (job.c) after the miss/-ENOENT checks, so the AFTER_EVENT hook gate fires (WAL append + replication) and the handlers' commit-barrier pattern sees 0. Handlers do_jobcontrol_op/do_queuecontrol_op/do_hostcontrol_op now run persist_commit_barrier(&c->persist, 0) on rc==0 before replying; handle_worker_join barriers between AFTER_EVENT and the success reply. All 12 test_commit_barrier_reply tests PASS; full controller suite 0 FAIL.", + "tags": [ + "production-gap", + "queue_ctrl", + "worker_ctrl", + "job_ctrl", + "key_find", + "wal", + "ha-commit-barrier-rollout", + "red-finding" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T11:00:00Z" + }, + { + "id": "bug-448", + "timestamp": "2026-06-12T10:05:12.464Z", + "error_message": "job_ctrl reference leak: reference(j) taken at job.c:317 but normal exit path (job.c:444-446) never calls dereference(j); only the workermgr-miss error path releases it", + "file": "src/controller/job.c", + "root_cause": "asymmetric reference/dereference across exit paths, predates ha-commit-barrier-rollout", + "fix": "NOT YET FIXED - discovered by task 3.2 adversarial review; add dereference(j) on the normal exit path", + "tags": [ + "wrong-value", + "c", + "controller", + "job", + "refcount", + "leak", + "pre-existing" + ], + "related_bugs": [ + "bug-447" + ], + "occurrences": 2, + "last_seen": "2026-06-12T10:30:00-07:00" + }, + { + "id": "bug-451", + "timestamp": "2026-06-12T11:05:00-07:00", + "error_message": "va_list type confusion: WAL_JOB_RUSAGE hook payload reads job_t fields from an xce_job_rusages_t*", + "file": "src/controller/job.c (caller) / src/controller/persist.c (consumer)", + "root_cause": "update_job_rusages fires AFTER_EVENT HOOK_DETAIL_JOB_RUSAGE_UPDATE with (cluster_t*, xce_job_rusages_t*) at job.c:1015-1016, but persist_job_hook_fun's default arm (persist.c:1354-1359) reads the second vararg as job_t* and dereferences j->job.state at offset 56 == sizeof(xce_job_rusages_t) \u2014 an out-of-bounds read on an exactly-sized allocation; the WAL_JOB_RUSAGE payload state field is garbage (jobid at offset 0 lines up by accident)", + "fix": "NOT FIXED (discovered while writing task 3.3 receipt-ACK verdict pins; production out of boundary). Test test_update_op_rusage_receipt_reply_facts_retained uses a padded union so the pin stays in-bounds under ASAN without masking the bug; fix candidates: dedicated RUSAGE arm in persist_job_hook_fun reading xce_job_rusages_t*, or fire the hook with (c, j) like the other job hooks", + "tags": [ + "pre-existing", + "type-confusion", + "va_list", + "wal", + "rusage", + "asan", + "oob-read", + "ha-commit-barrier-rollout" + ], + "related_bugs": [ + "bug-447", + "bug-448" + ], + "occurrences": 1, + "last_seen": "2026-06-12T11:05:00-07:00" + }, + { + "id": "bug-452", + "timestamp": "2026-06-12T15:03:38.962Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-12T15:03:38.962Z" + }, + { + "id": "bug-457", + "timestamp": "2026-06-13T00:18:17.080Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c->ha.standby_endpoints", + "fix": "Added guard clause: if (!c->ha.standby_endpoints)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T00:18:17.080Z" + }, + { + "id": "bug-459", + "timestamp": "2026-06-13T00:30:00Z", + "error_message": "configAck quorum fabricated success: counted acked_config_lsn > 0 (any previous upgrade's ack), denominator n_active with no self-count, and n_active==0 short-circuited to success; upgradeConfig also ACKed the admin tool unconditionally and requested restart even when the majority wait timed out (logged-only result)", + "file": "src/controller/repl.c, src/controller/config_repl.c", + "root_cause": "legacy connectivity-based counting on the configAck track predated the commit engine's per-push arithmetic; push_config_to_standbys returned the transport push count and only logged the barrier outcome, so do_upgrade_config_with_bytes could not observe failure", + "fix": "task 5.2: configAck branch counts acked_config_lsn >= push_tag (entry_lsn carries the tag), majority 1+count > n_peers/2 over engine n_peers, n_peers<=1 immediate 0, ce==NULL -EINVAL fail-loud; push_config_to_standbys returns the barrier result (push count via acked_out); do_upgrade_config_with_bytes sends send_upgrade_error and skips success ACK + restart on non-zero; committed_config_tag advance stays success-branch-only", + "tags": [ + "ha", + "config-repl", + "quorum", + "configAck", + "commit-barrier", + "upgradeConfig" + ], + "related_bugs": [ + "bug-396" + ], + "occurrences": 1, + "last_seen": "2026-06-13T00:30:00Z" + }, + { + "id": "bug-461", + "timestamp": "2026-06-13T01:11:49.673Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/config_repl.c", + "root_cause": "Condition was: if (n == 0)", + "fix": "Changed to: if (n_out)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T01:11:49.673Z" + }, + { + "id": "bug-465", + "timestamp": "2026-06-13T01:30:00Z", + "error_message": "Config-ACK signal never produced in production: repl_on_config_ack had zero production callers; standby replconfigpushreplymsg silently dropped in repl_ack_decode_and_dispatch (ha.c) and myreq_handler default case (cluster.c) -> every multi-member config upgrade barrier timed out (upgrades bricked)", + "file": "src/controller/ha.c", + "root_cause": "ha-config-bootstrap task 8.1's production ingestion half was never landed; the 5.2 barrier relied on a signal nothing produced (tests injected repl_on_config_ack directly)", + "fix": "Added replconfigpushreplymsg decode branch in repl_ack_decode_and_dispatch (ha.c) and an OPCode_reply ingestion case in cluster.c myreq_handler, both calling repl_on_config_ack(&c->persist.repl, ep, versionTag); wire-path integration test added (test_config_ack_wire_path_completes_barrier)", + "tags": [ + "ha", + "config-repl", + "barrier", + "capnp", + "wire-path", + "task-5.2" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T01:30:00Z" + }, + { + "id": "bug-466", + "timestamp": "2026-06-13T01:30:00Z", + "error_message": "Unconditional 500ms config barrier wait in push_config_to_standbys reachable from iomgr epoll thread (ha.c raft_cb_become_leader, ha_dispatch_repl_action_for_standby, legacy election path; join.c controller join) -> blocks all controller I/O; the awaited ACK arrives on the blocked thread (structural deadlock)", + "file": "src/controller/config_repl.c", + "root_cause": "task 5.2 made the barrier unconditional inside push_config_to_standbys while best-effort catch-up/election pushes on iomgr-reachable paths still called the same entry", + "fix": "Split the API: exported transport-only fan-out push_config_transport_only (no barrier, iomgr-safe) and switched all 5 ha.c sites + join.c:187 to it; push_config_to_standbys (transport+barrier) kept for the vtask-context upgrade path only; doc comments updated in config_repl.h", + "tags": [ + "ha", + "iomgr", + "deadlock", + "config-repl", + "api-split", + "task-5.2" + ], + "related_bugs": [ + "bug-465" + ], + "occurrences": 1, + "last_seen": "2026-06-13T01:30:00Z" + }, + { + "id": "bug-467", + "timestamp": "2026-06-13T01:38:08.308Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (rep_s.data_which == RequestReply_data_replconfigpu)", + "fix": "Changed to: if (0 /* REVIEWER RED CHECK */ && rep_s.data_which == )", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-13T01:38:43.310Z" + }, + { + "id": "bug-468", + "timestamp": "2026-06-13T03:56:09.185Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_reply.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T03:56:09.185Z" + }, + { + "id": "bug-469", + "timestamp": "2026-06-13T06:33:41.449Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_ha_cluster.c'", + "fix": "Changed to 'testsuite/test_ha_role_lifecycle.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T06:33:41.449Z" + }, + { + "id": "bug-470", + "timestamp": "2026-06-13T06:34:01.929Z", + "error_message": "Wrong reference: test_ha_cluster should be test_ha_role_lifecycle", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_ha_cluster\" instead of \"test_ha_role_lifecycle\"", + "fix": "Changed test_ha_cluster \u2192 test_ha_role_lifecycle", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-13T06:34:05.212Z" + }, + { + "id": "bug-471", + "timestamp": "2026-06-13T06:41:01.532Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "No early return/throw for edge case: pid < 0", + "fix": "Added guard clause: if (pid < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T06:41:01.532Z" + }, + { + "id": "bug-473", + "timestamp": "2026-06-13T07:32:43.287Z", + "error_message": "Wrong return value", + "file": "src/controller/persist.c", + "root_cause": "Was returning: WAL_MODE_LOCAL;", + "fix": "Now returns: WAL_MODE_RETAIN;", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T07:32:43.287Z" + }, + { + "id": "bug-476", + "timestamp": "2026-06-13T08:38:00.471Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: c->ha.hooks_registered", + "fix": "Added guard clause: if (c->ha.hooks_registered)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T08:38:00.471Z" + }, + { + "error_message": "AddressSanitizer: global-buffer-overflow in xce_wal_batch_req_raw <- repl_broadcast <- persist_wal_async <- ha_arm_own_term_gate <- ha_promote_to_primary, during test_promotion_registers_persist_hooks (test_ha_role_lifecycle.c case 3)", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", + "root_cause": "task-2.2's ha_promote_to_primary now appends a WAL_NOOP via persist_wal_async, which broadcasts to registered standbys. The lifecycle fixture (lifecycle_cluster_setup) registers fake `int g_lc_eps[]` endpoints via repl_add_standby and NEVER installs repl_broadcast_send_hook, so repl_broadcast calls xce_wal_batch_req_raw(s->ep) and reads ep->peer (offset 36) past the 4-byte int. IDENTICAL to the prior test_p2_integration tests 8/9 bug already in buglog.", + "fix": "Test-side fix (parent must apply, mirrors the documented p2_integration fix): in test_ha_role_lifecycle.c add `extern int (*repl_broadcast_send_hook)(void*, wal_catchup_entry_t**, int);` and a no-op drop hook, install it in lifecycle_cluster_setup before promotion (reset to NULL in teardown), and cast endpoints to (void*). Production ha.c is correct per design (persist_wal_async appends+broadcasts by design).", + "tags": [ + "ha-single-leader-role", + "task-2.2", + "test-fixture", + "asan", + "repl_broadcast", + "global-buffer-overflow", + "out-of-boundary" + ], + "id": "bug-477", + "timestamp": "2026-06-13", + "last_seen": "2026-06-13", + "occurrences": 1, + "related_bugs": [] + }, + { + "error_message": "task-2.1 lifecycle fixture off-by-one: case 5/6/7 assert writable_lsn==noop_lsn and current_lsn==noop_lsn+1, but lifecycle_cluster_setup primes current_lsn = appended_lsn + 1, so a real persist_wal_async appends the noop at appended_lsn+1 (current_lsn->appended_lsn+2), giving noop_lsn=appended_lsn+1 != asserted appended_lsn.", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c (fixture defect \u2014 OUT OF task 2.2 boundary)", + "root_cause": "persist_wal_async assigns hdr.lsn = current_lsn++ (entry takes the CURRENT value). The fixture comment assumes the next append takes current_lsn-1, which is off by one. To make the noop land at `appended_lsn`, the prime must be `current_lsn = appended_lsn`, not `appended_lsn + 1`.", + "fix": "Test-side fix (parent): set g_lc.persist.current_lsn = appended_lsn in lifecycle_cluster_setup. Production noop_lsn = current_lsn-1 (post-append) is correct per design \u00a7Promotion Rewrite / brief.", + "tags": [ + "ha-single-leader-role", + "task-2.2", + "test-fixture", + "off-by-one", + "lsn", + "out-of-boundary" + ], + "id": "bug-478", + "timestamp": "2026-06-13", + "last_seen": "2026-06-13", + "occurrences": 1, + "related_bugs": [] + }, + { + "id": "bug-480", + "timestamp": "2026-06-13T08:57:05.687Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T08:57:05.687Z" + }, + { + "id": "bug-482", + "timestamp": "2026-06-13T09:36:17.806Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_start_election_async.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T09:36:17.806Z" + }, + { + "id": "bug-483", + "timestamp": "2026-06-13T09:45:01.531Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T09:45:01.531Z" + }, + { + "id": "bug-484", + "timestamp": "2026-06-13T10:16:06.331Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-single-leader-role/tasks.md", + "root_cause": "Had `lifecycle_cluster_setup`", + "fix": "Changed to `ha_promote_to_primary`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T10:16:06.331Z" + }, + { + "id": "bug-493", + "timestamp": "2026-06-13T12:08:04.621Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (role == CTRL_PRIMARY || role == CTRL_PRIMARY_COMMI)", + "fix": "Changed to: if (role == CTRL_PRIMARY)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-13T12:09:17.994Z" + }, + { + "id": "bug-494", + "timestamp": "2026-06-13T12:10:08.938Z", + "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", + "file": "src/controller/ha.c", + "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", + "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T12:10:08.938Z" + }, + { + "id": "bug-496", + "timestamp": "2026-06-13T13:07:42.791Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Had \"winning election must land DIRECTLY in CTRL_PRIMA", + "fix": "Changed to \"winning election must land DIRECTLY in CTRL_PRIMA", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T13:07:42.791Z" + }, + { + "id": "bug-497", + "timestamp": "2026-06-13T13:10:05.452Z", + "error_message": "Wrong reference: CTRL_PRIMARY_COMMITTING should be CTRL_PRIMARY", + "file": "src/controller/testsuite/test_ha_connection_reuse.c", + "root_cause": "Used \"CTRL_PRIMARY_COMMITTING\" instead of \"CTRL_PRIMARY\"", + "fix": "Changed CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T13:10:05.452Z" + }, + { + "id": "bug-498", + "timestamp": "2026-06-13T13:10:16.164Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_connection_reuse.c", + "root_cause": "Had \"err_handler: ep must be removed from standby_endp", + "fix": "Changed to \"err_handler: ep must be removed from standby_endp", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T13:10:16.164Z" + }, + { + "id": "bug-500", + "timestamp": "2026-06-13T13:14:11.496Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Had \"test_dispatch_gate_committing_allows_repl_ops: PA", + "fix": "Changed to \"test_dispatch_gate_primary_no_quorum_allows_repl_", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T13:14:11.496Z" + }, + { + "id": "bug-501", + "timestamp": "2026-06-13T13:15:33.777Z", + "error_message": "Wrong reference: test_nonblocking_single_node_election_enters_committing should be test_nonblocking_single_node_election_enters_primary_directly", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Used \"test_nonblocking_single_node_election_enters_committing\" instead of \"test_nonblocking_single_node_election_enters_primary_directly\"", + "fix": "Changed test_nonblocking_single_node_election_enters_committing \u2192 test_nonblocking_single_node_election_enters_primary_directly", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T13:15:33.777Z" + }, + { + "id": "bug-503", + "timestamp": "2026-06-13T13:30:00.000Z", + "error_message": "test_dispatch_gate_primary_no_quorum_denies_job_ops: Expected -10013 Was 0 (jobsub allowed, expected -XCE_ERR_NOT_READY)", + "file": "src/controller/testsuite/test_ha_startup.c", + "root_cause": "Retargeting the dispatch-gate test from CTRL_PRIMARY_COMMITTING to CTRL_PRIMARY (ha-single-leader-role task 5.1) changed gate semantics: the deleted COMMITTING branch denied job ops UNCONDITIONALLY, but the CTRL_PRIMARY branch only denies job ops when ha_has_write_quorum() is false. ha_init leaves n_peers=0, so ha_has_write_quorum returns true (single-node = self-majority, ha.c:1437) and jobsub was ALLOWED (rc=0). A blind role swap does not reproduce the original deny.", + "fix": "Set ha.n_peers = 2 in both retargeted dispatch-gate tests so a multi-node primary with zero connected standbys genuinely lacks write quorum (connected 0 < n_peers/2=1), making the universal commit barrier deny job ops with -XCE_ERR_NOT_READY while still permitting replication ops.", + "tags": [ + "ha-single-leader-role", + "dispatch-gate", + "write-quorum", + "test-retarget", + "commit-barrier", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T13:30:00.000Z" + }, + { + "id": "bug-505", + "timestamp": "2026-06-13T14:01:32.553Z", + "error_message": "Wrong reference: uint32_t should be ha_config_t", + "file": "src/controller/ha.h", + "root_cause": "Used \"uint32_t\" instead of \"ha_config_t\"", + "fix": "Changed uint32_t \u2192 ha_config_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T14:01:32.553Z" + }, + { + "id": "bug-507", + "timestamp": "2026-06-13T14:43:54.194Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/ha-async-io.md", + "root_cause": "Had `CTRL_PRIMARY_COMMITTING \u2192 CTRL_PRIMARY`", + "fix": "Changed to 's own-term commit rule (\u00a75.4.2): an election winn", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T14:43:54.194Z" + }, + { + "id": "bug-508", + "timestamp": "2026-06-13T14:54:21.781Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-deploy-ha/SKILL.md", + "root_cause": "Had `\u2192 CTRL_PRIMARY_COMMITTING`", + "fix": "Changed to `HA FSM (raft): \u2192 CTRL_PRIMARY (term=...)`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-13T14:54:21.781Z" + }, + { + "id": "bug-509", + "timestamp": "2026-06-13T15:31:21.740Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_consistency_invariants.md", + "root_cause": "Had \"XCE HA Raft+CP umbrella \u2014 sub-specs 1&2 of 4 IMPL", + "fix": "Changed to \"XCE HA Raft+CP umbrella \u2014 sub-specs 1,2,3 of 4 IM", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-13T15:31:40.089Z" + }, + { + "id": "bug-510", + "timestamp": "2026-06-14T00:30:31.077Z", + "error_message": "Wrong reference: stdint should be stdbool", + "file": "src/controller/include/cluster.h", + "root_cause": "Used \"stdint\" instead of \"stdbool\"", + "fix": "Changed stdint \u2192 stdbool", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T00:30:31.077Z" + }, + { + "id": "bug-512", + "timestamp": "2026-06-14T00:34:38.745Z", + "error_message": "Wrong reference: xlist_destroy should be xlist_fini", + "file": "src/controller/testsuite/test_egress_filter.c", + "root_cause": "Used \"xlist_destroy\" instead of \"xlist_fini\"", + "fix": "Changed xlist_destroy \u2192 xlist_fini", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T00:34:38.745Z" + }, + { + "id": "bug-513", + "timestamp": "2026-06-14T00:35:13.711Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_egress_filter.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T00:35:13.711Z" + }, + { + "id": "bug-514", + "timestamp": "2026-06-14T00:37:31.926Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_egress_filter.c", + "root_cause": "No early return/throw for edge case: iter_make(&it, c->jobmgr", + "fix": "Added guard clause: if (iter_make(&it, c->jobmgr)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T00:37:31.926Z" + }, + { + "id": "bug-516", + "timestamp": "2026-06-14T01:46:09.600Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T01:46:09.600Z" + }, + { + "id": "bug-518", + "timestamp": "2026-06-14T01:50:24.429Z", + "error_message": "Missing guard clause", + "file": "src/controller/sched.c", + "root_cause": "No early return/throw for edge case: l->count > 0", + "fix": "Added guard clause: if (l->count > 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T01:50:24.429Z" + }, + { + "id": "bug-519", + "timestamp": "2026-06-14T01:52:56.833Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_egress_filter.c", + "root_cause": "Had \"\\n>> commit-gated scheduler egress tests (RED, ta", + "fix": "Changed to \"\\n>> commit-gated scheduler egress tests (GREEN, ", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T01:52:56.833Z" + }, + { + "id": "bug-521", + "timestamp": "2026-06-14T03:55:55.380Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "Had \"commit.h\"", + "fix": "Changed to \"cluster.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-14T03:58:21.222Z" + }, + { + "id": "bug-523", + "timestamp": "2026-06-14T04:21:18.071Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha_convergence.c", + "root_cause": "No early return/throw for edge case: ctx == NULL || ctx->c == NULL", + "fix": "Added guard clause: if (ctx == NULL || ctx->c == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T04:21:18.071Z" + }, + { + "id": "bug-343", + "timestamp": "2026-06-14T04:34:32+00:00", + "error_message": "AddressSanitizer: heap-use-after-free in ha_rejoin_reconcile_fn (ha_convergence.c:236) during test_rejoin_second_call_no_second_restart", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "dc_run_pending_task peeked tasks.head and ran entry(args) WITHOUT dequeuing. The once-guard correctly makes the 2nd ha_rejoin_reconcile schedule no new task, so head still pointed at the first (already-run) task whose body had freed its heap ctx (args). The 2nd dc_run_pending_task re-ran that task on the freed ctx. The convergence task bodies free their own args (production delaymgr killer frees only the delay_task_t node, never args; production execv never re-runs a one-shot).", + "fix": "Made dc_run_pending_task DEQUEUE via ordq_get(&mgr->tasks, NULL), run the body, then free(t)/free(it) \u2014 mirroring the real delaymgr worker (ordq_get -> runner_fn -> FREEUP for runcount==1). A 2nd call now returns DC_NO_TASK as production would. Production ha_rejoin_reconcile/ha_rejoin_reconcile_fn unchanged (the body-frees-ctx pattern is correct).", + "tags": [ + "asan", + "use-after-free", + "ha", + "rejoin", + "delaymgr", + "test-harness", + "ha-convergence-recovery", + "task-3.2" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T04:34:32+00:00" + }, + { + "id": "bug-526", + "timestamp": "2026-06-14T04:52:58.813Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-convergence-recovery/tasks.md", + "root_cause": "Had \"No space left on device\"", + "fix": "Changed to `/tmp/xce_*`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T04:52:58.813Z" + }, + { + "id": "bug-527", + "timestamp": "2026-06-14T04:57:22.178Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /*; * ha_walack_continuation_for_test: test-only entry point for", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-14T04:57:42.146Z" + }, + { + "id": "bug-532", + "timestamp": "2026-06-14T06:12:36.513Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha_convergence.c", + "root_cause": "No early return/throw for edge case: ha == NULL", + "fix": "Added guard clause: if (ha == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T06:12:36.513Z" + }, + { + "id": "bug-533", + "timestamp": "2026-06-14T06:37:35.904Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_convergence_verification.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: printf(\"\\n>> Regression pins: fan-out / snapshot fallback / ; \"re-inclusion (task 6.1 \u2014 PASS against current code)\\n\\n\"); | Also: /* \u2500\u2500 Fan-out counting send-hook (task 6.1 (a)/(d), umbrella; * repl_broadcast routes each enqueue through repl_broadcast_", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-14T06:40:03.882Z" + }, + { + "id": "bug-534", + "timestamp": "2026-06-14T06:38:38.792Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_convergence_verification.c", + "root_cause": "Had \"delay.h\"", + "fix": "Changed to \"controller.capnp.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T06:38:38.792Z" + }, + { + "id": "bug-536", + "timestamp": "2026-06-14T07:56:01.819Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_rdhup_integration.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T07:56:01.819Z" + }, + { + "id": "bug-537", + "timestamp": "2026-06-14T08:02:40.832Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T08:02:40.832Z" + }, + { + "id": "bug-538", + "timestamp": "2026-06-14", + "error_message": "config-less controller (my_config_tag==0) became PRIMARY after RDHUP and pushed empty config, breaking replication on a live 3-node cluster", + "file": "src/controller/ha.c", + "root_cause": "ha_on_primary_ctrl_fd_rdhup did not apply the no-config voter-only guard that ha_start_election_async has; after primary RDHUP it set CTRL_ELECTING, allocated elect_ctx and called raft_on_leader_lost even for tag=0 nodes. Two tag=0 nodes granted each other (freshness gate only denies LOWER tag; tag0==tag0 granted), electing a config-less primary.", + "fix": "Added an election-safety guard in ha_on_primary_ctrl_fd_rdhup after the WAL-truncate block and before elect_ctx alloc/hook/raft_on_leader_lost: if my_config_tag==0 && candidates_count>1, log + set role=CTRL_STANDBY and return (stays passive voter). Mirrors ha_start_election_async. Single-node (n_peers==1) unaffected.", + "tags": [ + "ha", + "election-safety", + "rdhup", + "config-bootstrap", + "Req2.8", + "Req10.5", + "voter-only-guard" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14" + }, + { + "id": "bug-542", + "timestamp": "2026-06-14T09:29:31.490Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_coldstart_lsn0.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T09:29:31.490Z" + }, + { + "id": "bug-543", + "timestamp": "2026-06-14T09:36:36.967406Z", + "error_message": "WAL HOLE on cold-start catch-up: a fresh EMPTY standby (standby_lsn=0) catching up to a primary holding [0,1] receives ONLY lsn 1, missing lsn 0", + "file": "src/controller/config_repl.c", + "root_cause": "ha_stream_wal_to_standby filter `if (hdr.lsn > standby_lsn)` excludes lsn 0 when standby_lsn==0 (0>0 is false). standby_lsn=0 is ambiguous: it means BOTH 'empty standby (applied nothing)' AND 'standby has applied lsn 0'. The streamer cannot distinguish, computes start_lsn=standby_lsn+1=1, and drops the genuine lsn-0 entry. Only non-cosmetic because lsn-0 entries can be REAL data (first-ever WAL entry of a fresh cluster is lsn 0 from a real write OR the promotion NOOP).", + "fix": "FIXED via reserve-lsn-0: persist_init() now sets current_lsn=1 on a fresh store (persist.c), so lsn 0 is reserved as the none/empty sentinel and the first-ever WAL entry (promotion NOOP) is lsn 1. The streamer filter hdr.lsn>standby_lsn now yields 1>0=true for an empty standby (standby_lsn=0), streaming the first entry \u2014 no hole. No streamer change needed; the reserved base makes the existing filter correct. Also fixes the uncommittable cold-start NOOP (now lsn 1, committable 0->1) and the own-term writable_lsn gate (now 1, effective). Test: test_coldstart_lsn0.c H1-H4 all green incl H4 no-hole. Base-shift fallout fixed in test_commit_engine, test_commit_barrier_rollout, test_commit_barrier_reply, test_p2_integration, test_egress_filter (removed now-redundant priming appends / wired engine).", + "tags": [ + "ha", + "replication", + "wal", + "cold-start", + "lsn-0", + "catch-up", + "correctness", + "off-by-one" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T10:00:00Z" + }, + { + "id": "bug-550", + "timestamp": "2026-06-14T11:38:09.614Z", + "error_message": "Wrong reference: test_pin_dispatch_gate_degraded_allowlist should be test_empty_standby_fresh_cluster_streams_not_snapshot", + "file": "src/controller/testsuite/test_convergence_verification.c", + "root_cause": "Used \"test_pin_dispatch_gate_degraded_allowlist\" instead of \"test_empty_standby_fresh_cluster_streams_not_snapshot\"", + "fix": "Changed test_pin_dispatch_gate_degraded_allowlist \u2192 test_empty_standby_fresh_cluster_streams_not_snapshot", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T11:38:09.614Z" + }, + { + "id": "bug-551", + "timestamp": "2026-06-14T11:45:02.411Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (standby_lsn < oldest_wal_lsn)", + "fix": "Changed to: if (standby_lsn + 1 < oldest_wal_lsn)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T11:45:02.411Z" + }, + { + "id": "bug-555", + "timestamp": "2026-06-14T12:23:52.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_repl_apply.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T12:23:52.351Z" + }, + { + "id": "bug-556", + "timestamp": "2026-06-14T12:25:37.723Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T12:25:37.723Z" + }, + { + "id": "bug-557", + "timestamp": "2026-06-14T12:26:04.137Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T12:26:04.137Z" + }, + { + "id": "bug-560", + "timestamp": "2026-06-14T19:54:55.428Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t \u2192 uint32_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T19:54:55.428Z" + }, + { + "id": "bug-561", + "timestamp": "2026-06-14T20:05:48.755409", + "error_message": "jobsub intermittent -ETIMEDOUT \"Connection timed out\": commit: WAIT deadline expired without commit (timeout_ms=50)", + "file": "src/include/config.h", + "root_cause": "DEFAULT_SYNC_TIMEOUT_MS=50 too tight for durable sync majority replication (each standby fdatasyncs before ACK); not configurable (controller always used persist_config_defaults).", + "fix": "Raised DEFAULT_SYNC_TIMEOUT_MS 50->500; added --sync-timeout-ms CLI flag plumbed arguments->myconf.sync_timeout_ms->pcfg.sync_timeout_ms (0=use default) in main.c/myconf.h/cluster.c.", + "tags": [ + "ha", + "commit-barrier", + "timeout", + "replication", + "jobsub", + "config" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T20:05:48.755409" + }, + { + "id": "bug-562", + "timestamp": "2026-06-14T23:55:00Z", + "error_message": "AddressSanitizer: heap-use-after-free in encode_RequestReply / strlen (lib/controller.capnp.c:13168) on ctrl-2 startup, via xce_reply->xce_encode from OPCode_nop handler (cluster.c:307). Also: job submission succeeds but joblist shows no jobs.", + "file": "src/lib/request.c", + "root_cause": "init_reply_by_req() only set seq+opcode, leaving the rest of the stack-declared xce_reply_t uninitialized. The Cap'n-C generated encode_RequestReply reads the common trailer field redirect_host UNCONDITIONALLY (gated only on != NULL), so a garbage stack pointer in reply.redirect_host was strlen'd. The garbage aliased a 4KB capnp segment freed moments earlier by the WAL-ack reply encode on the same thread -> UAF. Handlers that build a reply via init_reply_by_req and only set status/data_which/data (e.g. the NOP/state-report path) never cleared redirect_host. Empty joblist is downstream: ctrl-2 crash-looping on this UAF destabilizes replication/quorum, so a get request that fails over to the not-caught-up standby reads an empty jobmgr (standby serves get from its own committed state).", + "fix": "memset(reply,0,sizeof(*reply)) at the top of init_reply_by_req() (src/lib/request.c) before setting seq/opcode. All 30+ reply sites follow declare->init->populate, so zeroing in the shared initializer fixes the whole class (redirect_host/redirect_port/status/data garbage) at the source. Added regression test test_init_reply_by_req_zeroes_reply in src/lib/testsuite/test_request.c (poisons reply with 0xFF, asserts redirect_host==NULL etc).", + "tags": [ + "use-after-free", + "asan", + "capnp", + "encode_RequestReply", + "redirect_host", + "init_reply_by_req", + "ha", + "joblist", + "uninitialized-stack" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-14T23:55:00Z" + }, + { + "id": "bug-563", + "timestamp": "2026-06-15T09:16:53.679025Z", + "error_message": "jobsub returns ok (status 0, jobid) but joblist AND jobinfo show the job absent on a stable single primary (ctrl-1); reslist works.", + "file": "src/lib/request.c", + "root_cause": "SOLVED (reproduced locally via /xce-build + 3-node deploy-ha, ASAN). cluster_start() \u2014 the ONLY place that sets c->running=1 \u2014 is ORPHANED (zero callers; grep confirms). The HA startup path (myapp_run) starts iomgr/vtaskmgr/delaymgr/persist_start itself but never calls cluster_start, so c->running stays 0 (calloc default) on the primary. check_controller() (the streaming gate passed to xce_reply_get for joblist/workerlist) returns -EINVAL when !c->running; xce_reply_get then sets nc=0 (chunk count), shipping an EMPTY list to the client even though collect_jobs/key_find found the job. reslist passes checkf=NULL so it was immune. DIAG proof: 'reply_get send: table=2 pos=0 nc=0 nrows=1 checkstatus=-22' (pre-fix) -> 'nc=1 ... checkstatus=1' (post-fix). Single-node/legacy cluster_init path called cluster_start, which is why non-replicated worked.", + "fix": "Set c->running=1 at the TOP of cluster_run() (src/controller/cluster.c), symmetric with the existing c->running=0 on shutdown \u2014 the orphaned cluster_start() used to set it. c->running is a liveness/shutdown flag: check_controller() is polled PER CHUNK inside xce_reply_get()'s streaming loop so a long multi-chunk fetch (e.g. thousands of jobs) ABORTS mid-stream (nc=0) if a shutdown request arrives during it. The bug: running started 0 and was never set under HA, so the gate fired on every reply, not just during shutdown. Fix restores running=1 for the run-loop lifetime; the shutdown abort path (running=0 when cluster_run's cbuf_get loop breaks on the stop signal) is unchanged. Verified E2E on a local 3-node deploy-ha (ASAN): jobs now appear in joblist; DIAG showed checkstatus/nc flip -22/0 -> 1/1. Final fix is in cluster_run (not myapp_run \u2014 moved there so the 1<->0 lifecycle is co-located in the function that receives the shutdown signal).", + "tags": [ + "joblist", + "workerlist", + "jobsub", + "ha", + "check_controller", + "c-running", + "cluster_start", + "orphaned-function", + "xce_reply_get", + "SOLVED" + ], + "related_bugs": [ + "bug-562" + ], + "occurrences": 1, + "last_seen": "2026-06-15T09:16:53.679025Z" + }, + { + "id": "bug-570", + "timestamp": "2026-06-15T11:10:10.553Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/product.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to `src/lib/raft/`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-15T11:10:10.553Z" + }, + { + "id": "bug-572", + "timestamp": "2026-06-15T11:10:26.312Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/tech.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to `src/lib/raft/`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-15T11:10:26.312Z" + }, + { + "id": "bug-575", + "timestamp": "2026-06-16T00:10:49.295Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-prevote/research.md", + "root_cause": "Had 't understand PreVote (older binary) \u2014 does the ca", + "fix": "Changed to 's \"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T00:10:49.295Z" + }, + { + "id": "bug-576", + "timestamp": "2026-06-16T00:18:28.793Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-prevote/research.md", + "root_cause": "Had 's \"", + "fix": "Changed to `ha_heartbeat_is_fresh()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T00:18:28.793Z" + }, + { + "id": "bug-578", + "timestamp": "2026-06-16T03:00:48.818Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-prevote/design.md", + "root_cause": "Had `ha_elect_ctx_t`", + "fix": "Changed to `raft_on_vote_reply`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 9, + "last_seen": "2026-06-16T03:02:59.202Z" + }, + { + "id": "bug-579", + "timestamp": "2026-06-16T03:01:07.659Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/ha-prevote/design.md", + "root_cause": "Was returning: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", + "fix": "Now returns: codes (e.g. `RAFT_PRE_VOTE_WON`, `RAFT_PRE_VOTE_LO", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T03:01:07.660Z" + }, + { + "id": "bug-584", + "timestamp": "2026-06-16T04:27:50.039Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T04:27:50.039Z" + }, + { + "id": "bug-585", + "timestamp": "2026-06-16T04:36:28.796Z", + "error_message": "Missing guard clause", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "No early return/throw for edge case: rc != 0", + "fix": "Added guard clause: if (rc != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T04:36:28.796Z" + }, + { + "id": "bug-586", + "timestamp": "2026-06-16T04:50:38.855Z", + "error_message": "Missing error handling in struct", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 3.1 \u2014 voter-side PreVote dispatch gate (stickiness +; * no mutation, reply term unchanged) */", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-16T04:50:49.601Z" + }, + { + "id": "bug-591", + "timestamp": "2026-06-16T05:27:09.380Z", + "error_message": "Wrong reference: test_commit_engine should be test_ha_prevote", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_prevote\"", + "fix": "Changed test_commit_engine \u2192 test_ha_prevote", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-16T05:27:18.877Z" + }, + { + "id": "bug-592", + "timestamp": "2026-06-16T05:28:01.236Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c || !c->myconf", + "fix": "Added guard clause: if (!c || !c->myconf)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-16T05:30:07.209Z" + }, + { + "id": "bug-593", + "timestamp": "2026-06-16T05:44:57.430Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-prevote/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T05:44:57.430Z" + }, + { + "id": "bug-594", + "timestamp": "2026-06-16T05:50:25.729Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_commit_engine.c'", + "fix": "Changed to 'testsuite/test_ha_failover_prevote.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T05:50:25.730Z" + }, + { + "id": "bug-595", + "timestamp": "2026-06-16T05:50:35.123Z", + "error_message": "Wrong reference: test_commit_engine should be test_ha_failover_prevote", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_commit_engine\" instead of \"test_ha_failover_prevote\"", + "fix": "Changed test_commit_engine \u2192 test_ha_failover_prevote", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-16T05:50:39.775Z" + }, + { + "id": "bug-597", + "timestamp": "2026-06-16T05:59:43.939Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T05:59:43.939Z" + }, + { + "id": "bug-598", + "timestamp": "2026-06-16T06:33:05.644Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: role == CTRL_STANDBY && ha->raft.pre_vote_in_progress", + "fix": "Added guard clause: if (role == CTRL_STANDBY && ha->raft.pre_vot)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T06:33:05.644Z" + }, + { + "id": "bug-603", + "timestamp": "2026-06-16T08:18:48.567Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T08:18:48.567Z" + }, + { + "id": "bug-604", + "timestamp": "2026-06-16T15:55:00Z", + "error_message": "controller unit test TIMEOUT 300.01s killed by SIGTERM; suite exceeds 300s meson cap", + "file": "src/controller/persist.c", + "root_cause": "fsync_worker slept the full wal_sync_ms in one uninterruptible nanosleep() then checked fsync_stop. persist_stop sets the flag and immediately pthread_join()s, so the join blocked until the in-progress sleep returned \u2014 up to a full wal_sync_ms per teardown. Tests set wal_sync_ms=5000 ('slow fsync, no races'), so each persist_stop cost ~5s; across ~7 suites with many fixtures the cumulative teardown time blew past the 300s controller-test cap. The 'commit: WAIT deadline expired' warnings in the log were unrelated intentional negative tests (50-300ms).", + "fix": "Replaced the nanosleep with pthread_cond_timedwait on a new fsync_mutex/fsync_cond (added to persist_t, init in persist_init, destroyed in persist_fini). persist_stop now sets fsync_stop and pthread_cond_signal(fsync_cond) under fsync_mutex before join, so the worker wakes immediately. fdatasync still runs under p->lock, released before re-acquiring fsync_mutex (no lock nesting). Full controller suite now completes in ~10s (was >300s); 0 failures.", + "tags": [ + "controller", + "persist", + "fsync", + "thread-shutdown", + "test-timeout", + "pthread_join", + "nanosleep", + "condvar", + "ha" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T15:55:00Z" + }, + { + "id": "bug-605", + "timestamp": "2026-06-16T16:30:00Z", + "error_message": "3-node fresh HA deploy never elects a primary: all nodes stay CTRL_STANDBY, config-holder (ctrl-1) loops PreVote LOST every ~11s forever; config-less peers idle as voter-only. REPRODUCED via /xce-deploy-ha + clean restart 2026-06-16.", + "file": "src/controller/ha.c (ha_dispatch_gate); test: src/controller/testsuite/test_ha_role.c", + "root_cause": "ha_dispatch_gate() admitted OPCode_requestVote in CTRL_INIT/CTRL_ELECTING/CTRL_STANDBY but NEVER OPCode_preVoteRequest (opcode 38). ha-prevote added the new opcode + handler (cluster.c:440) but missed wiring it into the role-admission gate. So a STANDBY voter rejected an incoming PreVote with -XCE_ERR_STANDBY (-10014) BEFORE the handler ran -> no preVoteReply sent -> candidate's failed/empty exchange tallied as a denial (pre_vote_reply_decode_and_dispatch counts ok=0 as non-grant; vote_ep_err_handler synthesizes a denied vote). In a cold cluster (config-holder + config-less voter-only peers) every PreVote 'lost' with NO voter ever making a liveness decision (zero 'PreVote DENIED' logs), so no primary was ever elected and the config-holder looped PreVote forever. Confirmed via gdb: myreq_handler ha_dispatch_gate rc=-10014 on opcode 38. Design invariant: a PreVote (strictly non-mutating) must be at least as admissible as a real requestVote in every role.", + "fix": "Added OPCode_preVoteRequest alongside OPCode_requestVote in ha_dispatch_gate() for CTRL_INIT, CTRL_ELECTING, and the CTRL_STANDBY switch (src/controller/ha.c). PreVote now reaches the cluster.c handler and a proper preVoteReply is sent. Added TDD test test_dispatch_gate_allows_prevote_in_voter_roles (test_ha_role.c) asserting gate(preVoteRequest)==gate(requestVote)==0 across voter roles. VERIFIED: controller unit suite OK (new test PASS); fresh /xce-deploy-ha 3-node cold start now elects ctrl-1 PRIMARY (term=2), standbys establish primary channel + config committed, prevote-LOST=0, connections 96->8.", + "tags": [ + "ha", + "prevote", + "ha-dispatch-gate", + "opcode-38", + "cold-start", + "bootstrap-deadlock", + "FIXED", + "XCE_ERR_STANDBY" + ], + "related_bugs": [ + "bug-339" + ], + "occurrences": 1, + "last_seen": "2026-06-16T09:38:00Z" + }, + { + "id": "bug-606", + "timestamp": "2026-06-16T17:46:00Z", + "error_message": "After killing the primary (ctrl-1), the two surviving standbys (ctrl-2,ctrl-3) livelock in synchronized elections: PreVote win \u2192 CTRL_ELECTING \u2192 become_follower (term bumped) \u2192 ~10s \u2192 repeat, term climbing unbounded (3\u21929\u2192\u221e), no new primary ever elected. Reproduce: deploy 3-node HA, elect primary, submit 3-5 jobs, kill ctrl-1.", + "file": "src/controller/ha.c (ha_begin_failover, ha_election_stagger_ms, ha_failover_stagger_fn); src/controller/ha.h; test: src/controller/testsuite/test_ha_role.c", + "root_cause": "Dueling-candidates split-vote livelock \u2014 Raft \u00a75.2 randomized election timeout is missing on the FAILOVER path. Both survivors lose the primary at the same instant; their liveness polls fire on the identical fixed heartbeat_timeout, so they start PreVote+real-election in perfect lock-step (ms-aligned timestamps). Both win PreVote (neither sees a primary, so each grants the other \u2014 PreVote cannot break this tie), then both campaign at the same term, each votes for itself and denies the other (quorum 2, only 2 live nodes) \u2192 split vote \u2192 election times out (fixed election_timeout_ms, raft.c:105, NO jitter) \u2192 become_follower, term+1 \u2192 repeat forever. The priority/jitter config (election_jitter_max_ms, election_priority_step_ms) is applied ONLY at startup (ha_wait_for_heartbeat_or_timeout); the failover re-election path (liveness_poll_fn \u2192 ha_on_primary_ctrl_fd_rdhup \u2192 PreVote \u2192 ha_start_election_async \u2192 raft_start_election) has no de-synchronization. raft_on_leader_lost DOES have a priority stagger (raft.c:421) but the failover loop does not go through it.", + "fix": "Added ha_election_stagger_ms() (ha.c/ha.h): per-node failover delay = my_priority*election_priority_step_ms + random[0,election_jitter_max_ms], re-randomized each call (mirrors the startup stagger). ha_begin_failover() now schedules the PreVote start via delaymgr after that stagger (production only; c==NULL/no-delaymgr stays synchronous so existing tests are unaffected). The delaymgr callback ha_failover_stagger_fn guards with role==STANDBY && ha_heartbeat_is_fresh \u2192 skips PreVote if a primary connected during the stagger. Effect: two survivors no longer campaign in lock-step; the higher-priority survivor wins and its heartbeats reset the other. TDD: test_election_stagger_desyncs_by_priority (bounds + step>jitter ordering). VERIFIED on kill-primary repro: ctrl-2 (stagger 630ms) won \u2192 PRIMARY term=1; ctrl-3 (stagger 1072ms) received ctrl-2's ctrlhello first \u2192 STANDBY, then its stagger timer hit the guard and skipped PreVote ('a primary is alive \u2014 skipping PreVote'). No term inflation, no livelock. NOTE: optional raft.c:105 give-up-watchdog randomization NOT needed \u2014 the failover-start stagger alone resolves it; left as future defense-in-depth.", + "tags": [ + "ha", + "raft", + "election", + "split-vote", + "dueling-candidates", + "livelock", + "randomized-timeout", + "failover", + "stagger", + "jitter", + "FIXED" + ], + "related_bugs": [ + "bug-605" + ], + "occurrences": 1, + "last_seen": "2026-06-16T10:04:00Z" + }, + { + "id": "bug-608", + "timestamp": "2026-06-16T10:36:00Z", + "error_message": "libxce client never fails over after primary killed; xce_joblist returns 'Invalid argument' (EINVAL) indefinitely", + "file": "src/lib/request.c", + "root_cause": "Two gaps: (1) xce_request_with_retry() (request.c:1956) is implemented+unit-tested but NO production path calls it - every xce_*_req wrapper calls plain xce_request() (request.c:1385 etc). (2) The retry primitive keys on E_XCE_STANDBY + reply.redirect_host/port (request.c:2010), but the real controller emits a dedicated E_XCE_REDIRECT status with data_which=nodata (join.c:264-266) and never populates those trailer fields. Controller non-primary taxonomy is E_XCE_REDIRECT / E_XCE_NOMASTER / E_XCE_NOT_READY / E_XCE_STANDBY (errnox.h:22-25). So the client both bypasses the wrapper AND, even if wired, would not detect the controller's actual redirect signal.", + "fix": "Spec libxce-failover: wire all production request paths through transparent failover; classify on actual controller statuses (REDIRECT=fast-rotate to find existing primary, address-if-present else rotate master list; NOMASTER/NOT_READY=backoff then cluster-not-ready; STANDBY=rotate); realign the existing retry unit tests to the emitted statuses. Implementation pending (requirements phase).", + "tags": [ + "libxce", + "failover", + "ha", + "E_XCE_REDIRECT", + "retry", + "client", + "protocol-mismatch" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T10:36:00Z" + }, + { + "id": "bug-609", + "timestamp": "2026-06-16T10:52:01.800Z", + "error_message": "After primary killed, libxce client xce_joblist/jobsubmit returns 'Invalid argument' (-EINVAL) on every fresh call \u2014 transparent failover did not engage", + "file": "src/lib/client_failover.c", + "root_cause": "handle_disconnect leaves xce_ctx_t.state=XCE_CTX_ERR after the primary's connection drops; the per-op xceapi guard (xceapi.c:14) returns -EINVAL when state!=XCE_CTX_RUN, BEFORE the op reaches a primary. xcecli_retry_decide classified -EINVAL as TERMINAL (non-retriable), so the failover loop bailed without rotating/recovering. The ctx->state ERR->RUN recovery only ran AFTER a rotate, which a terminal classification never triggered. Surfaced by the live 3-node E2E (task 6.2); the unit tests fed status codes directly to decide() and bypassed the real guard, so they missed it.", + "fix": "In xcecli_retry_decide, before classifying *rc, detect ac->ctx.state==XCE_CTX_ERR (known-dead connection) and treat it as a prompt transport-style failover cause (XCE_FO_RETRY_NOW, no backoff) regardless of *rc, so the loop rotates+recovers. On budget exhaustion surface -E_XCE_NOT_READY (cluster-not-ready), never -EINVAL. Regression: test_decide_known_dead_ctx_retries_not_terminal.", + "tags": [ + "md", + "libxce", + "failover", + "EINVAL", + "XCE_CTX_ERR", + "reconnect", + "E2E" + ], + "related_bugs": [ + "bug-608" + ], + "occurrences": 2, + "last_seen": "2026-06-17T04:45:00Z" + }, + { + "id": "bug-610", + "timestamp": "2026-06-16T10:52:36.535Z", + "error_message": "ASan SEGV in endpoint_connect (netio.c:672) via xce_failover_rotate \u2192 xcecli_retry_next \u2192 xcecli_joblist during live master failover (NULL ep->iomgr deref)", + "file": "src/lib/client_failover.c", + "root_cause": "xce_failover_rotate reconnected via endpoint_fini + endpoint_init + endpoint_connect, but endpoint_init (do_endpoint_init) resets the endpoint struct including ep->iomgr, and the rotate never re-bound it. The client's endpoint gets ep->iomgr = &ctx->iomgr only via xce_ctx_set_endpoint (called once in xcecli_start). endpoint_connect derefs iomgr->ssl_ctx when ep->ssl==NULL (true after endpoint_fini frees the SSL) \u2192 NULL deref crash. Also the rotate never re-set owner/msg_handler/err_handler nor iomgr_monitor'd the new fd, so even without the crash the reconnected endpoint would not route replies. Pure unit tests could not exercise the live socket/iomgr path; only the live failover E2E hit it.", + "fix": "Added xce_ctx_reconnect(ctx,host,port) in xcectx.c (where static handle_msg/handle_disconnect live) that mirrors the connection-establishment of xcecli_start/xce_ctx_start without restarting iomgr/vtaskmgr or touching ctx->state: endpoint_fini \u2192 endpoint_init \u2192 endpoint_set_peer \u2192 re-bind ep->iomgr=&ctx->iomgr \u2192 endpoint_connect \u2192 re-set owner+msg/err handlers \u2192 iomgr_monitor. xce_failover_rotate now delegates the reconnect to it.", + "tags": [ + "async-fix", + "md", + "libxce", + "failover", + "crash", + "SEGV", + "endpoint_connect", + "iomgr", + "reconnect", + "E2E" + ], + "related_bugs": [ + "bug-609", + "bug-608" + ], + "occurrences": 2, + "last_seen": "2026-06-17T15:00:00Z" + }, + { + "id": "bug-613", + "timestamp": "2026-06-16T13:15:11.048Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T13:15:11.048Z" + }, + { + "id": "bug-614", + "timestamp": "2026-06-16T13:18:29.113Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Had `xce_ctx_call`", + "fix": "Changed to `xcecli_retry_next`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-16T13:18:35.874Z" + }, + { + "id": "bug-617", + "timestamp": "2026-06-16T23:38:24.000Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Had `redirect_host/port`", + "fix": "Changed to `host/port`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-16T23:38:53.585Z" + }, + { + "id": "bug-619", + "timestamp": "2026-06-16T23:40:22.003Z", + "error_message": "Wrong reference: address should be hint", + "file": ".kiro/specs/libxce-failover/tasks.md", + "root_cause": "Used \"address\" instead of \"hint\"", + "fix": "Changed address \u2192 hint", + "tags": [ + "auto-detected", + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-16T23:40:22.003Z" + }, + { + "id": "bug-623", + "timestamp": "2026-06-17T00:04:51.775Z", + "error_message": "Wrong reference: XCE_MAXHOST_LEN should be XCE_MAXHOSTNAME_LEN", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Used \"XCE_MAXHOST_LEN\" instead of \"XCE_MAXHOSTNAME_LEN\"", + "fix": "Changed XCE_MAXHOST_LEN \u2192 XCE_MAXHOSTNAME_LEN", + "tags": [ + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T00:04:51.775Z" + }, + { + "id": "bug-624", + "timestamp": "2026-06-17T02:35:29.695Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'dataq.c'", + "fix": "Changed to 'client_failover.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T02:35:29.695Z" + }, + { + "id": "bug-626", + "timestamp": "2026-06-17T02:48:27.547Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL || host_out == NULL || host_sz == 0 || port_out =", + "fix": "Added guard clause: if (ac == NULL || host_out == NULL || host_s)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T02:48:27.547Z" + }, + { + "id": "bug-629", + "timestamp": "2026-06-17T03:18:22.365Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: base >= (double", + "fix": "Added guard clause: if (base >= (double)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T03:18:22.365Z" + }, + { + "id": "bug-631", + "timestamp": "2026-06-17T03:31:29.120Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL", + "fix": "Added guard clause: if (ac == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T03:31:29.120Z" + }, + { + "id": "bug-633", + "timestamp": "2026-06-17T03:40:10.853Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL", + "fix": "Added guard clause: if (ac == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T03:40:10.853Z" + }, + { + "id": "bug-635", + "timestamp": "2026-06-17T03:51:08.767Z", + "error_message": "Wrong return value", + "file": "src/lib/client.c", + "root_cause": "Was returning: xce_reslist(&(ctx->ctx), res);", + "fix": "Now returns: XCECLI_FAILOVER_CALL(ctx, /*is_write=*/false,", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 8, + "last_seen": "2026-06-17T03:51:29.554Z" + }, + { + "id": "bug-637", + "timestamp": "2026-06-17T03:57:33.232Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/libxce-failover/design.md", + "root_cause": "Had `nop`", + "fix": "Changed to `jobinfo`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-17T03:57:40.348Z" + }, + { + "id": "bug-638", + "timestamp": "2026-06-17T04:02:05.572Z", + "error_message": "Missing guard clause", + "file": "src/lib/client_failover.c", + "root_cause": "No early return/throw for edge case: ac == NULL", + "fix": "Added guard clause: if (ac == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T04:02:05.572Z" + }, + { + "id": "bug-639", + "timestamp": "2026-06-17T04:11:59.759Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_p1_integration.c'", + "fix": "Changed to 'testsuite/test_join_redirect.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T04:11:59.759Z" + }, + { + "id": "bug-640", + "timestamp": "2026-06-17T04:12:01.203Z", + "error_message": "Wrong reference: test_p1_integration should be test_join_redirect", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_p1_integration\" instead of \"test_join_redirect\"", + "fix": "Changed test_p1_integration \u2192 test_join_redirect", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-17T04:12:02.464Z" + }, + { + "id": "bug-641", + "timestamp": "2026-06-17T04:19:58.697Z", + "error_message": "Wrong reference: test_recover_resets_err_state_to_run should be test_decide_jitter_staggers_many_clients", + "file": "src/lib/testsuite/test_libxce_retry.c", + "root_cause": "Used \"test_recover_resets_err_state_to_run\" instead of \"test_decide_jitter_staggers_many_clients\"", + "fix": "Changed test_recover_resets_err_state_to_run \u2192 test_decide_jitter_staggers_many_clients", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T04:19:58.697Z" + }, + { + "id": "bug-644", + "timestamp": "2026-06-17T08:15:04.978Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'testsuite/test_raft.c'", + "fix": "Changed to 'testsuite/test_netio_arm_failure.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T08:15:04.978Z" + }, + { + "id": "bug-646", + "timestamp": "2026-06-17T08:21:32.099Z", + "error_message": "xce_test jobsub hangs forever after a failover; main thread blocked in cbuf_get (xceapi.c:304) while io_worker sits idle in epoll_wait. Logs: 'rotate failed (rc=-111)' then 'endpoint_update_epoll rc1=-1 < 0'.", + "file": "src/lib/netio.c", + "root_cause": "endpoint_update_epoll() swallowed iomgr_update_poll() failures: it WARN'd 'rc1=-1' but returned 0 and left ep->flags unchanged, so the poll interest was never armed. iomgr_update_poll uses EPOLL_CTL_MOD, which fails with ENOENT when the fd is no longer registered (e.g. after a failed reconnect: xce_ctx_reconnect endpoint_fini's the old fd but only iomgr_monitor/EPOLL_CTL_ADDs on a SUCCESSFUL connect). The next endpoint_enq() send then MOD-failed silently, request bytes were never flushed, no reply arrived, and the synchronous client blocked forever in cbuf_get (no timeout) with no socket-drop event to wake it (handle_disconnect only fires on a broken socket). Server side was a red herring: DEFAULT_SYNC_TIMEOUT_MS=500 bounds the commit barrier, and ha_dispatch_gate already replies STANDBY/NOT_READY for non-serving roles.", + "fix": "endpoint_update_epoll() captures errno and RETURNS -errno (not 0) on iomgr_update_poll failure. endpoint_enq(): on negative arm result, run handle_error(ep) (iomgr_unmonitor + handle_disconnect -> state=ERR + inject -ECONNRESET into wbuf, waking the blocked cbuf_get) and return the error so the failover loop rotates. handle_output(): propagates the arm failure so io_handler runs the error path. TDD: test_netio_arm_failure.c drives EPOLL_CTL_MOD ENOENT via endpoint_enq on an unregistered fd; asserts loud rc<0 + err_handler fired (verified red without fix, green with).", + "tags": [ + "wrong-reference", + "c", + "libxce", + "failover", + "netio", + "epoll", + "EPOLL_CTL_MOD", + "ENOENT", + "silent-failure", + "hang", + "cbuf_get", + "reconnect" + ], + "related_bugs": [ + "bug-610", + "bug-609" + ], + "occurrences": 2, + "last_seen": "2026-06-17T16:30:00Z" + }, + { + "id": "bug-647", + "timestamp": "2026-06-17T08:21:56.735Z", + "error_message": "Wrong reference: 611 should be 646", + "file": "src/lib/netio.c", + "root_cause": "Used \"611\" instead of \"646\"", + "fix": "Changed 611 \u2192 646", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T08:21:56.735Z" + }, + { + "id": "bug-648", + "timestamp": "2026-06-17T17:30:00Z", + "error_message": "Peer-close/EOF on an SSL connection was swallowed: io_fsm_do() set fsm->state=FSM_STATE_ERROR but returned rc=0 on a clean EOF, so handle_input() returned >=0, handle_error/handle_disconnect never ran, and a blocked cbuf_get() reply-waiter was not woken on the first close event.", + "file": "src/lib/netio.c", + "root_cause": "io_fsm_do() computed rc = -errno when a closed/reset connection was detected. By OpenSSL 3.x contract (verified against docs + a live probe on OpenSSL 3.6.0), a peer close makes SSL_read() return 0 and is reported via SSL_get_error(), NOT via the return value or errno: clean close_notify -> SSL_ERROR_ZERO_RETURN (errno 0); unexpected EOF (FIN, no close_notify) -> SSL_ERROR_SSL since OpenSSL 3.0 (pre-3.0 it was SSL_ERROR_SYSCALL with errno 0 -- documented); RST -> SSL_ERROR_SYSCALL errno=ECONNRESET. errno is only meaningful for SSL_ERROR_SYSCALL; for SSL_ERROR_SSL the error queue is authoritative and any errno (we observed EPIPE=32) is a leaked side effect of OpenSSL's internal alert-write (write() on a broken connection returns EPIPE; read() never does). So -errno is 0 for the documented EOF cases -> the close looked like success. NOTE: on this 3.6 build a process-death FIN surfaced as SSL_ERROR_SSL/errno=32, which the OLD code propagated -- so this swallow is NOT necessarily the cause of the observed controller-kill permanent hang (that still implicates the reconnect/SSL-handshake path); it is a real, separate, errno-dependent non-determinism.", + "fix": "Extracted pure io_classify_io_error(ssl_err, ecode, io_rc): returns 0 only for WANT_READ/WANT_WRITE and EAGAIN/EWOULDBLOCK syscall; otherwise returns a NEGATIVE error ((ecode!=0)?-ecode:-ECONNRESET) so a detected close NEVER returns 0 regardless of the unreliable errno. io_fsm_do() now sets FSM_STATE_ERROR iff the classifier returns <0. TDD: test_netio_close_propagation.c asserts ZERO_RETURN/errno0 and SSL_ERROR_SSL/EOF map to <0 (verified red with old -ecode, green with fix). libxce suite 168/0; controller_unit_test links clean.", + "tags": [ + "libxce", + "netio", + "ssl", + "SSL_read", + "EOF", + "close_notify", + "openssl-3.x", + "SSL_ERROR_ZERO_RETURN", + "SSL_ERROR_SSL", + "silent-failure", + "hang" + ], + "related_bugs": [ + "bug-646", + "bug-610" + ], + "occurrences": 1, + "last_seen": "2026-06-17T17:30:00Z" + }, + { + "id": "bug-651", + "timestamp": "2026-06-17T10:44:37.350Z", + "error_message": "Wrong return value", + "file": "src/lib/request.c", + "root_cause": "Was returning: 0;", + "fix": "Now returns: (enq_rc < 0) ? enq_rc : 0;", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T10:44:37.350Z" + }, + { + "id": "bug-650", + "timestamp": "2026-06-17T18:30:00Z", + "error_message": "xce_test jobsub hangs in cbuf_get (xceapi.c:304) after failover even with bug-646/648 deployed. Log shows 'endpoint marked dead' (bug-646 firing) then a permanent hang; target primary (ctrl-2) is fully idle (no vtask in handle_jobsubmit) and the client fd is an unconnected TCP socket \u2014 the request was never transmitted yet the client waits for a reply forever.", + "file": "src/lib/request.c", + "root_cause": "xce_request() and the 5 other request.c senders (xce_reply, xce_replconfigpush_req_raw, xce_wal_batch_req_raw, xce_snapshot_chunk_req_raw, xce_replconfig_reply_raw) called endpoint_enq(ep,&pkt->link) and IGNORED its return, always `return 0`. So when the transport was dead, endpoint_enq() correctly returned <0 and ran handle_error() (bug-646 fix) \u2014 but xce_request() swallowed that <0 and reported success. xce_jobsubmit() (xceapi.c:299-304) then saw rc>=0 and fell through to the BLOCKING cbuf_get(); the request was never flushed (server idle) and when err_handler was NULL after a failed reconnect, handle_disconnect never injected -ECONNRESET, so nothing woke the waiter. The bug-646 netio fix was defeated one layer up: the request layer reported success on a failed send. This was the missing link making every prior netio fix insufficient for the live hang.", + "fix": "All six request.c senders propagate the enq result: `int enq_rc = endpoint_enq(ep,&pkt->link); return (enq_rc < 0) ? enq_rc : 0;`. A failed send now returns <0 through xce_jobsubmit_req -> xce_jobsubmit, which returns before cbuf_get; the failover loop classifies the transport error and rotates instead of blocking on a reply that never arrives. TDD: test_request_send_propagation.c builds an OPCode_nop request (encodes OK) on a dead endpoint (EPOLL_CTL_MOD ENOENT) and asserts xce_request returns <0 (verified red with old `return 0`, green with fix). libxce 169/0; controller_unit_test links clean.", + "tags": [ + "libxce", + "request", + "xce_request", + "failover", + "silent-failure", + "hang", + "cbuf_get", + "endpoint_enq", + "send-propagation" + ], + "related_bugs": [ + "bug-646", + "bug-648" + ], + "occurrences": 1, + "last_seen": "2026-06-17T18:30:00Z" + }, + { + "id": "bug-654", + "timestamp": "2026-06-17T11:21:51.571Z", + "error_message": "Wrong return value", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Was returning: a negative error and (2) invoke the endpoint's err", + "fix": "Now returns: a negative, RETRIABLE error so the caller propagat", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T11:21:51.571Z" + }, + { + "id": "bug-656", + "timestamp": "2026-06-17T11:22:17.916Z", + "error_message": "Wrong reference: test_enq_arm_failure_is_loud_and_fires_errhandler should be test_enq_arm_failure_is_loud_no_errhandler", + "file": "src/lib/testsuite/test_netio_arm_failure.c", + "root_cause": "Used \"test_enq_arm_failure_is_loud_and_fires_errhandler\" instead of \"test_enq_arm_failure_is_loud_no_errhandler\"", + "fix": "Changed test_enq_arm_failure_is_loud_and_fires_errhandler \u2192 test_enq_arm_failure_is_loud_no_errhandler", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T11:22:17.916Z" + }, + { + "id": "bug-652", + "timestamp": "2026-06-17T19:10:00Z", + "error_message": "Controller (primary) self-deadlocks on ha->lock after bug-646/650 deploy: jobsub now works, but the controller freezes. gdb shows Thread 3 in primary_heartbeat_fn holding ha->lock (owner=self) and re-acquiring it via xce_nop_req -> xce_request -> endpoint_enq -> handle_error -> vote_ep_err_handler (ha.c:3478); Thread 9 (io_worker) also blocked on the same ha->lock. Whole controller frozen.", + "file": "src/lib/netio.c", + "root_cause": "REGRESSION from bug-646: endpoint_enq() invoked handle_error() synchronously on the send path when the poll arm failed. handle_error() calls the endpoint's err_handler, which on a controller vote endpoint is vote_ep_err_handler() \u2014 it locks ha->lock. But endpoint_enq runs on the CALLER's thread, and the controller's primary_heartbeat_fn already holds ha->lock when it sends heartbeat NOPs. Non-recursive mutex -> self-deadlock. The err_handler is contractually an io_worker-thread callback (off an epoll error event, holding no caller locks); calling it from the synchronous send path violated that. With bug-650 now propagating the enq error up the return chain, the handle_error call in endpoint_enq was both redundant and harmful.", + "fix": "endpoint_enq() no longer calls handle_error(); on arm failure it just returns a RETRIABLE -ECONNRESET. The caller propagates it (request.c senders return it -> client failover classifies ECONNRESET as RETRY_NOW and rotates; controller heartbeat path simply gets the error, no lock re-entry). The real teardown stays on the io_worker thread (handle_input/handle_output -> io_handler -> handle_error), which holds no caller locks. Updated test_netio_arm_failure.c: endpoint_enq must return <0 AND must NOT invoke the err_handler (g_err_count==0). libxce 169/0; controller_unit_test links clean.", + "tags": [ + "libxce", + "netio", + "endpoint_enq", + "deadlock", + "self-deadlock", + "ha-lock", + "reentrancy", + "regression", + "controller", + "vote_ep_err_handler" + ], + "related_bugs": [ + "bug-646", + "bug-650" + ], + "occurrences": 1, + "last_seen": "2026-06-17T19:10:00Z" + }, + { + "id": "bug-658", + "timestamp": "2026-06-17T14:46:54.135Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (role == CTRL_STANDBY && ha_heartbeat_is_fresh(ha)", + "fix": "Changed to: if (role == CTRL_STANDBY && ha->primary_ctrl_fd >= 0 &)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T14:46:54.135Z" + }, + { + "id": "bug-659", + "timestamp": "2026-06-17T14:47:25.446Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (role == CTRL_STANDBY &&\n )", + "fix": "Changed to: if (role == CTRL_STANDBY &&\n )", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T14:47:25.446Z" + }, + { + "id": "bug-660", + "timestamp": "2026-06-17T22:00:00Z", + "error_message": "Failover election takes minutes (11+ PreVote rounds) to converge. A standby keeps denying a legitimately-higher-term candidate's PreVote with reason=fresh_heartbeat for ~2.5 min AFTER its primary died (ctrl-3 killed 21:44:14, ctrl-2 logs 'PreVote DENIED ... reason=fresh_heartbeat (my_term=3)' until 21:46:58).", + "file": "src/controller/ha.c", + "root_cause": "The PreVote/vote eligibility gate raft_cb_vote_eligible (ha.c:311) denied on `role==CTRL_STANDBY && ha_heartbeat_is_fresh(ha)` \u2014 checking freshness ALONE. ha_heartbeat_is_fresh derives from ha->last_activity, but ha_arm_heartbeat_timeout_vtask resets last_activity=now on EVERY watchdog (re-)arm, and ha_prevote_rearm_liveness re-arms it after every lost PreVote. So a node with NO live primary perpetually refreshes its own 'fresh heartbeat' each failover cycle and denies all other candidates (reason=fresh_heartbeat), blocking convergence until it happens to restart. The sibling gate in handle_request_vote (ha.c:2205) already required `primary_ctrl_fd >= 0 && is_fresh`; raft_cb_vote_eligible was missing the primary_ctrl_fd guard. The existing fresh-deny tests even documented the intent ('the deny gate requires a live connection AND a fresh heartbeat', set primary_ctrl_fd=999), but production didn't enforce it.", + "fix": "Added `ha->primary_ctrl_fd >= 0 &&` to the ha.c:311 eligibility gate (mirrors ha.c:2205). primary_ctrl_fd is -1 on init / after the primary RDHUPs / after the liveness watchdog fires, and a real fd only via config_repl.c:1451 (first ctrlhello from the live primary) \u2014 so a dead/absent primary no longer masquerades as fresh and the standby GRANTS, converging in ~1 round. Kept cluster.c:526 denial-reason logging in sync. TDD: test_pre_vote_dead_primary_grants_despite_fresh_activity (fresh last_activity + primary_ctrl_fd=-1 -> GRANT); verified RED (Expected 1 Was 0) without the guard, GREEN with it, and the 3 existing fresh-deny tests (fd=999) unaffected. Full controller suite + libxce 169/0, zero regressions.", + "tags": [ + "controller", + "ha", + "election", + "prevote", + "convergence", + "fresh_heartbeat", + "raft_cb_vote_eligible", + "primary_ctrl_fd", + "slow-election" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T22:00:00Z" + }, + { + "id": "bug-661", + "timestamp": "2026-06-17T23:30:00Z", + "error_message": "3-node failover test lost committed jobs: 7 earliest JobIds (WAL LSN<=20), majority-ACKed and survived first failover, permanently gone after a restarted node with near-empty WAL won election. joblist dropped from 10 to 3.", + "file": "src/controller/persist.c, src/controller/commit.c, src/controller/repl.c", + "root_cause": "'committed' does not imply 'durable'. WAL fsync is asynchronous (persist.c:173 fsync_worker, timer wal_sync_ms); persist_append and the standby WalAck path do NOT fdatasync before ACK. commit_on_ack (commit.c:303) advances committed_lsn on majority ACK = OS page cache only, durable on zero disks. On kill+restart, killed nodes lose the not-yet-fsynced tail; persist_recover hits CRC break at zeroed region (prev_crc=00000000, offset 40) and truncates replay to ~0 entries, yet the hollow node stays eligible, wins election under degraded required=1 quorum (PreVote+freshness gate cannot fence it because the committed entries were durable on nobody), self-promotes advertising committed_lsn=21 over an empty job table, and the node holding the real data converges DOWN. Violates Leader-Completeness assumption (majority-ACK != durable-on-majority) in src/controller/CLAUDE.md.", + "fix": "NOT YET FIXED. Discovery -> new spec ha-commit-durability (Path C, brief written). R1 fsync-before-ACK (standby fdatasync before WalAck, primary self-count only after local fdatasync, committed_lsn advances only on durable majority, group-commit to bound latency). R2 promotion fence (WAL CRC/replay failure => ineligible to lead until caught up, FATAL not silent truncate; never lead below known committed high-water even under degraded quorum). Live E2E reproduces via xce_test jobsub + primary kills on /xce-deploy-ha. DETERMINISTIC REPRO (task 1.3, 2026-06-18): `bash src/tools/test-commit-durability.sh --deterministic` launches the cluster with --wal-sync-ms 30000 (plumbed via deploy-ha.sh start.sh `$*` passthrough, EXTRA_CTRL_ARGS) so the primary's fsync_worker never fires in-window -> every committed entry is page-cache-only on the primary (majority_candidate_locked counts cap=highest_appended_lsn=current_lsn-1, NOT durable). It then drives rolling PRIMARY kills (each right after committing to that primary) + restart, so hollow nodes re-lead. RED CONFIRMED: 6-round run lost the 3 EARLIEST committed JobIds permanently (rounds 4 and 6 + final check), Results 5 passed/3 failed, exit=1 \u2014 matching the incident signature (earliest jobs gone). NOTE: a single double-kill does NOT repro \u2014 this codebase enforces strict majority (lone survivor will NOT self-elect: log shows 'no single-voter primary within 40s'); loss needs the ROLLING/term-accumulating path the incident used. Evidence log: dur-det-run.log in worktree.", + "tags": [ + "controller", + "ha", + "durability", + "fsync", + "wal", + "commit-engine", + "data-loss", + "failover", + "leader-completeness", + "promotion-fence", + "async-fsync", + "deterministic-repro", + "wal-sync-ms" + ], + "related_bugs": [ + "bug-339", + "bug-447", + "bug-396" + ], + "occurrences": 2, + "last_seen": "2026-06-18T00:00:00Z" + }, + { + "id": "bug-662", + "timestamp": "2026-06-17T23:34:32.115Z", + "error_message": "Function not marked async", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/MEMORY.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-17T23:34:32.115Z" + }, + { + "id": "bug-664", + "timestamp": "2026-06-17T23:59:44.577Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"${2:-6}\"", + "fix": "Changed to \"${2:-8}\"", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-17T23:59:50.630Z" + }, + { + "id": "bug-666", + "timestamp": "2026-06-18T02:32:07.670Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T02:32:07.670Z" + }, + { + "id": "bug-667", + "timestamp": "2026-06-18T02:35:08.157Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T02:35:08.157Z" + }, + { + "id": "bug-668", + "timestamp": "2026-06-18T03:12:26.259Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `persist.c:515`", + "fix": "Changed to `fsync_worker`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-18T03:15:00.270Z" + }, + { + "id": "bug-671", + "timestamp": "2026-06-18T03:15:13.287Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `fsync_worker`", + "fix": "Changed to `/kiro-validate-design`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:15:13.287Z" + }, + { + "id": "bug-673", + "timestamp": "2026-06-18T03:50:26.761Z", + "error_message": "Function not marked async", + "file": "src/controller/CLAUDE.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T03:50:26.761Z" + }, + { + "id": "bug-676", + "timestamp": "2026-06-18T04:00:06.872Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `src/include/persist.h`", + "fix": "Changed to `persist_wal_sync_append`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:00:06.872Z" + }, + { + "id": "bug-677", + "timestamp": "2026-06-18T04:39:24.460Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t \u2192 uint32_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:39:24.460Z" + }, + { + "id": "bug-678", + "timestamp": "2026-06-18T04:50:36.884Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'testsuite/test_commit_durability.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:50:36.884Z" + }, + { + "id": "bug-679", + "timestamp": "2026-06-18T04:51:30.430Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T04:51:30.430Z" + }, + { + "id": "bug-682", + "timestamp": "2026-06-18T05:02:07.556Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"$(mktemp /tmp/xce-durability-acked.XXXXXX)\"", + "fix": "Changed to \"$(mktemp \"", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:02:07.556Z" + }, + { + "id": "bug-684", + "timestamp": "2026-06-18T05:28:05.352Z", + "error_message": "Function not marked async", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "sh" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-18T05:29:29.811Z" + }, + { + "id": "bug-686", + "timestamp": "2026-06-18T05:43:02.761Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T05:45:15.732Z" + }, + { + "id": "bug-688", + "timestamp": "2026-06-18T05:54:46.908Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T05:54:55.010Z" + }, + { + "id": "bug-690", + "timestamp": "2026-06-18T05:55:33.474Z", + "error_message": "Wrong reference: else should be RUN_TEST", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Used \"else\" instead of \"RUN_TEST\"", + "fix": "Changed else \u2192 RUN_TEST", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T05:55:33.474Z" + }, + { + "id": "bug-692", + "timestamp": "2026-06-18T05:59:17.869Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-18T06:00:42.436Z" + }, + { + "id": "bug-693", + "timestamp": "2026-06-18T06:05:00.000Z", + "error_message": "durable_lsn advanced even when fdatasync FAILED (EIO/ENOSPC) at two of three sites (flusher_flush_once, persist_stop), letting a non-durable LSN count toward the commit majority", + "file": "src/controller/persist.c", + "root_cause": "ha-commit-durability task 2.1 flush sites called fdatasync(p->wal_fd) without capturing/checking the return code, then advanced durable_lsn unconditionally. On sync error the bytes are not on stable storage, violating Req 1.1/1.2 and the fail-loud preference.", + "fix": "Capture int rc = fdatasync(...) at every durable_lsn advance site (flusher_flush_once gating + fallback paths, persist_stop) and advance durable_lsn ONLY when rc == 0; on error leave durable_lsn unchanged and log loudly via ERROR(). persist_flush already gated on rc==0 (reference). Added XCE_TESTING fault-injection seam (test_force_fsync_fail + persist_test_set_force_fsync_fail + flusher_fsync_forced_fail helper, constant-false in production) and error-path test test_durable_lsn_holds_on_fsync_failure proving durable_lsn holds at baseline under injected failure and recovers when cleared.", + "tags": [ + "ha-commit-durability", + "durability", + "fdatasync", + "fail-loud", + "remediation", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:05:00.000Z" + }, + { + "id": "bug-694", + "timestamp": "2026-06-18T06:09:08.298Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", + "root_cause": "Had `/kiro-spec-init ha-commit-durability`", + "fix": "Changed to 't fence the stale node because the entries were d", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:09:08.298Z" + }, + { + "id": "bug-695", + "timestamp": "2026-06-18T06:12:59.924Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:12:59.924Z" + }, + { + "id": "bug-696", + "timestamp": "2026-06-18T06:13:32.891Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:13:32.891Z" + }, + { + "id": "bug-697", + "timestamp": "2026-06-18T06:13:40.659Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:13:40.659Z" + }, + { + "id": "bug-699", + "timestamp": "2026-06-18T06:14:16.158Z", + "error_message": "Function not marked async", + "file": "src/controller/commit.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:14:16.158Z" + }, + { + "id": "bug-701", + "timestamp": "2026-06-18T06:15:45.248Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Had \"config.h\"", + "fix": "Changed to \"commit.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:15:45.248Z" + }, + { + "id": "bug-704", + "timestamp": "2026-06-18", + "error_message": "controller suite regressions after capping commit self-contribution at durable_lsn: test_repl_majority/test_commit_barrier_rollout/test_coldstart_lsn0/test_demotion_convergence got -ETIMEDOUT / committed_lsn=0 (Expected N Was 0)", + "file": "src/controller/testsuite/{test_repl_majority,test_commit_barrier_rollout,test_coldstart_lsn0,test_demotion_convergence}.c", + "root_cause": "ha-commit-durability 2.2 changed majority_candidate_locked self-cap from highest_appended_lsn to persist_durable_lsn. Existing commit-path fixtures prime current_lsn (appended tip) but never set durable_lsn, so under the new cap self contributed 0 and committed_lsn never advanced.", + "fix": "In every commit-path stub/in-memory-primed fixture, set durable_lsn to the appended tip (appended_lsn / current_lsn-1) right after committed_lsn=0, preserving the pre-durability 'self covers every appended entry' model. The durability precondition itself is exercised by dedicated new tests.", + "tags": [ + "durable", + "commit", + "majority", + "self-cap", + "test-fixture", + "ha-commit-durability", + "task-2.2" + ], + "related_bugs": [ + "bug-661", + "bug-693" + ], + "occurrences": 1, + "last_seen": "2026-06-18" + }, + { + "id": "bug-705", + "timestamp": "2026-06-18T06:33:12.523Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T06:33:25.815Z" + }, + { + "id": "bug-707", + "timestamp": "2026-06-18T06:34:06.875Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p", + "fix": "Added guard clause: if (p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:34:06.875Z" + }, + { + "id": "bug-708", + "timestamp": "2026-06-18T06:34:36.356Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:34:36.356Z" + }, + { + "id": "bug-711", + "timestamp": "2026-06-18T06:50:49.987Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: written_lsn > durable", + "fix": "Added guard clause: if (written_lsn > durable)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T06:50:49.987Z" + }, + { + "id": "bug-714", + "timestamp": "2026-06-18T07:00:36.008Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: written_lsn > durable", + "fix": "Added guard clause: if (written_lsn > durable)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T07:00:36.008Z" + }, + { + "id": "bug-720", + "timestamp": "2026-06-18T07:52:20.595Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 3.2 \u2014 Boot Policy Decision Table skeletons (IGNORED,; * 3.3/4.3). Registered so each row is reported as IGNORED, n", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T07:52:29.952Z" + }, + { + "id": "bug-721", + "timestamp": "2026-06-18T08:04:18.857Z", + "error_message": "Missing error handling in enum", + "file": "src/include/persist.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Bounded catch-up window (ha-commit-durability task 3.3, R; * The FINITE deadline a recovery-truncated / behind member i", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T08:04:38.131Z" + }, + { + "id": "bug-726", + "timestamp": "2026-06-18T08:19:47.036Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:19:47.036Z" + }, + { + "id": "bug-728", + "timestamp": "2026-06-18T08:39:54.889Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had `durable_floor \u2264 applied_lsn`", + "fix": "Changed to `applied_lsn`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:39:54.889Z" + }, + { + "id": "bug-732", + "timestamp": "2026-06-18T08:51:34.635Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "No early return/throw for edge case: !mkdtemp(tmp", + "fix": "Added guard clause: if (!mkdtemp(tmp)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:51:34.635Z" + }, + { + "id": "bug-733", + "timestamp": "2026-06-18T08:52:12.840Z", + "error_message": "Wrong return value", + "file": "src/controller/ha.c", + "root_cause": "Was returning: persist_durable_floor(p);", + "fix": "Now returns: floor > 8ULL ? floor : 8ULL; /* BUG: pretends lear", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T08:52:12.840Z" + }, + { + "id": "bug-735", + "timestamp": "2026-06-18T09:08:09.215Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* \u2500\u2500 ha-commit-durability task 4.3: pre-promotion eligibili; * Refuse promotion BEFORE the node becomes authoritative CTR", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:08:55.973Z" + }, + { + "id": "bug-736", + "timestamp": "2026-06-18T09:09:28.348Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Empty first-boot WAL: durable=0, HWM=0 -> ELIGIBLE. A fr; * member is not behind anything and must be promotable (no f", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:10:32.435Z" + }, + { + "id": "bug-738", + "timestamp": "2026-06-18T09:45:40.163Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:50:15.505Z" + }, + { + "id": "bug-739", + "timestamp": "2026-06-18T09:46:18.019Z", + "error_message": "Function not marked async", + "file": "src/include/persist.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:46:36.395Z" + }, + { + "id": "bug-740", + "timestamp": "2026-06-18T09:46:46.814Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:47:04.913Z" + }, + { + "id": "bug-741", + "timestamp": "2026-06-18T09:49:58.840Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Test-injectable send hook (defined in repl.c under XCE_TE; * repl_broadcast from dereferencing the dummy endpoints as r", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T09:50:07.979Z" + }, + { + "id": "bug-743", + "timestamp": "2026-06-18T10:25:02.673Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.h", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ha_cancel_promotion_fence_timer: cancel a pending bounded; * (idempotent; no-op when none is armed). Called when the m", + "tags": [ + "auto-detected", + "error-handling", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T10:27:05.112Z" + }, + { + "id": "bug-745", + "timestamp": "2026-06-18T10:26:34.227Z", + "error_message": "Wrong operator: != should be ==", + "file": "src/controller/ha.c", + "root_cause": "Used \"!=\" instead of \"==\"", + "fix": "Changed operator != \u2192 ==", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:26:34.227Z" + }, + { + "id": "bug-746", + "timestamp": "2026-06-18T10:30:18.073Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_commit_durability.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* task 4.4 \u2014 bounded-window catch-up before FATAL (no synch; * deferred-timer FATAL only after a real bounded failure; fe", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-18T10:30:51.746Z" + }, + { + "id": "bug-755", + "timestamp": "2026-06-18T10:55:31.898Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/design.md", + "root_cause": "Had \"bounded window\"", + "fix": "Changed to `DEFAULT_RECOVERY_CATCHUP_WINDOW_MS`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T10:55:31.898Z" + }, + { + "id": "bug-759", + "timestamp": "2026-06-18T11:21:44.844Z", + "error_message": "Incorrect value in code", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_ha_commit_durability.md", + "root_cause": "Had `--wal-sync-ms`", + "fix": "Changed to `:FAIL:`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T11:21:44.844Z" + }, + { + "id": "bug-760", + "timestamp": "2026-06-18T13:00:00Z", + "error_message": "test_commit_barrier_rollout.c:446 test_fact_hook_append_is_retained_and_nonblocking FAIL: Expected 1 Was 0 \u2014 a covering ACK must advance committed_lsn over the retained fact (6/6 deterministic)", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "ha-commit-durability task 2.2 capped the primary commit self-contribution at durable_lsn (advanced asynchronously by the demand-driven group-commit flusher) and wired a durable-advance commit-wake callback. This adjacent-spec (ha-commit-barrier-rollout) test uses a REAL persist+flusher and checks committed_lsn SYNCHRONOUSLY the instant after repl_on_ack(fact_lsn); the flusher has not yet fdatasync'd LSN1, so durable_lsn==0, self-cap==0, majority candidate==0, committed_lsn stays 0. Pre-spec the test passed (self counted the appended tip immediately). Task 2.2 primed durable_lsn in the in-memory stub fixtures (barrier_setup_vcpus) but did NOT update this real-flusher test. The implementation-note claim that this is a pre-existing /tmp-overlay fdatasync-EIO artifact is FALSE in this env: fdatasync returns rc=0 on /tmp here, and the test passed pre-spec.", + "fix": "FIXED \u2014 test_commit_barrier_rollout.c:445: insert persist_flush(g_fp) + assert persist_durable_lsn>=fact_lsn BEFORE repl_on_ack, so the covering ACK (commit_on_ack runs synchronously inside repl_on_ack) deterministically completes the durable majority. Mirrors task 2.2's durable_lsn priming of the in-memory barrier_setup_vcpus stubs. Verified: was 6/6 deterministic FAIL, now fact_hook PASS across 4 runs + meson controller suite Ok:1 Fail:0, all groups 0 Failures. Production was never affected (RETAIN facts have no waiter; WAIT-mode replies block on commit barrier commit.c:334).", + "tags": [ + "ha-commit-durability", + "commit-barrier-rollout", + "durable_lsn", + "group-commit", + "test-race", + "async-durability", + "validation", + "NO-GO" + ], + "related_bugs": [ + "bug-661", + "bug-704" + ], + "occurrences": 6, + "last_seen": "2026-06-18T13:30:00Z" + }, + { + "id": "bug-761", + "timestamp": "2026-06-18T13:13:49.968Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_commit_barrier_rollout.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T13:13:49.968Z" + }, + { + "id": "bug-762", + "timestamp": "2026-06-18T13:18:43.638Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T13:18:43.638Z" + }, + { + "id": "bug-763", + "timestamp": "2026-06-18T14:54:25.478Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T14:54:25.478Z" + }, + { + "id": "bug-765", + "timestamp": "2026-06-18T14:30:00Z", + "error_message": "scheduler-controller-msg-gap marked 36/36 [x] but NOT truly implemented: schedUpdateJob notify path hits the bulk CREATE path (sched_update_existing_job is dead code) \u2192 corrupts/duplicates live job state, no -ENOENT; Req 3 broken at dequeue, Req 6 stores nothing (rusage never stored).", + "file": "src/scheduler/ (sched_update_existing_job dead; schedUpdateJob \u2192 CREATE path)", + "root_cause": "Tasks were checked [x] without the dequeue/update path and rusage storage actually being wired; gap surfaced only by cross-task re-audit (per-task review missed it).", + "fix": "FIXED 2026-06-18 (commits 5c1f715,93124c5,080ff78,276e26c): G-8/G-4 (route notify job-update to in-place sched_update_existing_job, all mutable fields, -ENOENT no-create); G-5 (store rusage on xce_sched_job_t via sched_apply_job_rusage + free_JobRusages_ptr); G-3 (free all 5 killer op payloads with generated deep-freers); G-7 (doc reconciliation \u2014 sched_op_name stays static in scheduler.c, lib must not depend on scheduler headers); G-6 (real white-box tests: test_handlers.c 6/6 for 9.2/9.3, test_job_update.c 10/10 for 9.4/9.5/9.7, registered as meson tests). Scheduler suite 31/31 GREEN, ASAN-clean, boundary-clean. validate-impl GO. RESIDUAL (documented, non-blocking): 9.6 dequeue worker-resvals APPLY unit test (dormant path R-3 \u2014 enqueue tested), 9.8 documented compile-time absence, 9.9 log-capture gated, 9.2 -ENOMEM fault-injection.", + "tags": [ + "scheduler", + "controller", + "msg-gap", + "validate-gap", + "not-implemented", + "HIGH", + "job-state-corruption" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-18T17:00:00Z" + }, + { + "id": "bug-766", + "timestamp": "2026-06-19T02:55:49.212Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "Was returning: `-ENOENT` if job not found", + "fix": "Now returns: `-ENOENT` if job not found. _Remediation (bug-765,", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T02:55:49.212Z" + }, + { + "id": "bug-768", + "timestamp": "2026-06-19T03:30:28.056Z", + "error_message": "Wrong condition in logic", + "file": "src/scheduler/src/schedobj.c", + "root_cause": "Condition was: if (job->rusages != NULL && job->rusages != rusages)", + "fix": "Changed to: if (job->rusages != rusages)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:30:28.056Z" + }, + { + "id": "bug-772", + "timestamp": "2026-06-19T03:49:23.624Z", + "error_message": "Wrong reference: xce_sched_op_name should be sched_op_name", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "Used \"xce_sched_op_name\" instead of \"sched_op_name\"", + "fix": "Changed xce_sched_op_name \u2192 sched_op_name", + "tags": [ + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T03:49:23.624Z" + }, + { + "id": "bug-773", + "timestamp": "2026-06-19T03:49:39.373Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/scheduler-controller-msg-gap/design.md", + "root_cause": "Had `xce_sched_op_name()`", + "fix": "Changed to `sched_op_name()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T03:49:46.161Z" + }, + { + "id": "bug-774", + "timestamp": "2026-06-19T03:50:12.460Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/scheduler-controller-msg-gap/tasks.md", + "root_cause": "Had `xce_sched_op_name()`", + "fix": "Changed to `sched_op_name()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T03:50:20.916Z" + }, + { + "id": "bug-777", + "timestamp": "2026-06-19T13:00:00Z", + "error_message": "3-node rolling-SIGKILL soak round 3: 'no new primary elected within 40s'; soak reports 'DATA LOSS 5/5 jobs vanished'. Both surviving nodes recovered RECOVER_TRUNCATED after SIGKILL (torn WAL tail) and stay permanently fenced/non-promotable even though durable_floor==cluster_hwm (they hold ALL committed data).", + "file": "src/controller/ha.c (raft_cb_become_leader promotion fence release) + persist.c (recovery_truncated verdict stickiness); design ha-commit-durability Req 2.2/2.6 + Boot Policy Decision Table", + "root_cause": "A SIGKILL tears the WAL tail; on restart replay_wal hits a CRC-break/short-read at the tail -> RECOVER_TRUNCATED (sticky flag). The promotion-fence release condition requires `!recovery_truncated`, but the truncated flag is NEVER cleared by catch-up \u2014 so a node whose intact durable prefix (durable_floor) already reaches/exceeds the cluster committed high-water (durable_floor>=hwm, i.e. holds ALL committed entries; the torn tail is entirely UNCOMMITTED) stays permanently non-promotable. The design assumed 'TRUNCATED => provably below HWM', which is FALSE for a torn TAIL above the committed prefix. When all survivors are in this state (after multiple rolling failovers), none can promote => election/availability WEDGE. Observed: ctrl-1 recovery_truncated=1 durable_floor=13 cluster_hwm=13, PreVote-DENY peers reason=catch_up, indefinitely.", + "fix": "DATA-LOSS dimension RESOLVED by the bug-778 fix (commit 64ab210): the cold-restart loss was the CRC-chain clobber; with the chain intact, cold restart recovers all committed jobs (soak 5/5, joblist 6/6 after cold restart). The AVAILABILITY-WEDGE dimension was a CONSEQUENCE of the spurious RECOVER_TRUNCATED from the broken chain \u2014 no longer triggered (nodes now recover CLEAN; round-3 re-elects normally). RESIDUAL HARDENING (latent, not currently triggered): the promotion fence keeps recovery_truncated sticky even when durable_floor>=HWM, so a GENUINE torn-tail that leaves a node RECOVER_TRUNCATED while holding all committed data could still over-fence. Optional follow-up: clear the truncated fence when durable_floor>=cluster_hwm (the torn tail is provably uncommitted). Not blocking \u2014 the systematic trigger is gone.", + "tags": [ + "ha", + "convergence", + "promotion-fence", + "recovery-truncated", + "torn-tail", + "SIGKILL", + "availability", + "election-wedge", + "ha-commit-durability", + "ha-convergence-recovery", + "live-E2E", + "false-positive-data-loss" + ], + "related_bugs": [ + "bug-661", + "bug-339" + ], + "occurrences": 2, + "last_seen": "2026-06-19T14:30:00Z" + }, + { + "id": "bug-778", + "timestamp": "2026-06-19T13:40:00Z", + "error_message": "persist_recover RECOVER_TRUNCATED: 'CRC chain break ... prev_crc=00000000 expected=' at the first replicated job entry on cold restart; committed jobs unrecoverable (joblist=0). ROOT CAUSE of bug-777's cold-restart data loss.", + "file": "src/controller/persist.c:1336 (apply_wal_entry) + src/controller/ha.c ha_apply_single_entry (~1570-1588)", + "root_cause": "On the standby replicated-apply path, ha_apply_single_entry first calls persist_wal_write_replicated() which correctly sets p->last_crc to the standby's SELF-computed chain crc (the stream carries no crc; the on-disk CRC chain is node-local, prev_crc==last_crc). It THEN builds a synthetic wal_entry_hdr with hdr.crc=0 and calls apply_wal_entry(), whose line 1336 `p->last_crc = hdr->crc` OVERWRITES the correct chain link with 0. The next replicated write computes prev_crc=(seg_offset==0?WAL_CRC_NONE:last_crc)=0 and stamps prev_crc=0 on disk. So the on-disk CRC chain breaks at the 2nd replicated entry. Invisible while live (standby serves in-memory entities; never replays its own WAL), detonates on COLD RESTART where replay_wal validates the chain -> RECOVER_TRUNCATED -> recovers only the clean prefix -> committed jobs lost + the truncated verdict drives the promotion-fence availability wedge (the other half of bug-777). NOT an fd-position/offset race (pwrite+p->lock are correct); it is shared CRC-chain state (p->last_crc) written by two paths where the logical-apply step clobbers the physical-writer's link with 0.", + "fix": "FIXED 2026-06-19 (commit 64ab210). apply_wal_entry is now LOGICAL entity-restore only (signature (p,type,lsn,payload,payload_len); no header); it advances the current_lsn high-water but NEVER touches p->last_crc. The CRC chain has a single owner: persist_wal_write_replicated (live) + replay_wal/persist_recover (recovery). ha_apply_single_entry drops the fabricated crc=0 header. Verified: RED->GREEN test test_replicated_apply_chain_survives_cold_restart (real ha_apply_wal_catchup_entry path, cold-replay -> RECOVER_CLEAN); controller suite 0 failures; 3-node rolling-SIGKILL soak on real ext2/3 fs 5/5 (was wedge+loss at round 3); full cold restart recovers all 6 jobs (was 0), CLEAN verdicts, normal election. Test callers updated: test_repl_apply, test_ha_role_lifecycle, test_egress_filter.", + "tags": [ + "ha", + "wal", + "crc-chain", + "last_crc", + "apply_wal_entry", + "persist_wal_write_replicated", + "cold-restart", + "data-loss", + "bug-661", + "standby-replication", + "recovery", + "root-caused" + ], + "related_bugs": [ + "bug-777", + "bug-661" + ], + "occurrences": 2, + "last_seen": "2026-06-19T14:30:00Z" + }, + { + "id": "bug-781", + "timestamp": "2026-06-19T14:02:49.235Z", + "error_message": "Wrong reference: hdr should be payload_len", + "file": "src/controller/persist.c", + "root_cause": "Used \"hdr\" instead of \"payload_len\"", + "fix": "Changed hdr \u2192 payload_len", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T14:02:49.235Z" + }, + { + "id": "bug-785", + "timestamp": "2026-06-19T14:08:12.866Z", + "error_message": "Wrong reference: NULL should be type", + "file": "src/controller/testsuite/test_ha_role_lifecycle.c", + "root_cause": "Used \"NULL\" instead of \"type\"", + "fix": "Changed NULL \u2192 type", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-19T14:09:08.241Z" + }, + { + "id": "bug-786", + "timestamp": "2026-06-19T15:00:00Z", + "error_message": "xce_test jobsub prints 'jobsubmit ok: JobId: 63-15--1-1015808' (uninitialized/garbage jobid) on repeated/mid-failover submits; controller then logs 'handle_jobsubmit fail to lock job 63-15--1-...'. Client claims success with a malformed id it never got a clean ack for.", + "file": "src/controller/testsuite/test.c (cmd_jobsub)", + "root_cause": "On the 2nd+ submit (and during failover windows) the jobsub reply/jobid is not cleanly populated, leaving an uninitialized jobid struct; cmd_jobsub still prints 'jobsubmit ok' with the garbage id instead of verifying the controller reply status and failing. Forces E2E scripts (test-commit-durability.sh) to strict-regex-filter the 4-field id to avoid chasing phantom jobs.", + "fix": "PENDING \u2014 jobsub must report success ONLY on a clean ack with a valid jobid; otherwise print a typed error (REDIRECT/NOMASTER/NOT_READY/TIMEOUT) and set non-zero status. Part of the xce-test-e2e-primitives work (exit-code-as-verdict + $jobid auto-var). | STRENGTHENED 2026-06-19 (commit 8609976): `jobid != 0` gate was insufficient \u2014 a stale non-zero reply id (63-15--1-1015808) from a non-jobid reply during failover slipped through. New gate requires the echoed jobid == the locally idgen-generated jobsubmit.jobid (controller commits/echoes exactly that id, job.c:616). Surfaced by the live durability soak (task 7.2).", + "tags": [ + "xce_test", + "jobsub", + "garbage-jobid", + "e2e", + "client", + "test-tooling", + "xce-test-e2e-primitives", + "stale-reply-union", + "live-soak" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-19" + }, + { + "id": "bug-788", + "timestamp": "2026-06-19T15:34:41.623Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (active_ctx->parent != NULL)", + "fix": "Changed to: if (i > 1)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T15:34:41.623Z" + }, + { + "id": "bug-792", + "timestamp": "2026-06-19T16:13:02.715Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"shutdown\"", + "fix": "Changed to \"expect-jobs\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-19T16:13:02.715Z" + }, + { + "id": "bug-795", + "timestamp": "2026-06-19", + "error_message": "Cold-restart-all (kill ALL 3 controllers + restart) non-deterministically loses a committed job: a job that returned 'jobsubmit ok' and survived every rolling single-node failover round vanishes after a full-cluster restart (expect-jobs expected N, actual N-2; expect-job not found). Recovery logs show WAL truncate.", + "file": "src/controller/{persist.c recovery/truncate, commit.c commit-gate, ha.c promotion-fence/election} \u2014 UPSTREAM ha-commit-durability Phase 6.2", + "root_cause": "Commit-durability race under TOTAL-cluster-outage recovery. Acked (committed) entry not durable on a fault-tolerant majority before all nodes die, OR recovery truncates committed entries on cold restart. Non-deterministic (passed 1 run, failed next). Surfaced on overlayfs sandbox (NOT a real fsync-able fs) \u2014 must be reconfirmed on real hardware. UPDATE: with the hardened (trustworthy) soak, the loss ALSO reproduces in ROLLING single-node failover with the cluster confirmed serving (~1/3 runs) \u2014 broader than cold-restart-all; bug-661 not fully closed for rolling failover. Separate finding: >60s post-kill election (liveness) every run, always recovers. CORRECTION: NOT an overlay-fdatasync artifact \u2014 reproduces OUTSIDE the container (user's real host) AND on an in-sandbox real ext2/3 fs with ZERO fdatasync EIO. Real controller durability bug + a coupled liveness stall (>60s / stuck election after killing ctrl-1). Phase 6.1 answered (REAL); now at 6.2 root-cause.", + "fix": "NOT FIXED HERE \u2014 out of boundary for xce-test-e2e-primitives (this spec changes no controller/replication code). Routed to spec ha-commit-durability owner. The converted durability soak (src/tools/test-commit-durability.sh, new run_cold_restart_recovery phase) correctly detects it via the exit-code verdict.", + "tags": [ + "ha-commit-durability", + "commit-durability", + "cold-restart", + "total-outage", + "wal-truncate", + "upstream", + "xce-test-e2e-primitives", + "needs-real-fs", + "rolling-failover", + "liveness", + "non-deterministic" + ], + "related_bugs": [ + "bug-661", + "bug-778", + "bug-786" + ], + "occurrences": 3, + "last_seen": "2026-06-19" + }, + { + "id": "bug-796", + "timestamp": "2026-06-20T00:32:43.144Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/spec.json", + "root_cause": "Had \"2026-06-18T04:00:00Z\"", + "fix": "Changed to \"2026-06-19T00:00:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T00:32:43.144Z" + }, + { + "id": "bug-799", + "timestamp": "2026-06-20T01:05:25.091Z", + "error_message": "Incorrect value in code", + "file": "src/tools/test-commit-durability.sh", + "root_cause": "Had \"$label: DATA LOSS \u2014 a committed job vanished (bug", + "fix": "Changed to \"$label: DATA LOSS \u2014 a committed job vanished from", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T01:05:25.091Z" + }, + { + "id": "bug-804", + "timestamp": "2026-06-20T03:29:22.678Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "Had \"none\"", + "fix": "Changed to \"unknown\"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-20T03:33:32.399Z" + }, + { + "id": "bug-805", + "timestamp": "2026-06-20T03:29:29.570Z", + "error_message": "Wrong return value", + "file": ".kiro/specs/controller-lsid/requirements.md", + "root_cause": "Was returning: a well-defined \"unknown\" or \"none\" value for that ", + "fix": "Now returns: a well-defined \"unknown\" value for that field rath", + "tags": [ + "auto-detected", + "return-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T03:29:29.570Z" + }, + { + "id": "bug-808", + "timestamp": "2026-06-20T04:08:49.684Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/design.md", + "root_cause": "Had `committed_config_tag`", + "fix": "Changed to `committedConfigVersion`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-20T04:10:27.793Z" + }, + { + "id": "bug-809", + "timestamp": "2026-06-20T04:40:36.648Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'commit.c'", + "fix": "Changed to 'ctrlstatus.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T04:40:36.648Z" + }, + { + "id": "bug-810", + "timestamp": "2026-06-20T04:45:48.985Z", + "error_message": "Wrong reference: Data should be Text", + "file": "src/schema/controller.capnp", + "root_cause": "Used \"Data\" instead of \"Text\"", + "fix": "Changed Data \u2192 Text", + "tags": [ + "auto-detected", + "wrong-reference", + "capnp" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T04:45:48.985Z" + }, + { + "id": "bug-811", + "timestamp": "2026-06-20T04:45:49.790Z", + "error_message": "Wrong reference: xce_data_t should be char", + "file": "src/include/typemap.h", + "root_cause": "Used \"xce_data_t\" instead of \"char\"", + "fix": "Changed xce_data_t \u2192 char", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T04:45:49.790Z" + }, + { + "id": "bug-812", + "timestamp": "2026-06-20T05:08:41.030Z", + "error_message": "Incorrect value in code", + "file": "src/lib/meson.build", + "root_cause": "Had 'raft/raft_sim.c'", + "fix": "Changed to 'testsuite/test_ctrlstatus.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T05:08:41.030Z" + }, + { + "id": "bug-813", + "timestamp": "2026-06-20T05:42:31.707Z", + "error_message": "Wrong operator: >= should be ==", + "file": "src/controller/testsuite/test.c", + "root_cause": "Used \">=\" instead of \"==\"", + "fix": "Changed operator >= \u2192 ==", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-20T05:42:46.894Z" + }, + { + "id": "bug-817", + "timestamp": "2026-06-19", + "error_message": "client op (joblist, ctrlstatus, wait-primary) returns EINVAL / 'Invalid argument' and fails to reach the live primary when a DEAD controller is FIRST in the masters list; with a live-only masters list the same op succeeds in ~46ms.", + "file": "src/lib/client_failover.c / client.c failover-rotation (libxce) \u2014 UPSTREAM, not controller-status", + "root_cause": "libxce client failover does not cleanly advance past a dead FIRST master to the live nodes; surfaces a typed EINVAL instead of rotating. Pre-existing (controller-status consumes failover unchanged). Surfaced by controller-status task 5.1 E2E (authoritative wait-primary timed out after a primary kill while a dead node led the masters list, even though a new primary was serving).", + "fix": "NOT fixed in controller-status (out of boundary). Route to libxce-failover. Confirmed affects plain joblist identically. Workaround: ensure the masters list does not lead with a known-dead node, or restart killed nodes (as the durability soak does).", + "tags": [ + "libxce", + "failover", + "master-rotation", + "dead-first-master", + "EINVAL", + "controller-status", + "upstream" + ], + "related_bugs": [ + "bug-646" + ], + "occurrences": 1, + "last_seen": "2026-06-19" + }, + { + "id": "bug-818", + "timestamp": "2026-06-20T06:11:49.084Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (!g_continue_on_error)", + "fix": "Changed to: if (!g_continue_on_error && !g_interactive)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-20T06:14:41.073Z" + }, + { + "id": "bug-820", + "timestamp": "2026-06-20T06:16:34.685Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/xce-test-e2e-primitives/tasks.md", + "root_cause": "Had `g_interactive = isatty(STDIN_FILENO)`", + "fix": "Changed to `-e`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T06:16:34.685Z" + }, + { + "id": "bug-821", + "timestamp": "2026-06-20T08:14:43.577Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/requirements.md", + "root_cause": "Had 's identity and role, the configured controller to", + "fix": "Changed to 's identity and role, the configured controller to", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T08:14:43.577Z" + }, + { + "id": "bug-824", + "timestamp": "2026-06-20T09:30:23.040Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/controller-status/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T09:30:23.040Z" + }, + { + "id": "bug-825", + "timestamp": "2026-06-20T09:31:24.076Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/controller-status/spec.json", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T09:31:24.076Z" + }, + { + "id": "bug-827", + "timestamp": "2026-06-20T18:30:00.000Z", + "error_message": "AddressSanitizer: global-buffer-overflow READ in xce_strerror_r (lib/utils.c:335), triggered from cmd_ctrlstatus (test.c:1273) when ctrlstatus failover exhausts and returns E_XCE_NOT_READY (-10013) on a primary-less cluster", + "file": "src/lib/utils.c", + "root_cause": "Enum/table drift: errnox.h E_XCE_* enum gained NOMASTER(10011)/REDIRECT(10012)/NOT_READY(10013)/STANDBY(10014), bumping E_XCE_MAX to 10015, but xce_err_str[] still had only 10 strings + a NULL (11 slots, 88 bytes). xce_strerror_r's only upper guard was errnum>=E_XCE_MAX, so 10011-10014 passed the gate and indexed past the table: NOMASTER->xce_err_str[10]==NULL (strncpy NULL deref); NOT_READY->xce_err_str[12] = 8 bytes past the 88-byte array (the ASAN OOB). Pre-existing libxce foundation bug; controller-status ctrlstatus was the first caller to format these HA codes on a failing cluster.", + "fix": "Filled xce_err_str[] with strings for all 14 codes (enum order); hardened xce_strerror_r to bound against the ACTUAL array size (idx>=nstr) + reject NULL slots (degrade to 'unknown error', never OOB) + null-terminate; added a _Static_assert tying sizeof(xce_err_str) to (E_XCE_MAX-XCE_ERRNO_BASE) so future enum additions without a string fail the build. RED: new test_strerror_all_xce_codes SEGV'd at utils.c:335 under ASAN; GREEN: lib suite Ok:1 Fail:0. Commit 1aa0761.", + "tags": [ + "global-buffer-overflow", + "asan", + "xce_strerror_r", + "errnox", + "enum-table-drift", + "libxce", + "controller-status", + "ctrlstatus", + "E_XCE_NOT_READY" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T18:30:00.000Z" + }, + { + "id": "bug-828", + "timestamp": "2026-06-20T10:19:56.678Z", + "error_message": "Clean single failover under-computes safe_committed_lsn: new leader logs 'safe_committed_lsn=1 ... (from 1 granted voters)' (self only) and truncates committed entries down to its own boot floor. Post-election safe-commit-from-votes plumbing was DEAD in production.", + "file": "src/controller/ha.c (vote_reply_dispatch), src/controller/persist.c (persist_wal_write_replicated)", + "root_cause": "vote_reply_decode_and_dispatch decoded the granting voter's advertised peerLastLogLsn (vr_lsn at ha.c ~4254) but vote_reply_dispatch DISCARDED it ('(void)peer_lsn; raft_on_vote_reply(...)'). raft_on_vote_reply only bumps s->votes_granted; it never records the peer LSN. The only writer of ctx->votes[idx].vote_granted/peer_lsn (handle_request_vote_reply) was, in production, called ONLY from the connect/error failure paths with (granted=0, lsn=0); the real-grant caller was gated behind a test-only hook (xce_vote_reply_hook, never set in production). So ctx->votes[*] stayed calloc'd zero -> ha_compute_safe_committed_lsn saw n_granted=1 (self) -> safe_committed_lsn = self_lsn. A standby promoting with its own durable_floor=1 then persist_truncate_wal_above(1) destroyed committed LSN 2..5.", + "fix": "PRIMARY: in vote_reply_dispatch production branch, route the decoded reply into the election context via a NEW record-only seam ha_record_vote_reply(ha,ctx,idx,term,granted,tag,lsn) BEFORE raft_on_vote_reply. Factored ha_record_vote_reply_locked out of handle_request_vote_reply: it records ctx->votes[idx] under ctx->lock with the idempotency guard but does NOT call ha_elect_evaluate_and_signal -> no second/racing legacy promotion path (raft_on_vote_reply remains the SOLE production promotion driver; the two quorum tallies (raft s->votes_granted / legacy ctx->votes_granted) and their guards (s->election_decided / ctx->done_posted) stay independent and idempotent). idx == vctx->peer_idx == the local ctx->votes[] index (absolute->local map done in raft_cb_send_vote_request), exactly the index ha_compute_safe_committed_lsn + the test hook use. SECONDARY: persist_wal_write_replicated now advances durable_lsn to the synced tip immediately after the inline fdatasync (mirror persist_flush; monotonic max; fail-loud rc-gated) so a standby advertises its own synced durable floor without waiting for the 10ms cadence flusher. VERIFIED: unit test_safe_commit_reflects_granting_voter_lsn RED (safe=1 'from 1 granted voters') -> GREEN (safe=5 'from 3 granted voters'); 177/177 controller_unit_test pass. NOTE: the LIVE clean_failover_preservation harness still FAILS, but the failure is PRE-EXISTING on baseline 14c41cb (verified by stash+rerun: baseline runs also diverge / lose jobs) \u2014 it is the SEPARATE bug-795/828 replication+catch-up gap (standbys' WALs physically contain only LSN 1..2, durable_floor=1; committed entries 2..5 never reach the standby WAL; primary advanced committed_lsn to 5 without a standby durable at 5). The fix DID change the live fingerprint from 'from 1 granted voters' to 'from 2 granted voters', proving the plumbing is now alive. safe-commit-from-votes is correct; the residual live job loss is out of scope (needs the catch-up/snapshot-backfill fix tracked under ha-commit-durability).", + "tags": [ + "async-fix", + "md", + "data-loss", + "failover", + "committed-job-loss", + "wal", + "catch-up", + "snapshot", + "ha-commit-durability", + "bug-795", + "architecture", + "NOT-FIXED", + "safety", + "safe-committed-lsn", + "vote-reply", + "leader-completeness", + "ha-repl-hardening", + "PARTIAL-FIX", + "c" + ], + "related_bugs": [ + "bug-795", + "bug-851" + ], + "occurrences": 3, + "last_seen": "2026-06-23T01:40:00.000Z" + }, + { + "id": "bug-829", + "timestamp": "2026-06-20T11:44:25.496Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'ctrlstatus.c'", + "fix": "Changed to 'testsuite/test_ha_heartbeat_committed.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T11:44:25.496Z" + }, + { + "id": "bug-830", + "timestamp": "2026-06-20T11:45:26.943Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T11:45:26.943Z" + }, + { + "id": "bug-831", + "timestamp": "2026-06-20T11:54:30.060Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_ha_heartbeat_committed.c", + "root_cause": "Had \"xce/cluster_def.h\"", + "fix": "Changed to \"typemap.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T11:54:30.060Z" + }, + { + "id": "bug-832", + "timestamp": "2026-06-20T11:56:28.240Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c || !hb", + "fix": "Added guard clause: if (!c || !hb)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T11:56:28.240Z" + }, + { + "id": "bug-834", + "timestamp": "2026-06-20T12:22:39.011Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", + "root_cause": "Had `heartbeatmsg`", + "fix": "Changed to `xce-build.sh controller`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T12:22:39.011Z" + }, + { + "id": "bug-835", + "timestamp": "2026-06-20T12:26:15.999Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'ctrlstatus.c'", + "fix": "Changed to 'testsuite/test_ha_vote_floor_heartbeat.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T12:26:15.999Z" + }, + { + "id": "bug-836", + "timestamp": "2026-06-20T12:31:51.359Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-heartbeat-committed-propagation/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T12:31:51.359Z" + }, + { + "id": "bug-837", + "timestamp": "2026-06-20T12:32:25.549Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-heartbeat-committed-propagation/spec.json", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T12:32:25.549Z" + }, + { + "id": "bug-840", + "timestamp": "2026-06-20T14:05:49.912Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"xce/cluster.h\"", + "fix": "Changed to \"waldump.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T14:05:49.912Z" + }, + { + "id": "bug-841", + "timestamp": "2026-06-20T14:06:11.393Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/testsuite/test.c", + "root_cause": "Condition was: if (active_ctx->parent != NULL)", + "fix": "Changed to: if (argc < 2)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T14:06:11.393Z" + }, + { + "id": "bug-843", + "timestamp": "2026-06-20T14:15:35.599Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `!`", + "fix": "Changed to `xce_test`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T14:15:35.599Z" + }, + { + "id": "bug-845", + "timestamp": "2026-06-20T14:30:00.000Z", + "error_message": "Every committed job appears as TWO consecutive WAL_JOB_SUBMIT entries with the SAME jobid (e.g. ctrl-1 LSN 2,3 = jobid 1868520205844480). Surfaced by waldump on /tmp/xce-ha-deploy.", + "file": "src/lib/hook.c (xce_add_hook idempotency); root path src/controller/{persist.c persist_start, ha.c ha_register_persist_hooks}", + "root_cause": "NOT array expansion \u2014 job_submit() assigns array members DISTINCT ids (base_id+i, job.c) and the Job capnp schema has a single jobid field, so identical-jobid pairs cannot be two array tasks. A single client submit is producing two WAL_JOB_SUBMIT appends of the same job. EXACTLY 2 entries per job (never 1 or 3), always at ADJACENT LSNs, for every job on every node. CONFIRMED via enhanced waldump (payload-CRC + subtime columns): each pair is BYTE-IDENTICAL (same PCRC, e.g. LSN 2,3 both 6d79074e; same subtime 1781959730) \u2014 proving a literal double-append of the SAME encoded job_t in one submit flow; a failover retry would re-stamp subtime=time(NULL) and differ, so retry is DEFINITIVELY ruled out. Remaining candidates: AFTER_EVENT jlist contains the job twice, or the JOB_NEW hook/handler persists twice per submit. So 'ctrl-1 has 12 submits' is really 6 jobs (halve when reconciling failover-test job totals). Doubles LSN consumption; consistent across nodes here (F@14,15 on both ctrl-1 and ctrl-3) so not itself divergence, but a latent cross-node LSN-divergence risk if the duplicate count ever varies by primary.", + "fix": "FIXED 2026-06-23 (src/lib/hook.c). Root cause: persist_job_hook_fun was registered on XCE_CONTROLLER_HOOK_JOB by TWO owners \u2014 persist_start() (unconditional, boot) AND ha_register_persist_hooks() (on PRIMARY promotion) \u2014 with the SAME persist_t closure. xce_add_hook did not dedup, xce_run_hooks fires ALL slots, so a promoted primary had 2 active slots => persist_job_hook_fun ran twice per submit => 2 byte-identical adjacent WAL_JOB_SUBMIT appends. Fix: made xce_add_hook idempotent on exact (fn,data) \u2014 re-registering an active (fn,data) is a no-op; same fn with DISTINCT data still registers (legitimate multi-closure). Preserves the persist_start test contract many unit tests rely on. Verified: WAL now single-entry per job (waldump); 5/5 hook unit tests; controller suite 343 tests 0 new failures; 3/3 post-fix HA soaks zero job loss. NOTE: an earlier post-fix soak hit the KNOWN intermittent failover durability race (bug-795/828 class, LSN-10 cross-node divergence) \u2014 confirmed pre-existing (baseline passes/fails on timing too), NOT caused by this dedup.", + "tags": [ + "double-register", + "duplicate", + "fixed", + "ha", + "hook-dedup", + "investigation", + "job-submit", + "persistence", + "wal", + "waldump" + ], + "related_bugs": [ + "bug-795", + "bug-828" + ], + "occurrences": 2, + "last_seen": "2026-06-23T23:30:00.000Z" + }, + { + "id": "bug-847", + "timestamp": "2026-06-20T14:37:08.863Z", + "error_message": "Catch-up WAL streaming silently DROPS entry payloads at scale: a rejoining controller (ctrl-1 restarted) received [1..14] with correct LSN/type/CRC but PLEN=0 / empty job blobs (waldump PCRC=00000000, jobid=?). Structurally-intact but content-empty WAL = silent data corruption. THE committed-job-loss mechanism.", + "file": "src/lib/request.c (xce_wal_batch_encode_raw / wal_entry_to_wire / xce_decode extract loop) \u2014 capnp multi-segment Data pointers", + "root_cause": "Multi-segment capnp bug. xce_wal_batch_encode_raw builds a WalCatchupEntry struct-list where each element's payload is a capnp Data (List(UInt8)) allocated via capn_new_list8(cs=root_seg,...). When the message outgrows its first segment (large batch), the per-element payload Data pointer is lost across the segment boundary \u2014 on decode extract_data_bytes(we.payload) reads CAPN_NULL and returns len 0. write_WalCatchupEntry uses capn_setp (far-pointer-capable) so the fault is in the overflow/segment handling as used here. REPRODUCED deterministically: src/lib/testsuite/test_wal_wire.c test_wal_batch_roundtrip_large_payloads (14 entries x 872B) FAILS 'Expected 872 Was 0'; the 2-entry test_wal_batch_roundtrip_payload_bytes PASSES. Threshold is size, not count. Live replication (repl_broadcast) carries payloads fine \u2014 ONLY the catch-up/rejoin path corrupts, which is why loss only appears after a node restarts and catches up.", + "fix": "FIXED \u2014 CONFIRMED via un-ignoring its own purpose-built RED reproducer (strongest possible evidence). Removed the TEST_IGNORE_MESSAGE guard from test_wal_batch_roundtrip_large_payloads (src/lib/testsuite/test_wal_wire.c); rebuilt and ran it 2x under ASAN with leak detection ON \u2014 clean PASS both times, all 14 entries' 872-byte payloads byte-exact, zero leaks (285/285 lib tests, 0 failures, 0 ignored). Root-cause investigation traced the encode path (xce_wal_batch_encode_raw / wal_entry_to_wire in src/lib/request.c) and the underlying c-capnproto far-pointer machinery (write_ptr/capn_setp in capn.c) \u2014 the cross-segment struct->Data far-pointer path is correct by construction (new_object relocates and re-tags .seg on segment overflow; write_ptr emits a proper far/double-far pointer when segments differ). capn-malloc.c's create() confirms genuine multi-segment allocation occurs at this test's scale (4096-byte segments, 12.2KB of payload), so the test IS exercising the real segment-boundary path, not a false negative from staying single-segment. No production code changed \u2014 whatever regressed this (library version bump via the documented nix flake repin, or an incidental request.c change) already fixed it; the TEST_IGNORE was simply never removed. This is the 6th bug in this sweep found to be already-fixed-but-untracked (846/855/1096/1147/1198/847) \u2014 buglog.json has significant staleness debt from fixes landing without their originating bug entries being closed.", + "tags": [ + "c", + "reproducer-unignored", + "corruption", + "ha", + "waldump", + "stale-report", + "reverified-open", + "catchup", + "data-loss", + "replication", + "capnp", + "critical", + "wal", + "fixed" + ], + "related_bugs": [ + "bug-828", + "bug-795", + "bug-661" + ], + "occurrences": 4, + "last_seen": "2026-07-03T03:15:00.000Z" + }, + { + "id": "bug-848", + "timestamp": "2026-06-20T14:40:02.211Z", + "error_message": "On a PRESERVED-DATA restart (no wipe \u2014 user confirmed), a controller recovers its WAL [1..7] from disk but rejoins advertising applied_lsn=0, so the primary re-streams the WHOLE log [1..committed] and O_TRUNC-overwrites the node's already-correct WAL. Logs: 'HA: learned committed_lsn=10 ... (was 0)'. THE root trigger that forces the full re-stream (then bug-847 empties it).", + "file": "src/controller/cluster.c (cluster_init_phase2, after persist_recover) + src/controller/ha.c (ha_init applied_lsn=0; state report sr->applied_lsn)", + "root_cause": "ha->applied_lsn is NEVER reconstructed from the recovered WAL on boot. ha_init zeros it (ha.c:1367); persist_recover (cluster.c:1352) replays the WAL and advances persist->current_lsn/durable_lsn, but nothing copies that recovered position into ha->applied_lsn (persist_t and ha_state_t are separate; the only setter besides ha_init is the SNAPSHOT receive path ha.c:1771 = checkpoint_lsn). So the standby state report advertises sr->applied_lsn = ha->applied_lsn = 0 (ha.c:1787) and the primary's ha_evaluate_repl_action streams [0+1 .. committed]. ASYMMETRY proving it's a bug: ELECTION advertise uses ha_candidate_advertise_lsn = persist_durable_floor (the RECOVERED position, correct), but CATCH-UP advertise uses the un-recovered applied_lsn=0. The system knows its durable tip for voting but not for catch-up. Defeats Raft incremental catch-up for every restarted node; also the upstream cause of the earlier missing-prefix cases (full re-stream from a trimmed primary -> prefix gap).", + "fix": "FIXED 2026-06-20 (HA-layer). Added ha_sync_applied_lsn_from_recovery(ha,p) (ha.c/ha.h) = monotonic store of persist_durable_floor(p) into ha->applied_lsn; wired into cluster_init_phase2 right after persist_recover, plus a diagnostic INFO log of recovered durable_floor/current_lsn/applied_lsn. TDD: test_recovered_member_advertises_real_applied_lsn (test_persist_recovery.c) write->recover->sync asserts applied_lsn==durable_floor>0 and monotonic. controller_unit_test 6/6 + adjacent HA 8/8, daemon builds. CAVEAT: live ctrl-1 also showed current_lsn=1 (seg O_TRUNC'd from LSN 1) implying persist_recover may not have loaded the WAL on that restart at all; if so this HA-layer fix is necessary but not sufficient. The new INFO log reveals on the next live restart whether persist_recover is effective (durable_floor>0) or not (bug-849 candidate: recovery not running/loading on the rejoin path).", + "tags": [ + "c", + "ha", + "raft", + "recovery", + "catchup", + "applied_lsn", + "data-loss", + "root-trigger", + "critical" + ], + "related_bugs": [ + "bug-847", + "bug-828", + "bug-795" + ], + "occurrences": 3, + "last_seen": "2026-06-20T15:50:00.000Z" + }, + { + "id": "bug-849", + "timestamp": "2026-06-20T14:42:09.400Z", + "error_message": "After bug-848 fix, restart still loses the committed prefix: restarted ctrl-1 ended with WAL [15..16] only \u2014 missing [1..14] (waldump 'GAP: missing prefix LSN 1..14'). Payloads now full (bug-848 era), but the prefix is gone. Loss mode changed (empty-payload -> missing-prefix), loss remains.", + "file": "src/controller/myapp.c (standby-rejoin path ~317-320) + src/controller/cluster.c cluster_init_phase2 (holds persist_recover)", + "root_cause": "CONFIRMED from live logs (/tmp/xce-ha-deploy/logs/ctrl-1.log). persist_recover (and the bug-848 applied_lsn sync, and catch-up) run ONLY inside cluster_init_phase2, which is ELECTION-GATED \u2014 its real callers are all ha.c election/promotion paths (ha.c:1021,1166,2943,3022) + cluster_init. First boot ran it (ctrl-1 won an election; logged 'persist_recover done \u2014 durable_floor=0 current_lsn=1'). The 11:07:20 RESTART came up as a follower ('myapp_run: CTRL_STANDBY \u2014 waiting for primary to connect and push config') with NO election \u2192 cluster_init_phase2 NEVER ran \u2192 NO 'persist_recover done' line for pid 304897. So the rejoining follower never recovered its on-disk [1..7], operated with current_lsn=1/applied_lsn=0, then received LIVE replication of the new job [15..16] (repl_broadcast), O_TRUNC'd seg-001 and wrote [15..16] there. [1..7] never recovered + [1..14] never caught up = permanent prefix gap. Ordering corollary: live replication advances applied past the gap so the primary's catch-up never backfills it.", + "fix": "FIXED 2026-06-21 (design B \u2014 boot-path-universal recovery). Refined root cause: phase2 is invoked ONLY from the election-result path; a rejoining follower that SKIPS its election (goto skip_election in myapp_run because a primary already exists) never reaches it. Fix: in myapp.c the skip-election standby branch now calls cluster_init_phase2(&c) (build resources + persist_recover + bug-848 ha_sync_applied_lsn) when !ha.phase2_done, BEFORE cluster_run starts serving \u2014 so every boot path recovers its own WAL and advertises its real applied_lsn before accepting replication. Idempotent via phase2_done (same once-per-process contract as the election callers); recovery decoupled from the primary config push (a restart already loaded config from disk pre-election). Daemon builds; controller suite incl. test_persist_recovery/test_ha_startup/test_repl_catchup/test_ha_failover 8/8 green. LIVE VALIDATION PENDING: redeploy + kill/restart ctrl-1; expect a 'HA: persist_recover done \u2014 durable_floor=7 current_lsn=8' line for the restarted pid and ctrl-1 keeping [1..7] + catching up only the suffix.", + "tags": [ + "wrong-value", + "md", + "ha", + "raft", + "recovery", + "boot", + "election-gated", + "catchup", + "data-loss", + "critical" + ], + "related_bugs": [ + "bug-848", + "bug-847", + "bug-828", + "bug-795" + ], + "occurrences": 2, + "last_seen": "2026-06-21T11:15:00.000Z" + }, + { + "id": "bug-850", + "timestamp": "2026-06-20T15:14:36.135Z", + "error_message": "A rejoining standby receives LIVE replication of new entries before its historical catch-up, advancing current_lsn past the gap so the idempotency guard skips the catch-up of the prefix. Explains ctrl-1 ending with [15..16] only (gap [1..14]). Distinct from bug-849; persists even after recovery.", + "file": "src/controller/repl.c repl_add_standby (line ~119) + repl_broadcast in_catch_up gate", + "root_cause": "repl_add_standby initializes a freshly-registered standby with in_catch_up=0, so it is IMMEDIATELY eligible for repl_broadcast (which correctly skips in_catch_up standbys at repl.c:27). Mechanism (confirmed from ctrl-1.log 11:07:21): ctrl-1 reconnects -> registered in_catch_up=0 -> the new job [15..16] is live-broadcast to it (not gated) -> persist_wal_write_replicated writes them, current_lsn jumps to 17 -> THEN the catch-up stream of [1..14] arrives but persist_wal_write_replicated's idempotency guard (lsn+1 <= current_lsn) skips ALL of them (1+1<=17 ... 14+1<=17) -> permanent prefix gap [1..14]. Independent of bug-849: even with recovery (current_lsn=8), a live [15..16] arriving first jumps current_lsn to 17 and the catch-up of [8..14] is skipped (gap [8..14]). Root architectural cause: catch-up streaming and live broadcast are SEPARATE unordered paths (unlike Raft AppendEntries which is one ordered per-follower stream).", + "fix": "FIXED 2026-06-21 via a CONTIGUITY GUARD at the persist layer (Raft log-matching), NOT in_catch_up. First tried defaulting in_catch_up=1 in repl_add_standby \u2014 REVERTED: it broke the whole controller suite (commit-WAIT tests TIMEOUT 300s because a fresh standby no longer counts toward write quorum until convergence, and the unit tests don't run the heartbeat dispatch that would clear it). Correct fix: persist_wal_write_replicated now REJECTS a forward gap (lsn > current_lsn) with -EAGAIN instead of writing it \u2014 so a live broadcast that arrives ahead of the in-order catch-up is DROPPED (no write, no ack, no current_lsn jump), and the ACK-driven catch-up continuation re-streams from current_lsn so the log fills in order. ha_apply_single_entry + ha_apply_wal_catchup_entry treat -EAGAIN as a quiet skip (TRACE, no ack, no ERROR spam). Combined with bug-848 (advertise durable_floor == current_lsn-1) the catch-up always starts AT current_lsn so it is never spuriously rejected; only genuine forward gaps are. Files: src/controller/persist.c (guard), src/controller/ha.c (quiet -EAGAIN). TDD: test_repl_write_rejects_forward_gap (test_repl_apply.c) \u2014 contiguous write ok, forward-gap lsn 5 @ current_lsn 2 -> -EAGAIN + current_lsn unchanged, fill 2..4 then 5 ok, recovery clean. FULL controller suite green (129/0/0, no timeout); daemon builds. LIVE VALIDATION PENDING with bug-848+849+850 together.", + "tags": [ + "wrong-value", + "c", + "ha", + "raft", + "replication", + "catchup", + "ordering", + "race", + "in_catch_up", + "data-loss", + "critical" + ], + "related_bugs": [ + "bug-849", + "bug-848", + "bug-847", + "bug-828", + "bug-795" + ], + "occurrences": 2, + "last_seen": "2026-06-21T12:30:00.000Z" + }, + { + "id": "bug-846", + "timestamp": "2026-06-20T15:20:00.000Z", + "error_message": "After the primary dies, xce_test client ops (ctrlstatus/joblist/jobsub) fail PERSISTENTLY with 'Invalid argument' (EINVAL) and wait-primary times out, even though a new primary was elected (ctrl-2 term 3). Blocks failover testing from the client.", + "file": "src/lib/xceapi.c (per-op state guard) + src/lib/client_failover.c (xce_failover_classify / xcecli_retry_decide)", + "root_cause": "Client-side, NOT server. ha_dispatch_gate ALLOWS OPCode_ctrlstatus in every role and rejects client writes with the RETRYABLE -XCE_ERR_NOT_READY (never EINVAL), so a reachable controller would answer. The EINVAL is the xceapi per-op guard: `if (ctx->state != XCE_CTX_RUN) return -EINVAL`. The failover loop only AUTO-RECOVERS ctx.state==XCE_CTX_ERR (xcecli_retry_decide rotates+recovers); xce_failover_classify maps EINVAL -> XCE_FO_TERMINAL ('everything else -> terminal'), so a bare EINVAL is surfaced and the sticky master cursor never advances to the live new primary. So once the ctx lands in a non-ERR non-RUN state (INIT/DOWN) \u2014 or yields a bare EINVAL while RUN \u2014 failover gives up permanently. Exact stuck state needs a live repro to confirm.", + "fix": "FIXED (verified 2026-07-02, no direct code trail to when \u2014 landed under libxce-failover). src/lib/client_failover.c xcecli_retry_decide() now checks `ac->ctx.state == XCE_CTX_ERR` BEFORE classifying rc, bypassing the -EINVAL misclassification entirely (comment there literally reads 'the reported bug'). xcecli_ctx_recover() resets ERR->RUN post-reconnect. This is exactly recommendation (B) from this bug's own original fix note.", + "tags": [ + "client", + "stale-report", + "einval", + "blocker", + "failover", + "ha", + "fixed", + "libxce" + ], + "related_bugs": [ + "bug-646" + ], + "occurrences": 1, + "last_seen": "2026-07-02T17:30:00.000Z" + }, + { + "id": "bug-853", + "timestamp": "2026-06-20T15:27:41.698Z", + "error_message": "Wrong reference: test_wal_batch_entry_without_payload should be test_wal_batch_roundtrip_large_payloads", + "file": "src/lib/testsuite/test_wal_wire.c", + "root_cause": "Used \"test_wal_batch_entry_without_payload\" instead of \"test_wal_batch_roundtrip_large_payloads\"", + "fix": "Changed test_wal_batch_entry_without_payload \u2192 test_wal_batch_roundtrip_large_payloads", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-20T15:27:41.698Z" + }, + { + "id": "bug-861", + "timestamp": "2026-06-21T00:20:55.974Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test_persist_recovery.c", + "root_cause": "Had \"persist.h\"", + "fix": "Changed to \"ha.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T00:20:55.974Z" + }, + { + "id": "bug-864", + "timestamp": "2026-06-21T03:39:07.483Z", + "error_message": "Missing guard clause", + "file": "src/controller/repl.c", + "root_cause": "No early return/throw for edge case: r->standbys[i].ep == ep", + "fix": "Added guard clause: if (r->standbys[i].ep == ep)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T03:39:07.483Z" + }, + { + "id": "bug-871", + "timestamp": "2026-06-21T04:01:07.089Z", + "error_message": "Wrong reference: test_repl_write_rejects_checkpoint should be test_repl_write_rejects_forward_gap", + "file": "src/controller/testsuite/test_repl_apply.c", + "root_cause": "Used \"test_repl_write_rejects_checkpoint\" instead of \"test_repl_write_rejects_forward_gap\"", + "fix": "Changed test_repl_write_rejects_checkpoint \u2192 test_repl_write_rejects_forward_gap", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T04:01:07.089Z" + }, + { + "id": "bug-851", + "timestamp": "2026-06-21T13:30:00.000Z", + "error_message": "CHAOTIC leadership churn (3x kill-the-primary) causes SILENT committed-prefix DIVERGENCE: all 3 nodes report committedLsn=54 with lsn range 1..54 / 0 gaps / CRC intact, but WAL CONTENT diverges at LSN 45-54 (ctrl-1/ctrl-2/ctrl-3 each hold DIFFERENT jobs at the same LSNs; fps bfeb26ad/216af434/865304ec). Violates Raft committed-prefix invariant (a committed entry must be byte-identical on every member). Originally seen as a liveness 'commit stall'; the deeper defect is divergence + acked-job loss under churn.", + "file": "src/controller/persist.c:588/726/1040 (hdr.term=p->term, p->term never set) + src/controller/ha.c (election term = local+1, PreVote no-persist; two leaders per term) + src/controller/ha_convergence.c (reconcile cannot (term,lsn) log-match)", + "root_cause": "ROOT-CAUSED 2026-06-21 (owned-cluster repro, see .kiro/specs/ha-commit-durability/repro-851/). THREE compounding defects: (1) Election term not monotonic across cluster: term = locally-persisted+1, PreVote does NOT persist/propagate ('PreVote round ... no persist'); under chaotic kills a leader wins term N, appends own-term noop+jobs, and is killed BEFORE term N propagates to a majority; a restarted node recovers a STALE term and elects at the SAME N+1 => TWO leaders share a term (logs: term3 led by ctrl-3@08:40 AND ctrl-1@08:47:39; term4 led by ctrl-2@08:47:52 AND ctrl-3@08:48:04), each appending its own-term noop at the SAME lsn 45. (2) WAL entry term field is DEAD (always 0): wal_entry_hdr_t.term exists (wal_types.h:30) but writer does hdr.term=p->term (persist.c:588/726/1040) and p->term is NEVER assigned from ha->term anywhere in the controller (zero matches) => every entry persists term=0; waldump confirms term=0 cluster-wide. (3) No (term,lsn) reconcile: with no usable per-entry term, ha_convergence.c reconcile cannot Raft-log-match, so a divergent same-lsn suffix (45@term3 vs 45@term4) looks already-present and is never truncated; ha_compute_safe_committed_lsn then advances committed 'from 1 granted voter' over content other members never received. NOTE: a CLEAN single failover fully converges (all WALs identical 1..22, 848/849/850 validated) \u2014 only CHAOTIC churn triggers it.", + "fix": "NOT FIXED (HA-core + persistence => full spec workflow per CLAUDE.md, not a quick patch). Keystone lever: stamp hdr.term from live ha->term on the primary append path + carry term through the repl envelope + persist it, THEN make ha_convergence reconcile truncate any local suffix whose (term,lsn) != leader's (standard Raft log-matching). MUST pair with durably-establishing the election term before a new leader acts (no two leaders per term: persist/propagate the won term to a majority before appending the own-term noop, or bump term beyond any (term,lsn) a voter advertises). Route to ha-consistency-invariants / ha-convergence-recovery as a new sub-spec.", + "tags": [ + "ha", + "failover", + "replication", + "convergence", + "raft-safety", + "term-monotonicity", + "wal-term", + "committed-divergence", + "data-loss", + "spec-needed" + ], + "related_bugs": [ + "bug-850", + "bug-849", + "bug-848", + "bug-646" + ], + "occurrences": 2, + "last_seen": "2026-06-21T09:10:00.000Z", + "root_caused": "2026-06-21T09:10:00.000Z" + }, + { + "id": "bug-854", + "timestamp": "2026-06-21T09:10:00.000Z", + "error_message": "Client primary-discovery is POSITIONAL, not failover-correct: with masters=[m0,m1,...] the client only finds the primary if m0 IS the primary. wait-primary: 9993(primary-first)=ok 45ms; 9992,9993(standby-first)=no primary; 9991,9993(dead-first)=no primary; 9993,9991=ok.", + "file": "src/lib/client.c:403 xcecli_ctrlstatus_primary + src/controller/ctrlstatus.c:104 (standby primaryName=\"\")", + "root_cause": "Two compounding defects in libxce: (1) XCECLI_FAILOVER_CALL classifies a standby's status-0 ctrlstatus reply as SUCCESS and never rotates the endpoint (ctrlstatus is answerable in every role, so ANY reachable standby answers 0 \u2014 the inner call never rotates off it). (2) xcecli_ctrlstatus_primary captures a redirect hint from the non-authoritative reply but does not force rotation to consume it \u2014 it re-queries the same standby until the retry budget exhausts, never traversing the master ring. No server-side primaryName source existed on a standby (voted_for_name is not the leader, and primary_host is vestigial/never-written). Verified: baseline (no fix) with standby-first masters does NOT find the primary (pins to standby).", + "fix": "FIXED (2026-06-23). xcecli_ctrlstatus_primary: force xce_failover_rotate(ctx) + xcecli_ctx_recover(ctx) under failover.lock on every non-authoritative reply (hint-first then next-master rotation). Loop floor raised to max(max_retries, nmasters+1). Verified: wait-primary 'ctrl-3' serving after 92 ms with standby-first masters list; end-to-end clean-failover discover pass. Deliberately NOT adding server-side primaryName source \u2014 it is only a latency optimization; the client rotation fix alone resolves the hang.", + "tags": [ + "libxce", + "client", + "failover", + "primary-discovery", + "ctrlstatus", + "ha", + "FIXED", + "client", + "rotation", + "xcecli_ctrlstatus_primary" + ], + "related_bugs": [ + "bug-846", + "bug-851", + "bug-646" + ], + "occurrences": 2, + "last_seen": "2026-06-23" + }, + { + "id": "bug-875", + "timestamp": "2026-06-21T10:04:37.876Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/requirements.md", + "root_cause": "Had `OPCode_replConfigPush`", + "fix": "Changed to `snapshot_transfer_s`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T10:04:37.876Z" + }, + { + "id": "bug-857", + "timestamp": "2026-06-21T10:30:00.000Z", + "error_message": "Config upgrade can leave members with DIVERGENT materialized state: do_upgrade_config_with_bytes pushes config bytes only then restart_requested; each node re-derives state by replaying its WAL under the NEW config, with no snapshot pinning an identical post-upgrade state. Replay is config-dependent (lost_and_found re-homes orphaned jobs), so nodes can diverge; amplified by bug-851 WAL tail divergence.", + "file": "src/controller/config_repl.c do_upgrade_config_with_bytes (~1001-1163)", + "root_cause": "Config is a separate consistency domain (committed_config_tag, no term, tag anti-entropy) outside both the WAL log and raft.c. Upgrade activation = config push + restart + per-node WAL replay under new config; no snapshot transfers an identical materialized state, and state is interpreted against config. Identified via TLA+ gap analysis (config should be log-entry/snapshot-anchored). NOT empirically reproduced yet.", + "fix": "NOT FIXED. Descoped 2026-06-21 from ha-term-safety R8 into the independent `config-upgrade-snapshot` spec (.kiro/specs/config-upgrade-snapshot/, requirements generated). Fix = upgrade materializes a FULL entity-bearing state snapshot at the config-change marker (term,lsn) boundary, embeds new config, installs on every member; no per-node replay-under-new-config. HARD dependency: entity-bearing-snapshots (snapshots are a 16-byte LSN sentinel today; COW xzarray_freeze/thaw built but dormant). Verify by reproducing bug-857 (upgrade during chaotic multi-kill, diff materialized state).", + "tags": [ + "config", + "snapshot", + "ha", + "needs-verification", + "spec-needed", + "descoped", + "config-upgrade-snapshot", + "entity-bearing-snapshots", + "state-divergence", + "raft-safety", + "upgrade" + ], + "related_bugs": [ + "bug-851", + "bug-828" + ], + "occurrences": 1, + "last_seen": "2026-06-21T10:30:00.000Z" + }, + { + "id": "bug-878", + "timestamp": "2026-06-21T10:34:30.593Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T10:34:30.593Z" + }, + { + "id": "bug-879", + "timestamp": "2026-06-21T10:59:29.212Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/TLA-gap-analysis.md", + "root_cause": "Had `raft.c`", + "fix": "Changed to 's split architecture, the chosen direction is to ", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T10:59:29.212Z" + }, + { + "id": "bug-880", + "timestamp": "2026-06-21T12:56:07.740Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/requirements.md", + "root_cause": "Had `snapshot_transfer_s`", + "fix": "Changed to `(term, lsn)`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T12:56:07.740Z" + }, + { + "id": "bug-881", + "timestamp": "2026-06-21T13:00:46.210Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/design.md", + "root_cause": "Had `xce_test`", + "fix": "Changed to `OPCode_replConfigPush`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-21T13:04:57.579Z" + }, + { + "id": "bug-883", + "timestamp": "2026-06-21T13:05:44.284Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/research.md", + "root_cause": "Had `config_repl.c:1857-1944`", + "fix": "Changed to `xzarray_freeze`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-21T13:06:07.689Z" + }, + { + "id": "bug-884", + "timestamp": "2026-06-21T15:20:00Z", + "error_message": "test_persistence_constants:FAIL Expected TRUE Was FALSE (DEFAULT_SYNC_TIMEOUT_MS==50)", + "file": "src/lib/testsuite/test_config.c", + "root_cause": "commit ca1d9ae raised DEFAULT_SYNC_TIMEOUT_MS 50->500 in config.h but did not update the test assertion; pre-existing red on HEAD, unrelated to raft", + "fix": "synced assertion to ==500 (commit 7b6cbb8)", + "tags": [ + "test", + "config", + "stale-assertion", + "ha-commit-engine" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:20:00Z" + }, + { + "id": "bug-885", + "timestamp": "2026-06-21T15:21:59.689Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: index < 1 || index > RAFT_SIM_MAX_LOG", + "fix": "Added guard clause: if (index < 1 || index > RAFT_SIM_MAX_LOG)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:21:59.689Z" + }, + { + "id": "bug-886", + "timestamp": "2026-06-21T15:41:42.176Z", + "error_message": "Wrong reference: test_raft_init_zero_peers_returns_einval should be test_raft_init_oversize_peers_returns_einval", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_raft_init_zero_peers_returns_einval\" instead of \"test_raft_init_oversize_peers_returns_einval\"", + "fix": "Changed test_raft_init_zero_peers_returns_einval \u2192 test_raft_init_oversize_peers_returns_einval", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:41:42.176Z" + }, + { + "id": "bug-887", + "timestamp": "2026-06-21T15:45:29.785Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", + "fix": "Added guard clause: if (s->role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:45:29.785Z" + }, + { + "id": "bug-888", + "timestamp": "2026-06-21T15:52:45.272Z", + "error_message": "Missing guard clause", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "No early return/throw for edge case: index >= 1 && index <= RAFT_SIM_MAX_LOG", + "fix": "Added guard clause: if (index >= 1 && index <= RAFT_SIM_MAX_LOG)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:52:45.272Z" + }, + { + "id": "bug-889", + "timestamp": "2026-06-21T15:53:14.276Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:53:14.276Z" + }, + { + "id": "bug-890", + "timestamp": "2026-06-21T15:59:59.295Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: rc != 0", + "fix": "Added guard clause: if (rc != 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T15:59:59.295Z" + }, + { + "id": "bug-891", + "timestamp": "2026-06-21T16:06:37.923Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T16:06:37.923Z" + }, + { + "id": "bug-893", + "timestamp": "2026-06-21T16:16:47.908Z", + "error_message": "Wrong reference: g_te_spy should be g_pe_spy", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"g_te_spy\" instead of \"g_pe_spy\"", + "fix": "Changed g_te_spy \u2192 g_pe_spy", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T16:16:47.908Z" + }, + { + "id": "bug-894", + "timestamp": "2026-06-21T16:20:39.535Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->cbs->on_trace", + "fix": "Added guard clause: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T16:20:39.535Z" + }, + { + "id": "bug-895", + "timestamp": "2026-06-21T16:29:35.778Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: c->nodes[src].partition || c->nodes[dst].partition", + "fix": "Added guard clause: if (c->nodes[src].partition || c->nodes[dst])", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T16:29:35.778Z" + }, + { + "id": "bug-899", + "timestamp": "2026-06-21T16:42:30.241Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", + "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T16:42:30.241Z" + }, + { + "id": "bug-900", + "timestamp": "2026-06-21T16:51:25.960Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: RUN_TEST(test_sim_9_1_replication_and_partition_catch_up);; RUN_TEST(test_sim_9_2_divergent_suffix_truncation);", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-21T16:51:35.044Z" + }, + { + "id": "bug-903", + "timestamp": "2026-06-21T17:04:17.830Z", + "error_message": "Wrong operator: <= should be >=", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"<=\" instead of \">=\"", + "fix": "Changed operator <= \u2192 >=", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T17:04:17.830Z" + }, + { + "id": "bug-904", + "timestamp": "2026-06-21T21:44:26.804Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: node_idx < 0 || node_idx >= c->n", + "fix": "Added guard clause: if (node_idx < 0 || node_idx >= c->n)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T21:44:26.804Z" + }, + { + "id": "bug-905", + "timestamp": "2026-06-21T21:49:10.796Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-log-replication/tasks.md", + "root_cause": "Had `initial_log_len`", + "fix": "Changed to `raft_sim_restart_node`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T21:49:10.796Z" + }, + { + "id": "bug-907", + "timestamp": "2026-06-21T23:14:37.837Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft.c", + "root_cause": "No early return/throw for edge case: s->role != RAFT_LEADER", + "fix": "Added guard clause: if (s->role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T23:14:37.837Z" + }, + { + "id": "bug-908", + "timestamp": "2026-06-21T23:14:55.547Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: node_idx >= 0 && node_idx < c->n", + "fix": "Added guard clause: if (node_idx >= 0 && node_idx < c->n)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T23:14:55.547Z" + }, + { + "id": "bug-909", + "timestamp": "2026-06-21T23:30:04.636Z", + "error_message": "Missing guard clause", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "No early return/throw for edge case: g_mock.get_entry_term_log_len != 0 &&\n index == g_moc", + "fix": "Added guard clause: if (g_mock.get_entry_term_log_len != 0 &&\n )", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T23:30:04.636Z" + }, + { + "id": "bug-911", + "timestamp": "2026-06-21T23:32:38.225Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (s->cbs->on_vote_eligible != NULL)", + "fix": "Changed to: if (!raft_log_ok(s, cand_last_log_term, cand_last_log_)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-21T23:32:38.225Z" + }, + { + "id": "bug-913", + "timestamp": "2026-06-22T00:19:56.768Z", + "error_message": "Wrong reference: TEST_ASSERT_NOT_EQUAL_INT should be TEST_ASSERT_NOT_EQUAL", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"TEST_ASSERT_NOT_EQUAL_INT\" instead of \"TEST_ASSERT_NOT_EQUAL\"", + "fix": "Changed TEST_ASSERT_NOT_EQUAL_INT \u2192 TEST_ASSERT_NOT_EQUAL", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T00:19:56.768Z" + }, + { + "id": "bug-914", + "timestamp": "2026-06-22T01:24:34.541Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/spec.json", + "root_cause": "Had \"design-generated\"", + "fix": "Changed to \"tasks-generated\"", + "tags": [ + "auto-detected", + "wrong-value", + "json" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T01:24:34.541Z" + }, + { + "id": "bug-855", + "timestamp": "2026-06-22T03:10:06.653962Z", + "error_message": "controller_unit_test: 6 test_wal_catchup_* tests FAIL (Expected ... Was -11/-EAGAIN or wrong applied_lsn) on the raft_tla checkpoint branch", + "file": "src/controller/testsuite/test_config_repl.c (tests) / src/controller/persist.c:705 (guard)", + "root_cause": "PRE-EXISTING on checkpoint 20e59d0, NOT caused by ha-term-safety 1.1/1.2. The bug-850 contiguity guard (persist.c:705 rejects lsn>current_lsn with -EAGAIN) is committed at HEAD, but the test_wal_catchup_* tests feed forward-gap LSNs (30, 42) into a FRESH persist (current_lsn~=1) and still expect them to apply -> guard returns -EAGAIN -> applied_lsn does not advance. ha-term-safety changes touch only vote/schema/typemap/stubs; git diff --stat shows ZERO changes to persist.c/config_repl.c/test_config_repl.c. Also a separate run TIMED OUT 300s (the bug-850 DNR /tmp fsync-EIO + commit-WAIT signature; environment artifact on overlayfs).", + "fix": "FIXED (verified 2026-07-02): ran controller_unit_test directly \u2014 all test_wal_catchup_* tests (append_term_fields_round_trip, entry_advances_applied_lsn, entry_error_skipped_no_crash, entry_null_entry_no_crash, batch_advances_applied_lsn, partial_batch, batch_error_entry_skipped, batch_zero_entries_fallback, batch_256_entries) PASS. Landed as part of ha-term-safety's WAL term-index work (owner this bug deferred to).", + "tags": [ + "controller-test", + "stale-report", + "ha-term-safety", + "wal-catchup", + "EAGAIN", + "not-a-regression", + "contiguity-guard", + "fixed", + "bug-850", + "pre-existing" + ], + "related_bugs": [ + "bug-850", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-07-02T17:30:00.000Z" + }, + { + "id": "bug-918", + "timestamp": "2026-06-22T03:30:00Z", + "error_message": "nix develop fails: hash mismatch for c-capnproto fixed-output derivation (expected sha256-9dzSc3w+yxYzsF/rZe3qIarKOpjt2zvIyjuS5FXngIM=, got sha256-9rTlcTia+s9ARGeTi+8auGvM0RkS+wZK7aIkzecaeSk=)", + "file": "src/flake.nix", + "root_cause": "flake pins c-capnproto to the MUTABLE github archive/main.tar.gz; upstream main branch advanced so the tarball content/hash drifted, breaking the fixed-output derivation. Only pre-cached nix stores still build.", + "fix": "RESOLVED: user refreshed src/flake.lock; fresh nix develop builds (capnproto 1.2.0). A clean controller rebuild made the full controller test suite green (Ok:1 Fail:0, 12.7s) \u2014 the earlier wal_catchup -EAGAIN/decode failures were a STALE incremental-build artifact (schema compiled against the drifted toolchain), not fsync or a source regression.", + "tags": [ + "nix", + "flake", + "c-capnproto", + "env", + "build", + "worktree-isolation" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T03:45:00Z" + }, + { + "id": "bug-920", + "timestamp": "2026-06-22T03:58:08.538Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_term_safety_adapter.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T03:58:08.538Z" + }, + { + "id": "bug-921", + "timestamp": "2026-06-22T03:58:19.392Z", + "error_message": "Wrong reference: test_waldump should be test_term_safety_adapter", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_waldump\" instead of \"test_term_safety_adapter\"", + "fix": "Changed test_waldump \u2192 test_term_safety_adapter", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T03:58:28.433Z" + }, + { + "id": "bug-922", + "timestamp": "2026-06-22T03:59:43.099Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T03:59:43.099Z" + }, + { + "id": "bug-923", + "timestamp": "2026-06-22T04:01:37.467Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/persist.c", + "root_cause": "Condition was: if (hdr.lsn <= from_lsn)", + "fix": "Changed to: if (hdr.type < WAL_CHECKPOINT)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:01:37.467Z" + }, + { + "id": "bug-924", + "timestamp": "2026-06-22", + "error_message": "controller_unit_test: ~63 'Expected Non-NULL' + test_config_repl catchup FAILs (Expected 0 Was -11 EAGAIN) during ha-term-safety 1.3 validation", + "file": "environment (/ overlay filesystem)", + "root_cause": "Disk filled to 100% (7466 leftover /tmp/xce_* WAL dirs from repeated ASAN test runs). make_fresh_persist's persist_init mkdir_p hit ENOSPC -> returned NULL -> cascade of 'Expected Non-NULL'. Also overlayfs returned fdatasync EIO under disk pressure. NOT a code regression: baseline full suite passes; test_config_repl catchup tests are order-dependent and fail in isolation on baseline too.", + "fix": "rm -rf /tmp/xce_* freed 327GB; full controller suite then Ok:1 Fail:0. For future runs: clean /tmp/xce_* between heavy ASAN test cycles.", + "tags": [ + "disk-space", + "enospc", + "overlayfs", + "fdatasync-eio", + "controller-tests", + "flaky", + "ha-term-safety" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22" + }, + { + "id": "bug-925", + "timestamp": "2026-06-22T04:42:28.897Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:42:28.897Z" + }, + { + "id": "bug-926", + "timestamp": "2026-06-22T04:45:07.018Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:45:07.018Z" + }, + { + "id": "bug-928", + "timestamp": "2026-06-22T04:55:58.789Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:55:58.789Z" + }, + { + "id": "bug-929", + "timestamp": "2026-06-22T04:57:03.605Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T04:57:15.567Z" + }, + { + "id": "bug-930", + "timestamp": "2026-06-22T04:57:25.141Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T04:57:25.141Z" + }, + { + "id": "bug-931", + "timestamp": "2026-06-22T05:45:09.373Z", + "error_message": "Function not marked async", + "file": "src/include/repl.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T05:45:09.373Z" + }, + { + "id": "bug-933", + "timestamp": "2026-06-22T05:47:50.037Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !d", + "fix": "Added guard clause: if (!d)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T05:47:50.037Z" + }, + { + "id": "bug-937", + "timestamp": "2026-06-22T06:39:51.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha || !ha->raft_c", + "fix": "Added guard clause: if (!ha || !ha->raft_c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T06:39:51.351Z" + }, + { + "id": "bug-941", + "timestamp": "2026-06-22T07:53:06.457Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:53:06.457Z" + }, + { + "id": "bug-942", + "timestamp": "2026-06-22T07:54:52.956Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:54:52.956Z" + }, + { + "id": "bug-943", + "timestamp": "2026-06-22T07:55:30.714Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:55:30.714Z" + }, + { + "id": "bug-944", + "timestamp": "2026-06-22T07:56:08.623Z", + "error_message": "Wrong reference: ha_arm_own_term_gate should be ha_arm_own_term_gate_for_test", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Used \"ha_arm_own_term_gate\" instead of \"ha_arm_own_term_gate_for_test\"", + "fix": "Changed ha_arm_own_term_gate \u2192 ha_arm_own_term_gate_for_test", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T07:56:08.623Z" + }, + { + "id": "bug-945", + "timestamp": "2026-06-22T08:09:41.508Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:09:41.508Z" + }, + { + "id": "bug-948", + "timestamp": "2026-06-22T08:21:57.062Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* ha-term-safety task 3.2 (R2.6 / G5): route the heartbeat ; * library as a 0-entry AppendEntries so the LIBRARY performs", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T08:23:05.006Z" + }, + { + "id": "bug-953", + "timestamp": "2026-06-22T08:48:45.792Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: rc == 0 && out_lsn", + "fix": "Added guard clause: if (rc == 0 && out_lsn)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T08:48:45.792Z" + }, + { + "id": "bug-957", + "timestamp": "2026-06-22T09:08:41.177Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ep_is_vote", + "fix": "Added guard clause: if (ep_is_vote)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T09:11:38.490Z" + }, + { + "id": "bug-960", + "timestamp": "2026-06-22T09:20:00.000Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "ha-term-safety", + "raft", + "peer-index", + "match_index", + "oob", + "false-quorum", + "task-3.3", + "remediation", + "c", + "auto-detected", + "guard-clause" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 2, + "last_seen": "2026-06-22T09:43:23.309Z" + }, + { + "id": "bug-959", + "timestamp": "2026-06-22T09:31:11.227Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !sha256 || !buf || !out_len", + "fix": "Added guard clause: if (!sha256 || !buf || !out_len)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:31:11.227Z" + }, + { + "id": "bug-961", + "timestamp": "2026-06-22T09:45:31.779Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c || !sha256", + "fix": "Added guard clause: if (!c || !sha256)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T09:45:31.779Z" + }, + { + "id": "bug-962", + "timestamp": "2026-06-22T10:04:06.960Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: committed == 0", + "fix": "Added guard clause: if (committed == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:04:06.960Z" + }, + { + "id": "bug-964", + "timestamp": "2026-06-22T10:05:22.007Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: acked_out", + "fix": "Added guard clause: if (acked_out)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:05:22.007Z" + }, + { + "id": "bug-966", + "timestamp": "2026-06-22T10:10:11.668Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p", + "fix": "Added guard clause: if (!p)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:10:11.668Z" + }, + { + "id": "bug-967", + "timestamp": "2026-06-22T10:10:17.811Z", + "error_message": "Wrong reference: persist_roll_segment should be persist_test_force_new_segment", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"persist_roll_segment\" instead of \"persist_test_force_new_segment\"", + "fix": "Changed persist_roll_segment \u2192 persist_test_force_new_segment", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:10:17.811Z" + }, + { + "id": "bug-970", + "timestamp": "2026-06-22T10:24:35.905518Z", + "error_message": "ha-term-safety task 4.2 rejected (review r1): live config-change path (do_upgrade_config_with_bytes) never appended WAL_CONFIG_MARKER \u2014 config_committed_marker had ZERO production callers (dead primitive); and checkpoint/compaction could trim a committed marker segment, regressing the derive to -ENOENT.", + "file": "src/controller/config_repl.c, src/controller/persist.c", + "root_cause": "Finding1: do_upgrade_config_with_bytes still drove the term-blind committed_config_tag push/ack/advance and never called config_append_marker, so the derive primitive was dead code. Finding2: persist_checkpoint->trim_wal_segments unlinks older segments and the snapshot sentinel carried no config reference, so a committed marker in a trimmed segment vanished and config_committed_marker regressed/returned -ENOENT.", + "fix": "Added config_commit_change() wired into do_upgrade_config_with_bytes step 7: appends WAL_CONFIG_MARKER as the consensus record (config_append_marker), then delegates byte delivery + the existing majority barrier to push_config_to_standbys (mechanism unchanged); committed_config_tag is now only a cache, config_committed_marker (committed-prefix derive) is authoritative. Compaction safety: persist_checkpoint scans the committed prefix for the latest marker {version_tag,sha256} (scan_latest_committed_config_ref) BEFORE trim and stamps it into the snapshot sentinel (41-byte optional tail); load_snapshot reads it back into persist.cfgref_*; config_committed_marker falls back to it when no marker survives the live WAL. Added integration test (drives config_commit_change) + checkpoint-trim test + persist_test_force_new_segment seam. cb5 upgrade pins made persist-backed so the marker append succeeds. RED confirmed (Expected 1 Was 0; Expected 0 Was -110); GREEN 0 failures direct-run.", + "tags": [ + "ha-term-safety", + "task-4.2", + "config-marker", + "wal", + "checkpoint", + "compaction", + "review-remediation", + "controller" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T10:24:35.905518Z" + }, + { + "id": "bug-971", + "timestamp": "2026-06-22T10:39:44.072Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: !mc", + "fix": "Added guard clause: if (!mc)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:39:44.072Z" + }, + { + "id": "bug-972", + "timestamp": "2026-06-22T10:40:31.131Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T10:40:31.131Z" + }, + { + "id": "bug-975", + "timestamp": "2026-06-22T11:00:42.661Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: !ha", + "fix": "Added guard clause: if (!ha)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:00:42.661Z" + }, + { + "id": "bug-976", + "timestamp": "2026-06-22T11:15:00.000Z", + "error_message": "ha-term-safety task 4.3 REJECTED r1: verify-before-activate BYPASSED on push path (handle_repl_config_push pre-activated via load_primary_config before the hash gate); recovery-activate had NO production call site (boot loaded blindly); serve-fence had NO production reader (a fenced member still served)", + "file": "src/controller/config_repl.c, src/controller/ha.c, src/controller/myapp.c", + "root_cause": "config_activate_committed (verify-before-activate + fence) was unit-correct but not integrated: (A) handle_repl_config_push called load_primary_config UNCONDITIONALLY before config_activate_committed, so mismatched bytes went live before the gate; (B) myapp.c boot step 7 + on_commit_advanced never called the consumer, so a recovering member loaded config blind against the committed marker; (C) ha_dispatch_gate / pre-promotion gate never read config_serve_fence, so a fenced member still served + could promote", + "fix": "(A) handle_repl_config_push now SAVES bytes (conf_save_config) then calls config_activate_committed ONLY \u2014 removed premature load_primary_config; activate is the sole activation point (hash-verify gate). (B) config_activate_committed -ENOENT branch falls back to load_primary_config (absent marker != mismatch, no false fence); myapp.c step 7 calls config_activate_committed (latest committed marker, verified). on_commit_advanced documents why activation is deferred off the iomgr thread (async-I/O invariant: scan/load do disk I/O). (C) ha_dispatch_gate CTRL_PRIMARY now refuses config-dependent state-changing ops when config_serve_fenced (replication/read-only pass), pre-promotion gate in raft_cb_become_leader refuses promotion when fenced via new ha_config_promotion_fenced; member stays ALIVE, never self-terminates; fence clears on correct bytes re-verify", + "tags": [ + "ha-term-safety", + "config-replication", + "verify-before-activate", + "serve-fence", + "dispatch-gate", + "promotion-gate", + "safety", + "task-4.3", + "remediation" + ], + "related_bugs": [ + "bug-851", + "bug-924" + ], + "occurrences": 1, + "last_seen": "2026-06-22T11:15:00.000Z" + }, + { + "id": "bug-977", + "timestamp": "2026-06-22T11:21:09.742Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !c", + "fix": "Added guard clause: if (!c)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:21:09.742Z" + }, + { + "id": "bug-980", + "timestamp": "2026-06-22T11:32:35.966Z", + "error_message": "Wrong reference: ClusterConfig should be cluster_config_t", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Used \"ClusterConfig\" instead of \"cluster_config_t\"", + "fix": "Changed ClusterConfig \u2192 cluster_config_t", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:32:35.966Z" + }, + { + "id": "bug-981", + "timestamp": "2026-06-22T11:35:00Z", + "error_message": "ha-term-safety 4.3 Finding D: on a standby, config MARKER (WAL) and config BYTES (out-of-band push) arrive on independent transports; on a 2nd+ upgrade the NEW bytes are saved before the NEW marker enters the committed prefix, so config_activate_committed hash-mismatches the OLDER committed marker and sets the serve-fence. When the NEW marker later commits, nothing re-runs activation (on_commit_advanced does no disk I/O on the iomgr ACK thread; primary's config-behind re-push net does not fire because conf_save_config already advanced the advertised config_version_tag) -> standby PERMANENTLY serve-fenced + non-promotable until reboot. Liveness violation of R7.9.", + "file": "src/controller/config_repl.c, src/controller/ha.c", + "root_cause": "No post-commit retry trigger for a config serve-fence set under the push-before-commit ordering. config_committed_marker returns the older committed marker until the new marker is in the committed prefix; the two activation call sites (boot, handle_repl_config_push) both run BEFORE the new marker commits, and ha_cb_commit_advanced_stub deliberately did nothing (async-I/O invariant forbids disk reads on the iomgr ACK thread).", + "fix": "Added config_schedule_activate_retry(c) (non-blocking ordq_put via delay_task_ms, no-op unless serve-fenced AND delaymgr running) and config_activate_retry_fn(arg) (delaymgr task body that re-runs config_activate_committed off the iomgr thread). ha_cb_commit_advanced_stub now schedules the retry whenever config_serve_fenced(c) is set after commit advances. Once the NEW marker is in the committed prefix and the saved bytes hash-verify, the retry clears the fence and activates; -ENOENT still falls back without fencing; no busy-spin (no-op once unfenced). RED captured by stubbing config_activate_retry_fn to a no-op (fence stayed set, NEW config not loaded); GREEN with the real off-thread retry.", + "tags": [ + "ha-term-safety", + "config-repl", + "serve-fence", + "liveness", + "async-io", + "delaymgr", + "commit-advance", + "push-before-commit", + "R7.9" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:35:00Z" + }, + { + "id": "bug-983", + "timestamp": "2026-06-22T11:53:36.582Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: commit_index", + "fix": "Added guard clause: if (commit_index)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T11:53:36.582Z" + }, + { + "id": "bug-984", + "timestamp": "2026-06-22T12:03:15.865Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:03:15.865Z" + }, + { + "id": "bug-985", + "timestamp": "2026-06-22T12:06:43.530Z", + "error_message": "Wrong return value", + "file": "src/controller/testsuite/waldump.c", + "root_cause": "Was returning: -EINVAL;", + "fix": "Now returns: waldump_path_visit(path, out, stats, NULL, NULL);", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:06:43.530Z" + }, + { + "id": "bug-986", + "timestamp": "2026-06-22T12:08:51.529Z", + "error_message": "Incorrect value in code", + "file": "src/controller/testsuite/test.c", + "root_cause": "Had \"waldump.h\"", + "fix": "Changed to \"walcompare.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:08:51.529Z" + }, + { + "id": "bug-987", + "timestamp": "2026-06-22T12:09:28.306Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test.c", + "root_cause": "No early return/throw for edge case: st.n_gaps > 0 || st.crc_broken", + "fix": "Added guard clause: if (st.n_gaps > 0 || st.crc_broken)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T12:09:28.306Z" + }, + { + "id": "bug-990", + "timestamp": "2026-06-22T13:24:07.476480", + "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term (e.g. primary LSN1..5 term=1, ctrl-2/ctrl-3 term=0); a clean failover's committed prefix is byte-identical in payload (pcrc) but NOT in term, so R5.1 'byte-identical WAL including terms' cannot pass. walcompare passes only via legacy-0 -> WAL_LEGACY_TERM tolerance, which masks it on a fresh cluster.", + "file": "src/controller/repl.c (live send repl_broadcast persist.c:747) + src/controller/ha.c (ha_apply_wal_catchup_entry term!=0 legacy direct-apply)", + "root_cause": "The live replication SEND path was deferred from tasks 3.2/3.3 and still uses the legacy term-0 repl_broadcast; the standby applies via the legacy direct-apply path, NOT through raft_on_append_entries_request, so it never receives/persists the leader's term. The raft library send loop (on_send_append_entries / raft_send_append_entries) has ZERO live callers. on_persist_entry stamps the real term on the PRIMARY only.", + "fix": "NOT FIXED (out of task 5.2 testsuite-only boundary). Remaining integration work: retire the term-0 repl_broadcast live SEND and drive raft_send_append_entries/on_send_append_entries on the live primary so standbys persist the leader's term via raft_on_append_entries_request. Built clean_failover_preservation.sh (XCE_FAILOVER_STRICT_TERM=1) as the ready-to-run assertion this retirement must satisfy.", + "tags": [ + "ha-term-safety", + "task-5.2", + "live-send-retirement", + "term-0", + "repl_broadcast", + "committed-prefix-divergence", + "bug-851", + "BLOCKED" + ], + "related_bugs": [ + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T13:24:07.476480" + }, + { + "id": "bug-991", + "timestamp": "2026-06-22T16:30:00.000Z", + "error_message": "Standby WAL catch-up advanced applied_lsn to lsn+1 (next-to-apply) instead of lsn (last-held). The standby then advertised one LSN too high; the leader's catch-up streamer streamed (lsn+1, committed] and SKIPPED the real next-needed entry lsn+1, wedging the library logOk gate forever (no WalAck -> match_index never reaches -> raft commit_index frozen -> committed_lsn pinned at 1). Catch-up-path-specific; live repl_broadcast (absolute-LSN, no skip) commits fine, so failure was nondeterministic.", + "file": "src/controller/ha.c (ha_apply_term_entry_via_library ~2115; batch ~2205; single-entry ~2225)", + "root_cause": "applied_lsn carried TWO conflicting semantics. Recovery (=durable_floor), checkpoint (=checkpoint_lsn), init (=0) and the leader's catch-up streamer + ha_evaluate_repl_action all treat it as 'last contiguously-held LSN' and stream (standby_lsn, committed]. But the three live/catch-up apply setters advanced to entry->lsn + 1 ('next-to-apply'). After a standby applied catch-up entry N it advertised N+1, so the leader's next stream began at N+2 and skipped N+1 -> logOk reject -> no WalAck.", + "fix": "fix-1: the three live-apply setters now advance applied_lsn to the LAST-HELD LSN (e->lsn / last_good_lsn / entry->lsn), not +1, so applied_lsn means 'last contiguously-held LSN' everywhere. Reader audit (all consistent with last-held, no +1 reader needed): state report (advertised LSN, ha.c:2461) now matches streamer contract; convergence reported_lsn>=committed_lsn (2536) correct; snapshot/stream boundary standby_lsn+1 routed through the library, so raft.log_len already == applied_lsn after fix-1; raft.log_len legitimately differs from applied_lsn on the legacy term-0 direct-apply and recovery paths, so re-sourcing would regress bug-848 incremental catch-up -> fix-1 alone delivers the value the streamer+logOk need. Unit RED->GREEN: test_catchup_applied_lsn_is_last_held_no_skip (Expected 1 Was 2 pre-fix, PASS post-fix). 7 legacy test_wal_catchup_* expected values updated lsn+1 -> last-held. NOTE: the LIVE clean_failover_preservation STRICT_TERM commit=5 target was NOT reachable in the sandbox: committed_lsn stalls at 1 IDENTICALLY with this fix stashed (baseline experiment) -> that stall is a SEPARATE upstream blocker (library raft.log_len / commit-advance not advancing past the own-term noop; ha-term-safety task 3.1 area), NOT the bug-991 applied_lsn skip. waldump confirms all 3 members hold LSN 1..5 byte-identical (term 1) -> the replication half (which this fix corrects) is sound.", + "tags": [ + "ha", + "controller", + "wal-catchup", + "applied_lsn", + "commit-stall", + "off-by-one", + "term-safety", + "bug-991", + "streamer-skip" + ], + "related_bugs": [ + "bug-851", + "bug-850", + "bug-848", + "bug-924", + "bug-960" + ], + "occurrences": 1, + "last_seen": "2026-06-22T16:30:00.000Z" + }, + { + "id": "bug-993", + "timestamp": "2026-06-22T15:10:39.717Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:10:39.717Z" + }, + { + "id": "bug-998", + "timestamp": "2026-06-22T15:27:10.703Z", + "error_message": "Wrong reference: test_repl_ack_currentterm_quorum_advances_commit should be test_live_broadcast_receive_persists_leader_term", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Used \"test_repl_ack_currentterm_quorum_advances_commit\" instead of \"test_live_broadcast_receive_persists_leader_term\"", + "fix": "Changed test_repl_ack_currentterm_quorum_advances_commit \u2192 test_live_broadcast_receive_persists_leader_term", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:27:10.703Z" + }, + { + "id": "bug-990-fix", + "timestamp": "2026-06-22T16:00:00.000000", + "error_message": "ha-term-safety 5.2 live failover: standbys persist per-entry WAL term=0 while primary persists the real term; clean-failover committed prefix not byte-identical in TERM (R5.1 term-identity gap). Original bug-990.", + "file": "src/controller/repl.c (repl_broadcast), src/controller/persist.c (persist_wal_async call site + entry_term_locked), src/controller/config_repl.c (ha_stream_wal_to_standby catch-up), src/controller/ha.c (ha_apply_wal_catchup_entry receive route), src/lib/request.c (wal_entry_to_wire per-entry prev tuple encode)", + "root_cause": "The LIVE replication SEND paths (repl_broadcast live append + ha_stream_wal_to_standby catch-up) built wal_catchup_entry_t via memset(0) and never set entry.term/prev_log_index/prev_log_term, so standbys received term=0 entries and applied them via the LEGACY term-0 direct-apply branch (persist_wal_write_replicated stamps the STANDBY's local p->term, not the leader's). The term-carrying library receive route (raft_on_append_entries_request, task 3.2) only fired for term!=0 entries, which the live send never produced. Additionally wal_entry_to_wire only encoded per-entry term, not the per-entry prev tuple, so multi-entry catch-up batches could not be routed through the library one-at-a-time.", + "fix": "Task 3.4 minimal cutover (chosen over driving raft_send_append_entries live, which would rip out the repl model and risk 848/849/850): (1) repl_broadcast gains a prev_log_term param and sets entry.term=hdr->term, entry.prev_log_index=hdr->lsn-1, entry.prev_log_term (sourced under p->lock via new entry_term_locked accessor in persist_wal_async). (2) ha_stream_wal_to_standby sets each catch-up entry's term + prev tuple (tracking the prior streamed entry's lsn/term in ascending order). (3) wal_entry_to_wire now encodes prevLogIndex/prevLogTerm/leaderCommit PER SUB-ENTRY so a multi-entry batch is library-routable. (4) ha_apply_wal_catchup_entry routes every term-carrying entry (single + each batch sub-entry) through new ha_apply_term_entry_via_library -> ha_consume_append_entries -> raft_on_append_entries_request -> on_persist_entry, which persist_set_term(leader_term) before persist_wal_write_replicated (bug-850 contiguity guard preserved beneath). The term-0 legacy direct-apply is retired for live/catch-up entries. ACK/commit path (3.3) unchanged.", + "tags": [ + "ha-term-safety", + "task-3.4", + "live-send-retirement", + "term-0", + "repl_broadcast", + "bug-990", + "FIXED", + "catch-up-term" + ], + "related_bugs": [ + "bug-990", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T16:00:00.000000", + "validation": "Unit RED->GREEN: test_broadcast_carries_leader_term_and_prev_tuple (repl: neuter term -> Expected 7 Was 0) + test_live_broadcast_receive_persists_leader_term (adapter: neuter receive route -> Expected 7 Was 3, i.e. standby persisted its LOCAL term). Full controller suite 175 Tests 0 Failures (clean /tmp direct binary) + meson Ok:1 Fail:0. Live STRICT_TERM gate (clean_failover_preservation.sh) NOT runnable in this overlayfs sandbox: baseline-confirmed no failover completes (split-vote no-promotion within 20s + fdatasync errno=5) \u2014 bug-924 env limitation, requires real-fs. The 7 test_wal_catchup_* (test_config_repl.c) + occasional durability FAILs on the direct binary are the SAME pre-existing bug-924 overlayfs fdatasync artifact (identical on baseline)." + }, + { + "id": "bug-1000", + "timestamp": "2026-06-22T15:44:38.466Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T15:44:38.466Z" + }, + { + "id": "bug-1001", + "timestamp": "2026-06-22T15:44:54.113Z", + "error_message": "Missing error handling in errno", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - _Blocked (live portion only \u2014 build+unit gate CONFIRMED GR", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T15:48:39.777Z" + }, + { + "id": "bug-1002", + "timestamp": "2026-06-22T21:10:31.345Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-term-safety/tasks.md", + "root_cause": "Had 's term and standbys persist it, proven at the int", + "fix": "Changed to 's REAL term (2) for the committed prefix (pre-3.4", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T21:10:31.345Z" + }, + { + "id": "bug-1003", + "timestamp": "2026-06-22T22:09:23.064Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (agreed == 0)", + "fix": "Changed to: if (s->cbs->on_trace)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T22:09:23.064Z" + }, + { + "id": "bug-1005", + "timestamp": "2026-06-22T22:31:03.315Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_term_safety_adapter.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: RUN_TEST(test_catchup_applied_lsn_is_last_held_no_skip);", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T22:31:08.791Z" + }, + { + "id": "bug-1006", + "timestamp": "2026-06-22T22:32:53.722Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/ha.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Advance applied_lsn to last_good_lsn after the full batch; * last contiguously-held LSN, NOT last_good_lsn + 1 \u2014 see th", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-22T22:33:00.066Z" + }, + { + "id": "bug-1009", + "timestamp": "2026-06-22T22:44:50.365Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-22T22:44:50.365Z" + }, + { + "id": "bug-1013", + "timestamp": "2026-06-22T00:00:00Z", + "error_message": "Stable HA cluster: committed_lsn pinned at 1 (< 5) while all 5 entries are durable on all 3 nodes \u2014 STRICT_TERM live failover gate fails 'committed_lsn>=5'", + "file": "src/controller/commit.c", + "root_cause": "ha-term-safety task 2.4 added a min(durable-majority, raft_commit_index) ceiling in majority_candidate_locked. raft_commit_index is fed only via the library ACK path (ha_raft_on_repl_ack -> raft_advance_commit -> on_commit_advanced), which advances from match_index[] driven by the asynchronous catch-up streamer and routinely LAGS the durable majority. When delivery goes via catch-up, raft_commit_index stays low and PINS committed_lsn below the safe durable watermark (a redundant, starvation-prone gate). The durable-majority barrier was already current-term-safe under single-writer + the own-term NOOP writable gate, making the ceiling redundant.", + "fix": "Option A: removed the raft_commit_index fold from majority_candidate_locked (commit.c). The durable-majority barrier + own-term writable_lsn gate are now authoritative for commit-advance. commit_set_raft_commit_index / ce->raft_commit_index made INERT-ADVISORY (recorded for observability/conformance-oracle only; still fed by on_commit_advanced so the library half keeps a sink). Kept durable self-cap (persist_durable_lsn), persist-before-ACK, and assert(cand<=persist_durable_lsn). Verified live: commit advances 4->5 via durable barrier (commit.c:184); walcompare OK committed prefix byte-identical across 3 nodes up to LSN 5.", + "tags": [ + "ha", + "commit", + "raft", + "term-safety", + "starvation", + "commit-ceiling", + "optA", + "liveness", + "catch-up" + ], + "related_bugs": [ + "bug-991", + "bug-661", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-06-22T00:00:00Z" + }, + { + "id": "bug-828-part2", + "timestamp": "2026-06-23T03:05:00.000Z", + "error_message": "Clean failover still truncates committed jobs: new primary logs 'safe_committed_lsn=1 (from 2 granted voters)' and walcompare reports committed-prefix DIVERGENCE at LSN 2; jobs at LSN 2..5 lost.", + "file": "src/controller/testsuite/test_ha_vote_gate.c (test added); ha.c/persist.c advertise path investigated (no change needed)", + "root_cause": "INVESTIGATED, NOT an advertisement-staleness bug. The advertise path already reflects the live durable tip: ha_candidate_advertise_lsn -> persist_durable_floor, and the voter floor = max(persist_durable_floor, learned_committed); persist_durable_floor == persist_durable_lsn (both read p->durable_lsn, the fsync watermark), and part-1's secondary fix advances durable_lsn after the standby's inline fdatasync in persist_wal_write_replicated. LIVE evidence (fix828c_r1..r3, STRICT_TERM clean failover x3) proves the surviving voters advertise 1 because they are GENUINELY durable only at LSN 1: their WAL physically holds 1..2 (LSN2 = their own term-3 NOOP), recovered tip=2, durable_floor=1. The original primary ctrl-1 (down at election time, WAL 1..5 tip=5) advanced committed_lsn 1->2->3->4->5 in ~250ms while standbys had only converged applied_lsn=1 and catch-up streaming was still in flight. So committed advanced past what the surviving majority durably held \u2014 the UPSTREAM replication / commit-barrier durability gap (bug-795/661), not the vote advertisement. safe_committed_lsn=1 is the CORRECT, SAFE Raft answer given the real durable state; advertising the true durable tip cannot conjure data never replicated.", + "fix": "No production code change required for part-2 \u2014 the durable-tip advertisement is already correct & safe after part-1. Added unit test test_live_durable_tip_advertised_over_lagging_learned (test_ha_vote_gate.c): a standby durable at LSN5 (real persist, flush -> durable_lsn=5) with learned_committed lagging at 1 must advertise 5 as BOTH candidate self-LSN (ha_candidate_advertise_lsn) and voter floor (ha_vote_self_freshness over persist_durable_floor, dominating learned=1), and ha_compute_safe_committed_lsn from two such granting voters must return 5 \u2014 locking in the in-scope contract. Explicitly did NOT fold learned_committed into the CANDIDATE advertise (the requested max(...,learned_committed)) because that re-opens the truncated-candidate-over-advertises-and-wins hole ha-commit-durability task 4.2 deliberately closed (candidate must advertise only a DURABLE position; learned_committed is volatile/not necessarily durable on this node). The residual live job loss is bug-795/661, out of scope.", + "tags": [ + "safety", + "failover", + "safe-committed-lsn", + "durable-tip", + "advertise", + "leader-completeness", + "bug-795", + "bug-661", + "commit-durability", + "INVESTIGATION", + "c" + ], + "related_bugs": [ + "bug-828", + "bug-795", + "bug-661", + "bug-851", + "bug-924" + ], + "occurrences": 1, + "last_seen": "2026-06-23T03:05:00.000Z" + }, + { + "id": "bug-1023", + "timestamp": "2026-06-23T03:45:00Z", + "error_message": "Clean single failover (STRICT_TERM) loses committed jobs: safe_committed_lsn=1 from votes despite a standby being genuinely durable at LSN 5; walcompare reports committed-prefix divergence at LSN 2 (jobs gone on the new leader).", + "file": "src/controller/ha.c", + "root_cause": "ha_enter_real_election() PRE-TRUNCATED the candidate's WAL above ha->last_known_committed_lsn BEFORE the vote. last_known_committed_lsn is the LOCALLY-LEARNED commit watermark, which LAGS the true cluster commit point (the primary commits an entry on a durable majority, then dies before a heartbeat carries that committed LSN to the standby). persist_truncate_wal_above clamps durable_lsn down to the cutoff (clamp_durable_lsn_locked). So a standby durably holding committed entries [committed+1..tip] discarded them, then advertised the stale-low cutoff as its durable floor (ha_candidate_advertise_lsn / ha_vote_self_freshness both read persist_durable_floor). Every voter doing the same collapsed safe-committed-from-votes to the stale watermark; the AUTHORITATIVE post-election truncate (raft_cb_become_leader -> persist_truncate_wal_above(safe_committed_lsn)) then kept only that stale prefix, losing committed jobs. Live trace (DBG828) PROVED the standbys wrote+fdatasync'd LSN 2-5 (durable_lsn=5) and only LOST them at the pre-election truncate clamp \u2014 NOT a durability/ack-before-fsync violation. Final bug-828/795 slice.", + "fix": "Removed the pre-election persist_truncate_wal_above(last_known_committed_lsn) in ha_enter_real_election (src/controller/ha.c). Raft model: a candidate does NOT truncate its log before campaigning \u2014 it keeps its full durable log and advertises its real durable tip so voters holding more committed entries deny a colder candidate and the winner has every committed entry (Leader Completeness). Divergent UNCOMMITTED tails are removed only by the authoritative post-election safe-commit truncate (above the majority-held point) and by the library's per-entry (term,lsn) reconcile (on_truncate_from) during AppendEntries. RED: live STRICT_TERM harness safe=1 + walcompare divergence at LSN 2. GREEN: every run now logs safe_committed_lsn=5 (from 2 granted voters) and walcompare 'committed prefixes identical'. Unit: new test_repl_catchup_durable_floor_survives_failover (test_repl_apply.c) + 177/177 unit GREEN. NOTE: live EXIT still 1 due to a PRE-EXISTING libxce client-failover bug (bug: 'client op fails to reach live primary when a DEAD controller is FIRST in the masters list') surfacing now that the durability bug no longer masks the post-walcompare expect-job/wait-primary step \u2014 committed jobs are physically present on disk (walcompare proves it).", + "tags": [ + "ha", + "raft", + "failover", + "leader-completeness", + "bug-828", + "bug-795", + "durability", + "truncate", + "election", + "durable_lsn" + ], + "related_bugs": [ + "bug-661", + "bug-795", + "bug-828", + "bug-851", + "bug-854", + "bug-339" + ], + "occurrences": 1, + "last_seen": "2026-06-23T03:45:00Z" + }, + { + "id": "bug-1025", + "timestamp": "2026-06-23T04:09:25.647Z", + "error_message": "Wrong reference: $MASTERS should be $MASTERS_LIVE", + "file": "src/controller/testsuite/clean_failover_preservation.sh", + "root_cause": "Used \"$MASTERS\" instead of \"$MASTERS_LIVE\"", + "fix": "Changed $MASTERS \u2192 $MASTERS_LIVE", + "tags": [ + "wrong-reference", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T04:09:25.647Z" + }, + { + "id": "bug-1028", + "timestamp": "2026-06-23", + "error_message": "Post-failover AVAILABILITY: after a clean failover the new primary never becomes available to serve clients (wait-primary times out). New primary logs 'ha_dispatch_repl_action_for_standby: standby[0] lsn=5 < committed_lsn=6 -> stream WAL batches' every ~3s indefinitely; the lagging standby never advances to the new primary's own-term tip, and the client cannot reach a serving primary. Surfaced only AFTER bug-828 was fixed (the committed-job-loss previously masked this by failing the harness earlier at walcompare).", + "file": "src/controller/ha.c (ha_dispatch_repl_action_for_standby / catch-up streamer on the NEW term), config_repl.c (ha_stream_wal_to_standby), persist.c (standby apply) -- NOT data loss; committed prefix is byte-identical on disk (walcompare OK).", + "root_cause": "ha_stream_wal_to_standby (config_repl.c) stamped each streamed entry's term and the AppendEntries prev_log_term from raw on-disk hdr.term (the WAL header), but the standby's logOk (raft_on_append_entries_request) and the library's own send path both source terms from persist_entry_term (the authoritative in-memory per-LSN term index), which maps a legacy-0 entry to WAL_LEGACY_TERM(1). So for a legacy-0 predecessor entry (raw hdr.term==0), the streamer sent prev_log_term=0 while the standby computed its own term as 1 \u2192 logOk mismatch \u2192 the new-term NOOP (the post-promotion first entry) is perpetually rejected \u2192 no WalAck \u2192 applied_lsn stuck at pre-promotion value \u2192 the heartbeat state report advertises the stale lsn \u2192 the primary re-streams [lsn+1..committed] every heartbeat cycle. The committed prefix stays byte-identical because only term metadata diverges, not data. CONFIRMED: baseline walcompare shows committed-prefix DIVERGENCE (term mismatch \u2192 CRC-chain breaks on some members); with the fix the committed prefix is IDENTICAL.", + "fix": "FIXED (2026-06-23). ha_stream_wal_to_standby gained a const persist_t* p parameter (decl in config_repl.h). All three hdr.term source sites in the streamer body (entry->term, prev_streamed_term on the main path, prev_streamed_term on the OOM-skip path) replaced with p ? persist_entry_term(p, hdr.lsn) : hdr.term (NULL-p path for count-only unit tests). Callers (ha_dispatch_repl_action_for_standby in ha.c, catchup_stream_range in config_repl.c) pass &c->persist. Unit test (test_wal_stream_term_from_index_not_raw_hdr) legacy-appends LSN 1 at term 0 + LSN 2 at term 3; streams standby_lsn=1,committed=2; asserts streamed entry term==3 and prev_log_term==WAL_LEGACY_TERM(1). RED\u2192GREEN proven (Expected 1 Was 0 \u2192 PASS). Live: standby converge + committed prefix identical.", + "tags": [ + "failover", + "availability", + "liveness", + "catch-up", + "new-primary", + "not-serving", + "ha-commit-durability", + "NOT-FIXED", + "FIXED", + "catch-up", + "ha_stream_wal_to_standby", + "persist_entry_term", + "logOk" + ], + "related_bugs": [ + "bug-828", + "bug-854", + "bug-795", + "bug-991" + ], + "occurrences": 2, + "last_seen": "2026-06-23" + }, + { + "id": "bug-1033", + "timestamp": "2026-06-23T08:05:43.916Z", + "error_message": "Incorrect value in code", + "file": "repro854.sh", + "root_cause": "Had 'set masters %s\\nwait-primary 6000\\nctrlstatus\\nqu", + "fix": "Changed to 'set masters %s\\nset myname failover-test\\nclient\\", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T08:05:43.916Z" + }, + { + "id": "bug-1035", + "timestamp": "2026-06-23T09:53:37.812Z", + "error_message": "Incorrect value in code", + "file": "src/lib/xcectx.c", + "root_cause": "Had \"xcectx.h\"", + "fix": "Changed to \"controller.capnp.h\"", + "tags": [ + "auto-detected", + "wrong-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T09:53:37.812Z" + }, + { + "id": "bug-1036", + "timestamp": "2026-06-23T10:44:48.074Z", + "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", + "file": "src/lib/netio.c", + "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", + "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T10:44:48.074Z" + }, + { + "id": "bug-1037", + "timestamp": "2026-06-23T10:48:18.493Z", + "error_message": "Wrong reference: handle_ssl_handshake should be endpoint_ssl_handshake", + "file": "src/controller/cluster.c", + "root_cause": "Used \"handle_ssl_handshake\" instead of \"endpoint_ssl_handshake\"", + "fix": "Changed handle_ssl_handshake \u2192 endpoint_ssl_handshake", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T10:48:18.493Z" + }, + { + "id": "bug-1040", + "timestamp": "2026-06-23T10:54:14.342Z", + "error_message": "Wrong reference: endpoint_ssl_handshake should be handle_ssl_handshake", + "file": "src/lib/netio.c", + "root_cause": "Used \"endpoint_ssl_handshake\" instead of \"handle_ssl_handshake\"", + "fix": "Changed endpoint_ssl_handshake \u2192 handle_ssl_handshake", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T10:54:14.342Z" + }, + { + "id": "bug-1045", + "timestamp": "2026-06-23T14:08:05.466Z", + "error_message": "Missing error handling in unknown", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: # \u2500\u2500 Steps 7\u201310 looped: kill primary, verify, submit, restar; #", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-23T14:08:49.394Z" + }, + { + "id": "bug-1049", + "timestamp": "2026-06-23T14:39:01.881Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (0 && recovered_tip > ha->raft.log_len)", + "fix": "Changed to: if (recovered_tip > ha->raft.log_len)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T14:39:01.881Z" + }, + { + "id": "bug-1051", + "timestamp": "2026-06-23T14:48:18.998Z", + "error_message": "Wrong operator: >= should be ==", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Used \">=\" instead of \"==\"", + "fix": "Changed operator >= \u2192 ==", + "tags": [ + "auto-detected", + "operator-fix", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T14:48:18.998Z" + }, + { + "id": "bug-1054", + "timestamp": "2026-06-23T16:02:42.489Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/SKILL.md", + "root_cause": "Had \"End-to-end HA failover soak test for an XCE contr", + "fix": "Changed to \"End-to-end HA failover soak test for an XCE contr", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-23T16:02:42.489Z" + }, + { + "id": "bug-1060", + "timestamp": "2026-06-24T12:05:00.000Z", + "error_message": "After soak test coverage maneuver, ctrl-1 and ctrl-2 enter permanent prevote deadlock: ctrl-1 denies ctrl-2 (freshness: LSN=45 < floor=47), ctrl-2 denies ctrl-1 (log_stale: cand_llt=12 < local_last_log_term=13). Neither can ever win.", + "file": "src/controller/ha.c (raft_cb_become_leader / safe_committed_lsn), src/controller/cluster.c (OPCode_preVoteRequest)", + "root_cause": "ctrl-3 computed safe_committed_lsn=44 from voter peerLastLogLsn (ctrl-3=47, ctrl-2=44 \u2192 majority at 44) and truncated committed entries 45-47 (term=12, acked by ctrl-3 itself as standby under ctrl-1). ctrl-3 then wrote leadership NOP at LSN=45 (term=13) and replicated it to ctrl-2. Result: ctrl-1 has LSN=45=JOB_SUBMIT/term=12, ctrl-2 has LSN=45=NOOP/term=13 \u2014 divergent WALs. The safe_committed_lsn formula ignores the candidate's own durable knowledge of committed entries, violating Leader Completeness: the new leader must never truncate entries it has that were committed by a prior leader.", + "fix": "IMPLEMENTED (2026-06-24): added pure helper ha_promotion_truncate_floor(safe_lsn, durable_tip)=max(safe_lsn,durable_tip) in ha.c/ha.h; raft_cb_become_leader now truncates at max(safe_committed_lsn, persist_durable_floor(&c->persist)) instead of bare safe_committed_lsn. Since the cutoff is never below the leader's own durable tip, persist_truncate_wal_above is a no-op on the leader's durable log \u2014 the new leader NEVER truncates entries it holds (Raft \u00a75.4 Leader Completeness). safe_committed_lsn is still the commit lower-bound and promotion-gate HWM (NOT clamped \u2014 clamping it would defeat the behind-node fence). The leadership NOP now appends at the true tip (current_lsn) so replication converges peers to the leader's WAL with zero job loss. Unit test test_promotion_never_truncates_own_committed_entries in test_ha_election.c (GREEN). Steering src/controller/CLAUDE.md updated (Leader Completeness section + Post-Election Safe Commit Point + verification checklist). Recovery of an already-diverged cluster still requires restarting the over-truncated node (loses the cut jobs); this fix prevents the divergence.", + "tags": [ + "prevote", + "deadlock", + "safe_committed_lsn", + "truncation", + "leader-completeness", + "data-loss", + "coverage-maneuver", + "FIXED" + ], + "related_bugs": [ + "bug-795", + "bug-828" + ], + "occurrences": 1, + "last_seen": "2026-06-24T18:00:00.000Z" + }, + { + "id": "bug-1066", + "timestamp": "2026-06-24T04:36:50.602Z", + "error_message": "Incorrect value in code", + "file": "src/controller/CLAUDE.md", + "root_cause": "Had `persist_truncate_wal_above(safe_committed_lsn)`", + "fix": "Changed to 's own durable log**: the cutoff is `", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-24T04:37:04.779Z" + }, + { + "id": "bug-1067", + "timestamp": "2026-06-24T00:00:00.000Z", + "error_message": "Cross-term WAL catch-up permanently stalls: ctrl-2 stuck at lsn=8, all walCatchupEntry rejected (ae_reject_stale + ae_reject_logmismatch)", + "file": "src/controller/config_repl.c, src/controller/ha.c", + "root_cause": "AE term conflation: entry->term is the WAL entry's stored term (e.g. 4 for entries written by ctrl-1 at term=4), but ha_consume_append_entries uses it as the AE header ae_term. For live appends this is correct (leader appends at current term), but for catch-up streaming of historical entries from prior terms, ae_term = entry_term < leader.current_term. The Raft AE stale-term check (if term < s->term \u2192 ae_reject_stale) correctly rejects, but it should never fire \u2014 the AE header term must always be the leader's current term. There is no separate leader_term field in wal_catchup_entry_t. Consequence: historical entries at entry_term < raft.term are permanently rejected; standby_reported_lsn never advances; streamer retransmits forever.", + "fix": "IMPLEMENTED (2026-06-24): separated the AppendEntries HEADER term (leader's CURRENT term) from the per-entry STORED term. Added wire field WalCatchupEntry.leaderCurrentTerm @8 (controller.capnp) + C field wal_catchup_entry_t.leader_current_term (typemap.h, hand-defined struct \u2014 capnp generates encode/decode against it). Encoder (request.c wal_entry_to_wire + top container) writes it; decode auto-populates (generated, scalar mapname field). The catch-up streamer ha_stream_wal_to_standby gained a leader_term param (callers pass atomic_load(&c->ha.term)) and stamps entry->leader_current_term=leader_term on every streamed entry. Receiver ha_consume_append_entries now uses ae_term = e->leader_current_term ? e->leader_current_term : entry_term (single-entry branch) \u2014 the leader term rides on the sub-entry e by reference through ha_apply_term_entry_via_library, so no extra threading needed. entry->term (@4) remains the per-entry stored term for logOk/on_persist_entry. 0 = unset (legacy/live sender) \u2192 falls back to stored term (live appends have entry_term==leader_term, so unchanged). repl.c live paths UNTOUCHED. Tests (test_term_safety_adapter.c): test_intake_crossterm_catchup_uses_leader_current_term (historical entry@term4 streamed by leader@term5 to follower@term5 is ACCEPTED, not stale-rejected) + test_intake_stale_leader_term_still_rejected (leader_current_term=3 < follower term 5 still REJECTED \u2014 safety preserved). GREEN: 185 PASS term-safety+config-repl, 6 PASS wal_wire wire round-trip, 80 PASS failover/repl/durability, 0 failures.", + "tags": [ + "catch-up", + "replication", + "raft", + "ae_reject_stale", + "ae_reject_logmismatch", + "walCatchupEntry", + "cross-term", + "FIXED" + ], + "related_bugs": [ + "bug-1028", + "bug-991" + ], + "occurrences": 1, + "last_seen": "2026-06-24T20:00:00.000Z" + }, + { + "id": "bug-1070", + "timestamp": "2026-06-24T08:18:27.510Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/config_repl.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: if (c && c->persist.wal_path[0]) {; /* bug-1067: pass the leader's CURRENT term so historical ca", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-24T08:18:42.316Z" + }, + { + "id": "bug-1075", + "timestamp": "2026-06-24T08:23:04.980Z", + "error_message": "Wrong reference: standby_lsn should be leader_term", + "file": "src/controller/testsuite/test_coldstart_lsn0.c", + "root_cause": "Used \"standby_lsn\" instead of \"leader_term\"", + "fix": "Changed standby_lsn \u2192 leader_term", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T08:23:04.980Z" + }, + { + "id": "bug-1077", + "timestamp": "2026-06-24T17:30:00.000Z", + "error_message": "AddressSanitizer: SEGV / stack-use-after-return in free_CtrlStatus (lib/controller.capnp.c:11638) via handle_reply -> free_CtrlStatus_ptr (lib/xcectx.c:91) on `ctrlstatus` after failover; READ derefs the dead-stack `getreply` of xce_get_with_condition (xceapi.c)", + "file": "src/lib/xcectx.c", + "root_cause": "ctx->reply.u is an UNTAGGED union shared across opcodes (debugmsg/get/jobid/clusterconf/ctrlstatus alias the same 8 bytes). Commit 61e1391 ('fix bug-1060 bug-1067', unrelated Raft work) added speculative free-guards `if (ctx->reply.u.ctrlstatus) free_CtrlStatus_ptr(&ctx->reply.u.ctrlstatus)` (and the clusterconf twin). A non-NULL slot does NOT imply a CtrlStatus*: a failover-aborted reslist/joblist leaves ctx->reply.u.get == &getreply (a dead STACK pointer) because xce_get_with_condition/xce_resval only NULL the slot on the success path, not the xce_get_req<0 / cbuf_get<0 early-returns. The next ctrlstatus reply (worker thread) sees the foreign non-NULL slot and free_CtrlStatus walks it -> stack-use-after-return SEGV.", + "fix": "1) Revert the two unsafe free-guards in handle_reply (untagged union slot can never be safely freed by type) \u2014 restores prior MOVE_PTR-overwrite behavior. 2) Clear ctx->reply.u.get = NULL on ALL early-return paths of xce_get_with_condition and xce_resval (kill the dangling-stack-pointer leak at its source; also prevents a late OPCode_get reply writing through the dead stack via handle_get_reply). 3) Defense-in-depth: handle_get_reply drops a reply when ctx->reply.u.get is NULL. Regression test: test_handle_reply_ctrlstatus_does_not_free_foreign_slot (exposes xce_handle_reply, proves a foreign heap value in the shared union is not freed). VERIFIED 2026-06-24 (Opus): re-confirmed 3-layer fix present in working tree; RED proven by re-injecting the ctrlstatus free-guard -> ASan double-free at test_ctrlstatus.c:384 (free_CtrlStatus_ptr at controller.capnp.c:11892), then restored -> GREEN. Full libxce suite 291 Tests / 0 Failures / 0 ASan errors under ASAN_OPTIONS=detect_leaks=0.", + "tags": [ + "asan", + "use-after-return", + "union-type-confusion", + "ha", + "failover", + "ctrlstatus", + "libxce", + "memory-safety", + "regression-61e1391", + "VERIFIED" + ], + "related_bugs": [ + "bug-646", + "bug-1060", + "bug-1067" + ], + "occurrences": 1, + "last_seen": "2026-06-24T18:30:00.000Z" + }, + { + "id": "bug-1078", + "timestamp": "2026-06-24T09:18:52.605Z", + "error_message": "Wrong reference: handle_reply should be xce_handle_reply", + "file": "src/lib/xcectx.c", + "root_cause": "Used \"handle_reply\" instead of \"xce_handle_reply\"", + "fix": "Changed handle_reply \u2192 xce_handle_reply", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T09:18:52.605Z" + }, + { + "id": "bug-1079", + "timestamp": "2026-06-24T09:27:59.772Z", + "error_message": "Wrong reference: libxce_failover_client_hang should be project_libxce_failover_hang", + "file": "../../../../../claude/.claude/projects/-home-rshen-projects-xce-xce/memory/project_bug1077_reply_union_type_confusion.md", + "root_cause": "Used \"libxce_failover_client_hang\" instead of \"project_libxce_failover_hang\"", + "fix": "Changed libxce_failover_client_hang \u2192 project_libxce_failover_hang", + "tags": [ + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T09:27:59.772Z" + }, + { + "id": "bug-1080", + "timestamp": "2026-06-24T10:24:07.439Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-sim-conformance/design.md", + "root_cause": "Had \"on_get_entry_term -> node.log[idx].term | 0\"", + "fix": "Changed to \"on_get_entry_term -> node.log[idx].term | 0\"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-24T10:24:15.191Z" + }, + { + "id": "bug-1081", + "timestamp": "2026-06-24T10:30:45.585Z", + "error_message": "Wrong reference: apply should be advanced", + "file": ".kiro/specs/raft-sim-conformance/design.md", + "root_cause": "Used \"apply\" instead of \"advanced\"", + "fix": "Changed apply \u2192 advanced", + "tags": [ + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-24T10:30:50.917Z" + }, + { + "id": "bug-1082", + "timestamp": "2026-06-24T10:30:56.237Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-sim-conformance/design.md", + "root_cause": "Had \"vote/AE requests + responses;
Drop/Duplicate/", + "fix": "Changed to \"vote/AE requests + responses,
Drop/Duplicate/", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T10:30:56.237Z" + }, + { + "id": "bug-1083", + "timestamp": "2026-06-24T11:45:52.473Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: n > sizeof(v", + "fix": "Added guard clause: if (n > sizeof(v)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-24T11:46:08.548Z" + }, + { + "id": "bug-1085", + "timestamp": "2026-06-24T12:00:01.924Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i != last", + "fix": "Added guard clause: if (i != last)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T12:00:01.924Z" + }, + { + "id": "bug-1086", + "timestamp": "2026-06-24T12:13:27.532Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i < 0 || i >= c->bag_count", + "fix": "Added guard clause: if (i < 0 || i >= c->bag_count)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T12:13:27.532Z" + }, + { + "id": "bug-1087", + "timestamp": "2026-06-24T12:22:22.094Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: sa.role != RAFT_LEADER", + "fix": "Added guard clause: if (sa.role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T12:22:22.094Z" + }, + { + "id": "bug-1088", + "timestamp": "2026-06-24T12:30:35.436Z", + "error_message": "Wrong reference: test_sim_check_election_safety_fail_on_two_leaders should be test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_check_election_safety_fail_on_two_leaders\" instead of \"test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix\"", + "fix": "Changed test_sim_check_election_safety_fail_on_two_leaders \u2192 test_sim_3_2_log_matching_shared_idx_term_requires_identical_prefix", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T12:30:35.436Z" + }, + { + "id": "bug-1089", + "timestamp": "2026-06-24T12:38:40.360Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: sl.role != RAFT_LEADER", + "fix": "Added guard clause: if (sl.role != RAFT_LEADER)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T12:38:40.360Z" + }, + { + "id": "bug-1090", + "timestamp": "2026-06-24T12:47:03.843Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: shared > RAFT_SIM_MAX_LOG", + "fix": "Added guard clause: if (shared > RAFT_SIM_MAX_LOG)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T12:47:03.843Z" + }, + { + "id": "bug-1091", + "timestamp": "2026-06-24T13:04:27.057Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: c->trace_len >= RAFT_SIM_MAX_STEPS", + "fix": "Added guard clause: if (c->trace_len >= RAFT_SIM_MAX_STEPS)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T13:04:27.057Z" + }, + { + "id": "bug-1094", + "timestamp": "2026-06-24T13:22:05.858Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP", + "fix": "Added guard clause: if (c->bag[i].kind != RAFT_SIM_MSG_VOTE_RSP)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T13:22:05.858Z" + }, + { + "id": "bug-1095", + "timestamp": "2026-06-24", + "error_message": "Election Safety violation (two leaders at same term) surfaced by raft-sim-conformance step driver (task 4.2): a duplicated VOTE_RSP grant double-counts a single peer's vote", + "file": "src/lib/raft/raft.c", + "root_cause": "raft_on_vote_reply tallies grants with a COUNTER (s->votes_granted++) and explicitly ignores from_idx ((void)from_idx;), so DuplicateMessage of one peer's grant counts that peer twice and a candidate reaches quorum with a single real vote. Diverges from raft.tla where votesGranted is a SET (idempotent). Proven: 3-node, single peer grant duplicated -> candidate becomes LEADER with votes_granted=2.", + "fix": "NOT FIXED (out of task 4.2 boundary = raft_sim.c/test_raft.c). Library fix: dedupe per-peer in raft_on_vote_reply (track a granted-by bitset/array keyed on from_idx, or guard against re-counting the same peer in the current term). The conformance harness correctly CATCHES this on adversarial seeds.", + "tags": [ + "raft", + "vote-reply", + "idempotency", + "election-safety", + "split-brain", + "conformance", + "needs-library-fix", + "bug-851-family", + "dedup" + ], + "related_bugs": [ + "bug-1096", + "bug-851" + ], + "occurrences": 2, + "last_seen": "2026-06-24T13:30:00.000Z" + }, + { + "id": "bug-1096", + "timestamp": "2026-06-24", + "error_message": "raft_on_vote_reply counts a stale-term vote-reply (term < currentTerm) toward the current election instead of dropping it", + "file": "src/lib/raft/raft.c", + "root_cause": "No DropStaleResponse: a reply with term < s->term fell through to replies_received++/votes_granted++, corrupting quorum accounting (raft.tla DropStaleResponse discards mterm < currentTerm).", + "fix": "FIXED (verified 2026-07-02): src/lib/raft/raft.c:832 has `if (reply_term != s->term)` guard before tallying, with an on_trace(\"vote_reply_stale_drop\", ...) at raft.c:455 \u2014 exactly the fix this bug's own second entry described. Landed under raft-sim-conformance / raft-commit-safety (both implemented, 18/18 + 23/23 tasks).", + "tags": [ + "raft", + "vote-reply", + "stale-term", + "election-safety", + "split-brain", + "conformance", + "needs-library-fix", + "stale-report", + "fixed", + "bug-851-family", + "stale-response" + ], + "related_bugs": [ + "bug-1095", + "bug-851" + ], + "occurrences": 2, + "last_seen": "2026-07-02T17:30:00.000Z" + }, + { + "id": "bug-1099", + "timestamp": "2026-06-24T14:08:44.250Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: failing != RAFT_SIM_INV_NONE", + "fix": "Added guard clause: if (failing != RAFT_SIM_INV_NONE)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T14:08:44.250Z" + }, + { + "id": "bug-1104", + "timestamp": "2026-06-24T14:28:33.169Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Condition was: if (c->n >= 2 && c->nodes[0].commit_index >= 1 && c->n)", + "fix": "Changed to: if (c->n >= 2)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T14:28:33.169Z" + }, + { + "id": "bug-1108", + "timestamp": "2026-06-24T14:38:06.432Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: chk < 0", + "fix": "Added guard clause: if (chk < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T14:38:06.432Z" + }, + { + "id": "bug-1110", + "timestamp": "2026-06-24T14:51:30.855Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i == leader", + "fix": "Added guard clause: if (i == leader)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T14:51:30.855Z" + }, + { + "id": "bug-1111", + "timestamp": "2026-06-24T15:13:15.333Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: p == leader", + "fix": "Added guard clause: if (p == leader)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T15:13:15.333Z" + }, + { + "id": "bug-1112", + "timestamp": "2026-06-24T15:15:03.882281Z", + "error_message": "ld: undefined reference to 'raft_sim_scenario_drop_dup_reorder' at test_raft.c:8288 \u2014 lib test binary fails to link", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Commit e445404 (task 5.3 drop/dup/reorder) added the raft.h declaration + the test_sim_5_3 test but FORGOT to add the raft_sim_scenario_drop_dup_reorder implementation to raft_sim.c. Symbol is declared+referenced but never defined, breaking the link for the whole xcelib_test binary.", + "fix": "Pre-existing, outside task 5.4 boundary. Surfaced while validating task 5.4 (figure8). Needs task 5.3 implementation added to raft_sim.c. NOT fixed by 5.4.", + "tags": [ + "raft-sim-conformance", + "link-error", + "task-5.3", + "undefined-reference", + "pre-existing" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T15:15:03.882281Z" + }, + { + "id": "bug-1113", + "timestamp": "2026-06-24T15:16:52.683Z", + "error_message": "Missing error handling in unknown", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /*; * sim_scen_catch_up_commit: drive bounded AppendEntries-to-a | Also: * on a CONNECTED quorum, yet raft_advance_commit must NOT co; * count alone \u2014 the current-term Figure-8 rule (raft.c G4) f", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-24T15:20:15.882Z" + }, + { + "id": "bug-1115", + "timestamp": "2026-06-24T15:18:59.248Z", + "error_message": "Wrong operator: != should be ==", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Used \"!=\" instead of \"==\"", + "fix": "Changed operator != \u2192 ==", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T15:18:59.248Z" + }, + { + "id": "bug-1116", + "timestamp": "2026-06-24T15:25:17.886Z", + "error_message": "Missing guard clause", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "No early return/throw for edge case: i == leader", + "fix": "Added guard clause: if (i == leader)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T15:25:17.886Z" + }, + { + "id": "bug-1117", + "timestamp": "2026-06-24T15:32:34.085Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft.c", + "root_cause": "Condition was: if (s->cbs->on_get_entry_term(s->ctx, agreed)", + "fix": "Changed to: if (0 && s->cbs->on_get_entry_term(s->ctx, agreed)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-24T15:33:53.327Z" + }, + { + "id": "bug-1120", + "timestamp": "2026-06-24T16:09:04.749Z", + "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_1_fault_switch_mechanism", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_1_fault_switch_mechanism\"", + "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_1_fault_switch_mechanism", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T16:09:04.749Z" + }, + { + "id": "bug-1123", + "timestamp": "2026-06-24T16:22:58.427Z", + "error_message": "Wrong reference: test_sim_6_1_seed_sweep should be test_sim_7_2_negative_both_directions", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_6_1_seed_sweep\" instead of \"test_sim_7_2_negative_both_directions\"", + "fix": "Changed test_sim_6_1_seed_sweep \u2192 test_sim_7_2_negative_both_directions", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T16:22:58.427Z" + }, + { + "id": "bug-1125", + "timestamp": "2026-06-24T16:42:32.261Z", + "error_message": "Wrong reference: test_sim_4_2_step_liveness_and_coverage should be test_sim_8_1_export_trace_named_and_gated", + "file": "src/lib/testsuite/test_raft.c", + "root_cause": "Used \"test_sim_4_2_step_liveness_and_coverage\" instead of \"test_sim_8_1_export_trace_named_and_gated\"", + "fix": "Changed test_sim_4_2_step_liveness_and_coverage \u2192 test_sim_8_1_export_trace_named_and_gated", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T16:42:32.261Z" + }, + { + "id": "bug-1126", + "timestamp": "2026-06-24T16:43:39.427Z", + "error_message": "Wrong condition in logic", + "file": "src/lib/raft/raft_sim.c", + "root_cause": "Condition was: if (node_idx < 0 || node_idx >= c->n)", + "fix": "Changed to: if (getenv(\"RAFT_SIM_EXPORT\")", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-24T16:43:39.427Z" + }, + { + "id": "bug-1127", + "timestamp": "2026-06-25T03:17:45.530Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T03:17:45.530Z" + }, + { + "id": "bug-1128", + "timestamp": "2026-06-25T03:20:00.000Z", + "error_message": "test_demotion_replay_yields_committed_only flakes ~16% on overlayfs: boot recovery over surviving WAL must yield the committed prefix ONLY", + "file": "src/controller/testsuite/test_demotion_convergence.c", + "root_cause": "TWO compounding non-determinism sources, both fixture artifacts not product bugs: (1) overlayfs metadata incoherence \u2014 a freshly-opened persist_t fd over a just-ftruncate'd WAL intermittently reads the STALE pre-truncate size, so the cross-context fresh recovery replays the already-removed suffix; (2) DEEPER, fs-independent: the fixture wires a LIVE commit engine and on a single node (n_peers=1) the persist durable-advance callback (fsync worker -> on_durable_advance) commits the whole durable log, so persist_flush asynchronously raced committed_lsn UP from the test-set committed_boundary (5) to the appended tip (8) AFTER the test pinned it; the demotion seam reads committed_lsn at task time -> truncated to 8 not 5 -> current_lsn=9 > committed_boundary+1. DIAG proof: committed_boundary=5 live_committed=8 live_tip=9 on failing runs.", + "fix": "Test-only, two parts. (a) Detach the durable-advance commit wake (g_dc.persist.on_durable_advance = NULL) and re-pin committed_lsn = committed_boundary right before driving the demotion task, killing the commit-engine race deterministically (helps BOTH fs paths). (b) Hybrid gate by fs type via static dc_path_is_overlayfs() (statfs f_type == OVERLAYFS_SUPER_MAGIC 0x794c7630, defaults to real-fs path on statfs error): on a real fs keep the strong cross-context fresh boot-recovery assertion (fresh->current_lsn <= committed_boundary+1); on overlayfs skip that fd-incoherent step and assert the LIVE-context truncate effect (g_dc.persist.current_lsn <= committed_boundary+1, coherent under p->lock). Both branches prove the same property. Production persist.c truncate (ftruncate+fdatasync) untouched.", + "fix_validation": "Full controller suite 20/20 runs: 0 demotion_replay failures, 0 total failures. Group-only 60/60: 0 failures. Baseline (pre-fix) was 4/20. /tmp confirmed overlayfs (statfs magic 0x794c7630), so the overlayfs branch is the one exercised here.", + "tags": [ + "test-flake", + "demotion-convergence", + "persist", + "overlayfs", + "ha-convergence-recovery", + "quiescence", + "wal-truncate", + "sandbox", + "commit-engine-race", + "deterministic", + "c" + ], + "related_bugs": [ + "bug-343", + "bug-704", + "bug-795", + "bug-828", + "bug-9799", + "bug-1060" + ], + "occurrences": 2, + "last_seen": "2026-06-25T04:30:00.000Z" + }, + { + "id": "bug-1133", + "timestamp": "2026-06-25T07:12:01.309Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-log-replication/spec.json", + "root_cause": "Had \"2026-06-21T11:00:00Z\"", + "fix": "Changed to \"2026-06-25T00:00:00Z\"", + "tags": [ + "auto-detected", + "wrong-value", + "json" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-25T07:12:25.418Z" + }, + { + "id": "bug-1134", + "timestamp": "2026-06-25T08:18:43.915Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T08:18:43.915Z" + }, + { + "id": "bug-1135", + "timestamp": "2026-06-25T09:26:49.441Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/facts-in-raft.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T09:26:49.441Z" + }, + { + "id": "bug-1136", + "timestamp": "2026-06-25T09:27:00.312Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-apply-decoupling/brief.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T09:27:00.312Z" + }, + { + "id": "bug-1138", + "timestamp": "2026-06-25T11:36:52.942Z", + "error_message": "Function not marked async", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T11:36:52.942Z" + }, + { + "id": "bug-1139", + "timestamp": "2026-06-25T14:02:51.225Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/research.md", + "root_cause": "Had `VTASK_IO`", + "fix": "Changed to `endpoint_block_tasks()`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-25T14:09:24.614Z" + }, + { + "id": "bug-1141", + "timestamp": "2026-06-25T15:12:16.193Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T15:12:16.193Z" + }, + { + "id": "bug-1142", + "timestamp": "2026-06-25T15:13:00.445Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `.kiro/specs/raft-drive-async/brief.md`", + "fix": "Changed to `delay_task_ms`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T15:13:00.445Z" + }, + { + "id": "bug-1143", + "timestamp": "2026-06-25T23:37:38.579Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - Commit-advance under pool load (R4.5): with the vtask pool; - Snapshot/checkpoint concurrency (R1.1/1.3/1.5): concurrent", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-25T23:38:18.782Z" + }, + { + "id": "bug-1144", + "timestamp": "2026-06-25T23:37:49.697Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had `persist_checkpoint`", + "fix": "Changed to `wait_all_with_timeout`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T23:37:49.697Z" + }, + { + "id": "bug-1145", + "timestamp": "2026-06-25T23:38:08.746Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-25T23:38:08.746Z" + }, + { + "id": "bug-1147", + "timestamp": "2026-06-26T03:46:38.904419+00:00", + "error_message": "WAL trim_wal_segments deletes any closed segment regardless of standby catch-up cursor \u2192 lagging/empty-rejoin standby loses its needed log range with no snapshot backfill (permanent committed-prefix gap)", + "file": "src/controller/persist.c:1374 (trim_wal_segments)", + "root_cause": "trim_wal_segments bounds deletion ONLY by active_seg_lsn (the currently-open segment): `if (seg_start_lsn < active_seg_lsn) unlink()`. It consults no standby acked_lsn, no committed_lsn, no in-flight snapshot boundary. Strategy-A (cursor-bounded retention) discipline is absent. Strategy-B (entity-bearing snapshot install) is also not live \u2014 write_snapshot (persist.c:1247) writes only a 16-byte P0 sentinel, not materialized state. So a standby that lags by more than ~one open segment (down across segment rolls, slow link, >64MB lag, or empty rejoin from cursor=0) hits a log range that exists nowhere and cannot catch up.", + "fix": "FIXED (verified 2026-07-02): src/controller/persist.c trim_wal_segments() computes `floor = repl_trim_lsn(&p->repl, active_seg_lsn, p->config.snap_interval_s)` (min of active segment, all standby acked_lsn, disconnected-slot guards) and only unlinks segments below that floor \u2014 comment inline explicitly cites 'bug-1147 guard'. Landed under raft-drive-async task 3.2 (all leaf tasks complete).", + "tags": [ + "durability", + "permanent-gap", + "stale-report", + "trim", + "bug-828", + "raft-drive-async", + "latent", + "catch-up", + "wal", + "fixed", + "snapshot", + "bug-795" + ], + "related_bugs": [ + "bug-795", + "bug-828", + "bug-1060" + ], + "occurrences": 1, + "last_seen": "2026-07-02T17:30:00.000Z" + }, + { + "id": "bug-1148", + "timestamp": "2026-06-26T03:46:51.038Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had 's boundary or any standby'", + "fix": "Changed to `persist.c:1374`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T03:46:51.038Z" + }, + { + "id": "bug-1149", + "timestamp": "2026-06-26T03:48:48.222Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `/kiro-spec-design raft-drive-async`", + "fix": "Changed to `approved: false`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T03:49:01.467Z" + }, + { + "id": "bug-1151", + "timestamp": "2026-06-26T04:31:53.879Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Had `commit_submit`", + "fix": "Changed to `commit.c`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T04:36:52.562Z" + }, + { + "id": "bug-1152", + "timestamp": "2026-06-26T04:32:20.960Z", + "error_message": "Wrong reference: PeerCtrlDispatch should be LivenessAdmission", + "file": ".kiro/specs/raft-drive-async/design.md", + "root_cause": "Used \"PeerCtrlDispatch\" instead of \"LivenessAdmission\"", + "fix": "Changed PeerCtrlDispatch \u2192 LivenessAdmission", + "tags": [ + "wrong-reference", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:32:20.960Z" + }, + { + "id": "bug-1153", + "timestamp": "2026-06-26T04:33:10.286Z", + "error_message": "Incorrect value in code", + "file": ".kiro/steering/roadmap.md", + "root_cause": "Had `COMMIT_WAIT_RESERVED=2`", + "fix": "Changed to `commit.c:16`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:33:10.287Z" + }, + { + "id": "bug-1154", + "timestamp": "2026-06-26T04:37:18.052Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `VTASK_LIVENESS_RESERVED`", + "fix": "Changed to `R = clamp(n_peers \u2212 1, 1, n_vcpus \u2212 1)`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T04:42:01.556Z" + }, + { + "id": "bug-1155", + "timestamp": "2026-06-26T04:47:49.937Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_drive_async_config.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:47:49.937Z" + }, + { + "id": "bug-1156", + "timestamp": "2026-06-26T04:48:34.246Z", + "error_message": "Function not marked async", + "file": "src/include/config.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:48:34.246Z" + }, + { + "id": "bug-1157", + "timestamp": "2026-06-26T04:48:50.005Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:48:50.005Z" + }, + { + "id": "bug-1158", + "timestamp": "2026-06-26T04:49:04.568Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:49:04.568Z" + }, + { + "id": "bug-1159", + "timestamp": "2026-06-26T04:49:44.821Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:49:44.821Z" + }, + { + "id": "bug-1160", + "timestamp": "2026-06-26T04:57:57.877Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T04:57:57.877Z" + }, + { + "id": "bug-1161", + "timestamp": "2026-06-26T05:02:36.720Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_slowlane.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:02:36.720Z" + }, + { + "id": "bug-1162", + "timestamp": "2026-06-26T05:06:30.148Z", + "error_message": "Function not marked async", + "file": "src/include/xce/cluster_def.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:06:30.148Z" + }, + { + "id": "bug-1163", + "timestamp": "2026-06-26T05:06:42.680Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/cluster.c", + "root_cause": "Condition was: if (ha_is_primary(&c->ha)", + "fix": "Changed to: if (c->slowlane.workers == NULL)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:06:42.680Z" + }, + { + "id": "bug-1164", + "timestamp": "2026-06-26T05:06:53.129Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T05:06:53.129Z" + }, + { + "id": "bug-1165", + "timestamp": "2026-06-26T06:51:00.808Z", + "error_message": "Wrong reference: test_request_send_propagation should be test_netio_bounded_connect", + "file": "src/lib/testsuite/test.c", + "root_cause": "Used \"test_request_send_propagation\" instead of \"test_netio_bounded_connect\"", + "fix": "Changed test_request_send_propagation \u2192 test_netio_bounded_connect", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:51:00.808Z" + }, + { + "id": "bug-1166", + "timestamp": "2026-06-26T06:51:57.649Z", + "error_message": "Function not marked async", + "file": "src/lib/netio.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:51:57.649Z" + }, + { + "id": "bug-1168", + "timestamp": "2026-06-26T06:52:21.608Z", + "error_message": "Function not marked async", + "file": "src/include/netio.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T06:52:21.608Z" + }, + { + "id": "bug-1169", + "timestamp": "2026-06-26T07:16:36.377Z", + "error_message": "Function not marked async", + "file": "src/include/vtaskmgr.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T07:17:06.235Z" + }, + { + "id": "bug-1171", + "timestamp": "2026-06-26T07:18:02.307Z", + "error_message": "Function not marked async", + "file": "src/lib/vtaskmgr.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:18:02.307Z" + }, + { + "id": "bug-1172", + "timestamp": "2026-06-26T07:19:14.857Z", + "error_message": "Function not marked async", + "file": "src/controller/include/commit.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T07:19:30.679Z" + }, + { + "id": "bug-1173", + "timestamp": "2026-06-26T07:19:45.059Z", + "error_message": "Function not marked async", + "file": "src/controller/commit.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:20:29.751Z" + }, + { + "id": "bug-1174", + "timestamp": "2026-06-26T07:20:07.233Z", + "error_message": "Wrong operator: >= should be !=", + "file": "src/controller/commit.c", + "root_cause": "Used \">=\" instead of \"!=\"", + "fix": "Changed operator >= \u2192 !=", + "tags": [ + "auto-detected", + "operator-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:20:07.233Z" + }, + { + "id": "bug-1175", + "timestamp": "2026-06-26T07:20:41.448Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:20:41.448Z" + }, + { + "id": "bug-1177", + "timestamp": "2026-06-26T07:47:19.731Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:47:39.153Z" + }, + { + "id": "bug-1178", + "timestamp": "2026-06-26T07:48:00.209Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T07:48:57.917Z" + }, + { + "id": "bug-1179", + "timestamp": "2026-06-26T07:51:01.491Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: deadline.tv_nsec >= 1000000000L", + "fix": "Added guard clause: if (deadline.tv_nsec >= 1000000000L)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:51:01.491Z" + }, + { + "id": "bug-1180", + "timestamp": "2026-06-26T07:51:10.764Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:51:10.764Z" + }, + { + "id": "bug-1183", + "timestamp": "2026-06-26T07:57:35.484Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T07:57:35.484Z" + }, + { + "id": "bug-1184", + "timestamp": "2026-06-26T08:12:52.624Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T08:12:59.952Z" + }, + { + "id": "bug-1185", + "timestamp": "2026-06-26T08:13:19.511Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T08:15:04.588Z" + }, + { + "id": "bug-1186", + "timestamp": "2026-06-26T08:15:44.315Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (cur_sctx && cur_sctx->peer_idx == sctx->peer_idx)", + "fix": "Changed to: if (0 && cur_sctx && cur_sctx->peer_idx == sctx->peer_)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T08:16:44.785Z" + }, + { + "id": "bug-1188", + "timestamp": "2026-06-26T10:20:33.560Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T10:20:50.537Z" + }, + { + "id": "bug-1189", + "timestamp": "2026-06-26T10:21:12.319Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-26T10:23:40.982Z" + }, + { + "id": "bug-1190", + "timestamp": "2026-06-26T10:24:21.473Z", + "error_message": "Function not marked async", + "file": "src/controller/cluster.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:24:21.473Z" + }, + { + "id": "bug-1191", + "timestamp": "2026-06-26T10:27:00.557Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: atomic_load(&g_pc_gate_on", + "fix": "Added guard clause: if (atomic_load(&g_pc_gate_on)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:27:00.557Z" + }, + { + "id": "bug-1192", + "timestamp": "2026-06-26T10:27:14.358Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:27:14.358Z" + }, + { + "id": "bug-1193", + "timestamp": "2026-06-26T10:56:43.461Z", + "error_message": "Function not marked async", + "file": "src/include/vtaskmgr.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:56:43.461Z" + }, + { + "id": "bug-1194", + "timestamp": "2026-06-26T10:57:18.280Z", + "error_message": "Function not marked async", + "file": "src/lib/vtaskmgr.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T10:57:18.280Z" + }, + { + "id": "bug-1195", + "timestamp": "2026-06-26T10:57:44.750Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 4, + "last_seen": "2026-06-26T10:59:54.657Z" + }, + { + "id": "bug-1196", + "timestamp": "2026-06-26T11:00:15.213Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T11:00:15.213Z" + }, + { + "id": "bug-1197", + "timestamp": "2026-06-26T00:00:00.000Z", + "error_message": "raft-drive-async finding-1 (FLAG-ON ONLY, dormant): enabling peerctrl_dispatch creates an endpoint UAF \u2014 raft_cb_become_follower runs on a pool worker and endpoint_fini()s all peer vote-eps (frees ep->data) while the single io_worker can concurrently read ep_j->data in peer_ctrl_msg_handler_inline for a same-batch queued event", + "file": "src/controller/ha.c", + "root_cause": "Moving peer-control off the sequential io_worker onto the pool co-locates teardown (frees ep->data) and inbound delivery (reads ep->data) on different threads with no unmonitor-before-free ordering. Safe today only because g_drive_async_peerctrl_dispatch_enabled defaults OFF.", + "fix": "SUPERSEDED 2026-06-30 by bug-1298 endpoint reference counting (commit 6e05dc1). The pool-reachable teardown-frees-peer-ep UAF class is now governed generally by endpoint_ref/endpoint_unref (refcnt + heap_alloc gate) acquired per ep-bound VTASK_IO in vtask_enq and released in run_task/taskq_item_killer \u2014 a queued vtask holds a ref so the io_worker cannot free the ep out from under the dispatcher. Re-validate when raft-drive-async flips g_drive_async_peerctrl_dispatch ON.", + "tags": [ + "endpoint-quiescence", + "fixed", + "c", + "dispatch_inflight", + "task-3.4a", + "tearing_down", + "wrong-reference", + "raft-drive-async", + "task-3.1", + "use-after-free", + "flag-on-hazard", + "deferred", + "review-finding" + ], + "related_bugs": [ + "bug-1198" + ], + "occurrences": 3, + "last_seen": "2026-06-30T14:30:00Z" + }, + { + "id": "bug-1199", + "timestamp": "2026-06-26T00:00:00.000Z", + "error_message": "raft-drive-async task 3.1: peer-control vtask enqueued-but-not-executed at vtaskmgr_fini leaks its io_packet_t (only the executed entry transfers ownership via iod->pkt=NULL)", + "file": "src/lib/vtaskmgr.c", + "root_cause": "taskq_item_killer called vtask_fini (frees task->data wrapper only), never the carried io_packet_t. The generic killer could not free the pkt unconditionally because the libxce client path (xcectx.c vtask_handle_msg) does NOT clear ep->fsm_in.pkt \u2014 its pkt lifetime is the endpoint's, so an unconditional free would double-free.", + "fix": "Added int owns_io_pkt to vtask_t (default 0 in vtask_init). taskq_item_killer frees the pkt ONLY when owns_io_pkt && vtype==VTASK_IO && pkt!=NULL. Controller peer_ctrl_dispatch_to_pool sets owns_io_pkt=1. libxce path keeps owns_io_pkt=0 -> unchanged, no double-free. Verified: leak count 808 unchanged vs baseline, 0 double-free under ASan.", + "tags": [ + "raft-drive-async", + "task-3.1", + "memory-leak", + "vtaskmgr", + "packet-ownership", + "review-finding" + ], + "related_bugs": [ + "bug-1077" + ], + "occurrences": 1, + "last_seen": "2026-06-26T00:00:00.000Z" + }, + { + "id": "bug-1198", + "timestamp": "2026-06-26T00:00:00.000Z", + "error_message": "raft-drive-async finding-2 (FLAG-ON ONLY, dormant): peer_ctrl_classify tags all peer-control work VTASK_LIVENESS, but the CTRL_PRIMARY repl-ack path can reach HA_REPL_SEND_SNAPSHOT->persist_checkpoint (blocking fdatasync) inline \u2014 a blocking op under the never-gated liveness reservation breaks deadlock-freedom", + "file": "src/controller/ha.c", + "root_cause": "repl_ack_decode_and_dispatch -> ha_dispatch_repl_action_for_standby can run the snapshot send inline. LIVENESS is only correct once that long op is offloaded off the pool.", + "fix": "FIXED/CLOSED (verified 2026-07-02): src/controller/ha.c:5647 carries an explicit in-code 'CLOSED' banner comment \u2014 the bug-1197 endpoint_ref/endpoint_unref bracket around the lane dispatch means the repl-ack path's snapshot send is ALWAYS offloaded before peer_ctrl_classify can tag it VTASK_LIVENESS, so bug-1198 cannot recur. Both flags (peerctrl dispatch + snapshot offload) default ON since bug-1376 was fixed.", + "tags": [ + "task-3.1", + "review-finding", + "stale-report", + "raft-drive-async", + "deferred", + "flag-on-hazard", + "fixed", + "deadlock-freedom", + "liveness-admission" + ], + "related_bugs": [ + "bug-1197" + ], + "occurrences": 1, + "last_seen": "2026-07-02T17:30:00.000Z" + }, + { + "id": "bug-1202", + "timestamp": "2026-06-26T11:33:28.328Z", + "error_message": "Missing error handling in task", + "file": "src/controller/testsuite/test_e2e_checkpoint.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Helper: check if a file exists in a directory ; static int file_exists_in(const char* dir, const char* name)", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T11:34:53.637Z" + }, + { + "id": "bug-1204", + "timestamp": "2026-06-26T11:35:37.597Z", + "error_message": "Wrong reference: file_exists_for_trim_test should be seg_file_exists", + "file": "src/controller/testsuite/test_e2e_checkpoint.c", + "root_cause": "Used \"file_exists_for_trim_test\" instead of \"seg_file_exists\"", + "fix": "Changed file_exists_for_trim_test \u2192 seg_file_exists", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T11:35:37.597Z" + }, + { + "id": "bug-1205", + "timestamp": "2026-06-26T11:39:53.357Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T11:41:58.418Z" + }, + { + "id": "bug-1206", + "timestamp": "2026-06-26T15:16:40.996Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T15:16:40.996Z" + }, + { + "id": "bug-1207", + "timestamp": "2026-06-26T15:28:19.885Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_repl_send_snapshot_hook != NULL", + "fix": "Added guard clause: if (ha_repl_send_snapshot_hook != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T15:28:19.885Z" + }, + { + "id": "bug-1210", + "timestamp": "2026-06-26", + "error_message": "raft-drive-async task 3.2 snapshot slow-lane offload flag g_drive_async_snapshot_offload defaulted ON (=1); header doc also said default ON", + "file": "src/controller/ha.c, src/controller/ha.h", + "root_cause": "Snapshot offload borrows a standby endpoint shared with the io_worker epoll loop (not owned, unlike reconnect_lane_fn). With the flag ON the send runs on a slow-lane thread while io_worker can RDHUP-free that endpoint => cross-thread UAF (bug-1197 family). Defaulting ON would ship that UAF; the borrowed-endpoint lifetime discipline is not established until task 3.4.", + "fix": "Default the flag OFF (dormant landing, same protocol as task 3.1 peerctrl dispatch). Inline path runs on io_worker, serialized with teardown -> no overlap. Rewrote both the ha.c definition comment and the ha.h accessor doc to state OFF is the production default and 3.4 is the gate that flips it ON. bug-1198 stays open by design until then.", + "tags": [ + "raft-drive-async", + "task-3.2", + "snapshot", + "slow-lane", + "UAF", + "feature-flag", + "bug-1197", + "bug-1198", + "liveness" + ], + "related_bugs": [ + "bug-1197", + "bug-1198", + "bug-1147" + ], + "occurrences": 1, + "last_seen": "2026-06-26" + }, + { + "id": "bug-1211", + "timestamp": "2026-06-26T15:49:46.434Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.h", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "h" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-26T15:49:59.034Z" + }, + { + "id": "bug-1212", + "timestamp": "2026-06-26T15:50:27.084Z", + "error_message": "Function not marked async", + "file": "src/controller/ha.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-26T15:52:41.021Z" + }, + { + "id": "bug-1213", + "timestamp": "2026-06-26T15:51:29.747Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ha_configpush_probe != NULL", + "fix": "Added guard clause: if (ha_configpush_probe != NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T15:51:29.747Z" + }, + { + "id": "bug-1214", + "timestamp": "2026-06-26T15:54:05.463Z", + "error_message": "Function not marked async", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T15:54:05.463Z" + }, + { + "id": "bug-1216", + "timestamp": "2026-06-26T16:54:57.384Z", + "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", + "file": "src/include/netio.h", + "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", + "fix": "Changed ssl_handshake_done \u2192 tearing_down", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T16:54:57.384Z" + }, + { + "id": "bug-1217", + "timestamp": "2026-06-26T16:55:15.332Z", + "error_message": "Wrong reference: ssl_handshake_done should be tearing_down", + "file": "src/lib/netio.c", + "root_cause": "Used \"ssl_handshake_done\" instead of \"tearing_down\"", + "fix": "Changed ssl_handshake_done \u2192 tearing_down", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T16:55:15.332Z" + }, + { + "id": "bug-1218", + "timestamp": "2026-06-26T16:55:48.690Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: ectx", + "fix": "Added guard clause: if (ectx)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T16:55:48.690Z" + }, + { + "id": "bug-1222", + "timestamp": "2026-06-26T17:44:25.670Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_ha_dispatch_integration.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-26T17:44:25.670Z" + }, + { + "id": "bug-1224", + "timestamp": "2026-06-26", + "error_message": "raft-drive-async 3.4b integration tests (test_ha_dispatch_integration.c) hang intermittently \u2014 controller_unit_test never completes (timeout-killed)", + "file": "src/controller/testsuite/test_ha_dispatch_integration.c", + "root_cause": "Test teardown race: P1/P2/P3 released a parked vtask body (gate_open) then immediately called vtaskmgr_fini, which tears down workers via pthread_cancel. The tests polled a run-counter incremented MID-body, so fini raced the just-released body still unwinding (esp. P2 REQ_BLOCK bodies in vtaskmgr_blocking_release racing blk_lock teardown) \u2192 flaky deadlock. slowlane_fini is graceful (join) so the lane side was safe; vtaskmgr_fini (cancel) was the hazard.", + "fix": "Add end-of-body completion counters (g_pc_completed on both pc_probe paths incl. the park path; g_block_done at the end of block_task_entry) and poll them to the full expected count AFTER gate_open and BEFORE vtaskmgr_fini, so every pool body has fully returned (worker idle in cbuf_get) before the cancel-based fini. Verified deterministic 5/5 isolated runs + full suite EXIT=0, ASan-clean.", + "tags": [ + "raft-drive-async", + "task-3.4b", + "test", + "flaky", + "vtaskmgr_fini", + "pthread_cancel", + "teardown-race", + "deadlock" + ], + "related_bugs": [ + "bug-924" + ], + "occurrences": 1, + "last_seen": "2026-06-26" + }, + { + "id": "bug-1225", + "timestamp": "2026-06-26", + "error_message": "/xce-test-ha live failover soak (ASAN build, real fs ~/tmp): FAIL round 4 \"cluster did not reconverge to 3-of-3 after restarting ctrl-2\"", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh (wait_for_full_convergence)", + "root_cause": "Pre-existing HA-core instability under 6-round CHAOTIC repeated failover, NOT a raft-drive-async regression. BASELINE-CONFIRMED: the same soak against pre-Phase-3 binaries (commit 21e26a1, release/non-ASAN) ALSO fails at round 4 (baseline symptom: no-new-primary election stall; current symptom: round-4 reconverge + committed-prefix divergence at LSN 9). Both builds preserve committed JOBS (zero job loss in all completed rounds). The committed-prefix divergence is the documented open bug-851/795 family (chaotic-failover silent divergence; separate HA-core spec). raft-drive-async Phase 3 flags are default-OFF (inert); the live trim-guard only preserves more WAL.", + "fix": "Not a raft-drive-async bug (baseline-confirmed). 4.2 regression sweep PASSES for this spec: no NEW failure introduced. Pre-existing bug-851 chaotic-failover divergence + bug-1042 reconverge flakiness remain open for a separate HA-core effort.", + "tags": [ + "ASAN", + "baseline-confirmed", + "bug-851", + "false-negative", + "not-a-regression", + "raft-drive-async", + "reconverge", + "soak", + "task-4.2", + "xce-test-ha" + ], + "related_bugs": [ + "bug-1042", + "bug-924" + ], + "occurrences": 1, + "last_seen": "2026-06-26" + }, + { + "id": "bug-1226", + "timestamp": "2026-06-27T02:53:39.273Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Had \"$REPO/src/xce.build/controller/xce_controller\"", + "fix": "Changed to \"${XCE_HA_BIN:-$REPO/src/xce.build/controller/xce_", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T02:53:39.273Z" + }, + { + "id": "bug-1227", + "timestamp": "2026-06-27T03:04:05.891Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T03:04:05.891Z" + }, + { + "id": "bug-1228", + "timestamp": "2026-06-27T04:13:35.943Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/design.md", + "root_cause": "Had `; no silent retention of a stale-term entry.\n- In", + "fix": "Changed to `; no silent retention of a stale-term entry.\n- In", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-27T04:13:47.424Z" + }, + { + "id": "bug-1233", + "timestamp": "2026-06-27T04:59:57.805Z", + "error_message": "Missing error handling in unknown", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: # \u2500\u2500 Chaotic-mode primitives (count outcomes into the verdic; # Run `walcompare --report` across all three members and ADD | Also: # \u2500\u2500 Chaotic mode: overlapping-kill rounds \u2192 isolated verdic; # Opt-in (--chaotic / XCE_HA_MODE=chaotic). Kills land insid", + "tags": [ + "auto-detected", + "error-handling", + "sh" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-27T05:00:54.066Z" + }, + { + "id": "bug-1234", + "timestamp": "2026-06-27T05:00:06.726Z", + "error_message": "Function not marked async", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T05:00:06.726Z" + }, + { + "id": "bug-1236", + "timestamp": "2026-06-27T09:27:45.080Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/design.md", + "root_cause": "Had `raft_on_append_entries_request`", + "fix": "Changed to `research.md`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-27T09:31:54.096Z" + }, + { + "id": "bug-1237", + "timestamp": "2026-06-27T09:28:07.944Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-ha-core-safety/design.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - **Technical debt addressed:** the silent empty-payload shi | Also: - `.claude/skills/xce-test-ha/run-ha-soak.sh` (and `SKILL.md; - `src/controller/` log init (only if the client-probe appro | Also: | Catch-up Payload Integrity | Adapter | Stream a non-empty ; | Persist Conflict Guard | Persist | `(term,lsn,payload-crc) | Also: - The live-repro gate (task 1.3) has **already run and refut; - A green synthetic unit test over a still-red soak is not a", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 5, + "last_seen": "2026-06-27T09:32:08.493Z" + }, + { + "id": "bug-1243", + "timestamp": "2026-06-27T10:02:56.820Z", + "error_message": "Wrong operator: == should be <<", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Used \"==\" instead of \"<<\"", + "fix": "Changed operator == \u2192 <<", + "tags": [ + "auto-detected", + "operator-fix", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T10:02:56.820Z" + }, + { + "id": "bug-1244", + "timestamp": "2026-06-27T10:04:31.979Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Had \"no primary elected within 45s (check $TOP/logs)\"", + "fix": "Changed to \"no primary elected within ~90s via client probe (", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T10:04:31.979Z" + }, + { + "id": "bug-1246", + "timestamp": "2026-06-27T10:07:37.692Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/SKILL.md", + "root_cause": "Had `no primary elected within 45s`", + "fix": "Changed to `no primary elected within ~90s via client probe`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T10:07:37.692Z" + }, + { + "id": "bug-1247", + "timestamp": "2026-06-27T10:25:38.366Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || lsn == 0", + "fix": "Added guard clause: if (!p || lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T10:25:38.366Z" + }, + { + "id": "bug-1249", + "timestamp": "2026-06-27T10:45:00.686Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/persist.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* (a) non-empty incoming whose payload differs at an existi; * \u2014 a genuine payload conflict. Branch on tip vs below-tip.", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-27T10:45:20.669Z" + }, + { + "id": "bug-1250", + "timestamp": "2026-06-27T10:45:34.063Z", + "error_message": "Function not marked async", + "file": "src/controller/persist.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T10:45:34.063Z" + }, + { + "id": "bug-1252", + "timestamp": "2026-06-27T10:47:06.628Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/persist.c", + "root_cause": "Condition was: if (lsn != tip_lsn)", + "fix": "Changed to: if (trc0 != 0)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T10:47:06.628Z" + }, + { + "id": "bug-1254", + "timestamp": "2026-06-27T11:14:27.133Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_core_safety.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: * TASK 2.3 \u2014 cross-term catch-up streamer payload integrity ; * | Also: /* Task 2.3 cross-term catch-up streamer payload integrity:; * CORE RED \u2014 the streamer must never ship a non-empty cro", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-06-27T11:18:03.504Z" + }, + { + "id": "bug-1255", + "timestamp": "2026-06-27T11:28:45.351Z", + "error_message": "Missing guard clause", + "file": "src/controller/config_repl.c", + "root_cause": "No early return/throw for edge case: !pbuf", + "fix": "Added guard clause: if (!pbuf)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-27T11:28:45.351Z" + }, + { + "id": "bug-1264", + "timestamp": "2026-06-29T04:13:02.187Z", + "error_message": "Incorrect value in code", + "file": "src/controller/meson.build", + "root_cause": "Had 'testsuite/test_waldump.c'", + "fix": "Changed to 'testsuite/test_ha_partition.c'", + "tags": [ + "auto-detected", + "wrong-value", + "build" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-29T04:13:02.187Z" + }, + { + "id": "bug-1266", + "timestamp": "2026-06-29T04:25:39.111Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_partition.c", + "root_cause": "No early return/throw for edge case: peer_idx == g_partition_peer_idx && g_reconnect_gate", + "fix": "Added guard clause: if (peer_idx == g_partition_peer_idx && g_re)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-29T04:25:39.111Z" + }, + { + "id": "bug-1269", + "timestamp": "2026-06-30T01:47:07.635Z", + "error_message": "Missing await", + "file": "src/controller/ha.c", + "root_cause": "Async call without await \u2014 returned Promise instead of value", + "fix": "Added await to async call", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T01:47:07.635Z" + }, + { + "id": "bug-1271", + "timestamp": "2026-06-30T02:25:32.055Z", + "error_message": "Missing error handling in unknown", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: - _Depends: 1.5, 6.2, 5.2, 5.3_; - _Blocked: first soak run (HEAD 01419e1, Phase-2+5 fixes) i", + "tags": [ + "auto-detected", + "error-handling", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-30T02:26:08.789Z" + }, + { + "id": "bug-1272", + "timestamp": "2026-06-30T02:25:00Z", + "error_message": "raft-ha-core-safety 7.2 chaotic soak: round-4 'NO new primary within window (possible election stall)' after 3 overlapping kills", + "file": "src/controller/ha.c (election/PreVote path)", + "root_cause": "After 3 overlapping kills the only fully-up-to-date node is killed; the two survivors are both restarted-and-lagging (tips 15 and 11; leader was 19). Both hold the full committed prefix [1..11] (safe candidates) but their PreVote rounds repeatedly time out with no grant quorum (prospective terms frozen, never bumped). Distinct from the 5.2 spurious-failover flap (that is fixed and confirmed). Contributors: freshness mutual-denial between lagging survivors, in_catch_up-denies-all-votes (Req 2.5) with no live primary to catch up from, and/or peer vote connection not established between two nodes that only held a control channel to the dead primary.", + "fix": "FIXED at unit level by task 5.3 (commit bf88681): gate the in_catch_up PreVote deny in raft_cb_vote_eligible on a LIVE primary (primary_ctrl_fd>=0 && ha_heartbeat_is_fresh), so a catching-up survivor with a dead primary can grant/win a fresh PreVote (logOk preserves Leader Completeness). CORRECTION: this in_catch_up deadlock is a real latent defect (unit RED\u2192GREEN) but is NOT the soak's round-4 blocker. The soak stall is bug-924 (WAL fdatasync\u2192EINVAL wedges a survivor so it stops answering PreVotes; presents as an election stall). 7.2 live acceptance is blocked on that env defect (out of scope; owned by ha-commit-durability), not on consensus.", + "tags": [ + "prevote", + "ha", + "liveness", + "chaotic-soak", + "environment", + "election", + "bug-924", + "corrected", + "7.2", + "fdatasync", + "raft", + "5.3", + "in_catch_up" + ], + "related_bugs": [ + "bug-924", + "bug-851", + "bug-1042", + "bug-1060" + ], + "occurrences": 2, + "last_seen": "2026-06-30T02:55:00Z" + }, + { + "id": "bug-1273", + "timestamp": "2026-06-30T02:41:41.017Z", + "error_message": "Missing error handling in unknown", + "file": "src/controller/testsuite/test_ha_core_safety.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 5.3 lagging-but-safe survivor electability under ove; * CORE RED \u2014 a catching-up survivor with NO live primary m", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-06-30T02:41:53.665Z" + }, + { + "id": "bug-1275", + "timestamp": "2026-06-30T02:55:23.808Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T02:55:23.808Z" + }, + { + "id": "bug-1276", + "timestamp": "2026-06-30T02:55:37.693Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `research.md`", + "fix": "Changed to 's persist flusher jams on `", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T02:55:37.693Z" + }, + { + "id": "bug-1281", + "timestamp": "2026-06-30T03:25:00Z", + "error_message": "Restart-stability bug: a restarted xce_controller becomes unable to participate at chaotic round 4. Release build SIGSEGVs ('Segmentation fault (core dumped) --myname ctrl-3'); under gdb the SAME corruption manifests as a WEDGED inbound-accept/iomgr path (last accept 03:42:03 while the election timer keeps firing) so two restarted survivors can't exchange PreVotes -> permanent round-N election stall.", + "file": "src/controller (restart/recovery/election path \u2014 exact site unknown, no backtrace)", + "root_cause": "ROOT-CAUSED via gdb SIGQUIT all-thread bt: an ABBA lock-order deadlock in src/lib async dispatch. io_worker (netio.c:672) holds ep->lock then blocks on q->lock (dataq_enq); vdispatcher dataq_deq (dataq.c:65) holds q->lock then blocks on ep->lock via the vtask_eligible->endpoint_is_task_blocked (netio.c:1001) match callback. io_worker dies -> node stops accepting/answering PreVotes -> round-4 stall. In release the same ep/q race corrupts -> SIGSEGV. See bug-1282 (the actual dispatch defect).", + "fix": "The crash/hang IS bug-1282 (UPSTREAM raft-drive-async dispatch deadlock). raft-ha-core-safety 7.2 stays blocked until bug-1282 is fixed upstream, then re-run the gate. No raft-ha-core-safety code change.", + "tags": [ + "7.2", + "iomgr", + "gdb", + "election", + "crash", + "5.4", + "restart", + "recovery", + "heisenbug", + "accept-hang", + "restart-stability", + "segfault", + "chaotic-soak" + ], + "related_bugs": [ + "bug-1272", + "bug-1282", + "bug-924" + ], + "occurrences": 3, + "last_seen": "2026-06-30T04:00:00Z" + }, + { + "id": "bug-1279", + "timestamp": "2026-06-30T03:33:06.273Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had 's persist flusher jams on `", + "fix": "Changed to `xce_controller`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T03:33:06.273Z" + }, + { + "id": "bug-1280", + "timestamp": "2026-06-30T03:57:32.784Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had \"Task 7.2 on ~/tmp\"", + "fix": "Changed to \"Task 7.2/5.4 \u2014 UNIFIED root cause\"", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T03:57:32.785Z" + }, + { + "id": "bug-1282", + "timestamp": "2026-06-30T04:17:08.329Z", + "error_message": "ABBA lock-order deadlock under chaotic churn: io_worker (ep->lock then dataq_enq wants q->lock) vs vdispatcher dataq_deq (q->lock then vtask_eligible->endpoint_is_task_blocked wants ep->lock). Wedges io_worker -> controller stops accepting connections; in release the race corrupts -> SIGSEGV.", + "file": "src/lib/dataq.c:65 (dataq_deq holds q->lock across match cb) + src/lib/vtaskmgr.c:255 (vtask_eligible) + src/lib/netio.c:1001 (endpoint_is_task_blocked takes ep->lock) + src/lib/netio.c:672 (io_worker holds ep->lock across vtask_enq->dataq_enq)", + "root_cause": "dataq_deq runs its match callback while holding q->lock, and the callback (vtask_eligible->endpoint_is_task_blocked) acquires ep->lock \u2014 inverting the ep->lock->q->lock order io_worker uses. Two-thread ABBA deadlock, deterministic under round-4 chaotic churn. Confirmed via gdb all-thread backtrace on two independently-wedged nodes.", + "fix": "FIXED (commit 25e741d): promoted endpoint_t.busy from a 1-bit bitfield to _Atomic int; endpoint_block_tasks/endpoint_unblock_tasks/endpoint_is_task_blocked now lockless (release/acquire), removing the ep->lock acquire from the vtask-eligibility path that ran under q->lock. Verified: post-fix gdb soak has ZERO __lll_lock_wait threads + no endpoint_is_task_blocked in any lock path; controller_unit_test all green. Mirrors dispatch_inflight (bug-1197). NOTE: a SEPARATE release-only restart segfault (bug-1283) remains and still blocks 7.2.", + "tags": [ + "deadlock", + "lock-order", + "abba", + "vtaskmgr", + "dataq", + "netio", + "io_worker", + "vdispatcher", + "raft-drive-async", + "async-io", + "upstream", + "async-fix", + "md" + ], + "related_bugs": [ + "bug-1281" + ], + "occurrences": 3, + "last_seen": "2026-06-30T04:20:00Z" + }, + { + "id": "bug-1283", + "timestamp": "2026-06-30T04:17:58.579Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `xce_controller`", + "fix": "Changed to `src/lib`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T04:17:58.579Z" + }, + { + "id": "bug-1285", + "timestamp": "2026-06-30T08:24:48.860Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `vtask_eligible`", + "fix": "Changed to `endpoint_is_task_blocked`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T08:24:48.860Z" + }, + { + "id": "bug-1286", + "timestamp": "2026-06-30T05:30:00Z", + "error_message": "xce_controller SIGSEGV in ha_dispatch_gate during chaotic round-4 restart \u2014 Thread received SIGSEGV at ha.c:3194 (atomic_load &ha->role) with ha=0x3ff0 (garbage). The release-only crash that blocks raft-ha-core-safety 7.2; DISTINCT from the bug-1282 deadlock (which is fixed).", + "file": "src/controller/cluster.c:251 (myreq_handler) + src/controller/ha.c:3194 (ha_dispatch_gate)", + "root_cause": "NULL ep->owner. myreq_handler does `cluster_t* c = (cluster_t*)ep->owner;` then `ha_dispatch_gate(&c->ha, opcode)`. On a chaotic restart an inbound endpoint dispatches a walCatchupEntry (op 37) vtask while ep->owner==NULL, so c=NULL and &c->ha = NULL+offsetof(cluster_t,ha) = 0x3ff0 \u2014 a garbage pointer ha_dispatch_gate dereferences (atomic_load &ha->role, ha.c:3194) -> SIGSEGV. Backtrace (debug -O0 -g3 under gdb): ha_dispatch_gate(ha=0x3ff0,opcode=37) <- myreq_handler cluster.c:251 <- vtask_entry cluster.c:693 <- run_task/vcpu_entry. ep->owner is NULL because the accept/dispatch path runs a request before owner is wired (or owner cleared mid-flight during teardown).", + "fix": "FIXED 2026-06-30 (commit 5db734a). NULL-guard `c = ep->owner` in myreq_handler (cluster.c): a chaotic-restart walCatchupEntry (op 37) vtask ran with ep->owner==NULL \u2192 &c->ha==0x3ff0 deref crash at ha.c:3194. Guard returns early when owner unset. Verified: round-4 restart SIGSEGV gone; controller_unit_test green.", + "tags": [ + "segfault", + "null-deref", + "ep-owner", + "myreq_handler", + "cluster.c", + "ha_dispatch_gate", + "restart", + "7.2", + "ROOT-CAUSED" + ], + "related_bugs": [ + "bug-1282", + "bug-1281", + "bug-1197" + ], + "occurrences": 3, + "last_seen": "2026-06-30T14:00:00Z" + }, + { + "id": "bug-1290", + "timestamp": "2026-06-30T09:15:12.861Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had 't crash) \u2014 STILL blocks the gate; likely an endpo", + "fix": "Changed to 't crash) \u2014 is now **ROOT-CAUSED** (via the new `", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T09:15:12.861Z" + }, + { + "id": "bug-1292", + "timestamp": "2026-06-30T09:33:08.918Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `1+2*njobs`", + "fix": "Changed to `c = ep->owner`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T09:33:08.918Z" + }, + { + "id": "bug-1295", + "timestamp": "2026-06-30T13:19:42.706Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: jmax > 0", + "fix": "Added guard clause: if (jmax > 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T13:19:42.706Z" + }, + { + "id": "bug-1296", + "timestamp": "2026-06-30T13:21:00.000Z", + "error_message": "10-round chaotic soak FAIL at round 7: all 3 nodes in RDHUP\u2192PreVote\u2192timeout\u2192repeat livelock \u2014 zero pre_vote_reply_in events (not denials, silence). ctrl-3 sent 20+ PreVote rounds with NO reply received; ctrl-1 and ctrl-2 same loop. Rounds 1-6 PASS clean (all elections + job-verification succeeded); round 7 kills ctrl-2 (term=9 primary), restarts ctrl-3 (no convergence wait \u2014 chaotic). 2 surviving nodes (ctrl-1, ctrl-3) both start PreVote; neither processes the other's request \u2192 mutual timeout \u2192 RDHUP fires again \u2192 repeat forever.", + "file": "src/controller/ha.c", + "root_cause": "TASK 5.6 dueling-candidate PreVote cross-denial \u2014 DISTINCT from the earlier PreVote-WINS + real-election split-vote (which had both nodes winning PreVote then deadlocking in RequestVote). Here PreVote itself starves: each node's PreVote handler likely drops/ignores inbound PreVote requests while in its own PreVote round (no reply sent, not even denial). The task 5.6 RDHUP guard (commit 25e741d+) only prevents restarting failover while in CTRL_ELECTING \u2014 it does NOT fix the core problem where two nodes enter simultaneous PreVote rounds and each starves the other's reply. The priority stagger (bug-606) helps when 3 nodes are alive (one grants to break symmetry), but with only 2 survivors (the third restarting without convergence in chaotic mode), both deny each other and the stagger is insufficient.", + "fix": "FIXED 2026-06-30 (subsumed by tasks 5.6 commit 81cb09a + 5.7 commit 74784b2). The RDHUP->PreVote->timeout liveloc was the same fixed-watchdog resonance as bug covered by 5.7: per-campaign election watchdog re-randomized to [T,2T] (Ongaro 3.4) + don't-preempt-in-flight-election (5.6). CONFIRMED by a 10-round chaotic soak: 10/10 rounds elect a new primary in a single term increment (4->13), zero term escalation, round 7+ (the old liveloc point) all clean.", + "tags": [ + "raft-ha-core-safety", + "election-liveness", + "prevote-cross-denial", + "dueling-candidates", + "task-5.6", + "chaotic-soak", + "round-7-stall" + ], + "related_bugs": [ + "bug-1281", + "bug-1282", + "bug-1286", + "bug-1292" + ], + "occurrences": 1, + "last_seen": "2026-06-30T14:30:00Z" + }, + { + "id": "bug-1297", + "timestamp": "2026-06-30T13:28:01.170Z", + "error_message": "Missing guard clause", + "file": "src/controller/ha.c", + "root_cause": "No early return/throw for edge case: jmax > 0", + "fix": "Added guard clause: if (jmax > 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T13:28:01.170Z" + }, + { + "id": "bug-1298", + "timestamp": "2026-06-30T13:36:00-07:00", + "error_message": "AddressSanitizer: heap-use-after-free READ size 4 at endpoint_is_task_blocked netio.c:1008 (vtask dispatcher T3) \u2014 endpoint freed by endpoint_killer cluster.c:813 (io_worker T2); ctrl-3 ABORTs mid-chaotic-soak round 3\u21924, dropping the cluster to 1/3 (no quorum) so round 4 'NO new primary within window'", + "file": "src/lib/netio.c / src/lib/vtaskmgr.c / src/controller/cluster.c", + "root_cause": "Teardown-vs-dispatch race: io_worker (T2) tears down a connection on error \u2014 handle_endpoint_error \u2192 zarray_del \u2192 endpoint_killer free()s the endpoint_t \u2014 while the vtask dispatcher (T3) is concurrently scanning the dataq (dataq_deq_first_match \u2192 vtask_eligible \u2192 endpoint_is_task_blocked) and reads ep->busy on that just-freed endpoint. The endpoint is freed while still referenced by a pending vtask in the dataq; nothing drains/guards the ep's queued vtasks before the free, and no refcount keeps it alive across the dispatcher read. Same netio surface as bug-1282 (ABBA, ep->busy made lockless) and the bug-1197 dispatch_inflight precedent; the bug-1282 lockless read removed the (incidental) lock contention so the dispatcher reaches the read faster, exposing the latent UAF.", + "fix": "FIXED (commit 6e05dc1): added endpoint_t reference count (refcnt _Atomic + heap_alloc flag). endpoint_get starts refcnt=1 (owner); vtask_enq takes a per-vtask ref before publishing the task to the dispatcher, run_task/taskq_item_killer release it; every heap-ep teardown site drops its owner ref via endpoint_unref (last unref fini+frees). The dispatcher can now read ep->busy on a queued vtask without the io_worker freeing it underneath. Embedded eps (heap_alloc=0) never freed by unref. Validated: 182 unit tests/0 fail; 6-round ASAN chaotic soak ZERO ASan reports, ctrl-3 survives round 4, all rounds elect.", + "tags": [ + "netio", + "vtaskmgr", + "use-after-free", + "asan", + "endpoint-teardown", + "concurrency", + "raft-ha-core-safety", + "soak", + "quorum", + "bug-1282-family", + "FIXED" + ], + "related_bugs": [ + "bug-1282", + "bug-1197", + "bug-1286" + ], + "occurrences": 1, + "last_seen": "2026-06-30T14:25:00-07:00" + }, + { + "id": "bug-1299", + "timestamp": "2026-06-30T13:39:09.194Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T13:39:09.194Z" + }, + { + "id": "bug-1304", + "timestamp": "2026-06-30T14:24:36.874Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/research.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T14:24:36.874Z" + }, + { + "id": "bug-1306", + "timestamp": "2026-06-30T14:31:00-07:00", + "error_message": "6-round chaotic-failover soak FAIL: committed_job_loss=4 committed_divergence=7 (rounds 4-6). Committed job 0-0-1782829181-1 permanently absent on reachable primary from round 4 on, across 3 elected primaries; rounds 1-3 clean.", + "file": "src/lib/hook.c (xce_run_hooks); job.c:603 gate is correct", + "root_cause": "SHARPENED 2026-06-30 (waldump+log bisect of the 10-round soak): NOT the catch-up prefix-gap/entity-snapshot mechanism. xce_run_hooks (src/lib/hook.c:134-157) returns only the LAST-registered hook's rc (rc overwritten each loop iter; break is BEFORE_EVENT-only). The durable-majority gate works: a term-9 leader's job at LSN 30 timed out commit (peers mid-restart, repl_wait_majority_ack -> commit_submit -> -ETIMEDOUT), persist_job_hook_fun returned -ETIMEDOUT, but a later-registered JOB AFTER_EVENT hook (sched/apiserver) returned 0 and CLOBBERED it -> handle_jobsubmit saw hook_rc==0 -> replied 'jobsubmit ok' for an uncommitted entry -> term-10 leader legally overwrote LSN 30 (term-10 NOOP) -> committed-job-loss. Violates the Commit Durability invariant (reply-success => majority-durable). Evidence: all 3 WALs gapless/CRC-intact/full payloads; lost job only on ctrl-1@LSN30 term9 vs term10 NOOP on majority; ctrl-1 log has the WAIT-deadline-expired WARNs and NO 'rejecting job' line.", + "fix": "FIXED 2026-06-30 (raft-apply-decoupling item #1 down payment). Extracted the WAL append + majority-commit barrier out of persist_job_hook_fun's WAIT branch into an explicit persist_submit_wal_and_commit(p, jlist); handle_jobsubmit now calls it directly and reads the commit rc DIRECTLY (rejects on <0 before notifying), so the -ETIMEDOUT can no longer be clobbered by a later notify hook's 0 in xce_run_hooks. The JOB_NEW AFTER hook is now notification-only. PROVEN: 10-round chaotic soak committed_job_loss 5 -> 0 (same election code, A/B isolates the durability fix). Unit: +test_job_hook_new_is_notify_only; persist_hooks 19/0, e2e_hooks 3/0, commit_barrier_reply 22/0, commit_durability 38/0. Remaining soak FAIL is committed_divergence (transient catch-up lag, present=2) + a separate round-4 election escalation tail (5.7 class) \u2014 neither is job loss.", + "tags": [ + "ha-commit-durability", + "pre-existing", + "not-a-regression", + "soak", + "task-5.9", + "committed-job-loss", + "commit-durability", + "term-boundary-divergence", + "catch-up-gap", + "failover", + "hook-rc-clobber", + "ack-before-majority", + "data-loss" + ], + "related_bugs": [ + "bug-795", + "bug-828", + "bug-851", + "bug-1298" + ], + "occurrences": 1, + "last_seen": "2026-06-30T17:40:00Z" + }, + { + "id": "bug-1307", + "timestamp": "2026-06-30T15:05:00.000Z", + "error_message": "Task 5.7 election-resonance jitter (74784b2) introduced data-safety REGRESSION: 6-round chaotic soak shows 7 committed divergences (LSNs 8,12,16,19) + 4 committed job losses (job 0-0-1782831451-1). Previous run (pre-5.7, commit 81cb09a) had ZERO divergences and ZERO job losses through 6 rounds. LSN 19 divergence is present=3 (ALL 3 members have different content at same LSN \u2014 silent divergence, textbook bug-851). Election liveness improved (all 6 rounds complete vs stalled at round 7), but the jitter defeats the election-safety balance: faster-starting node wins before catching up on committed prefix \u2192 Leader Completeness violation.", + "file": "src/controller/ha.c", + "root_cause": "Task 5.7 jittered watchdog ([T,2T] randomized) lets a less-complete node start and win an election before catching up on the full committed prefix from the dead primary. With the original fixed watchdog, both survivors fired RDHUP simultaneously \u2192 cross-denied \u2192 the library's priority stagger resolved the tie safely (the fresher node won after deterministic desynchronization). With jitter, one node can fire much earlier (near T) and the other much later (near 2T) \u2192 the early node campaigns with an incomplete log \u2192 voters grant based on stale view \u2192 winner truncates committed entries it never held. The 5.3 in_catch_up release (which drops the vote-deny when the primary is dead) opens a window where a still-catching-up candidate can win \u2014 with the new jitter timings this window is hit frequently.", + "fix": "CONFIRMED FIXED via live re-soak (2026-07-03): ran the exact reproducing scenario \u2014 6-round chaotic soak (overlapping kill-the-primary, real ext4 at ~/tmp, ASAN controller binary) \u2014 that originally showed 7 committed divergences + 4 job losses. This run: 21 jobs submitted, 0 committed_divergence, 0 committed_job_loss, PASS across all 6 rounds (terms 3->9, all 3 nodes rotated through primary). Matches the earlier circumstantial evidence (raft-ha-core-safety Phase 5 fully complete + prior soak-clean sessions) with a fresh, direct confirmation run \u2014 no residual doubt.", + "tags": [ + "data-safety-regression", + "task-5.7", + "chaotic-soak", + "leader-completeness-violation", + "confirmed-via-soak", + "committed-job-loss", + "election-liveness", + "bug-1296-followup", + "committed-divergence", + "fixed" + ], + "related_bugs": [ + "bug-1296", + "bug-1281", + "bug-1282", + "bug-1286", + "bug-851" + ], + "occurrences": 1, + "last_seen": "2026-07-03T03:15:00.000Z" + }, + { + "id": "bug-1308", + "timestamp": "2026-06-30T21:22:32.991Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Had `research.md`", + "fix": "Changed to 't-preempt-in-flight-election; reduced term churn)", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-06-30T21:22:32.991Z" + }, + { + "id": "bug-1310", + "timestamp": "2026-06-30T18:40:00Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: p == NULL || jlist == NULL", + "fix": "Added guard clause: if (p == NULL || jlist == NULL)", + "tags": [ + "auto-detected", + "guard-clause", + "c", + "election-liveness", + "rdhup", + "term-escalation", + "debounce", + "5.10b", + "raft" + ], + "related_bugs": [ + "bug-1296", + "bug-1306" + ], + "occurrences": 2, + "last_seen": "2026-07-01T00:24:06.562Z" + }, + { + "id": "bug-1312", + "timestamp": "2026-07-01T00:29:56.872Z", + "error_message": "Wrong reference: test_job_hook_start_writes_wal_entry should be test_job_hook_new_is_notify_only", + "file": "src/controller/testsuite/test_persist_hooks.c", + "root_cause": "Used \"test_job_hook_start_writes_wal_entry\" instead of \"test_job_hook_new_is_notify_only\"", + "fix": "Changed test_job_hook_start_writes_wal_entry \u2192 test_job_hook_new_is_notify_only", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T00:29:56.872Z" + }, + { + "id": "bug-1313", + "timestamp": "2026-07-01T00:42:13.591Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T00:42:13.591Z" + }, + { + "id": "bug-1315", + "timestamp": "2026-07-01T01:26:38.341Z", + "error_message": "Wrong reference: unistd should be time", + "file": "src/controller/testsuite/test_ha_rdhup_integration.c", + "root_cause": "Used \"unistd\" instead of \"time\"", + "fix": "Changed unistd \u2192 time", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T01:26:38.341Z" + }, + { + "id": "bug-1319", + "timestamp": "2026-07-01T04:04:13.051Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-entity-snapshots/design.md", + "root_cause": "Had \"collect+encode jobs\\n(RTTI_LOCK_READ on jobmgr)\"", + "fix": "Changed to \"collect+encode jobs\\n(table READ lock + per-job R", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-07-01T04:04:56.893Z" + }, + { + "id": "bug-1320", + "timestamp": "2026-07-01T04:47:19.662Z", + "error_message": "Wrong reference: test_ha_partition should be test_persist_snapshot_entities", + "file": "src/controller/testsuite/controller_test.c", + "root_cause": "Used \"test_ha_partition\" instead of \"test_persist_snapshot_entities\"", + "fix": "Changed test_ha_partition \u2192 test_persist_snapshot_entities", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T04:47:19.662Z" + }, + { + "id": "bug-1326", + "timestamp": "2026-07-01T06:17:52.049Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_persist_snapshot_entities.c", + "root_cause": "No early return/throw for edge case: fd < 0", + "fix": "Added guard clause: if (fd < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T06:17:52.049Z" + }, + { + "id": "bug-1327", + "timestamp": "2026-07-01T06:18:39.877Z", + "error_message": "Wrong reference: pthread should be fcntl", + "file": "src/controller/testsuite/test_persist_snapshot_entities.c", + "root_cause": "Used \"pthread\" instead of \"fcntl\"", + "fix": "Changed pthread \u2192 fcntl", + "tags": [ + "auto-detected", + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T06:18:39.877Z" + }, + { + "id": "bug-1328", + "timestamp": "2026-07-01T06:33:25.117Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_persist_snapshot_entities.c", + "root_cause": "No early return/throw for edge case: fd < 0", + "fix": "Added guard clause: if (fd < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T06:33:25.117Z" + }, + { + "id": "bug-1330", + "timestamp": "2026-07-01T07:24:22.541Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_ha_liveness.c", + "root_cause": "No early return/throw for edge case: cleared == 0", + "fix": "Added guard clause: if (cleared == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T07:24:22.541Z" + }, + { + "id": "bug-1331", + "timestamp": "2026-07-01T07:46:42.046Z", + "error_message": "Wrong return value", + "file": "src/controller/persist.c", + "root_cause": "Was returning: load_snapshot(p, path, lsn);", + "fix": "Now returns: rc;", + "tags": [ + "auto-detected", + "return-value", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T07:46:42.046Z" + }, + { + "id": "bug-1340", + "timestamp": "2026-07-01T09:55:17.967Z", + "error_message": "Missing guard clause", + "file": "src/controller/persist.c", + "root_cause": "No early return/throw for edge case: !p || checkpoint_lsn == 0", + "fix": "Added guard clause: if (!p || checkpoint_lsn == 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T09:55:17.967Z" + }, + { + "id": "bug-1343", + "timestamp": "2026-07-01T13:27:42.519Z", + "error_message": "Missing error handling in every", + "file": "src/controller/testsuite/test_ha_core_safety.c", + "root_cause": "Code path had no error handling \u2014 exceptions would propagate uncaught", + "fix": "Added try/catch block | Also: /* Task 6.1 integration: leader->standby replication carries; * payload) faithfully end-to-end across BOTH live in-term re", + "tags": [ + "auto-detected", + "error-handling", + "c" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-01T13:27:51.887Z" + }, + { + "id": "bug-1348", + "timestamp": "2026-07-01T14:10:48.987Z", + "error_message": "Wrong reference: test_pre_vote_catch_up_denies should be test_pre_vote_catch_up_grants_when_primary_unreachable", + "file": "src/controller/testsuite/test_ha_vote_gate.c", + "root_cause": "Used \"test_pre_vote_catch_up_denies\" instead of \"test_pre_vote_catch_up_grants_when_primary_unreachable\"", + "fix": "Changed test_pre_vote_catch_up_denies \u2192 test_pre_vote_catch_up_grants_when_primary_unreachable", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T14:10:48.987Z" + }, + { + "id": "bug-1349", + "timestamp": "2026-07-01T14:24:21.310Z", + "error_message": "Wrong condition in logic", + "file": "src/controller/ha.c", + "root_cause": "Condition was: if (atomic_load_explicit(&ha->in_catch_up, memory_orde)", + "fix": "Changed to: if (0 && atomic_load_explicit(&ha->in_catch_up, memory)", + "tags": [ + "auto-detected", + "logic-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-01T14:24:21.310Z" + }, + { + "id": "bug-1355", + "timestamp": "2026-07-02T03:03:21.985Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t \u2192 uint32_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T03:03:21.985Z" + }, + { + "id": "bug-1357", + "timestamp": "2026-07-02T03:03:51.842Z", + "error_message": "Function not marked async", + "file": "src/controller/main.c", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T03:03:51.842Z" + }, + { + "id": "bug-1362", + "timestamp": "2026-07-02T03:50:14.650Z", + "error_message": "Wrong reference: persist_config_t should be uint32_t", + "file": "src/include/persist.h", + "root_cause": "Used \"persist_config_t\" instead of \"uint32_t\"", + "fix": "Changed persist_config_t \u2192 uint32_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T03:50:14.650Z" + }, + { + "id": "bug-1363", + "timestamp": "2026-07-02T03:50:42.583Z", + "error_message": "Wrong reference: WAL_SEGMENT_SIZE_BYTES should be p", + "file": "src/controller/persist.c", + "root_cause": "Used \"WAL_SEGMENT_SIZE_BYTES\" instead of \"p\"", + "fix": "Changed WAL_SEGMENT_SIZE_BYTES \u2192 p", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 3, + "last_seen": "2026-07-02T03:50:52.480Z" + }, + { + "id": "bug-1364", + "timestamp": "2026-07-02T03:52:09.933Z", + "error_message": "Wrong reference: myconf_t should be uint32_t", + "file": "src/include/myconf.h", + "root_cause": "Used \"myconf_t\" instead of \"uint32_t\"", + "fix": "Changed myconf_t \u2192 uint32_t", + "tags": [ + "wrong-reference", + "h" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T03:52:09.933Z" + }, + { + "id": "bug-1369", + "timestamp": "2026-07-02T03:57:52.302Z", + "error_message": "Incorrect value in code", + "file": ".claude/skills/xce-test-ha/run-ha-soak.sh", + "root_cause": "Had \"MODE=force-snapshot \u2014 forcing one full-checkpoint", + "fix": "Changed to \"MODE=force-snapshot \u2014 forcing one full-checkpoint", + "tags": [ + "auto-detected", + "wrong-value", + "sh" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T03:57:52.302Z" + }, + { + "id": "bug-1370", + "timestamp": "2026-07-02T04:10:10.430Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/raft-ha-core-safety/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T04:10:10.430Z" + }, + { + "id": "bug-1372", + "timestamp": "2026-07-02T04:39:29.527Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: fd < 0", + "fix": "Added guard clause: if (fd < 0)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T04:39:29.527Z" + }, + { + "id": "bug-1376", + "timestamp": "2026-07-02T05:10:00.000Z", + "error_message": "raft-drive-async: cluster hangs silently (no crash, no ASAN report) during a --force-snapshot soak with g_drive_async_snapshot_offload + g_drive_async_peerctrl_dispatch_enabled both ON: a freshly-wiped empty node cold-starts, self-elects, then goes silent after 'peer_ctrl_dispatch_to_pool: vtask pool not wired -- running inline' is immediately followed by cluster_init_phase2 wiring the pool while raft_cb_become_follower's deferred-teardown vtasks (bug-1197's mechanism) are being enqueued -- the process never logs another line and is eventually killed by the test harness timeout.", + "file": "src/controller/ha.c", + "root_cause": "peer_ctrl_msg_handler (ha.c) held ep->dispatch_inflight incremented across the ENTIRE peer_ctrl_dispatch_to_pool call, including its inline fallback (the common cold-start shape when ha->pool is not yet wired). That inline body can reach raft_cb_become_follower, which -- now that cluster_init_phase2 ran synchronously earlier in the SAME call chain and wired the pool -- tears down its own loser vote-ep (the very ep whose reply triggered this call) via the deferred path (vote_ep_teardown_async), whose endpoint_mark_tearing_down spins waiting for dispatch_inflight==0 on that SAME ep. That count is held only by this exact call, on this exact thread, and cannot reach 0 until the spin itself returns -- a permanent self-deadlock. Confirmed live via a gdb thread backtrace (GDB_WRAP debug seam added to run-ha-soak.sh, launching the target node directly under `gdb --batch` so it becomes ptrace-legal as gdb's own direct child, then SIGINT to gdb once the log went stale for a bounded interrupt + full backtrace): frame `peer_ctrl_dispatch_to_pool` showed pool=0x0 at dispatch time; the same ep pointer appeared in the nested vote_ep_teardown_async -> endpoint_mark_tearing_down frame, spinning on sched_yield().", + "fix": "Moved the ep->dispatch_inflight release into peer_ctrl_dispatch_to_pool itself, at the exact right point per exit path: released BEFORE calling the inline body on every fallback path (pool==NULL, OOM, enqueue failure) -- safe because io_worker is single-threaded and the inline path never enqueues anything, so there is no pool-worker race left to protect against once inline execution is committed to -- and only AFTER the (non-blocking) enqueue completes on the success path, preserving the original protection for the genuinely concurrent pool-worker case. New unit test test_vep_self_teardown_does_not_deadlock (test_ha_vep_teardown.c) uses a bounded-timeout pthread_timedjoin_np so a regression fails loud instead of hanging the suite; RED confirmed via a temporary manual revert (Expected 0 Was 110 == ETIMEDOUT) before restoring the fix. Live-verified with both g_drive_async_snapshot_offload and g_drive_async_peerctrl_dispatch_enabled flipped ON: --force-snapshot soak (the exact repro shape) completes without hanging across repeated runs, confirmed real snapshot installs, committed_divergence=0, committed_job_loss=0; a 6-round --chaotic soak with both flags ON also passed clean; full controller suite green throughout. Both flags now default ON in production (raft-drive-async tasks 3.1/3.2 closed).", + "tags": [ + "raft-drive-async", + "cold-start", + "vtask-pool", + "self-deadlock", + "fixed" + ], + "related_bugs": [ + "bug-1197", + "bug-1298", + "bug-1310" + ], + "occurrences": 1, + "last_seen": "2026-07-02T00:20:00.000Z" + }, + { + "id": "bug-1380", + "timestamp": "2026-07-02T07:16:35.420Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `, differential-verified: this specific soak scena", + "fix": "Changed to `, differential-verified). (4) Re-ran `", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-02T07:17:36.171Z" + }, + { + "id": "bug-1381", + "timestamp": "2026-07-02T00:35:00.000Z", + "error_message": "raft-drive-async task 3.3 (config-push slow-lane offload) blocked: push_config_transport_only has a pre-existing endpoint-lifetime race, independent of the g_drive_async_configpush_offload flag -- it grabs raw endpoint_t* pointers for every registered standby under c->ha.lock, releases the lock, then spawns one pthread per standby (push_thread) that reads/writes those endpoints unprotected for up to CONF_PUSH_TIMEOUT_MS. If a standby RDHUP-disconnects during that window, the io_worker's teardown path (endpoint_killer -> endpoint_unref, the bug-1298 refcount system) can free the endpoint while a push_thread is still using it -- a UAF. This is NOT specific to the lane offload: the transport fan-out spawns its own independent OS threads regardless of which caller (inline drive-thread or lane worker) invokes it, so the documented 'runs inline on io_worker so teardown and push are serialized on one thread' rationale for keeping the flag OFF does not actually hold -- the race exists in the inline path today too.", + "file": "src/controller/config_repl.c", + "root_cause": "push_config_transport_only (config_repl.c ~line 1366) takes c->ha.lock only long enough to copy standby ep pointers into a push_ctx_t array, then releases the lock before pthread_create-ing one push_thread per standby. Those threads read/write the raw endpoint_t* for the full duration of the send + wait_all_with_timeout(CONF_PUSH_TIMEOUT_MS), with no ref taken on the endpoints and no coordination with concurrent teardown (handle_endpoint_error -> zarray_del -> endpoint_killer -> endpoint_unref). Unlike the snapshot-lane fix (bug-1197/bug-1376 session), no endpoint_ref/endpoint_unref bracket protects this fan-out.", + "fix": "FIXED. push_config_transport_only (config_repl.c) now takes endpoint_ref() on each standby ep while the pointer is snapshotted under c->ha.lock, mirroring the snapshot-lane fix (bug-1197/bug-1376). Every ref has exactly one matching endpoint_unref(): from push_thread after it finishes using ctx->ep (new close in push_thread itself), or from push_config_transport_only inline if pthread_create fails for that standby, or from a cleanup loop if the threads[] allocation fails entirely. A concurrent RDHUP teardown (endpoint_killer -> endpoint_unref) can now only drop the OWNER reference while a push is in flight -- it can no longer free ep out from under push_thread. New regression test test_configpush_transport_holds_endpoint_ref_across_send (test_config_repl.c) uses a new push_thread_send_hook test seam (same pattern as xce_wal_stream_request_hook) to park a push_thread mid-send, simulate the owner teardown, and assert ep survives with refcnt intact. RED->GREEN verified via temporary revert (Expected 2 Was 1 with the ref-take removed); full controller suite green (230/230) after restore.", + "tags": [ + "raft-drive-async", + "configpush", + "endpoint-lifetime", + "uaf-suspect", + "fixed" + ], + "related_bugs": [ + "bug-1197", + "bug-1298", + "bug-1376" + ], + "occurrences": 1, + "last_seen": "2026-07-02T00:35:00.000Z" + }, + { + "id": "bug-1382", + "timestamp": "2026-07-02T07:23:50.910Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `snapshot_lane_fn`", + "fix": "Changed to `configpush_offload`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-02T07:24:12.475Z" + }, + { + "id": "bug-1383", + "timestamp": "2026-07-02T07:31:28.737Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: waited >= timeout_ms", + "fix": "Added guard clause: if (waited >= timeout_ms)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T07:31:28.737Z" + }, + { + "id": "bug-1384", + "timestamp": "2026-07-02T07:37:33.427Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had ` (test_ha_liveness.c). Blocked on **bug-1381** (o", + "fix": "Changed to ` (test_ha_liveness.c). **bug-1381 CLOSED 2026-07-", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-02T07:37:48.507Z" + }, + { + "id": "bug-1386", + "timestamp": "2026-07-02T08:15:00.000Z", + "error_message": "/xce-test-ha --force-snapshot round: after a wiped ctrl-2 rejoins empty and the primary sends a full-checkpoint snapshot, ctrl-2 sometimes never logs a completed install (\"HA: snapshot loaded/legacy\" line absent) though the cluster still reconverges to 3-of-3 -- OR the cluster fails to reconverge to 3-of-3 at all. Reproduced with g_drive_async_configpush_offload BOTH ON (0/3 pass) and OFF (1/2 pass) on the same freshly-built xce.build binary, so it is independent of that flag -- discovered while validating the bug-1381 configpush flip.", + "file": "src/controller/config_repl.c", + "root_cause": "ha_send_snapshot_to_standby (config_repl.c, ~line 2596) loops over up to 12 65536-byte chunks calling xce_snapshot_chunk_req_raw(ep, &chunk, buf, chunk_len) WITHOUT checking or propagating its return value. xce_snapshot_chunk_req_raw -> endpoint_enq can return -ECONNRESET (e.g. if the standby endpoint is mid-teardown or reconnecting during the empty-rejoin window) but the loop keeps incrementing offset/chunk_index regardless, and the function still logs \"transfer complete\" and returns rc=0 at the end. Confirmed via ctrl-2 log evidence: primary logs start+complete for a 735546-byte (12-chunk) snapshot within ~2ms, but ctrl-2 only ever receives a handful of leading chunks (2 in one run, 5 in another) before the stream silently stops -- no ERROR on either side.", + "fix": "FIXED 2026-07-02 (config_repl.c): ha_send_snapshot_to_standby's chunk loop now captures the return value of xce_snapshot_chunk_req_raw (production path) and xce_snapshot_chunk_hook (XCE_TESTING path) into send_rc, breaks the loop on send_rc != 0, and propagates it as the function's rc instead of silently continuing. The 'transfer complete' INFO log is now strictly gated on rc==0; a failure path now logs a WARN with checkpoint_lsn/total_size/sent-offset/chunk_index/rc for observability. No receive-side timeout/re-request mechanism was added (out of scope, no existing seam) \u2014 the existing per-peer snapshot_inflight[] flag already clears unconditionally on send-thread exit (snapshot_lane_fn, ha.c) and the next heartbeat's SEND_SNAPSHOT decision naturally re-triggers a fresh transfer to a peer whose applied_lsn never advanced, so the existing retry cadence recovers once the endpoint is healthy again. New RED->GREEN unit test test_snapshot_transfer_stops_and_propagates_send_failure (test_config_repl.c) injects a mid-loop -ECONNRESET via the existing xce_snapshot_chunk_hook seam (new snaptest_fail_after_hook) and asserts rc == -ECONNRESET and exactly 2 chunks attempted (not the false-'complete' 3). RED confirmed via temporary revert ('Expected -104 Was 0'); GREEN restored, full controller_unit_test suite clean (0 failures across all groups).", + "tags": [ + "raft-drive-async", + "snapshot-transfer", + "silent-failure", + "force-snapshot-soak", + "fixed" + ], + "related_bugs": [ + "bug-1197", + "bug-1376", + "bug-1381" + ], + "occurrences": 1, + "last_seen": "2026-07-02T09:30:00.000Z" + }, + { + "id": "bug-1388", + "timestamp": "2026-07-02T08:19:04.015Z", + "error_message": "Missing guard clause", + "file": "src/controller/testsuite/test_config_repl.c", + "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16", + "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)", + "tags": [ + "auto-detected", + "guard-clause", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T08:19:04.015Z" + }, + { + "id": "bug-1389", + "timestamp": "2026-07-02T09:05:00.000Z", + "error_message": "/xce-test-ha --force-snapshot: after bug-1386's fix (sender now correctly reports success only on true completion), the force-snapshot soak still FAILs. ctrl-2's log shows all 12 chunks of a 735546-byte snapshot received, then `ha_handle_snapshot_chunk: persist_load_snapshot failed rc=-5` / `handle_snapshot_transfer: chunk 11 rc=-5` (rc=-5=-EIO). ctrl-1 (sender) correctly logs 'transfer complete' both times \u2014 the send genuinely succeeded, full byte count. This is NOT bug-1386 (that was a chunk DROP producing a truncated/short transfer). Byte-comparing the primary's source checkpoint file against the exact bytes ctrl-2 assembled into snap.part -> snap-{lsn}.bin (same run, two separate checkpoint LSNs 0x75 and 0x76, both reproduced) shows: identical total size (735546==735546) but DIFFERENT CONTENT \u2014 the received file diverges from the sender's file starting exactly at a SNAPSHOT_CHUNK_SIZE (65536-byte) chunk boundary, then the content is a byte-SHIFTED copy of the correct data (same repeating job-frame strings, same occurrence count, just offset) that re-synchronizes with the source at a later offset. The shift direction/magnitude differs between the two independent attempts (-912 bytes resyncing at 262144 vs +4684 bytes resyncing at ~458752) \u2014 nondeterministic, not a fixed structural bug.", + "file": "src/lib/request.c (suspected: xce_snapshot_chunk_encode_raw capnp Data encode) or src/controller/ha.c (ha_handle_snapshot_chunk per-chunk append)", + "root_cause": "NOT YET PINNED to an exact line \u2014 root cause is a chunk-CONTENT-corruption bug in the OPCode_snapshotTransfer chunk pipeline, most likely in the capnp raw-Data encode/decode path for the 64KB `payload :Data` field (xce_snapshot_chunk_encode_raw, request.c:630-671, uses the same capn_new_list8/capn_setv8 pattern as bug-847 \u2014 'WAL Catch-Up Streaming Silently Corrupts Payloads at capnp Segment Boundary' \u2014 a proven prior case of this exact capnp pattern corrupting raw-byte payloads at a segment boundary). Ruled out as NOT the cause: (1) receiver-side concurrent chunk processing \u2014 vtaskmgr's per-endpoint FIFO (endpoint_block_tasks/endpoint_unblock_tasks) guarantees chunk N+1 cannot dispatch until chunk N's ha_handle_snapshot_chunk (including its fopen/fwrite/fclose) fully returns; (2) sender buf-reuse race \u2014 xce_snapshot_chunk_encode_raw copies bytes into the capnp segment synchronously before ha_send_snapshot_to_standby's loop re-fread()s the shared buf; (3) fixed-size packet-pool truncation \u2014 alloc_packet (netio.c:416-432) mallocs fresh per packet, sized to capn_size(&c), no shared buffer to overflow into. persist_load_snapshot (persist.c:2440 -> load_snapshot at 2325 -> load_snapshot_entities at 2222) is NOT the bug \u2014 it correctly rejects the corrupted file via one of its length-sanity -EIO guards (most likely persist.c:2240-2241, the job-frame payload_len-vs-file_size guard); format_version/job_count field-order misparse was considered and ruled out (load_snapshot consumes and validates format_version at persist.c:2399-2413 before ever calling load_snapshot_entities).", + "fix": "FIXED 2026-07-02 (ha.c, ha.h). Root cause pinned via live crc32 instrumentation (TRACE logs at both ha_send_snapshot_to_standby's send point and ha_handle_snapshot_chunk's receive point, correlated by snapshot_lsn+chunk_index): every chunk's CONTENT crossed the wire byte-for-byte intact (crc32 matched exactly for all 12 chunks) \u2014 the corruption was never in capnp encode/decode/transport at all. The real bug: OPCode_snapshotTransfer chunks of one transfer are dispatched across the vtask pool and are NOT guaranteed to arrive/execute in chunk_index order (live soak log showed arrival order 0,1,3,4,2,5,6,7,8,9,10,11 across cpu#0-3 workers), but ha_handle_snapshot_chunk appended every chunk's bytes to snap.part in ARRIVAL order (fopen \"ab\"/fwrite) instead of at the chunk's own chunk_offset (a field already carried on the wire but never used for anything). Reordered chunks landed at the wrong file position: same total byte count, wrong content from the first reordered chunk's offset onward \u2014 exactly the observed 'byte-shifted, resyncing later, nondeterministic magnitude' signature. Fix: (1) write each chunk via pwrite(fd, payload, len, chunk->chunk_offset) instead of append \u2014 order-independent and safe for concurrent writers (positioned writes to disjoint byte ranges of the same fd don't race). (2) Track completion via a new ha_state_t.snapshot_recv_count/snapshot_recv_total pair (guarded by the existing ha->lock) instead of finalizing on 'this call happens to carry is_last' \u2014 under out-of-order arrival, is_last can be processed before an earlier chunk; finalize now fires only once every chunk 0..N-1 has actually been written, and snapshot_recv_total is claimed (reset to 0) by whichever thread crosses the completion threshold so a late straggler chunk can never double-finalize. chunk_index==0 still resets both the stale-file unlink and the new counters (Req 4.4 unchanged). New RED->GREEN unit test test_snapshot_recv_out_of_order_chunks_assemble_correctly (test_config_repl.c) replays the exact non-sequential delivery order and asserts byte-identical assembly; RED confirmed via temporary revert (Expected 0 Was -22, job_count field corrupted by the misordered chunk boundary); GREEN restored, full controller suite clean (0 failures). Also added 2 new deterministic full-size (65536-byte) capnp encode/decode round-trip tests (test_wal_wire.c) that definitively cleared the capnp/transport layer before the real cause was found \u2014 permanent regression coverage, the prior test only covered a 300-byte payload. New xce_test snapverify/snapdiff subcommands (built as part of this investigation) were instrumental in diagnosing this live. Live-validated: 3/3 --force-snapshot rounds PASS (genuine 'HA: snapshot loaded ... full job/queue entity restoration' installs confirmed, zero divergence/loss) + 6-round --chaotic PASS.", + "tags": [ + "raft-drive-async", + "snapshot-transfer", + "data-corruption", + "capnp", + "force-snapshot-soak", + "fixed" + ], + "related_bugs": [ + "bug-1386", + "bug-847", + "bug-1197" + ], + "occurrences": 2, + "last_seen": "2026-07-02T12:10:00.000Z" + }, + { + "id": "bug-1393", + "timestamp": "2026-07-02T12:05:51.415Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `~/tmp/xce-ha-deploy/controller/ctrl-{1,2}/data/sn", + "fix": "Changed to `ha_send_snapshot_to_standby`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T12:05:51.415Z" + }, + { + "id": "bug-1395", + "timestamp": "2026-07-02T13:25:19.081Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/raft-drive-async/tasks.md", + "root_cause": "Had `[~]`", + "fix": "Changed to `[x]`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-02T13:25:28.568Z" + }, + { + "id": "bug-1396", + "timestamp": "2026-07-02T14:00:00.000Z", + "error_message": "ha-commit-durability task 6.1 (cold-restart-all real-fs reproduction, src/tools/test-commit-durability.sh on a genuine ext4 deploy dir, ~/tmp not overlay): run 5 of 5 reported `expect-jobs: expected 19, actual 20` after a full cold-stop-all + cold-start-all cycle \u2014 the cluster had ONE MORE committed job than the harness's own ACKED_FILE (deduplicated via `sort -u`) tracked. All 19 tracked job IDs individually verified present (`expect-job` loop all `ok`); the 20th job's identity was not captured (cluster had already been torn down by the script's EXIT trap by the time this was investigated). This is the OPPOSITE of bug-661/795's symptom (loss) \u2014 an unexplained EXTRA job, not a missing one. 4 of the other 5 runs (and every prior round within run 5 itself, rounds 1-8) showed correct exact-count durability with zero anomalies.", + "file": "src/tools/test-commit-durability.sh (harness) or controller job-recovery path (unconfirmed which)", + "root_cause": "NOT INVESTIGATED \u2014 deprioritized because it doesn't match the loss scenario task 6.1 was probing for, and reproducing it requires re-running with a 'joblist' capture at the exact expect-jobs failure point (the script doesn't dump the full list on a count mismatch, only the count). Two live hypotheses, unconfirmed: (a) harness-side race \u2014 ACKED_FILE's write (submit_n) and run_cold_restart_recovery's read (`committed=$(sort -u ACKED_FILE | grep -c .)`) are not synchronized against a submission whose ack arrives asynchronously after the harness moves on, so a genuinely-committed job simply never got recorded in ACKED_FILE \u2014 a test-harness bookkeeping gap, not a controller bug; (b) a genuine job-duplication bug on the controller's recovery/replay path creating an extra job entry with a fresh ID. Also observed in this run's ctrl-1.log: repeated unrelated `handle_jobsubmit fail to lock job 63-15--1-1015808[32751]` errors and one `job not durable (rc=-110) \u2014 rejecting submission` for a DIFFERENTLY-FORMATTED job ID (not this harness's `0-0--0` convention) \u2014 almost certainly a stray unrelated process hitting the same cluster ports in the shared sandbox, not this test's own traffic, but flagged here in case it's connected.", + "fix": "NOT YET ROOT-CAUSED (still needs a live re-run to capture the actual extra job's identity). Made the harness able to diagnose it next time it fires: src/tools/test-commit-durability.sh's assert_job_count() now dumps the live `joblist` output AND the ACKED_FILE ledger (sorted, deduplicated) on any count mismatch, instead of only printing the count as before. This was the bug's own recorded 'next step'. Bash syntax-checked (bash -n); not yet re-run against a live cluster in this session (that requires reproducing the original 5-round real-fs soak, which is long-running).", + "tags": [ + "ha-commit-durability", + "low-priority", + "cold-restart-all", + "job-count-mismatch", + "harness-instrumented", + "unconfirmed" + ], + "related_bugs": [ + "bug-661" + ], + "occurrences": 1, + "last_seen": "2026-07-02T16:00:00.000Z" + }, + { + "id": "bug-1397", + "timestamp": "2026-07-02T13:57:56.610Z", + "error_message": "Function not marked async", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Function uses await but wasn't declared async", + "fix": "Added async modifier", + "tags": [ + "auto-detected", + "async-fix", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T13:57:56.610Z" + }, + { + "id": "bug-1398", + "timestamp": "2026-07-02T13:58:18.885Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/ha-commit-durability/tasks.md", + "root_cause": "Had `/kiro-validate-impl`", + "fix": "Changed to `src/tools/test-commit-durability.sh ~/tmp/xce-dur", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T13:58:18.885Z" + }, + { + "id": "bug-1399", + "timestamp": "2026-07-02T14:03:20.814Z", + "error_message": "Incorrect value in code", + "file": "src/controller/CLAUDE.md", + "root_cause": "Had `xce-test-e2e-primitives`", + "fix": "Changed to `ha-commit-durability`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T14:03:20.814Z" + }, + { + "id": "bug-1400", + "timestamp": "2026-07-02T14:20:00.000Z", + "error_message": "xce_test cmd_jobsub SEGVs (ASAN) on `jobsub sleep 1` \u2014 positional args instead of required getopt flags (-u/-q/-c). getopt silently ignores unrecognized positional args, leaving submit.command/queue etc. NULL, and something downstream derefs a NULL field.", + "file": "src/controller/testsuite/test.c (cmd_jobsub, ~line 1748)", + "root_cause": "CONFIRMED STALE for the reported SEGV: git blame shows NULLSTR guards for submit.user/submit.queue/submit.start_info->cmdline were added 2025-11-05 (commit 41f56fa6), long before this bug was logged 2026-07-02 \u2014 'jobsub sleep 1' getopt-returns -1 immediately on the non-option 'sleep' token, then hits the -EINVAL guard at test.c:1838 before any NULL deref. No SEGV path exists in current code; the buglog entry was stale (fix landed under a different/untracked change, buglog never updated). A REAL, separate defect was found in the same function: submit.start_info (calloc'd test.c:1767) was never freed on any of the 4 early-return validation paths (getopt default case + the 3 NULLSTR checks) \u2014 a ~128-byte leak per malformed jobsub invocation.", + "fix": "SEGV claim: not applicable, already fixed pre-existing. Leak: refactored cmd_jobsub (test.c) to `goto cleanup` on all 4 early-return validation failures instead of a bare `return -EINVAL`, reusing the existing FREEUP(submit.user/queue/resreq/start_info->cmdline/exdata/start_info) block via a new `cleanup:` label \u2014 mirrors the `goto clean` idiom already used in queue_ctrl() (queue.c). Verified: controller ASAN build + full controller_unit_test suite green (0 failures) after the change.", + "tags": [ + "stale-report", + "low-priority", + "test-client-only", + "input-validation", + "crash", + "cmd_jobsub", + "fixed", + "xce_test" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T16:00:00.000Z" + }, + { + "id": "bug-1401", + "timestamp": "2026-07-02T14:25:00.000Z", + "error_message": "ASAN LeakSanitizer: every ctrlstatus reply leaks memory in decode_CtrlStatus_ptr (lib/controller.capnp.c:11873) \u2014 observed during ha-heartbeat-committed-propagation task 4.1 live validation: 120 bytes/1 alloc after a single ctrlstatus call, 12360 bytes/103 allocs after ~a dozen calls (roughly proportional to call count).", + "file": "src/lib/controller.capnp.c (decode_CtrlStatus_ptr, generated codec) or its caller in request.c/xcecli", + "root_cause": "CONFIRMED via codegraph + generated codec trace: src/include/client.h:119 defined `xce_free_ctrlstatus(s)` as `free_CtrlStatus(s)` \u2014 the generated free_CtrlStatus() only frees the FIELDS INSIDE the xce_ctrlstatus_t (clusterName/primaryName/responderName/configChecksum/peers list), matching decode_CtrlStatus(). But the real client reply path (xce_handle_reply -> MOVE_PTR from the decoded RequestReply -> xce_ctrlstatus) populates the struct via decode_CtrlStatus_PTR(), which calloc's the xce_ctrlstatus_t itself (controller.capnp.c:11873) before calling decode_CtrlStatus() to fill it. Only free_CtrlStatus_ptr() (which calls free_CtrlStatus() THEN free()s the struct and nulls the caller's pointer) matches that allocation. free_RequestReply/free_Request/MOVE_PTR were all confirmed symmetric and correct \u2014 the bug was isolated to this one macro definition. Existing unit tests never caught it because they call free_CtrlStatus(&out) directly on a stack struct, never exercising the xce_free_ctrlstatus() macro used by real client code.", + "fix": "src/include/client.h:119 changed to `#define xce_free_ctrlstatus(s) free_CtrlStatus_ptr(&(s))`. Added regression test test_xce_free_ctrlstatus_frees_the_struct_itself (src/lib/testsuite/test_ctrlstatus.c), which decodes via decode_CtrlStatus_ptr (the real allocation path, not the stack-struct helper the other roundtrip tests use) and frees via the xce_free_ctrlstatus macro. RED confirmed: reverting the fix reproduces the exact reported leak signature (LeakSanitizer: 120 bytes in 1 object, decode_CtrlStatus_ptr calloc). GREEN after fix: 285 lib tests / 0 failures / 0 leaks. Controller ASAN build + controller_unit_test also green post-fix (client.h is shared).", + "tags": [ + "asan", + "fixed", + "low-priority", + "controller-status", + "ctrlstatus", + "memory-leak" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-02T16:00:00.000Z" + }, + { + "id": "bug-1402", + "timestamp": "2026-07-02T14:21:35.080Z", + "error_message": "Incorrect value in code", + "file": ".kiro/specs/controller-status/tasks.md", + "root_cause": "Had `/xce-deploy-ha`", + "fix": "Changed to `~/tmp`", + "tags": [ + "auto-detected", + "wrong-value", + "md" + ], + "related_bugs": [], + "occurrences": 2, + "last_seen": "2026-07-02T14:22:28.117Z" + }, + { + "id": "bug-1405", + "timestamp": "2026-07-03T00:08:25.320Z", + "error_message": "Wrong reference: test_xcecli_ctrlstatus_null_ctx_typed_failure should be test_xce_free_ctrlstatus_frees_the_struct_itself", + "file": "src/lib/testsuite/test_ctrlstatus.c", + "root_cause": "Used \"test_xcecli_ctrlstatus_null_ctx_typed_failure\" instead of \"test_xce_free_ctrlstatus_frees_the_struct_itself\"", + "fix": "Changed test_xcecli_ctrlstatus_null_ctx_typed_failure \u2192 test_xce_free_ctrlstatus_frees_the_struct_itself", + "tags": [ + "wrong-reference", + "c" + ], + "related_bugs": [], + "occurrences": 1, + "last_seen": "2026-07-03T00:08:25.320Z" + } +] diff --git a/.wolf/hooks/_session.json b/.wolf/hooks/_session.json index 6c8acb1..3b74701 100644 --- a/.wolf/hooks/_session.json +++ b/.wolf/hooks/_session.json @@ -13,7 +13,7 @@ "first_read": "2026-07-02T23:58:46.397Z" }, "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.c": { - "count": 3, + "count": 8, "tokens": 0, "first_read": "2026-07-02T23:58:53.910Z" }, @@ -46,6 +46,46 @@ "count": 3, "tokens": 4804, "first_read": "2026-07-03T00:07:50.578Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c": { + "count": 3, + "tokens": 4273, + "first_read": "2026-07-03T02:58:53.775Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/request.c": { + "count": 4, + "tokens": 15919, + "first_read": "2026-07-03T02:59:46.887Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/src/schema/controller.capnp": { + "count": 1, + "tokens": 9344, + "first_read": "2026-07-03T03:00:35.357Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.h": { + "count": 1, + "tokens": 0, + "first_read": "2026-07-03T03:00:48.039Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/ha-single-leader-role/spec.json": { + "count": 1, + "tokens": 142, + "first_read": "2026-07-03T03:19:29.123Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/ha-convergence-recovery/spec.json": { + "count": 1, + "tokens": 142, + "first_read": "2026-07-03T03:19:43.313Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/ha-consistency-invariants/spec.json": { + "count": 1, + "tokens": 259, + "first_read": "2026-07-03T03:19:58.314Z" + }, + "/home/rshen/projects/xce/worktrees/raft_tla/.gitignore": { + "count": 1, + "tokens": 131, + "first_read": "2026-07-03T03:23:24.225Z" } }, "files_written": [ @@ -126,17 +166,52 @@ "action": "edit", "tokens": 38, "at": "2026-07-03T02:40:28.333Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c", + "action": "edit", + "tokens": 9, + "at": "2026-07-03T03:07:07.144Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/ha-single-leader-role/spec.json", + "action": "edit", + "tokens": 135, + "at": "2026-07-03T03:19:38.097Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/ha-convergence-recovery/spec.json", + "action": "edit", + "tokens": 148, + "at": "2026-07-03T03:19:53.139Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/.kiro/specs/ha-consistency-invariants/spec.json", + "action": "edit", + "tokens": 416, + "at": "2026-07-03T03:20:11.299Z" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/.gitignore", + "action": "edit", + "tokens": 88, + "at": "2026-07-03T03:23:30.976Z" } ], "edit_counts": { "src/include/client.h": 1, "src/controller/testsuite/test.c": 9, "src/tools/test-commit-durability.sh": 1, - "src/lib/testsuite/test_ctrlstatus.c": 2 + "src/lib/testsuite/test_ctrlstatus.c": 2, + "src/lib/testsuite/test_wal_wire.c": 1, + ".kiro/specs/ha-single-leader-role/spec.json": 1, + ".kiro/specs/ha-convergence-recovery/spec.json": 1, + ".kiro/specs/ha-consistency-invariants/spec.json": 1, + ".gitignore": 1 }, - "anatomy_hits": 1, - "anatomy_misses": 8, - "repeated_reads_warned": 11, + "anatomy_hits": 8, + "anatomy_misses": 9, + "repeated_reads_warned": 21, "cerebrum_warnings": 0, - "stop_count": 2 + "stop_count": 6 } \ No newline at end of file diff --git a/.wolf/memory.md b/.wolf/memory.md index dbe220f..fe8ff48 100644 --- a/.wolf/memory.md +++ b/.wolf/memory.md @@ -3040,3 +3040,12 @@ | 19:39 | Edited src/controller/testsuite/test.c | added 3 condition(s) | ~523 | | 19:40 | Edited src/controller/testsuite/test.c | modified if() | ~200 | | 19:40 | Edited src/controller/testsuite/test.c | 4→5 lines | ~36 | +| 19:52 | Session end: 13 writes across 4 files (client.h, test.c, test-commit-durability.sh, test_ctrlstatus.c) | 9 reads | ~35584 tok | +| 20:01 | Session end: 13 writes across 4 files (client.h, test.c, test-commit-durability.sh, test_ctrlstatus.c) | 13 reads | ~65120 tok | +| 20:07 | Edited src/lib/testsuite/test_wal_wire.c | removed 5 lines | ~9 | +| 20:13 | Session end: 14 writes across 5 files (client.h, test.c, test-commit-durability.sh, test_ctrlstatus.c, test_wal_wire.c) | 13 reads | ~65129 tok | +| 20:16 | Session end: 14 writes across 5 files (client.h, test.c, test-commit-durability.sh, test_ctrlstatus.c, test_wal_wire.c) | 13 reads | ~65129 tok | +| 20:19 | Edited .kiro/specs/ha-single-leader-role/spec.json | 18→19 lines | ~135 | +| 20:19 | Edited .kiro/specs/ha-convergence-recovery/spec.json | 18→19 lines | ~148 | +| 20:20 | Edited .kiro/specs/ha-consistency-invariants/spec.json | modified spec() | ~416 | +| 20:23 | Edited .gitignore | expanded (+6 lines) | ~82 | diff --git a/.wolf/token-ledger.json b/.wolf/token-ledger.json index ba0165a..e0940e9 100644 --- a/.wolf/token-ledger.json +++ b/.wolf/token-ledger.json @@ -2,14 +2,14 @@ "version": 1, "created_at": "2026-05-21T04:35:43.577Z", "lifetime": { - "total_tokens_estimated": 89778026, - "total_reads": 14386, - "total_writes": 25240, + "total_tokens_estimated": 90008988, + "total_reads": 14434, + "total_writes": 25294, "total_sessions": 187, - "anatomy_hits": 9755, - "anatomy_misses": 4438, - "repeated_reads_blocked": 44514, - "estimated_savings_vs_bare_cli": 782237017 + "anatomy_hits": 9768, + "anatomy_misses": 4473, + "repeated_reads_blocked": 44583, + "estimated_savings_vs_bare_cli": 783187916 }, "sessions": [ { @@ -224186,6 +224186,632 @@ "repeated_reads_blocked": 10, "anatomy_lookups": 1 } + }, + { + "id": "session-2026-07-02-1655", + "started": "2026-07-02T23:55:20.704Z", + "ended": "2026-07-03T02:52:02.231Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 28321, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/utils.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/xcectx.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/utils.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/client.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 4804, + "was_repeated": true, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 136, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 288, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 566, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 52, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 90, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 57, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 100, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 106, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 560, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 33125, + "output_tokens_estimated": 2459, + "reads_count": 9, + "writes_count": 13, + "repeated_reads_blocked": 11, + "anatomy_lookups": 1 + } + }, + { + "id": "session-2026-07-02-1655", + "started": "2026-07-02T23:55:20.704Z", + "ended": "2026-07-03T03:01:24.343Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 28321, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/utils.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/xcectx.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/utils.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/client.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 4804, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c", + "tokens_estimated": 4273, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/request.c", + "tokens_estimated": 15919, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/schema/controller.capnp", + "tokens_estimated": 9344, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 136, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 288, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 566, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 52, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 90, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 57, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 100, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 106, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 560, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 62661, + "output_tokens_estimated": 2459, + "reads_count": 13, + "writes_count": 13, + "repeated_reads_blocked": 16, + "anatomy_lookups": 4 + } + }, + { + "id": "session-2026-07-02-1655", + "started": "2026-07-02T23:55:20.704Z", + "ended": "2026-07-03T03:13:37.476Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 28321, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/utils.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/xcectx.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/utils.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/client.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 4804, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c", + "tokens_estimated": 4273, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/request.c", + "tokens_estimated": 15919, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/schema/controller.capnp", + "tokens_estimated": 9344, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 136, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 288, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 566, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 52, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 90, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 57, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 100, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 106, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 560, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c", + "tokens_estimated": 9, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 62661, + "output_tokens_estimated": 2468, + "reads_count": 13, + "writes_count": 14, + "repeated_reads_blocked": 21, + "anatomy_lookups": 4 + } + }, + { + "id": "session-2026-07-02-1655", + "started": "2026-07-02T23:55:20.704Z", + "ended": "2026-07-03T03:16:34.020Z", + "reads": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 28321, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/utils.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.c", + "tokens_estimated": 0, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/xcectx.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/utils.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/client.c", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 4804, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c", + "tokens_estimated": 4273, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/request.c", + "tokens_estimated": 15919, + "was_repeated": true, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/schema/controller.capnp", + "tokens_estimated": 9344, + "was_repeated": false, + "anatomy_had_description": false + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/build/lib/controller.capnp.h", + "tokens_estimated": 0, + "was_repeated": false, + "anatomy_had_description": false + } + ], + "writes": [ + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/include/client.h", + "tokens_estimated": 136, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/tools/test-commit-durability.sh", + "tokens_estimated": 288, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 566, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_ctrlstatus.c", + "tokens_estimated": 52, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 90, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 57, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 100, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 106, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 560, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 214, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/controller/testsuite/test.c", + "tokens_estimated": 38, + "action": "edit" + }, + { + "file": "/home/rshen/projects/xce/worktrees/raft_tla/src/lib/testsuite/test_wal_wire.c", + "tokens_estimated": 9, + "action": "edit" + } + ], + "totals": { + "input_tokens_estimated": 62661, + "output_tokens_estimated": 2468, + "reads_count": 13, + "writes_count": 14, + "repeated_reads_blocked": 21, + "anatomy_lookups": 4 + } } ], "daemon_usage": [],