finalize configuration replication

2026-06-09 16:40:13 +08:00 · 2026-06-09 16:40:13 +08:00 · fa66d1ba9c
commit fa66d1ba9c
parent e1517bbf28
30 changed files with 8867 additions and 1560 deletions
--- a/.claude/settings.json
+++ b/.claude/settings.json
@ -1,6 +1,40 @@
 {
  "hooks": {
    "PostToolUse": [
      {
        "matcher": "Edit|Write|Bash",
        "hooks": []
      },
      {
        "matcher": "Read",
        "hooks": [
          {
            "type": "command",
            "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/post-read.js\"",
            "timeout": 5
          }
        ]
      },
      {
        "matcher": "Write|Edit|MultiEdit",
        "hooks": [
          {
            "type": "command",
            "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/post-write.js\"",
            "timeout": 10
          }
        ]
      }
    ],
    "SessionStart": [
      {
        "matcher": "",
        "hooks": []
      },
      {
        "matcher": "",
        "hooks": []
      },
      {
        "matcher": "",
        "hooks": [
@ -34,28 +68,6 @@
        ]
      }
    ],
    "PostToolUse": [
      {
        "matcher": "Read",
        "hooks": [
          {
            "type": "command",
            "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/post-read.js\"",
            "timeout": 5
          }
        ]
      },
      {
        "matcher": "Write|Edit|MultiEdit",
        "hooks": [
          {
            "type": "command",
            "command": "node \"$CLAUDE_PROJECT_DIR/.wolf/hooks/post-write.js\"",
            "timeout": 10
          }
        ]
      }
    ],
    "Stop": [
      {
        "matcher": "",
--- a/.claude/skills/xce-deploy-ha/SKILL.md
+++ b/.claude/skills/xce-deploy-ha/SKILL.md
@ -58,11 +58,11 @@ bash src/tools/deploy-ha.sh "$BINARY" "$DATADIR" "$TOP" "$PEERS"
 ```
 This:
- Creates `$TOP/ctrl-{1..N}/conf`, `ctrl-{1..N}/data`, `ctrl-{1..N}/certs`, `logs/`
+- Creates `$TOP/certs/`, `$TOP/controller/ctrl-{1..N}/conf`, `$TOP/controller/ctrl-{1..N}/data`, `$TOP/logs/`
- Copies `xce.crt` and `xce.key` into each controller's `certs/` dir
+- Copies `xce.crt` and `xce.key` into the shared `$TOP/certs/` dir
- Places `cluster.bin.<tag>` + `conf/current` in `ctrl-1/conf/` only
+- Places `cluster.bin.<tag>` + `conf/current` in `controller/ctrl-1/conf/` only
  (standbys receive config from primary via `OPCode_replConfigPush`)
- Generates `$TOP/scripts/start.sh` and `cleanup.sh`
+- Generates `$TOP/scripts/start.sh` (with `--workdir $TOP/controller --certdir $TOP/certs`) and `cleanup.sh`
 ## Step 4 — Start all nodes
@ -88,7 +88,7 @@ Wait ~15 seconds for election and config replication, then check:
 grep -i "CTRL_PRIMARY\|push_config\|Config committed" "$TOP/logs/ctrl-1.log" | grep -v DEBUG3
 # 2. Confirm ctrl-2 and ctrl-3 received config
-find "$TOP/ctrl-2/conf" "$TOP/ctrl-3/conf" -type f | sort
+find "$TOP/controller/ctrl-2/conf" "$TOP/controller/ctrl-3/conf" -type f | sort
 # 3. Confirm standbys set primary_ctrl_fd
 grep "primary control channel established" "$TOP/logs/ctrl-2.log" "$TOP/logs/ctrl-3.log"
@ -97,7 +97,7 @@ grep "primary control channel established" "$TOP/logs/ctrl-2.log" "$TOP/logs/ctr
 **Healthy cluster signs:**
 - `ctrl-1.log`: `→ CTRL_PRIMARY_COMMITTING` then `→ CTRL_PRIMARY`
 - `ctrl-1.log`: `Config committed on majority (tag=..., pushed=2/2 standbys)`
- `ctrl-2/conf/cluster.bin.*` and `ctrl-3/conf/cluster.bin.*` exist
+- `controller/ctrl-2/conf/cluster.bin.*` and `controller/ctrl-3/conf/cluster.bin.*` exist
 - `ctrl-2.log` / `ctrl-3.log`: `handle_repl_config_push: config saved and loaded`
 ## Step 6 — Raft trace (DEBUG3 logs with -vvv)
@ -120,8 +120,8 @@ Stop all instances, clean up state, and restart without redeploying:
 kill -9 $(pgrep xce_controller) 2>/dev/null || true
 # 2. Run the generated cleanup script
-#    Keeps: ctrl-1/conf/* (primary config + current pointer), all certs
+#    Keeps: controller/ctrl-1/conf/* (primary config + current pointer), $TOP/certs/
-#    Wipes: ctrl-{1..N}/data/*, ctrl-{2..N}/conf/*, logs/*.log
+#    Wipes: controller/ctrl-{1..N}/data/*, controller/ctrl-{2..N}/conf/*, logs/*.log
 bash "$TOP/scripts/cleanup.sh"
 # 3. Restart all nodes
@ -133,13 +133,12 @@ VERBOSE=-vvv "$TOP/scripts/start.sh" ctrl-3 &
 ```
 **What cleanup.sh preserves:**
- `ctrl-1/certs/` — TLS cert and key
+- `$TOP/certs/` — shared TLS cert and key
- `ctrl-1/conf/cluster.bin.*` + `conf/current` — primary config (standbys re-receive via push)
+- `controller/ctrl-1/conf/cluster.bin.*` + `conf/current` — primary config (standbys re-receive via push)
 - `ctrl-{2,3}/certs/` — TLS cert and key
 **What cleanup.sh removes:**
- All `data/` dirs (WAL, snapshots, `ha-term.bin`) — forces fresh election next start
+- All `controller/ctrl-{1..N}/data/` dirs (WAL, snapshots, `ha-term.bin`) — forces fresh election next start
- `ctrl-{2,3}/conf/` — standbys rebuild config from primary's `OPCode_replConfigPush`
+- All `controller/ctrl-{2..N}/conf/` — standbys rebuild config from primary's `OPCode_replConfigPush`
 - All `logs/*.log` — clean log slate
 If you need a full redeploy (different peers or new binary), run steps 3–5 instead.
@ -149,7 +148,7 @@ If you need a full redeploy (different peers or new binary), run steps 3–5 ins
 | Symptom | Cause | Fix |
 |---|---|---|
 | `cluster_open_listener: no port` | Peers missing from `--peers` | Ensure `name=host:port` format in generated start.sh |
-| `ctrl-2/3/conf` empty | `push_config_to_standbys` bug or `OPCode_replConfigPush` not handled | Verify binary is latest build |
+| `controller/ctrl-2/3/conf` empty | `push_config_to_standbys` bug or `OPCode_replConfigPush` not handled | Verify binary is latest build |
 | `repl_ack_decode_and_dispatch: unexpected opcode` | Heartbeat NOP on promoted ep | Fixed in current build; update binary |
 | `election_timeout` loops without primary | All nodes have `config_tag=0` (cold-start) or network issue | Check peers string matches actual listen ports |
 | `ha-term.bin` causes "already voted" on restart | Old state from previous run | Run clean restart above |
--- a/.claude/ucai-formatter-cache.local.json
+++ b/.claude/ucai-formatter-cache.local.json
@ -1 +0,0 @@
 {"detected":false,"exe":null,"args":null,"extensions":[]}
--- a/.kiro/settings/mcp.json
+++ b/.kiro/settings/mcp.json
@ -1,4 +0,0 @@
 {
  "mcpServers": {
  }
 }
--- a/.kiro/specs/election-iomgr-refactor/research.md
+++ b/.kiro/specs/election-iomgr-refactor/research.md
@ -263,3 +263,13 @@ None of these affect correctness or test pass rate. Safe to clean up in a single
 ## Conclusion
 The `election-iomgr-refactor` spec is **fully implemented and verified**. No blocking issues remain. The spec phase is `implemented` and can be closed. The one deferred item (DNS blocking) is tracked under `ha-peer-connection-reuse`.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Status: CLOSED — No remaining gaps
 All 15 implementation tasks (1.1–5.3) and all 5 validation tasks (6.1–6.5) confirmed done. The spec is in `implemented` phase. Deferred cleanup items (dead code: `am_i_primary`, `ha_wait_for_heartbeat_or_timeout`, `start_heartbeat_listener`/`stop_heartbeat_listener` stubs) are cosmetic and tracked as tech debt.
--- a/.kiro/specs/ha-config-bootstrap/manual-tests.md
+++ b/.kiro/specs/ha-config-bootstrap/manual-tests.md
@ -58,9 +58,9 @@ All tests use the `/xce-deploy-ha` skill which automates deployment:
 ```
 This deploys to **`/tmp/xce-deploy/`** with:
- `ctrl-{1,2,3}/certs/` — TLS cert and key (copied from `ha/data/`)
+- `certs/` — shared TLS cert and key (copied from `ha/data/`)
- `ctrl-1/conf/` — initial config (primary only; standbys receive via push)
+- `controller/ctrl-1/conf/` — initial config (primary only; standbys receive via push)
- `ctrl-{1,2,3}/data/` — WAL, snapshots, `ha-term.bin`
+- `controller/ctrl-{1,2,3}/data/` — WAL, snapshots, `ha-term.bin`
 - `logs/ctrl-{1,2,3}.log` — per-node log files
 - `scripts/start.sh` and `scripts/cleanup.sh` — lifecycle helpers
@ -78,7 +78,7 @@ VERBOSE=-vvv /tmp/xce-deploy/scripts/start.sh ctrl-3 &
 # Wait 15s for election + config replication, then verify
 grep -i "CTRL_PRIMARY\|raft.*leader\|Config committed" /tmp/xce-deploy/logs/ctrl-1.log | grep -v DEBUG3
-find /tmp/xce-deploy/ctrl-2/conf /tmp/xce-deploy/ctrl-3/conf -type f | sort
+find /tmp/xce-deploy/controller/ctrl-2/conf /tmp/xce-deploy/controller/ctrl-3/conf -type f | sort
 ```
 ### Clean restart (preserves config, wipes data)
@ -89,8 +89,8 @@ bash /tmp/xce-deploy/scripts/cleanup.sh
 # Then restart nodes as above
 ```
-**What cleanup.sh preserves:** `ctrl-1/conf/*`, all `certs/`
+**What cleanup.sh preserves:** `controller/ctrl-1/conf/*`, `$TOP/certs/`
-**What cleanup.sh removes:** all `data/` dirs, `ctrl-{2,3}/conf/*`, all `logs/*.log`
+**What cleanup.sh removes:** all `controller/ctrl-{1..N}/data/` dirs, `controller/ctrl-{2..N}/conf/*`, all `logs/*.log`
 ### Manual restart with config preserved (for failover testing)
@ -98,8 +98,8 @@ To restart without wiping standby configs (so standbys retain `config_tag > 0`):
 ```bash
 kill -9 $(pgrep xce_controller) 2>/dev/null || true
-rm -rf /tmp/xce-deploy/ctrl-{1,2,3}/data /tmp/xce-deploy/logs/*.log
+rm -rf /tmp/xce-deploy/controller/ctrl-{1,2,3}/data /tmp/xce-deploy/logs/*.log
-mkdir -p /tmp/xce-deploy/ctrl-{1,2,3}/data
+mkdir -p /tmp/xce-deploy/controller/ctrl-{1,2,3}/data
 # Then restart nodes as above
 ```
@ -124,8 +124,8 @@ mkdir -p /tmp/xce-deploy/ctrl-{1,2,3}/data
 - ctrl-2/ctrl-3: `handle_repl_config_push: config saved and loaded (tag=<hex>)`
 - Config files appear in both standby workdirs:
  ```bash
-  ls /tmp/xce-deploy/ctrl-2/conf/
+  ls /tmp/xce-deploy/controller/ctrl-2/conf/
-  ls /tmp/xce-deploy/ctrl-3/conf/
+  ls /tmp/xce-deploy/controller/ctrl-3/conf/
  # Expected: cluster.bin.<hex_tag> and current
  ```
@ -133,8 +133,8 @@ mkdir -p /tmp/xce-deploy/ctrl-{1,2,3}/data
 - [ ] Exactly ctrl-1 becomes `CTRL_PRIMARY`
 - [ ] ctrl-2 and ctrl-3 become `CTRL_STANDBY`
 - [ ] No crash or ASAN error
- [ ] Config replicated to ctrl-2: `cluster.bin.*` exists in `/tmp/xce-deploy/ctrl-2/conf/`
+- [ ] Config replicated to ctrl-2: `cluster.bin.*` exists in `/tmp/xce-deploy/controller/ctrl-2/conf/`
- [ ] Config replicated to ctrl-3: `cluster.bin.*` exists in `/tmp/xce-deploy/ctrl-3/conf/`
+- [ ] Config replicated to ctrl-3: `cluster.bin.*` exists in `/tmp/xce-deploy/controller/ctrl-3/conf/`
 - [ ] Config tag on standbys matches primary's tag
 ---
@ -207,7 +207,7 @@ CTRL_STANDBY
 **Scenario:** ctrl-3 starts with no local cluster config (config_tag=0); ctrl-1 and ctrl-2 already hold a valid config from Test 11.1.
 1. Start ctrl-1 and ctrl-2 first (let one become primary).
-2. Ensure ctrl-3 has no conf: `rm -rf /tmp/xce-deploy/ctrl-3/conf/* 2>/dev/null` (or rely on fresh deploy where only ctrl-1 has conf).
+2. Ensure ctrl-3 has no conf: `rm -rf /tmp/xce-deploy/controller/ctrl-3/conf/* 2>/dev/null` (or rely on fresh deploy where only ctrl-1 has conf).
 3. Start ctrl-3.
 **Expected in ctrl-3 log:**
@ -217,14 +217,14 @@ myapp_run: CTRL_STANDBY — waiting for primary to connect and push config
 handle_repl_config_push: config saved and loaded (tag=...)
 ```
-**Expected in `/tmp/xce-deploy/ctrl-3/conf/`:** a `cluster.bin.<tag>` and `current` pointer file appear after primary push.
+**Expected in `/tmp/xce-deploy/controller/ctrl-3/conf/`:** a `cluster.bin.<tag>` and `current` pointer file appear after primary push.
 **Checklist:**
 - [ ] ctrl-3 detects config_tag=0, participates as voter only
 - [ ] ctrl-3 yields election to the existing primary (fresher config)
 - [ ] ctrl-3 receives config via `handle_repl_config_push` from primary
 - [ ] ctrl-3 enters CTRL_STANDBY with correct config
- [ ] Config files appear in `/tmp/xce-deploy/ctrl-3/conf/` after push
+- [ ] Config files appear in `/tmp/xce-deploy/controller/ctrl-3/conf/` after push
 ---
@ -237,7 +237,7 @@ handle_repl_config_push: config saved and loaded (tag=...)
 1. Run 3-node cluster from Test 11.1, wait for election to complete.
 2. Check that `ha-term.bin` was written on the primary:
 ```bash
-xxd /tmp/xce-deploy/ctrl-1/data/ha-term.bin | head -4
+xxd /tmp/xce-deploy/controller/ctrl-1/data/ha-term.bin | head -4
 # Expected: first 8 bytes = 0xBEBAFECAEFBEADDE (magic), next 8 bytes = term > 0
 ```
 3. Kill ctrl-1.
@ -246,15 +246,15 @@ xxd /tmp/xce-deploy/ctrl-1/data/ha-term.bin | head -4
 **Expected in startup log:**
 ```
-HA: restored term=N from /tmp/xce-deploy/ctrl-1/data/ha-term.bin
+HA: restored term=N from /tmp/xce-deploy/controller/ctrl-1/data/ha-term.bin
 ```
 (See `cluster_init_phase1` log line)
 **Checklist:**
- [ ] `ha-term.bin` file exists in `/tmp/xce-deploy/ctrl-1/data/` after election
+- [ ] `ha-term.bin` file exists in `/tmp/xce-deploy/controller/ctrl-1/data/` after election
 - [ ] Magic bytes are `0xDEADBEEFCAFEBABE` (appears as `beba feca efbe adde` in `xxd` little-endian)
 - [ ] Term field (bytes 8–15) is > 0 after a vote was granted
- [ ] On restart: log shows `HA: restored term=N from /tmp/xce-deploy/ctrl-1/data/ha-term.bin`
+- [ ] On restart: log shows `HA: restored term=N from /tmp/xce-deploy/controller/ctrl-1/data/ha-term.bin`
 - [ ] Node re-joins correctly without double-voting
 ---
@ -312,8 +312,8 @@ Build: /xce-build controller completed successfully? YES
  - Time to election: ~16s (first boot: term=1 standby → phase2 → term=2 leader → term=3 PRIMARY)
  - Two standbys confirmed: YES
 - Config replication result: PASS
-  - cluster.bin exists in /tmp/xce-deploy/ctrl-2/conf/: YES (cluster.bin.4986624fd309)
+  - cluster.bin exists in /tmp/xce-deploy/controller/ctrl-2/conf/: YES (cluster.bin.4986624fd309)
-  - cluster.bin exists in /tmp/xce-deploy/ctrl-3/conf/: YES (cluster.bin.4986624fd309)
+  - cluster.bin exists in /tmp/xce-deploy/controller/ctrl-3/conf/: YES (cluster.bin.4986624fd309)
  - Config tag on standbys matches primary: YES (4986624fd309142c)
 - Log excerpt:
  ctrl-1: `raft_cb_become_leader → CTRL_PRIMARY_COMMITTING (term=2)`
@ -339,8 +339,8 @@ Build: /xce-build controller completed successfully? YES
 ### 11.3 Unconfigured Node Bootstrap (ctrl-2/ctrl-3 join ctrl-1+cluster)
 - Result: PASS
- cluster.bin created in /tmp/xce-deploy/ctrl-2/conf/? YES
+- cluster.bin created in /tmp/xce-deploy/controller/ctrl-2/conf/? YES
- cluster.bin created in /tmp/xce-deploy/ctrl-3/conf/? YES
+- cluster.bin created in /tmp/xce-deploy/controller/ctrl-3/conf/? YES
 - Log excerpt:
  ctrl-2: `ha_start_election_async: no local config (tag=0, n_peers=3) — participating as voter only`
  ctrl-2: `myapp_run: CTRL_STANDBY — waiting for primary to connect and push config`
@ -356,7 +356,7 @@ Build: /xce-build controller completed successfully? YES
  Term: 2 ✓
  voted_for: "ctrl-1" ✓
 - Restart log (term restore line):
-  `cluster_init_phase1: HA: restored term=2 from /tmp/xce-deploy/ctrl-1/data/ha-term.bin`
+  `cluster_init_phase1: HA: restored term=2 from /tmp/xce-deploy/controller/ctrl-1/data/ha-term.bin`
 - Result: PASS
 ### 11.5 ASAN Build
--- a/.kiro/specs/ha-config-bootstrap/research.md
+++ b/.kiro/specs/ha-config-bootstrap/research.md
@ -71,6 +71,19 @@ No new code needed for either spec.
 - **Recommendation**: Run the 5 integration tests from `manual-tests.md`
  against a 3-node cluster, then mark the spec `implemented`.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **ha-config-bootstrap** is **~95% complete**. All 10 requirements have verified implementation assets. All 22 leaf subtasks (1.1–10.2) are done.
 - **Remaining work**: Tasks 11.1–11.5 (manual integration tests documented in `manual-tests.md`).
 - **Cross-spec verification**: `election-iomgr-refactor` superseded the blocking-pthread election path while preserving all ha-config-bootstrap requirements. The two specs are consistent. `ha-repl-hardening` extends Req 5 (majority-commit WAL) and Req 8 (majority-commit config) with heartbeat-driven convergence — no conflicts.
 - **Cleanup debt**: `am_i_primary()` (ha.c:869), `ha_wait_for_heartbeat_or_timeout`, and `start_heartbeat_listener()`/`stop_heartbeat_listener()` stubs are dead code — safe to delete in a dedicated cleanup commit (tracked in election-iomgr-refactor/research.md).
 ## Requirement-to-Asset Map — Final State
 | Requirement | Verified Key Assets | Status |
@ -446,7 +459,7 @@ Add all new stubs with `-ENOSYS` first, then wire each one. Matches the existing
 ### Gap 1: Task 11.1 — Concurrent startup and config-fresh election
 - **What's needed**: 3-node cluster starts simultaneously; verify only one elects primary; config-fresh node wins.
- **How to run**: See `manual-tests.md` — Test 1 (3-node clean start, verify ctrl-a primary).
+- **How to run**: See `manual-tests.md` — Test 1 (3-node clean start, verify ctrl-1 primary).
 - **Infrastructure**: `deploy-ha.sh` + `start.sh` scripts; log grep for `CTRL_PRIMARY`.
 - **Effort**: ~30 min.
--- a/.kiro/specs/ha-heartbeat-liveness/research.md
+++ b/.kiro/specs/ha-heartbeat-liveness/research.md
@ -155,3 +155,15 @@ liveness path, not a new subsystem.
  `OPCode_walCatchupEntry` sites in `ha.c` for full Req 1.3 literal compliance.
 - Run task 4.5 integration test (SIGSTOP/SIGCONT) when a 3-node cluster is
  available.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **ha-heartbeat-liveness** is **~97% complete**. All core implementation is done. The `hb_timeout_thread` has been replaced with delaymgr-based `liveness_poll_fn`.
 - **Remaining gaps**: (1) Req 1.3 partial — `OPCode_replEntry` and `OPCode_walCatchupEntry` dispatch sites in ha.c do not call `ha_on_heartbeat_received`. Low risk: active WAL replication keeps `last_push_time` fresh on primary, suppressing NOPs, and ctrlhello resets the standby's deadline. Adding 2 `ha_on_heartbeat_received` calls in the dispatch sites would achieve full literal compliance. (2) Task 4.5 — SIGSTOP/SIGCONT missed-heartbeat tolerance is an optional manual integration test (script exists at `src/tools/test-heartbeat-tolerance.sh`).
 - **Cross-spec relationship**: `ha-repl-hardening` heartbeat-driven convergence tracking relies on this spec's liveness infrastructure. The `primary_heartbeat_fn` idle-triggered NOP suppression is correct and does not interfere with ha-repl-hardening's per-standby state reporting via heartbeat replies.
--- a/.kiro/specs/ha-peer-connection-reuse/research.md
+++ b/.kiro/specs/ha-peer-connection-reuse/research.md
@ -221,3 +221,17 @@ All seams appear to be in place. The tests should pass, but a build run (`/xce-b
 | Task 7.3 integration test | M (1–2 days) | Medium — requires promoted ep + WAL harness |
 **Overall**: The core feature is implemented. The remaining work is test validation + one latent correctness issue in the old election path.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **ha-peer-connection-reuse** is **~98% complete**. All 7 task groups (1–7) are `[x]` in tasks.md.
 - **Prior CRITICAL gap RESOLVED**: The `fetch_config_from_primary` call in `ha_elect_evaluate_and_signal`'s STANDBY path was removed (confirmed: only comments remain in ha.c). The standby is fully passive — no outbound TCP connections from `CTRL_STANDBY`.
 - **Integration tests (7.2, 7.3)**: Both marked `[x]`. The two-node integration test and WAL ACK test are done.
 - **Remaining**: Build verification (`/xce-build controller`) to confirm all unit tests in `test_ha_connection_reuse.c` pass under ASAN.
 - **Cross-spec relationships**: `ha-repl-hardening` commit `0762577` (task 5.1: "reduce handle_ctrlhello to connection setup only") directly depends on this spec's `peer_ctrl_msg_handler` + promoted connection model. The two specs are consistent — ha-repl-hardening moves all replication decisions to the heartbeat decision loop, which works over the promoted connections established by this spec.
--- a/.kiro/specs/ha-repl-hardening/research.md
+++ b/.kiro/specs/ha-repl-hardening/research.md
@ -133,3 +133,18 @@ Add `OPCode_stateReport` / `OPCode_stateAck` to capnp schema:
 - **Capnp schema design**: Whether to extend `RequestReply` with state fields, add a new `stateReport` struct to existing heartbeat messages, or create a new opcode. Evaluate against existing capnp codegen patterns.
 - **Heartbeat reply processing**: Currently fire-and-forget; need to add a reply handler on the primary side. Determine best insertion point in the dispatch path.
 - **Snapshot wire format**: How to serialize `persist_checkpoint` output for transfer. COW zarray vs raw file transfer.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **ha-repl-hardening** is **~98% complete**. All 6 task groups (1–6), 27 subtasks (1.1–6.7) are `[x]` in tasks.md. Recent commits (aab41fa through e1517bb) confirm active implementation. The git log shows all tasks through 6.7 implemented with bug fixes applied.
 - **Remaining gaps**: Task groups 1–4 and 5–6 all show `[ ]` as section headers, but all subtasks within them are `[x]`. The section checkboxes are cosmetic.
 - **Deferred items**: None. The original research's "Research Needed" items (capnp schema design, heartbeat reply processing, snapshot wire format) were resolved during the design phase.
 - **Cross-spec relationships**: This spec extends `ha-config-bootstrap` Req 5 (majority-commit WAL) and Req 8 (majority-commit config) with heartbeat-driven convergence. It depends on `ha-heartbeat-liveness` for the idle-triggered NOP policy on the primary and activity-based liveness detection on the standby. It depends on `ha-peer-connection-reuse` for the promoted connection model — `handle_ctrlhello` was reduced to connection setup only (task 5.1), with all replication decisions moved to the heartbeat decision loop.
 - **Single-node mode**: Verified working — `n_peers=1`, immediate convergence, `CTRL_PRIMARY` with no heartbeat wait.
 - **Recommendation**: Run `/xce-build controller` to confirm ASAN-clean build, then mark the spec `implemented`.
--- a/.kiro/specs/raft-library/research.md
+++ b/.kiro/specs/raft-library/research.md
@ -36,3 +36,17 @@
 - Added on_trace callback to raft_callbacks_t for full protocol observability
 - Simulator dump log enables post-mortem analysis of split-brain scenarios
 - raft_snapshot() provides safe point-in-time state read for crash/failure logging
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **raft-library** is **~100% complete**. All 7 task groups (1–7), all subtasks (1–2.2, 3.1–3.5, 4.1–4.2, 5.1–5.4, 6.1–6.4, 7) are `[x]` in tasks.md. The `ready_for_implementation` flag is set in spec.json.
 - **No remaining gaps**: The library (`src/lib/raft/raft.h`, `raft.c`, `raft_sim.c`) exposes `raft_state_t` + `raft_callbacks_t` with 9 callback hooks. The deterministic simulator (`raft_sim.c`) verifies split-brain freedom across 100 randomized rounds. The controller adapter in ha.c delegates all election state management (term, voted_for, quorum evaluation) to the raft library.
 - **Cross-spec integration**: `election-iomgr-refactor` uses `raft_start_election` instead of the old `am_i_primary_quorum`. The adapter callbacks in ha.c wire `on_send_vote_request` → iomgr async connect, `on_become_leader` → `cluster_init_phase2` + config push, `on_become_follower` → `CTRL_STANDBY`. The `on_vote_eligible` callback implements the freshness gate (config_tag + applied_lsn comparison) and role gate (`CTRL_PRIMARY_COMMITTING` → deny).
 - **Test coverage**: 34+ Unity test cases across 5 groups (A–H) covering basic election correctness, voted_for invariant, eligibility policy, quorum evaluation, election timeout, RDHUP failover, and split-brain prevention. All pass under ASAN.
 - **Recommendation**: The spec can be marked `implemented`.
--- a/.kiro/specs/resval-id-decoupling/research.md
+++ b/.kiro/specs/resval-id-decoupling/research.md
@ -152,3 +152,17 @@ Ruled out: does not fix the design — just moves the source of truth without el
 4. **Removal of `collect_resval_with_idx`**: **Remove entirely.** No cross-subsystem callers will remain after this change, and intra-subsystem callers (e.g., `worker.c` reading from `sharedinst` for API get-resval) access the zarray directly by `r->idx` without going through this function. Delete `collect_resval_with_idx`, `collect_resval_with_idx_`, and remove the declaration from `src/controller/include/cluster.h`.
 5. **`free_name_val_list` utility**: **Add to `utils.c`/`utils.h`.** A new `free_name_val_list(xce_name_val_t***, int)` function mirroring `free_resval_with_idx_list` is needed. The capnp-generated `free_NameValue_list` is an option but couples cleanup to the serialization layer — a standalone utility in `utils.c` is cleaner and consistent with the existing pattern.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **resval-id-decoupling** is **~100% complete**. All 8 task groups, 22 subtasks (1.1–8.3) are `[x]` in tasks.md.
 - **No remaining gaps**: `collect_resval_with_idx` and `free_resval_with_idx_list` are removed (verified by grep: zero matches). `collect_resval` (name-keyed, iterates resmgr not zarray) replaces it. The capnp schema (`Conf4Sched.resvals @6`, `Worker.resvals @5`) now use `List(NameValue)`. The scheduler resolves by name via `key_find(sched->resmgr, name)` → `r->idx`. The Go API server has `cNameValToWorkerResval` in `convert.go` for the new `xce_name_val_t**` type.
 - **Task approval gap**: spec.json shows `tasks.approved: false`. All tasks are implemented but the tasks document was never formally approved. This is a process artifact — the implementation matches the approved design.
 - **Cross-spec impact**: `scheduler-controller-msg-gap` extends the same scheduler handlers (`sched_init_sharedInst`, `make_sched_worker`) that were modified by this spec. The two specs are consistent — both use name-based resource resolution.
 - **Recommendation**: Mark tasks as approved in spec.json, or run `/kiro-spec-status resval-id-decoupling` to sync.
--- a/.kiro/specs/scheduler-controller-msg-gap/research.md
+++ b/.kiro/specs/scheduler-controller-msg-gap/research.md
@ -95,4 +95,18 @@ All 19 NotifyData event tags:
 4. **Plugin hook for rusage:** Add `plugin_job_rusage_hook_t` typedef and `plugin_job_rusage_update()` global function following existing patterns.
-5. **Helper function:** Add `xce_notify_event_name(int)` and `xce_sched_op_name(xce_sched_op_t)` to `src/lib/helper.c` and `src/include/helper.h`.
+5. **Helper function:** Add `xce_notify_event_name(int)` and `xce_sched_op_name(xce_sched_op_t)` to `src/lib/helper.c` and `src/include/helper.h`.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **scheduler-controller-msg-gap** is **~100% complete**. All 9 task groups, 28 subtasks (1.1–9.9) are `[x]` (9.9 optional).
 - **No remaining gaps**: All 8 requirements have verified implementation. The 6 previously stubbed notify/dataset handlers (`schedUpdateJob`, `schedUpdateQueue`, `schedUpdateResvals`, `schedJobRusages`, `Data_data_resources`, `Data_data_resvals`) are wired with full business logic. Dead code (`pushmsg` branches, `sched_handle_jobpush`, `sched_handle_workerpush`) removed. Symbolic logging via `xce_notify_event_name()` and `xce_sched_op_name()` is in place.
 - **Optional gap**: Task 9.9 (verify WARN log when handler returns non-zero) is marked optional — requires log capture infrastructure not yet present in the test harness.
 - **Cross-spec relationship**: `resval-id-decoupling` uses the same pattern (name-based resource lookup via `sched->resmgr`) in `sched_init_sharedInst`. The `XCE_SCHED_OP_UPDATE_RESVALS_FULL` handler added in this spec also resolves by name — consistency is maintained.
 - **Recommendation**: The spec can be marked `implemented`. Task 9.9 can be closed as deferred.
--- a/.kiro/specs/worker-join-refactor/research.md
+++ b/.kiro/specs/worker-join-refactor/research.md
@ -190,3 +190,17 @@ Design phase is **already approved**. Proceed directly to implementation:
 5. Run `make test` in controller testsuite (task 5 validation)
 **No further research needed.** All unknowns from the original research log are resolved in `design.md`.
 ---
 # Cross-Spec Gap Audit (2026-06-08)
 > Batch `/kiro-validate-gap` across 9 specs on persistent_ha branch.
 ## Analysis Summary
 - **worker-join-refactor** is **~100% complete**. All 5 task groups, all subtasks (1.1–5) are `[x]` in tasks.md.
 - **Prior gaps RESOLVED**: `init_worker` (worker.c:339), `rejoin_worker` (worker.c:366), and `apply_join_state` (static helper) are all implemented. `add_worker` declaration removed from `worker.h`. The double-lock deadlock (`handle_worker_join` held `workermgr` WRITE lock while `makeWorker` re-acquired it) is eliminated.
 - **No remaining gaps**: `handle_worker_join` now has explicit `key_find` → branch → `init_worker`/`rejoin_worker` → `apply_join_state` flow visible at the call site. `BEFORE_EVENT`/`AFTER_EVENT` hooks fire at the same event points. `on_worker_join` called after successful join. Admin status bits (`LOCKEDU | CLOSED | HIDDEN | CLOSED_BY_ADMIN`) preserved in `rejoin_worker`.
 - **Build verification**: Run `/xce-build controller` to confirm `test_worker_join.c` (7 test cases) pass.
 - **Recommendation**: The spec can be marked `implemented`.
--- a/.wolf/anatomy.md
+++ b/.wolf/anatomy.md
@ -1,7 +1,7 @@
 # anatomy.md
-> Auto-maintained by OpenWolf. Last scanned: 2026-06-04T12:21:31.476Z
+> Auto-maintained by OpenWolf. Last scanned: 2026-06-09T08:36:14.668Z
-> Files: 589 tracked | Anatomy hits: 0 | Misses: 0
+> Files: 590 tracked | Anatomy hits: 0 | Misses: 0
 ## ../../../../../claude/.claude/plans/
@ -233,7 +233,7 @@
 - `design.md` — Design Document: election-iomgr-refactor (~6671 tok)
 - `requirements.md` — Requirements Document (~2695 tok)
- `research.md` — Research & Design Decisions (~5073 tok)
+- `research.md` — Research & Design Decisions (~5189 tok)
 - `spec.json` (~129 tok)
 - `tasks.md` — Implementation Plan (~3693 tok)
@ -244,7 +244,7 @@
 - `election-trace.md` — Election Trace: 3-Node Cold Start (2026-06-04 00:15:58) (~2907 tok)
 - `manual-tests.md` — Manual Integration Tests: ha-config-bootstrap (~3677 tok)
 - `requirements.md` — Requirements: HA Config Bootstrap & Raft-Style Quorum Election (~3890 tok)
- `research.md` — Cross-Spec Gap Audit (2026-06-03) (~8150 tok)
+- `research.md` — Cross-Spec Gap Audit (2026-06-03) (~8392 tok)
 - `spec.json` (~198 tok)
 - `tasks.md` — Implementation Plan: HA Config Bootstrap & Raft-Style Quorum Election (~5619 tok)
@ -253,7 +253,7 @@
 - `design.md` — Design Document: ha-heartbeat-liveness (~5409 tok)
 - `manual-tests.md` — Manual Integration Tests: ha-heartbeat-liveness — Task 4.5 (~1726 tok)
 - `requirements.md` — Requirements Document (~2054 tok)
- `research.md` — Gap Analysis: ha-heartbeat-liveness (~2001 tok)
+- `research.md` — Gap Analysis: ha-heartbeat-liveness (~2290 tok)
 - `spec.json` (~129 tok)
 - `tasks.md` — Implementation Plan (~2227 tok)
@ -261,7 +261,7 @@
 - `design.md` — Design Document: ha-peer-connection-reuse (~5369 tok)
 - `requirements.md` — Requirements Document (~1605 tok)
- `research.md` — Research & Design Decisions: ha-peer-connection-reuse (~3517 tok)
+- `research.md` — Research & Design Decisions: ha-peer-connection-reuse (~3801 tok)
 - `spec.json` (~130 tok)
 - `tasks.md` — Implementation Plan: ha-peer-connection-reuse (~2022 tok)
@ -270,15 +270,15 @@
 - `design.md` — Design: HA Replication Hardening (~5514 tok)
 - `raft-tla-audit.md` — Raft TLA+ Specification Audit (~1591 tok)
 - `requirements.md` — Requirements Document (~2817 tok)
- `research.md` — Gap Analysis: HA Replication Hardening (~2381 tok)
+- `research.md` — Gap Analysis: HA Replication Hardening (~2756 tok)
 - `spec.json` (~128 tok)
- `tasks.md` — Implementation Plan: HA Replication Hardening (~3502 tok)
+- `tasks.md` — Implementation Plan: HA Replication Hardening (~3816 tok)
 ## .kiro/specs/raft-library/
 - `design.md` — Design Document: raft-library (~7756 tok)
 - `requirements.md` — Requirements Document (~2115 tok)
- `research.md` — Research & Design Decisions: raft-library (~579 tok)
+- `research.md` — Research & Design Decisions: raft-library (~949 tok)
 - `spec.json` (~127 tok)
 - `tasks.md` — Implementation Plan (~4312 tok)
@ -286,7 +286,7 @@
 - `design.md` — Design: resval-id-decoupling (~3914 tok)
 - `requirements.md` — Requirements: resval-id-decoupling (~2360 tok)
- `research.md` — Gap Analysis: resval-id-decoupling (~3180 tok)
+- `research.md` — Gap Analysis: resval-id-decoupling (~3505 tok)
 - `spec.json` — Declares is (~229 tok)
 - `tasks.md` — Implementation Tasks: resval-id-decoupling (~2424 tok)
@ -294,7 +294,7 @@
 - `design.md` — Design: scheduler-controller-msg-gap (~4756 tok)
 - `requirements.md` — Requirements: scheduler-controller-msg-gap (~2712 tok)
- `research.md` — Research Log: scheduler-controller-msg-gap (~1540 tok)
+- `research.md` — Research Log: scheduler-controller-msg-gap (~1854 tok)
 - `spec.json` (~184 tok)
 - `tasks.md` — Tasks: scheduler-controller-msg-gap (~2324 tok)
@ -302,7 +302,7 @@
 - `design.md` — Design Document: worker-join-refactor (~4709 tok)
 - `requirements.md` — Requirements Document (~1401 tok)
- `research.md` — Research Log: worker-join-refactor (~3079 tok)
+- `research.md` — Research Log: worker-join-refactor (~3357 tok)
 - `spec.json` (~129 tok)
 - `tasks.md` — Implementation Plan (~1658 tok)
@ -455,13 +455,13 @@
 - `apiserver.c` — include <errno.h> (~5303 tok)
 - `CLAUDE.md` — XCE Controller (~2625 tok)
- `cluster.c` — include "config.h" (~11414 tok)
+- `cluster.c` — include "config.h" (~12219 tok)
- `config_repl.c` — define _GNU_SOURCE (~13868 tok)
+- `config_repl.c` — define _GNU_SOURCE (~14728 tok)
- `config_repl.h` — pragma once (~2069 tok)
+- `config_repl.h` — pragma once (~2481 tok)
 - `ctrl_metrics.c` — include "ctrl_metrics.h" (~767 tok)
 - `dataex.c` — include <errno.h> (~2466 tok)
- `ha.c` — define _GNU_SOURCE (~30494 tok)
+- `ha.c` — define _GNU_SOURCE (~39651 tok)
- `ha.h` — ifndef HA_H (~6474 tok)
+- `ha.h` — ifndef HA_H (~9764 tok)
 - `job.c` — include <errno.h> (~8259 tok)
 - `join.c` — include "config.h" (~2347 tok)
 - `main.c` — include <argp.h> (~1996 tok)
@ -471,7 +471,7 @@
 - `myconf.c` — define _GNU_SOURCE (~1100 tok)
 - `opget.c` — include "config.h" (~1725 tok)
 - `opupd.c` — include "config.h" (~714 tok)
- `persist.c` — define _GNU_SOURCE (~13035 tok)
+- `persist.c` — define _GNU_SOURCE (~13088 tok)
 - `queue.c` — include <errno.h> (~1260 tok)
 - `repl.c` — include <assert.h> (~3964 tok)
 - `reply.c` — include "config.h" (~590 tok)
@ -490,7 +490,7 @@
 - `controller_test.c` — include <argp.h> (~1305 tok)
 - `test_cluster_persist.c` — define _GNU_SOURCE (~1062 tok)
- `test_config_repl.c` — define _GNU_SOURCE (~8338 tok)
+- `test_config_repl.c` — define _GNU_SOURCE (~22128 tok)
 - `test_ctrl_metrics.c` — define _GNU_SOURCE (~940 tok)
 - `test_e2e_capnp.c` — include <stdint.h> (~1690 tok)
 - `test_e2e_checkpoint.c` — define _GNU_SOURCE (~1793 tok)
@ -506,7 +506,7 @@
 - `test_ha_ctrlhello.c` (~1322 tok)
 - `test_ha_durability.c` — Declares char (~4075 tok)
 - `test_ha_election.c` — Declares char (~11144 tok)
- `test_ha_evaluate_committing.c` — Declares checks (~4401 tok)
+- `test_ha_evaluate_committing.c` — Declares checks (~7656 tok)
 - `test_ha_failover.c` — define _GNU_SOURCE (~3781 tok)
 - `test_ha_liveness_sigstop.sh` — test_ha_liveness_sigstop.sh — Task 4.5 integration test: missed-heartbeat (~2606 tok)
 - `test_ha_liveness.c` — Declares directly (~6006 tok)
@ -514,7 +514,7 @@
 - `test_ha_redirect.c` — include <stdatomic.h> (~905 tok)
 - `test_ha_repl_ack.c` (~3093 tok)
 - `test_ha_repl_push.c` — Declares is (~2818 tok)
- `test_ha_role.c` — include <signal.h> (~1962 tok)
+- `test_ha_role.c` — include <signal.h> (~9651 tok)
 - `test_ha_start_election_async.c` — Declares returns (~1957 tok)
 - `test_ha_startup.c` — Declares char (~8594 tok)
 - `test_ha_vote_gate.c` — define _GNU_SOURCE (~2679 tok)
@ -547,6 +547,7 @@
 - `apiserver-backend-integration-test-plan.md` — Integration Test Plan — P0 + P1 Features (~5693 tok)
 - `apiserver-backend-reviews.md` — Final Cross-Perspective Review — API Server Backend Enhancement Plan (~6046 tok)
 - `apiserver-backend-tasks.md` — API Server Backend Enhancement — Task Tracker (~2318 tok)
 - `cluster-config-split-analysis.md` — ClusterConfig Schema Split Analysis (~3448 tok)
 - `config-replication-design.md` — XCE Controller: Configuration Replication Design (~31261 tok)
 - `config-replication-gap-analysis.md` — Config Replication — Gap Analysis: Design vs Implementation (~2674 tok)
 - `config-replication-impl-status.md` — Config Replication — Implementation Status (~2258 tok)
@ -919,14 +920,14 @@
 - `netio.h` — if !defined(_NETIO_H_) (~1293 tok)
 - `objs.h` — if !defined(_OBJS_H_) (~36 tok)
 - `ordq.h` — if !defined(_ORDQ_H_) (~266 tok)
- `persist.h` — pragma once (~1837 tok)
+- `persist.h` — pragma once (~1914 tok)
 - `preason.h` — if !defined(_PREASON_H_) (~108 tok)
 - `qobj.h` — if !defined(_QOBJ_H_) (~41 tok)
 - `repl.h` — pragma once (~979 tok)
 - `request.h` — if !defined(_REQUEST_H_) (~2108 tok)
 - `resobj.h` — if !defined(_RESOBH_H_) (~46 tok)
 - `resreq.h` — if !defined(_RESREQ_H_) (~3223 tok)
- `typemap.h` — if !defined(_TYPEMAP_H_) (~3378 tok)
+- `typemap.h` — if !defined(_TYPEMAP_H_) (~3593 tok)
 - `utils.h` — if !defined(_UTILS_H_) (~1574 tok)
 - `vtaskmgr.h` — if !defined(_VTASKMGR_H_) (~442 tok)
 - `wal_types.h` — pragma once (~679 tok)
@ -950,7 +951,7 @@
 ## src/lib/
- `delay.c` — include <errno.h> (~1619 tok)
+- `delay.c` — include <errno.h> (~1798 tok)
 - `helper.c` — include <errno.h> (~2956 tok)
 - `meson.build` (~1513 tok)
 - `netio.c` — include <arpa/inet.h> (~5716 tok)
@ -976,7 +977,7 @@
 ## src/schema/
- `controller.capnp` — Declares OPCode (~7861 tok)
+- `controller.capnp` — Declares OPCode (~8223 tok)
 ## src/tools/
--- a/.wolf/buglog.json
+++ b/.wolf/buglog.json
@ -4469,6 +4469,829 @@
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T11:54:56.114Z"
    },
    {
      "id": "bug-280",
      "timestamp": "2026-06-04T13:13:54.348Z",
      "error_message": "Significant refactor of ",
      "file": "src/schema/controller.capnp",
      "root_cause": "3 lines replaced/restructured",
      "fix": "Rewrote 4→4 lines (3 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "capnp"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T13:13:54.348Z"
    },
    {
      "id": "bug-281",
      "timestamp": "2026-06-04T13:38:04.971Z",
      "error_message": "Significant refactor of ",
      "file": "src/include/typemap.h",
      "root_cause": "4 lines replaced/restructured",
      "fix": "Rewrote 5→7 lines (4 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "h"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T13:38:04.971Z"
    },
    {
      "id": "bug-282",
      "timestamp": "2026-06-04T13:48:05.580Z",
      "error_message": "Significant refactor of ",
      "file": "src/schema/controller.capnp",
      "root_cause": "2 lines replaced/restructured",
      "fix": "Rewrote 4→13 lines (2 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "capnp"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T13:48:05.580Z"
    },
    {
      "id": "bug-283",
      "timestamp": "2026-06-04T13:48:16.712Z",
      "error_message": "Significant refactor of ",
      "file": "src/include/typemap.h",
      "root_cause": "2 lines replaced/restructured",
      "fix": "Rewrote 4→8 lines (2 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "h"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T13:48:16.712Z"
    },
    {
      "id": "bug-284",
      "timestamp": "2026-06-04T14:33:03.464Z",
      "error_message": "Missing error handling in XCE_TESTING",
      "file": "src/controller/testsuite/test_config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block | Also: /* ── Task 2.1: OPCode_walCatchupEntry handler ─────────────; #ifdef XCE_TESTING | Also: /* Task 2.1: OPCode_walCatchupEntry handler */; RUN_TEST(test_wal_catchup_entry_advances_applied_lsn);",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 3,
      "last_seen": "2026-06-04T14:33:51.476Z"
    },
    {
      "id": "bug-285",
      "timestamp": "2026-06-04T14:41:35.073Z",
      "error_message": "Missing error handling in XCE_TESTING",
      "file": "src/controller/cluster.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block | Also: case OPCode_walCatchupEntry:; /* Task 2.1: standby applies a single WAL catch-up entry fro",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-04T14:41:48.239Z"
    },
    {
      "id": "bug-286",
      "timestamp": "2026-06-04T14:44:41.114Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/cluster.c",
      "root_cause": "57 lines replaced/restructured",
      "fix": "Rewrote 82→16 lines (57 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T14:44:41.114Z"
    },
    {
      "id": "bug-287",
      "timestamp": "2026-06-04T14:45:42.730Z",
      "error_message": "Missing error handling in unknown",
      "file": "src/controller/ha.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T14:45:42.730Z"
    },
    {
      "id": "bug-288",
      "timestamp": "2026-06-04T14:46:44.652Z",
      "error_message": "Missing error handling in unknown",
      "file": "src/controller/ha.h",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block | Also: #include \"persist.h\"      /* persist_t, apply_wal_entry */; #include \"typemap.h\"      /* wal_catchup_entry_t */",
      "tags": [
        "auto-detected",
        "error-handling",
        "h"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-04T14:47:53.528Z"
    },
    {
      "id": "bug-289",
      "timestamp": "2026-06-04T15:09:54.280Z",
      "error_message": "Missing error handling in unknown",
      "file": "src/controller/testsuite/test_config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T15:09:54.280Z"
    },
    {
      "id": "bug-290",
      "timestamp": "2026-06-04T15:22:45.178Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "22 lines replaced/restructured",
      "fix": "Rewrote 58→114 lines (22 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T15:22:45.178Z"
    },
    {
      "id": "bug-291",
      "timestamp": "2026-06-04T15:51:04.816Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/cluster.c",
      "root_cause": "3 lines replaced/restructured",
      "fix": "Rewrote 10→15 lines (3 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T15:51:04.816Z"
    },
    {
      "id": "bug-292",
      "timestamp": "2026-06-04T16:15:43.743Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/ha.c",
      "root_cause": "No early return/throw for edge case: peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES",
      "fix": "Added guard clause: if (peer_idx < 0 || peer_idx >= MAX_HA_CANDI)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T16:15:43.743Z"
    },
    {
      "id": "bug-293",
      "timestamp": "2026-06-04T16:19:17.173Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "4 lines replaced/restructured",
      "fix": "Rewrote 44→89 lines (4 removed) | Also: /* Decode stateReport and call convergence tracker; memset(&sr, 0, sizeof(sr));",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-04T16:21:09.851Z"
    },
    {
      "id": "bug-294",
      "timestamp": "2026-06-04T16:45:25.724Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/ha.c",
      "root_cause": "No early return/throw for edge case: standby_tag < committed_tag",
      "fix": "Added guard clause: if (standby_tag < committed_tag)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T16:45:25.724Z"
    },
    {
      "id": "bug-295",
      "timestamp": "2026-06-04T16:57:59.213Z",
      "error_message": "Incorrect value in code",
      "file": "src/controller/testsuite/test_ha_role.c",
      "root_cause": "Had \"test_evaluate_repl_action_stream_wal_from_zero: P",
      "fix": "Changed to \"test_evaluate_repl_action_send_snapshot_lsn_zero:",
      "tags": [
        "auto-detected",
        "wrong-value",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T16:57:59.213Z"
    },
    {
      "id": "bug-296",
      "timestamp": "2026-06-04T16:58:16.627Z",
      "error_message": "Wrong reference: test_evaluate_repl_action_stream_wal_from_zero should be test_evaluate_repl_action_send_snapshot_lsn_zero",
      "file": "src/controller/testsuite/test_ha_role.c",
      "root_cause": "Used \"test_evaluate_repl_action_stream_wal_from_zero\" instead of \"test_evaluate_repl_action_send_snapshot_lsn_zero\"",
      "fix": "Changed test_evaluate_repl_action_stream_wal_from_zero → test_evaluate_repl_action_send_snapshot_lsn_zero",
      "tags": [
        "auto-detected",
        "wrong-reference",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T16:58:16.627Z"
    },
    {
      "id": "bug-297",
      "timestamp": "2026-06-04T17:08:18.571Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/ha.c",
      "root_cause": "No early return/throw for edge case: !ha || peer_idx < 0 || peer_idx >= MAX_HA_CANDIDATES",
      "fix": "Added guard clause: if (!ha || peer_idx < 0 || peer_idx >= MAX_H)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T17:08:18.571Z"
    },
    {
      "id": "bug-298",
      "timestamp": "2026-06-04T17:34:23.150Z",
      "error_message": "Missing error handling in XCE_TESTING",
      "file": "src/controller/testsuite/test_config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T17:34:23.150Z"
    },
    {
      "id": "bug-299",
      "timestamp": "2026-06-04T17:36:34.006Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "4 lines replaced/restructured",
      "fix": "Rewrote 71→93 lines (4 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T17:36:34.006Z"
    },
    {
      "id": "bug-300",
      "timestamp": "2026-06-04T17:40:05.798Z",
      "error_message": "Missing error handling in XCE_TESTING",
      "file": "src/controller/config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T17:40:05.798Z"
    },
    {
      "id": "bug-301",
      "timestamp": "2026-06-04T23:09:43.701Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/testsuite/test_config_repl.c",
      "root_cause": "No early return/throw for edge case: g_snaptest_chunk_count >= 16",
      "fix": "Added guard clause: if (g_snaptest_chunk_count >= 16)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T23:09:43.701Z"
    },
    {
      "id": "bug-302",
      "timestamp": "2026-06-04T23:10:44.426Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/config_repl.c",
      "root_cause": "No early return/throw for edge case: !snap_dir",
      "fix": "Added guard clause: if (!snap_dir)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T23:10:44.426Z"
    },
    {
      "id": "bug-303",
      "timestamp": "2026-06-04T23:11:48.274Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "2 lines replaced/restructured",
      "fix": "Rewrote 6→15 lines (2 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T23:11:48.274Z"
    },
    {
      "id": "bug-304",
      "timestamp": "2026-06-04T23:21:45.737Z",
      "error_message": "Missing error handling in XCE_TESTING",
      "file": "src/controller/config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T23:21:45.737Z"
    },
    {
      "id": "bug-305",
      "timestamp": "2026-06-04T23:22:03.572Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/testsuite/test_config_repl.c",
      "root_cause": "2 lines replaced/restructured",
      "fix": "Rewrote 9→12 lines (2 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-04T23:22:03.572Z"
    },
    {
      "id": "bug-306",
      "timestamp": "2026-06-05T01:16:16.713Z",
      "error_message": "Significant refactor of ",
      "file": ".kiro/specs/ha-repl-hardening/tasks.md",
      "root_cause": "9 lines replaced/restructured",
      "fix": "Rewrote 9→35 lines (9 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "md"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T01:16:16.713Z"
    },
    {
      "id": "bug-307",
      "timestamp": "2026-06-05T01:18:21.265Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "4 lines replaced/restructured",
      "fix": "Rewrote 7→10 lines (4 removed) | Also: /* Task 3.5: snapshot transfer.; * Creates a checkpoint in the delaymgr thread (not",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-05T01:18:36.333Z"
    },
    {
      "id": "bug-308",
      "timestamp": "2026-06-05T01:19:20.356Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/config_repl.c",
      "root_cause": "No early return/throw for edge case: !d",
      "fix": "Added guard clause: if (!d)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T01:19:20.356Z"
    },
    {
      "id": "bug-309",
      "timestamp": "2026-06-05T01:33:20.146Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/ha.c",
      "root_cause": "No early return/throw for edge case: !ha || !p || !snap_dir || !chunk",
      "fix": "Added guard clause: if (!ha || !p || !snap_dir || !chunk)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T01:33:20.146Z"
    },
    {
      "id": "bug-310",
      "timestamp": "2026-06-05T01:34:06.660Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/cluster.c",
      "root_cause": "No early return/throw for edge case: !chunk",
      "fix": "Added guard clause: if (!chunk)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T01:34:06.660Z"
    },
    {
      "id": "bug-311",
      "timestamp": "2026-06-05T02:21:15.033Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/ha.c",
      "root_cause": "No early return/throw for edge case: ua > ub",
      "fix": "Added guard clause: if (ua > ub)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T02:21:15.033Z"
    },
    {
      "id": "bug-312",
      "timestamp": "2026-06-05T02:47:32.237Z",
      "error_message": "Wrong reference: uintptr_t should be void",
      "file": "src/controller/testsuite/test_ha_evaluate_committing.c",
      "root_cause": "Used \"uintptr_t\" instead of \"void\"",
      "fix": "Changed uintptr_t → void",
      "tags": [
        "auto-detected",
        "wrong-reference",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T02:47:32.237Z"
    },
    {
      "id": "bug-313",
      "timestamp": "2026-06-05T03:35:34.699Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/config_repl.c",
      "root_cause": "149 lines replaced/restructured",
      "fix": "Rewrote 207→38 lines (149 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T03:35:34.699Z"
    },
    {
      "id": "bug-314",
      "timestamp": "2026-06-05T04:01:58.721Z",
      "error_message": "Missing guard clause",
      "file": "src/controller/ha.c",
      "root_cause": "No early return/throw for edge case: !ha",
      "fix": "Added guard clause: if (!ha)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T04:01:58.721Z"
    },
    {
      "id": "bug-315",
      "timestamp": "2026-06-05T04:52:32.206Z",
      "error_message": "Wrong reference: 10000 should be 60000",
      "file": "src/controller/ha.h",
      "root_cause": "Used \"10000\" instead of \"60000\"",
      "fix": "Changed 10000 → 60000",
      "tags": [
        "auto-detected",
        "wrong-reference",
        "h"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-05T04:52:51.260Z"
    },
    {
      "id": "bug-316",
      "timestamp": "2026-06-05T04:53:20.573Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "5 lines replaced/restructured",
      "fix": "Rewrote 24→42 lines (5 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T04:53:20.573Z"
    },
    {
      "id": "bug-317",
      "timestamp": "2026-06-05T04:53:55.034Z",
      "error_message": "Wrong reference: 10000 should be HA_DEFAULT_COMMITTING_TIMEOUT_MS",
      "file": "src/controller/testsuite/test_ha_evaluate_committing.c",
      "root_cause": "Used \"10000\" instead of \"HA_DEFAULT_COMMITTING_TIMEOUT_MS\"",
      "fix": "Changed 10000 → HA_DEFAULT_COMMITTING_TIMEOUT_MS",
      "tags": [
        "auto-detected",
        "wrong-reference",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T04:53:55.034Z"
    },
    {
      "id": "bug-318",
      "timestamp": "2026-06-05T05:10:13.646Z",
      "error_message": "Missing error handling in unknown",
      "file": "src/controller/config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T05:10:13.646Z"
    },
    {
      "id": "bug-319",
      "timestamp": "2026-06-05T05:39:48.707Z",
      "error_message": "Missing error handling in unknown",
      "file": "src/controller/testsuite/test_config_repl.c",
      "root_cause": "Code path had no error handling — exceptions would propagate uncaught",
      "fix": "Added try/catch block",
      "tags": [
        "auto-detected",
        "error-handling",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T05:39:48.707Z"
    },
    {
      "id": "bug-320",
      "timestamp": "2026-06-05T06:50:00Z",
      "error_message": "3-node startup: majority convergence never fires; cluster falls through to committing_timeout degraded-mode WARN",
      "file": "src/controller/ha.c",
      "root_cause": "ha->n_peers was set to candidates_count (3, total including self) in ha_start_election_async. ha_check_majority_convergence loops i=0..n_peers-1 checking standby_converged[], but standby_converged[] is indexed by votes_idx (relative, self excluded: 0..n_other-1). With n_peers=3, it checked standby_converged[2] which is never set, so n_converged (2) < n_peers (3) always — majority convergence path could never succeed.",
      "fix": "Changed ha->n_peers = n_peers to ha->n_peers = n_other (= n_peers-1 = other-node count). Moved the assignment after n_other is computed. ha_repl_ack_received quorum formula (ha->n_peers+2)/2 still yields correct majority for both old and new value.",
      "tags": [
        "ha",
        "majority-convergence",
        "n_peers",
        "off-by-one",
        "integration-test"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T06:50:00Z"
    },
    {
      "id": "bug-321",
      "timestamp": "2026-06-05T07:31:54.806Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "2 lines replaced/restructured",
      "fix": "Rewrote 7→12 lines (2 removed) | Also: /* Disarm any existing poll ctx */; free(old);",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-05T07:32:06.251Z"
    },
    {
      "id": "bug-006",
      "timestamp": "2026-06-05T07:23:00.000Z",
      "error_message": "AddressSanitizer: attempting double-free on sctx in ha_connect_to_peer",
      "file": "src/controller/ha.c",
      "root_cause": "endpoint_fini calls FREEUP(ep->data) which frees sctx, then ha_connect_to_peer also calls free(sctx) — double-free at lines 3594-3595 and 3600-3601",
      "fix": "Set ep->data = NULL before endpoint_fini so FREEUP becomes a no-op; then free(sctx) explicitly in the error paths",
      "tags": [
        "double-free",
        "asan",
        "ha_connect_to_peer",
        "endpoint_fini",
        "integration-test"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T07:23:00.000Z"
    },
    {
      "id": "bug-007",
      "timestamp": "2026-06-05T07:23:00.000Z",
      "error_message": "AddressSanitizer: heap-use-after-free in liveness_poll_fn at ha.c:3293 — reading ctx->cancelled after ha_arm_heartbeat_timeout_vtask freed the old ctx",
      "file": "src/controller/ha.c",
      "root_cause": "ha_arm_heartbeat_timeout_vtask freed the old liveness_poll_ctx_t immediately after setting cancelled=1, but the delaymgr task was already in flight reading ctx->cancelled=0 and then continued to read ctx->ha after free",
      "fix": "ha_arm_heartbeat_timeout_vtask no longer frees old ctx; liveness_poll_fn now frees ctx when it fires with cancelled=1 (guarded by #ifndef XCE_TESTING to avoid bad-free on stack-allocated test ctxs)",
      "tags": [
        "heap-use-after-free",
        "asan",
        "liveness_poll_fn",
        "ha_arm_heartbeat_timeout_vtask",
        "race-condition",
        "integration-test"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T07:23:00.000Z"
    },
    {
      "id": "bug-324",
      "timestamp": "2026-06-05T08:26:02.778Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "2 lines replaced/restructured",
      "fix": "Rewrote 41→87 lines (2 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-05T08:26:02.778Z"
    },
    {
      "id": "bug-325",
      "timestamp": "2026-06-06T11:00:00.057Z",
      "error_message": "Significant refactor of ",
      "file": "src/controller/ha.c",
      "root_cause": "6 lines replaced/restructured",
      "fix": "Rewrote 8→5 lines (6 removed)",
      "tags": [
        "auto-detected",
        "refactor",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-06T11:00:00.057Z"
    },
    {
      "id": "bug-326",
      "timestamp": "2026-06-06T11:06:46.509Z",
      "error_message": "Wrong reference: DEBUG2 should be INFO",
      "file": "src/controller/ha.c",
      "root_cause": "Used \"DEBUG2\" instead of \"INFO\"",
      "fix": "Changed DEBUG2 → INFO",
      "tags": [
        "auto-detected",
        "wrong-reference",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-06T11:06:46.509Z"
    },
    {
      "id": "bug-327",
      "timestamp": "2026-06-06T12:11:21.422Z",
      "error_message": "Incorrect value in code",
      "file": "src/controller/ha.c",
      "root_cause": "Had \"HA: heartbeat tick (role=%d, n_standbys=%d)\"",
      "fix": "Changed to \"HA: primary heartbeat armed\"",
      "tags": [
        "auto-detected",
        "wrong-value",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-06T12:11:21.422Z"
    },
    {
      "id": "bug-328",
      "timestamp": "2026-06-06T13:02:49.052Z",
      "error_message": "Missing guard clause",
      "file": "src/lib/delay.c",
      "root_cause": "No early return/throw for edge case: t1_is_ms && t2_is_ms",
      "fix": "Added guard clause: if (t1_is_ms && t2_is_ms)",
      "tags": [
        "auto-detected",
        "guard-clause",
        "c"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-06T13:02:49.052Z"
    },
    {
      "id": "bug-329",
      "timestamp": "2026-06-08T13:12:54.436Z",
      "error_message": "Function not marked async",
      "file": ".kiro/specs/raft-library/research.md",
      "root_cause": "Function uses await but wasn't declared async",
      "fix": "Added async modifier",
      "tags": [
        "auto-detected",
        "async-fix",
        "md"
      ],
      "related_bugs": [],
      "occurrences": 1,
      "last_seen": "2026-06-08T13:12:54.436Z"
    },
    {
      "id": "bug-330",
      "timestamp": "2026-06-09T08:33:08.522Z",
      "error_message": "Significant refactor of ",
      "file": "src/docs/cluster-config-split-analysis.md",
      "root_cause": "8 lines replaced/restructured",
      "fix": "Rewrote 11→21 lines (8 removed) | Also: ### Phase 1: Schema split (wire-compatible); 4. **Keep the old `ClusterConfig` fields intact** ",
      "tags": [
        "auto-detected",
        "refactor",
        "md"
      ],
      "related_bugs": [],
      "occurrences": 2,
      "last_seen": "2026-06-09T08:35:24.717Z"
    }
  ]
 }
--- a/.wolf/cerebrum.md
+++ b/.wolf/cerebrum.md
@ -34,6 +34,8 @@
 <!-- Mistakes made and corrected. Each entry prevents the same mistake recurring. -->
 <!-- Format: [YYYY-MM-DD] Description of what went wrong and what to do instead. -->
 - [2026-06-04] capnpc-c 1.0.1 has a bug with `:group` inside a `union` when `$C.codecgen` is active: the encode/decode generator emits `d->FIELDNAME` (flat) instead of `d->data.GROUPNAME.FIELDNAME`, AND places the case as dead code after the prior `break`. Do NOT use inline `:group` in a union that has `$C.codecgen`. Instead, define a named outer struct (`struct Foo $C.mapname("foo_t")`) and reference it as a pointer union member. The named struct generates correct `encode_Foo_ptr`/`decode_Foo_ptr` calls. The `read_RequestReply`/`write_RequestReply` wire layer works correctly with groups but the codec layer does not.
 ## Decision Log
 <!-- Significant technical decisions with rationale. Why X was chosen over Y. -->
--- a/.wolf/cron-state.json
+++ b/.wolf/cron-state.json
@ -1,5 +1,5 @@
 {
-  "last_heartbeat": "2026-06-02T10:04:11.396Z",
+  "last_heartbeat": "2026-06-08T10:03:14.767Z",
  "engine_status": "running",
  "execution_log": [
    {
@ -161,6 +161,12 @@
      "status": "success",
      "timestamp": "2026-06-02T09:00:00.411Z",
      "duration_ms": 3
    },
    {
      "task_id": "memory-consolidation",
      "status": "success",
      "timestamp": "2026-06-08T09:00:00.899Z",
      "duration_ms": 4
    }
  ],
  "dead_letter_queue": [],
--- a/.wolf/daemon.log
+++ b/.wolf/daemon.log
@ -81,3 +81,13 @@
 [2026-06-02T07:58:22.561Z] [INFO] WebSocket client connected
 [2026-06-02T09:00:00.408Z] [INFO] Executing task: Consolidate old memory
 [2026-06-02T09:00:00.412Z] [INFO] Task Consolidate old memory completed in 3ms
 [2026-06-08T08:03:14.764Z] [INFO] Scheduled task: Full anatomy rescan (0 */6 * * *)
 [2026-06-08T08:03:14.766Z] [INFO] Scheduled task: Consolidate old memory (0 2 * * *)
 [2026-06-08T08:03:14.766Z] [INFO] Scheduled task: Token audit report (0 0 * * 1)
 [2026-06-08T08:03:14.766Z] [INFO] Scheduled task: Cerebrum reflection (0 3 * * 0)
 [2026-06-08T08:03:14.766Z] [INFO] Scheduled task: AI suggestions (0 4 * * 1)
 [2026-06-08T08:03:14.767Z] [INFO] File watcher started on .wolf/
 [2026-06-08T08:03:14.768Z] [INFO] OpenWolf daemon started
 [2026-06-08T08:03:14.768Z] [INFO] Dashboard server listening on port 18791
 [2026-06-08T09:00:00.895Z] [INFO] Executing task: Consolidate old memory
 [2026-06-08T09:00:00.900Z] [INFO] Task Consolidate old memory completed in 4ms
--- a/.wolf/hooks/_session.json
+++ b/.wolf/hooks/_session.json
@ -1,158 +1,80 @@
 {
-  "session_id": "session-2026-06-04-0446",
+  "session_id": "session-2026-06-09-2117",
-  "started": "2026-06-04T11:46:27.379Z",
+  "started": "2026-06-09T04:17:13.321Z",
  "files_read": {
    "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening": {
      "count": 1,
      "tokens": 0,
      "first_read": "2026-06-04T11:47:59.019Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/kiro-validate-design/rules/design-review.md": {
      "count": 1,
      "tokens": 1090,
      "first_read": "2026-06-04T11:47:59.216Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/spec.json": {
      "count": 2,
      "tokens": 128,
      "first_read": "2026-06-04T11:48:22.897Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/requirements.md": {
      "count": 2,
      "tokens": 2817,
      "first_read": "2026-06-04T11:48:23.086Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/design.md": {
      "count": 5,
      "tokens": 5514,
      "first_read": "2026-06-04T11:48:34.750Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/tasks.md": {
      "count": 2,
      "tokens": 2749,
      "first_read": "2026-06-04T11:49:26.515Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/kiro-spec-tasks/rules/tasks-generation.md": {
      "count": 1,
      "tokens": 2773,
      "first_read": "2026-06-04T12:10:52.924Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/kiro-spec-tasks/rules/tasks-parallel-analysis.md": {
      "count": 1,
      "tokens": 633,
      "first_read": "2026-06-04T12:10:53.319Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/kiro-spec-tasks/templates/specs/tasks.md": {
      "count": 1,
      "tokens": 318,
      "first_read": "2026-06-04T12:10:53.806Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/xce-build/SKILL.md": {
      "count": 1,
      "tokens": 112,
      "first_read": "2026-06-04T12:20:51.454Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/kiro-impl/templates/implementer-prompt.md": {
      "count": 1,
      "tokens": 1659,
      "first_read": "2026-06-04T12:20:52.075Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/.claude/skills/kiro-impl/templates/reviewer-prompt.md": {
      "count": 1,
      "tokens": 1352,
      "first_read": "2026-06-04T12:20:52.620Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/schema/controller.capnp": {
      "count": 28,
      "tokens": 7861,
      "first_read": "2026-06-04T12:22:19.810Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/xce.build/lib/controller.capnp.h": {
      "count": 1,
-      "tokens": 0,
+      "tokens": 8223,
-      "first_read": "2026-06-04T12:25:41.822Z"
+      "first_read": "2026-06-09T06:41:14.575Z"
    },
-    "/home/rshen/projects/xce/worktrees/persistent_ha/src/xce.build/lib/controller.capnp.c": {
+    "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/cluster.c": {
-      "count": 4,
+      "count": 2,
-      "tokens": 134780,
+      "tokens": 12219,
-      "first_read": "2026-06-04T12:26:31.693Z"
+      "first_read": "2026-06-09T06:43:58.698Z"
    },
-    "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/typemap.h": {
+    "/home/rshen/projects/xce/worktrees/persistent_ha/src/include/clusterconfig.h": {
      "count": 6,
      "tokens": 3378,
      "first_read": "2026-06-04T12:28:56.960Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/schema/c.capnp": {
      "count": 1,
-      "tokens": 0,
+      "tokens": 474,
-      "first_read": "2026-06-04T12:30:22.347Z"
+      "first_read": "2026-06-09T06:46:31.070Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/config_repl.c": {
      "count": 1,
      "tokens": 14728,
      "first_read": "2026-06-09T06:47:59.696Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/myconf.c": {
      "count": 1,
      "tokens": 1100,
      "first_read": "2026-06-09T07:31:53.498Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/testsuite/test.c": {
      "count": 1,
      "tokens": 18744,
      "first_read": "2026-06-09T07:31:53.527Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/controller/startup_validation.c": {
      "count": 1,
      "tokens": 1423,
      "first_read": "2026-06-09T08:04:51.033Z"
    },
    "/home/rshen/projects/xce/worktrees/persistent_ha/src/docs/cluster-config-split-analysis.md": {
      "count": 1,
      "tokens": 2796,
      "first_read": "2026-06-09T08:32:42.474Z"
    }
  },
  "files_written": [
    {
-      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/design.md",
+      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/docs/cluster-config-split-analysis.md",
      "action": "edit",
-      "tokens": 194,
+      "tokens": 308,
-      "at": "2026-06-04T11:54:56.112Z"
+      "at": "2026-06-09T08:33:08.519Z"
    },
    {
-      "file": "/home/claude/.claude/plans/functional-tumbling-squirrel.md",
+      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/docs/cluster-config-split-analysis.md",
      "action": "create",
      "tokens": 1056,
      "at": "2026-06-04T11:55:29.391Z"
    },
    {
      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/design.md",
      "action": "edit",
-      "tokens": 99,
+      "tokens": 577,
-      "at": "2026-06-04T11:56:57.713Z"
+      "at": "2026-06-09T08:33:34.327Z"
    },
    {
-      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/design.md",
+      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/docs/cluster-config-split-analysis.md",
      "action": "edit",
-      "tokens": 161,
+      "tokens": 618,
-      "at": "2026-06-04T11:57:25.385Z"
+      "at": "2026-06-09T08:35:24.714Z"
    },
    {
-      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/design.md",
+      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/src/docs/cluster-config-split-analysis.md",
      "action": "edit",
-      "tokens": 157,
+      "tokens": 165,
-      "at": "2026-06-04T11:57:49.314Z"
+      "at": "2026-06-09T08:36:14.677Z"
    },
    {
      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/design.md",
      "action": "edit",
      "tokens": 117,
      "at": "2026-06-04T11:58:02.934Z"
    },
    {
      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/tasks.md",
      "action": "create",
      "tokens": 4002,
      "at": "2026-06-04T12:14:41.929Z"
    },
    {
      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/spec.json",
      "action": "create",
      "tokens": 128,
      "at": "2026-06-04T12:15:05.654Z"
    },
    {
      "file": "/home/rshen/projects/xce/worktrees/persistent_ha/.kiro/specs/ha-repl-hardening/spec.json",
      "action": "create",
      "tokens": 128,
      "at": "2026-06-04T12:21:31.484Z"
    }
  ],
  "edit_counts": {
-    ".kiro/specs/ha-repl-hardening/design.md": 5,
+    "src/docs/cluster-config-split-analysis.md": 4
    "../../../../../claude/.claude/plans/functional-tumbling-squirrel.md": 1,
    ".kiro/specs/ha-repl-hardening/tasks.md": 1,
    ".kiro/specs/ha-repl-hardening/spec.json": 2
  },
-  "anatomy_hits": 14,
+  "anatomy_hits": 8,
-  "anatomy_misses": 3,
+  "anatomy_misses": 0,
-  "repeated_reads_warned": 42,
+  "repeated_reads_warned": 1,
  "cerebrum_warnings": 0,
-  "stop_count": 4
+  "stop_count": 3
 }
--- a/.wolf/memory.md
+++ b/.wolf/memory.md
--- a/.wolf/token-ledger.json
+++ b/.wolf/token-ledger.json
--- a/CLAUDE.md
+++ b/CLAUDE.md
@ -97,56 +97,6 @@ The primary sends `OPCode_nop` to each standby every `heartbeat_interval_ms` via
 - `src/scheduler/` — See `src/scheduler/CLAUDE.md`
 - `src/lib/` — See `src/lib/CLAUDE.md`
 <!-- code-review-graph MCP tools -->
 ## MCP Tools: code-review-graph
 **IMPORTANT: This project has a knowledge graph. ALWAYS use the
 code-review-graph MCP tools BEFORE using Grep/Glob/Read to explore
 the codebase.** The graph is faster, cheaper (fewer tokens), and gives
 you structural context (callers, dependents, test coverage) that file
 scanning cannot.
 ### When to use graph tools FIRST
 - **Exploring code**: `semantic_search_nodes` or `query_graph` instead of Grep
 - **Understanding impact**: `get_impact_radius` instead of manually tracing imports
 - **Code review**: `detect_changes` + `get_review_context` instead of reading entire files
 - **Finding relationships**: `query_graph` with callers_of/callees_of/imports_of/tests_for
 - **Architecture questions**: `get_architecture_overview` + `list_communities`
 Fall back to Grep/Glob/Read **only** when the graph doesn't cover what you need.
 ### Key Tools
 | Tool | Use when |
 |------|----------|
 | `detect_changes` | Reviewing code changes — gives risk-scored analysis |
 | `get_review_context` | Need source snippets for review — token-efficient |
 | `get_impact_radius` | Understanding blast radius of a change |
 | `get_affected_flows` | Finding which execution paths are impacted |
 | `query_graph` | Tracing callers, callees, imports, tests, dependencies |
 | `semantic_search_nodes` | Finding functions/classes by name or keyword |
 | `get_architecture_overview` | Understanding high-level codebase structure |
 | `refactor_tool` | Planning renames, finding dead code |
 ### Workflow
 1. The graph auto-updates on file changes (via hooks).
 2. Use `detect_changes` for code review.
 3. Use `get_affected_flows` to understand impact.
 4. Use `query_graph` pattern="tests_for" to check coverage.
 ## graphify
 This project has a graphify knowledge graph at graphify-out/.
 Rules:
 - Before answering architecture or codebase questions, read graphify-out/GRAPH_REPORT.md for god nodes and community structure
 - If graphify-out/wiki/index.md exists, navigate it instead of reading raw files
 - For cross-module "how does X relate to Y" questions, prefer `graphify query "<question>"`, `graphify path "<A>" "<B>"`, or `graphify explain "<concept>"` over grep — these traverse the graph's EXTRACTED + INFERRED edges instead of scanning files
 - After modifying code files in this session, run `graphify update .` to keep the graph current (AST-only, no API cost)
 # Agentic SDLC and Spec-Driven Development
 Kiro-style Spec-Driven Development on an agentic SDLC
--- a/src/.claude/ucai-formatter-cache.local.json
+++ b/src/.claude/ucai-formatter-cache.local.json
@ -1 +0,0 @@
 {"detected":false,"exe":null,"args":null,"extensions":[]}
--- a/src/controller/CLAUDE.md
+++ b/src/controller/CLAUDE.md
@ -160,7 +160,7 @@ VERBOSE=-vvv /tmp/xce-deploy/scripts/start.sh ctrl-3 &
 # Primary elected and config pushed
 grep -i "CTRL_PRIMARY\|Config committed" /tmp/xce-deploy/logs/ctrl-1.log | grep -v DEBUG3
 # Config files present on standbys
-find /tmp/xce-deploy/ctrl-2/conf /tmp/xce-deploy/ctrl-3/conf -type f
+find /tmp/xce-deploy/controller/ctrl-2/conf /tmp/xce-deploy/controller/ctrl-3/conf -type f
 ```
 **Clean restart** (keeps ctrl-1 config + all certs, wipes data/standby-conf/logs):
--- a/src/controller/ha.c
+++ b/src/controller/ha.c
@ -3544,6 +3544,9 @@ static void on_standby_connected(endpoint_t *ep)
        pthread_mutex_unlock(&ha->lock);
    }
    /* Reset failure counter so the next disconnect restarts logging from attempt 1. */
    ha->peer_connect_fail_count[sctx->peer_idx] = 0;
    /* Record last_push_time before any send so the first heartbeat tick after
     * connect does not immediately send a redundant NOP (Req 3.3, task 2.2). */
    {
@ -3639,10 +3642,18 @@ static void ha_connect_to_peer(cluster_t *c, int peer_idx)
    endpoint_set_on_connected(ep, on_standby_connected);
    endpoint_set_err_handler(ep, standby_ep_err_handler_primary);
    int n = ++ha->peer_connect_fail_count[peer_idx];
    int should_log = (n == 1) || (n % HA_CONNECT_LOG_THROTTLE == 0);
    if (should_log)
        INFO("HA: peer[%d] (%s) not connected — initiating outbound connect (attempt %d)",
             peer_idx, peer->hostname, n);
    int rc = endpoint_connect(ep, peer->hostname, peer->port);
    if (rc < 0) {
-        WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
+        if (should_log)
-             peer_idx, peer->hostname, peer->port, rc);
+            WARN("HA: connect to peer[%d] (%s:%u) failed: %d",
                 peer_idx, peer->hostname, peer->port, rc);
        /* endpoint_fini calls FREEUP(ep->data) which frees sctx — do not
         * double-free sctx here. */
        ep->data = NULL;
@ -3754,11 +3765,8 @@ static int primary_heartbeat_fn(void *arg)
                pthread_mutex_unlock(&ha->lock);
            }
-            if (!found) {
+            if (!found)
                INFO("HA: peer[%d] (%s) not connected — initiating outbound connect",
                     i, pcfg->candidates[i].name ? pcfg->candidates[i].name : "?");
                ha_connect_to_peer(c, i);
            }
        }
    }
--- a/src/controller/ha.h
+++ b/src/controller/ha.h
@ -123,6 +123,8 @@ typedef struct {
 #define HA_DEFAULT_HEARTBEAT_TIMEOUT_MS      10000
 #define HA_DEFAULT_ELECTION_TIMEOUT_MS       5000
 #define HA_DEFAULT_PROBE_CONNECT_TIMEOUT_MS  500
 /* Log the first connect failure per peer, then every Nth attempt. */
 #define HA_CONNECT_LOG_THROTTLE              10
 #define HA_DEFAULT_MAX_PROMOTION_RETRIES     5
 #define HA_DEFAULT_ELECTION_JITTER_MAX_MS    200
 #define HA_DEFAULT_ELECTION_PRIORITY_STEP_MS 500
@ -189,6 +191,7 @@ typedef struct {
    uint64_t         standby_reported_tag[MAX_HA_CANDIDATES]; /* Last config_tag reported by each standby */
    uint64_t         standby_reported_lsn[MAX_HA_CANDIDATES]; /* Last applied_lsn reported by each standby */
    bool             standby_converged[MAX_HA_CANDIDATES];    /* Whether standby[i] has converged with primary */
    int              peer_connect_fail_count[MAX_HA_CANDIDATES]; /* throttle log spam on repeated connect failure */
 } ha_state_t;
 /* ── Public API ────────────────────────────────────────────── */
--- a/src/controller/main.c
+++ b/src/controller/main.c
@ -28,6 +28,7 @@ const char* argp_program_bug_address = BUG_EMAIL;
 #define OPT_CONF_KEEP_N        257
 #define OPT_CONF_STALE_TIMEOUT 258
 #define OPT_VCPUS              259
 #define OPT_CERTDIR            260
 struct arguments {
    char* myname;
@ -38,6 +39,7 @@ struct arguments {
    int   conf_keep_n;
    long  conf_stale_timeout;
    int   vcpus;
    char* certdir;
 };
 static struct argp_option options[] = {
@ -54,6 +56,8 @@ static struct argp_option options[] = {
      "Seconds before saved config is flagged stale (0=disabled, default: 86400)" },
    { "vcpus",              OPT_VCPUS,               "N",    0,
      "Number of virtual CPUs for the task manager (must be >= 1)" },
    { "certdir",            OPT_CERTDIR,  "DIR", 0,
      "Directory containing xce.crt and xce.key (default: <workdir>/certs)" },
    { 0 }
 };
@ -93,6 +97,9 @@ static error_t parse_opt(int key, char* arg, struct argp_state* state)
        if (validate_vcpus(arguments->vcpus) != 0)
            argp_error(state, "--vcpus must be >= 1 (got: %s)", arg);
        break;
    case OPT_CERTDIR:
        arguments->certdir = arg;
        break;
    default:
        return ARGP_ERR_UNKNOWN;
    }
@ -147,8 +154,15 @@ int main(int argc, char* argv[])
    INFO("My name is %s", arguments.myname);
-    snprintf(key_file, XCE_MAXPATH_LEN, "%s/certs/xce.key", arguments.workdir);
+    const char *certdir = arguments.certdir;
-    snprintf(cert_file, XCE_MAXPATH_LEN, "%s/certs/xce.crt", arguments.workdir);
+    char certdir_buf[XCE_MAXPATH_LEN + 1];
    if (!certdir) {
        snprintf(certdir_buf, sizeof(certdir_buf),
                 "%s/certs", arguments.workdir);
        certdir = certdir_buf;
    }
    snprintf(key_file, XCE_MAXPATH_LEN, "%s/xce.key", certdir);
    snprintf(cert_file, XCE_MAXPATH_LEN, "%s/xce.crt", certdir);
    myconf.cluster_config    = NULL;
    myconf.conffile          = NULL; /* set by load_primary_config() or fetch_config_from_primary() */
@ -180,10 +194,10 @@ int main(int argc, char* argv[])
            return 1;
        }
-        /* CA cert derived from workdir — same infrastructure as certfile/keyfile */
+        /* CA cert from certdir (same certificate as certfile) */
        snprintf(myconf.peers_cfg.ca_cert_path,
                 sizeof(myconf.peers_cfg.ca_cert_path),
-                 "%s/certs/xce.crt", arguments.workdir);
+                 "%s/xce.crt", certdir);
        strncpy(myconf.peers_cfg.identity, arguments.myname,
                sizeof(myconf.peers_cfg.identity) - 1);
        INFO("HA mode: %d controller candidates, identity=%s",
--- a/src/docs/cluster-config-split-analysis.md
+++ b/src/docs/cluster-config-split-analysis.md
@ -0,0 +1,272 @@
 # ClusterConfig Schema Split Analysis
 > Analysis date: 2026-06-08. Branch: persistent_ha.
 > Source schema: `src/schema/controller.capnp:680`, struct `ClusterConfig`.
 ## Summary
 The `ClusterConfig` struct (20 fields) mixes three distinct configuration concerns:
 cluster-wide business config, HA peer membership, and per-controller performance
 tunables. These should be split into separate objects so each can evolve
 independently without forcing a re-push of the full config blob to all standbys.
 ## Current State Discovery
 ### Peer list is already structurally separate
 The `--peers` CLI flag (`main.c:178-210`) parses into `myconf.peers_cfg`
 (`xce_cluster_config_t`), which is a **separate struct** from the config blob's
 `cluster_config_t`. At startup, `cluster_init_phase1` uses `peers_cfg.candidates`
 to determine `my_name`, `my_port`, `my_priority`, and the peer list — all before
 the config blob is loaded (`cluster.c:865-880`). The config blob's `candidates`
 field is **not consulted** for HA peer discovery; it is carried for backward
 compatibility in the blob but superseded by `--peers`.
 ### HA tunables are hardcoded at phase1
 `cluster_init_phase1` uses compile-time defaults for all HA timing parameters
 (`cluster.c:854-863`):
 ```c
 ha_config_t ha_cfg = {
    .heartbeat_interval_ms     = HA_DEFAULT_HEARTBEAT_INTERVAL_MS,
    .heartbeat_timeout_ms      = HA_DEFAULT_HEARTBEAT_TIMEOUT_MS,
    .election_timeout_ms       = HA_DEFAULT_ELECTION_TIMEOUT_MS,
    .committing_timeout_ms     = HA_DEFAULT_COMMITTING_TIMEOUT_MS,
    ...
 };
 ```
 The `heartbeatInterval` and `inactiveHeartbeatCount` fields in
 `ClusterConfig` are consumed later (e.g., `cluster.c:581-585` writes them into
 a config key-value store for runtime access), but the HA init path does not
 read them. The heartbeat timeout validation in `startup_validation.c:144` works
 against the hardcoded defaults, not the config blob values.
 ### vcpus is already deprecated
 Field `@7 vcpus` is marked deprecated in the schema comment: "use --vcpus CLI
 flag; kept for config blob backward compatibility." `cluster_init_phase2`
 reads `c->myconf->vcpus` (set from CLI), not the blob's `vcpus` field
 (`cluster.c:1045`).
 ## Proposed Split: Three Configuration Objects
 ```
                         ┌──────────────────────────────┐
                         │    Current ClusterConfig      │
                         │         (20 fields)           │
                         └──────────────┬───────────────┘
                                        │
                    ┌───────────────────┼───────────────────┐
                    ▼                   ▼                   ▼
        ┌───────────────────┐ ┌─────────────────┐ ┌──────────────────────┐
        │  ClusterConfig     │ │  PeersConfig     │ │  ControllerConfig    │
        │  (cluster-wide)   │ │  (HA membership) │ │  (per-controller)    │
        └───────────────────┘ └─────────────────┘ └──────────────────────┘
 ```
 ### Group 1: Cluster-Wide Business Config (keep in `ClusterConfig`)
 These fields define the cluster's behavior, resources, and policy. They **must
 be identical** across all controllers. When modified via `OP_upgradeConfig`,
 the new blob is pushed to all standbys and committed on majority.
 | Field | Ord | Type | Rationale |
 |-------|-----|------|-----------|
 | `clusterName` | @0 | Text | Cluster identity — must be identical cluster-wide |
 | `resources` | @8 | List(Resource) | Resource catalog — shared state across all nodes |
 | `queues` | @9 | List(QueueConf) | Queue definitions — cluster scheduling policy |
 | `ugroups` | @10 | List(GroupConfig) | User group membership — access control |
 | `hgroups` | @11 | List(GroupConfig) | Host group membership — scheduling constraints |
 | `hostConfigs` | @12 | List(HostConfig) | Per-host overrides — cluster-wide host knowledge |
 | `hproviders` | @18 | List(ScalerConfig) | Autoscaler provider config — cluster infrastructure |
 | `limits` | @19 | List(LimitConfig) | Resource limits — cluster policy enforcement |
 | `configGeneration` | @20 | UInt64 | Monotonic version counter — replication consistency |
 **How consumed**: `cluster_init_phase2` (after election and config load)
 iterates these to populate `c->resmgr`, `c->jobmgr`, queue tables, host
 groups, etc.
 ### Group 2: HA Peer Membership (extract to `PeersConfig`)
 These fields define **who** is in the cluster. They change independently when
 controllers are added, removed, or re-IP'd — without any change to the
 business config.
 | Field | Ord | Type | Rationale |
 |-------|-----|------|-----------|
 | `candidates` | @2 | List(MasterCandidate) | HA peer list — changes independently of business config |
 | `port` | @1 | UInt32 | Fallback listener port — derived from candidates; per-controller in practice |
 **Already structurally separate**: The `--peers` CLI flag populates
 `myconf.peers_cfg` (`xce_cluster_config_t` in `src/include/clusterconfig.h`),
 a struct distinct from the config blob's `cluster_config_t`. The candidates
 list in the blob is carried for backward compatibility but is not read for HA
 peer discovery at startup. Making the separation formal means:
 - `PeersConfig` travels via its own capnp message type and replication path
 - Cluster config upgrades via `OP_upgradeConfig` no longer need to carry the candidate list
 - Peer list changes (add/remove controllers) are a dedicated operation with its own quorum semantics
 **Candidate list has special quorum requirements**: Changing the peer set is a
 Raft cluster membership change. It must not use the same
 `repl_wait_majority_ack` path as business config because the majority
 denominator itself is changing. A separate message type enables a two-phase
 membership change protocol (joint consensus) in the future.
 **Impact of removing `candidates` from `ClusterConfig` (verified against codebase):**
 `cluster_build_config()` (`cluster.c:575–609`) is the **only** place that reads
 `cluster_config->candidates` and `cluster_config->ncandidates` — it copies them
 into the runtime KV store as `ncandidates` and `candidate_N_{name,host,port}`.
 No other component (scheduler, apiserver, opget, sched) reads those KV keys.
 The live peer list used by `ha.c:3266–3272` and `cluster_open_listener` comes
 exclusively from `peers_cfg.candidates[]` (parsed by `--peers`). Removing
 candidates from the blob is therefore a **zero-blast-radius** change.
 | File | Location | Change |
 |------|----------|--------|
 | `src/controller/cluster.c` | 587–606 | Remove `ncandidates` + `candidate_N_*` KV loop from `cluster_build_config()` |
 | `src/include/clusterconfig.h` | 34–35 | Remove `int ncandidates` and `master_candidate_t** candidates` from `cluster_config_t` |
 | `src/schema/controller.capnp` | 683 | Remove `candidates @2: List(MasterCandidate)` from `ClusterConfig` |
 **`MasterCandidate` fields**:
 ```
 struct MasterCandidate {
  name  @0: Text;    // controller identity (e.g., "ctrl-1")
  host  @1: Text;    // hostname or IP
  port  @2: UInt32;  // peer listener port
 }
 ```
 ### Group 3: Per-Controller Performance Tunables (extract to `ControllerConfig`)
 These fields configure **how** an individual controller process behaves. They
 can legitimately differ across controllers (e.g., a lower-memory node with
 fewer vcpus, or a WAN-connected standby with a longer heartbeat timeout).
 | Field | Ord | Type | Rationale |
 |-------|-----|------|-----------|
 | `heartbeatInterval` | @3 | UInt32 | Primary heartbeat send interval — can be tuned per primary |
 | `inactiveHeartbeatCount` | @4 | UInt32 | Missed-heartbeat tolerance multiplier |
 | `fetchJobInterval` | @5 | UInt32 | Job fetch poll interval — local scheduler interaction |
 | `netioTimeout` | @6 | UInt32 | Network I/O timeout — environment-specific |
 | `vcpus` | @7 | UInt32 | **Already deprecated** — use `--vcpus` CLI flag; remove from schema |
 | `jobmgr` | @13 | HashZArrayConfig | Job manager hash table sizing (nHashSlots, delta) |
 | `workermgr` | @14 | HashZArrayConfig | Worker manager hash table sizing |
 | `qmgr` | @15 | HashZArrayConfig | Queue manager hash table sizing |
 | `resmgr` | @16 | ResmgrConfig | Resource manager hash + instance table sizing |
 | `xstrings` | @17 | HashZArrayConfig | String interning table sizing |
 **How consumed**: 
 - `jobmgr`/`workermgr`/`qmgr`/`resmgr`/`xstrings` are read once at
  `cluster_init_phase2` to construct the internal key-value stores
  (`cluster.c:803-808`). After construction, the config values are not referenced again.
 - `heartbeatInterval`/`inactiveHeartbeatCount` are read at runtime by the
  heartbeat and liveness tasks (`cluster.c:1180, 1195`).
 - `fetchJobInterval` drives the scheduler polling loop.
 - `netioTimeout` is used by the iomgr layer for TCP timeouts.
 **Why these can differ per controller**: A controller on a high-memory machine
 can be configured with larger hash tables. A controller on a high-latency WAN
 link can have a longer netioTimeout. These differences do not affect cluster
 correctness — they only affect local process behavior.
 **`vcpus` removal**: Already deprecated. The `--vcpus` CLI flag
 (`main.c:1045`) is the canonical source. Remove field @7 in the next
 schema revision.
 ## Fields With Dual Usage (require care)
 ### `port` (@1)
 **Decision: Remove. No fallback bind port.**
 `cluster_config->port` currently serves as a fallback in `cluster_open_listener`
 (`cluster.c:953–975`) when `--peers` does not specify a port for the matching
 candidate. This fallback is removed: the `--peers` port is mandatory.
 **Affected code (all changes mechanical):**
 | File | Location | Change |
 |------|----------|--------|
 | `src/controller/cluster.c` | 953–975 | Remove the `if (my_port == 0) { cluster_config->port }` branch; return `EINVAL` immediately if no port found in `peers_cfg` |
 | `src/controller/myconf.c` | 74 | Remove `SET_DEFAULT_IF_UNSET(c->port, 0, 9990)` |
 | `src/controller/config_repl.c` | 55 | Remove `SET_DEFAULT_IF_UNSET(c->port, 0, 9990)` |
 | `src/include/clusterconfig.h` | 33 | Remove `int port` from `cluster_config_t` |
 | `src/schema/controller.capnp` | 682 | Remove `port @1: UInt32` from `ClusterConfig` |
 Note: `"port"` in `testsuite/test.c:124,266` is the scheduler/worker
 connect-to-primary port (via `get_setting("port")`), not the listener bind
 port. It is unaffected.
 ## Implementation Approach
 ### Phase 1: Remove `port` and `candidates` from `ClusterConfig` (immediate)
 No backward-compat transition needed for these two fields — the live code
 already ignores both (peer discovery uses `peers_cfg`; port fallback is being
 deleted). All changes are mechanical.
 1. `src/schema/controller.capnp` — Remove `port @1` and `candidates @2` from
   `ClusterConfig`; mark or remove `MasterCandidate` struct if no longer
   referenced elsewhere
 2. `src/include/clusterconfig.h` — Remove `int port`, `int ncandidates`,
   `master_candidate_t** candidates` from `cluster_config_t`
 3. `src/controller/cluster.c` — Remove `if (my_port == 0)` fallback block in
   `cluster_open_listener` (lines 967–975); remove `ncandidates` + `candidate_N_*`
   KV loop in `cluster_build_config` (lines 587–606)
 4. `src/controller/myconf.c` and `src/controller/config_repl.c` — Remove
   `SET_DEFAULT_IF_UNSET(c->port, 0, 9990)` from `fill_default_cluster_config`
 5. Regenerate capnp bindings
 ### Phase 2: Schema split (wire-compatible)
 1. Define `PeersConfig` and `ControllerConfig` structs in `controller.capnp`
 2. Add them as new fields in the top-level `ConfigData` union alongside
   the existing `clusterconfig @2`
 3. Regenerate capnp bindings
 4. **Keep the remaining `ClusterConfig` fields intact** during transition —
   new fields are additive
 ### Phase 3: Controller reads from new structs
 1. `cluster_init_phase2`: read `ControllerConfig` for tunables
 2. `cluster_config_t` retains business config fields only
 3. Fall back to old `ClusterConfig` fields when new structs are absent
   (backward compatibility for existing `cluster.bin` files)
 ### Phase 4: Remove superseded tunables from `ClusterConfig`
 1. Remove `heartbeatInterval`, `inactiveHeartbeatCount`, `fetchJobInterval`,
   `netioTimeout`, `vcpus` from `ClusterConfig`
 2. Remove `jobmgr`, `workermgr`, `qmgr`, `resmgr`, `xstrings` from
   `ClusterConfig`
 ### Phase 5 (future): Peer list replication
 1. Define `OPCode_peersUpdate` / `OPCode_peersUpdateAck` for peer list
   replication with joint-consensus semantics
 2. Cluster membership changes become a dedicated operation with its own
   quorum protocol
 ## Effort and Risk
 | Phase | Effort | Risk | Justification |
 |-------|--------|------|---------------|
 | 1 — Remove port + candidates | XS (½d) | Low | No transition needed; dead code removal + schema field removal |
 | 2 — Schema split | S (1d) | Low | Additive capnp change; old fields preserved |
 | 3 — Read from new structs | M (2-3d) | Medium | Touches cluster_init paths; backward compat needed |
 | 4 — Remove superseded tunables | S (1d) | Low | Mechanical removal after Phase 3 is validated |
 | 5 — Peer replication | L (1-2wk) | High | New protocol; joint consensus; extensive testing |
 ## References
 - `src/schema/controller.capnp:680-700` — `ClusterConfig` definition
 - `src/schema/controller.capnp:253` — `ConfigData.clusterconfig @2`
 - `src/controller/main.c:178-213` — `--peers` CLI parsing into `peers_cfg`
 - `src/controller/cluster.c:854-880` — `ha_config_t` hardcoded defaults + priority derivation
 - `src/controller/cluster.c:945-977` — port derivation from candidates vs config blob
 - `src/controller/cluster.c:803-808` — internal manager construction
 - `src/controller/ha.h:134` — `ha_config_t.heartbeat_interval_ms` default via `ha_init`
 - `src/controller/startup_validation.c:133-163` — heartbeat config validation
 - `src/include/clusterconfig.h` — `xce_cluster_config_t` (separate peers struct)
--- a/src/tools/deploy-ha.sh
+++ b/src/tools/deploy-ha.sh
@ -22,8 +22,8 @@ mkdir -p "$TOP/certs"
 mkdir -p "$TOP/logs"
 mkdir -p "$TOP/scripts"
 for i in $(seq 1 $N); do
-    mkdir -p "$TOP/ctrl-$i/conf"
+    mkdir -p "$TOP/controller/ctrl-$i/conf"
-    mkdir -p "$TOP/ctrl-$i/data"
+    mkdir -p "$TOP/controller/ctrl-$i/data"
 done
 # Copy certificates
@ -32,8 +32,8 @@ cp "$DATADIR/xce.key" "$TOP/certs/"
 # Tag and copy cluster.bin
 TAG=$(sha256sum "$DATADIR/cluster.bin" | cut -c1-12)
-cp "$DATADIR/cluster.bin" "$TOP/ctrl-1/conf/cluster.bin.$TAG"
+cp "$DATADIR/cluster.bin" "$TOP/controller/ctrl-1/conf/cluster.bin.$TAG"
-echo "cluster.bin.$TAG" > "$TOP/ctrl-1/conf/current"
+echo "cluster.bin.$TAG" > "$TOP/controller/ctrl-1/conf/current"
 # Build peers string: ctrl-1=host1:port1,ctrl-2=host2:port2,...
 PEERS_STR=""
@ -52,12 +52,12 @@ if [ -z "\$NAME" ]; then
    echo "Usage: \$0 <ctrl-name>"
    exit 1
 fi
-CTRL_DIR=$TOP/\$NAME
+CTRL_DIR=$TOP/controller/\$NAME
 if [ ! -d "\$CTRL_DIR/conf" ] || [ ! -d "\$CTRL_DIR/data" ]; then
    echo "Error: \$CTRL_DIR/conf or \$CTRL_DIR/data does not exist"
    exit 1
 fi
-$BINARY --myname \$NAME --peers "$PEERS_STR" --workdir $TOP --logfile $TOP/logs/\$NAME.log --vcpus 4 \$VERBOSE
+$BINARY --myname \$NAME --peers "$PEERS_STR" --workdir $TOP/controller --certdir $TOP/certs --logfile $TOP/logs/\$NAME.log --vcpus 4 \$VERBOSE
 EOF
 chmod +x "$TOP/scripts/start.sh"
@ -67,10 +67,10 @@ cat > "$TOP/scripts/cleanup.sh" << EOF
 set -e
 rm -f $TOP/logs/*.log
 for i in \$(seq 1 $N); do
-    rm -rf $TOP/ctrl-\$i/data/*
+    rm -rf $TOP/controller/ctrl-\$i/data/*
 done
 for i in \$(seq 2 $N); do
-    rm -rf $TOP/ctrl-\$i/conf/*
+    rm -rf $TOP/controller/ctrl-\$i/conf/*
 done
 EOF
 chmod +x "$TOP/scripts/cleanup.sh"
		`@ -1 +0,0 @@`
			`{"detected":false,"exe":null,"args":null,"extensions":[]}`